From c22bc0b91b8fc9a6cb51c9dc9807ac98014e289e Mon Sep 17 00:00:00 2001 From: Leon Burgard Date: Wed, 4 Aug 2021 16:47:06 +0200 Subject: [PATCH 1/3] add Documentation for write_to_csv --- R/parse.R | 8 ++++++++ man/lookup_speaker.Rd | 24 ++++++++++++++++++++++++ man/repair.Rd | 9 +++++++-- man/write_to_csv.Rd | 18 ++++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 man/lookup_speaker.Rd create mode 100644 man/write_to_csv.Rd diff --git a/R/parse.R b/R/parse.R index db99eb5..d203780 100644 --- a/R/parse.R +++ b/R/parse.R @@ -205,6 +205,14 @@ parse_speakerliste <- function(speakerliste_xml) { rolle_lang = d["rolle_lang",]) } +#' Write to CSV-Document +#' +#' Put all tables from the entered list in separate CSV documents. +#' +#' @param tables The List of Tables you want to convert into a CSV-Document. +#' @param path The path where to put the CSV-Document. +#' @param create Set TRUE if the path does not exist yet and you want to create it. +#' #' @export write_to_csv <- function(tables, path="csv/", create=F) { check_directory(path, create) diff --git a/man/lookup_speaker.Rd b/man/lookup_speaker.Rd new file mode 100644 index 0000000..0888e14 --- /dev/null +++ b/man/lookup_speaker.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/repair.R +\name{lookup_speaker} +\alias{lookup_speaker} +\title{Lookup name in speakers table} +\usage{ +lookup_speaker(tb, speaker, name_variable) +} +\arguments{ +\item{tb}{tibble} + +\item{name_variable}{name + +Tries to match the name_variable column with speaker names + +returns a lookup table} + +\item{redner}{tibble} +} +\description{ +Tries to find the correct speaker id given a name. +This is sufficient since every prename lastname combination in the bundestag is +unique (luckily :D) +} diff --git a/man/repair.Rd b/man/repair.Rd index c7e0342..285412b 100644 --- a/man/repair.Rd +++ b/man/repair.Rd @@ -4,8 +4,13 @@ \alias{repair} \title{Repair parsed tables} \usage{ -repair(parse_output) +repair(parse_output, repair_comments = FALSE) } \description{ -Repair parsed tables +TODO: Explain repair_comments argument +(if TRUE, we try to lookup redner names in redner table) +} +\details{ +Possible test: check identical(repair(res), repair(repair(res))) == TRUE +Since repaired tables should be a fixpoint of repair. } diff --git a/man/write_to_csv.Rd b/man/write_to_csv.Rd new file mode 100644 index 0000000..6c05a74 --- /dev/null +++ b/man/write_to_csv.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/parse.R +\name{write_to_csv} +\alias{write_to_csv} +\title{Write to CSV-Document} +\usage{ +write_to_csv(tables, path = "csv/", create = F) +} +\arguments{ +\item{tables}{The List of Tables you want to convert into a CSV-Document.} + +\item{path}{The path where to put the CSV-Document.} + +\item{create}{Set TRUE if the path does not exist yet and you want to create it.} +} +\description{ +Put all tables from the entered list in separate CSV documents. +} From 1433159e097776b05b034f2fa5fcbdaec16ce8e4 Mon Sep 17 00:00:00 2001 From: Leon Burgard Date: Thu, 5 Aug 2021 13:44:19 +0200 Subject: [PATCH 2/3] create genderequality, add documentation read_from_csv --- R/analyze.R | 2 + R/parse.R | 4 ++ man/read_from_csv.Rd | 14 ++++++ vignettes/genderequality.Rmd | 94 ++++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 man/read_from_csv.Rd create mode 100644 vignettes/genderequality.Rmd diff --git a/R/analyze.R b/R/analyze.R index fd728cd..43aa990 100644 --- a/R/analyze.R +++ b/R/analyze.R @@ -12,6 +12,8 @@ join_speaker <- function(tb, res, fraction_only = F) { else joined } +#'Assignment of the official colors to the parties +#' #' @export party_colors <- c( AfD="#1A9FDD", diff --git a/R/parse.R b/R/parse.R index d203780..aa7ceb6 100644 --- a/R/parse.R +++ b/R/parse.R @@ -223,6 +223,10 @@ write_to_csv <- function(tables, path="csv/", create=F) { write.table(tables$applause, str_c(path, "applause.csv")) } +#'Create a Tibble out of the CSV-Document +#' +#'@param path The path that shows which Documents you want to use. +#' #' @export read_from_csv <- function(path="csv/") { list(speaker = read.table(str_c(path, "speaker.csv")) %>% diff --git a/man/read_from_csv.Rd b/man/read_from_csv.Rd new file mode 100644 index 0000000..92aedec --- /dev/null +++ b/man/read_from_csv.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/parse.R +\name{read_from_csv} +\alias{read_from_csv} +\title{Create a Tibble out of the CSV-Document} +\usage{ +read_from_csv(path = "csv/") +} +\arguments{ +\item{path}{The path that shows which Documents you want to use.} +} +\description{ +Create a Tibble out of the CSV-Document +} diff --git a/vignettes/genderequality.Rmd b/vignettes/genderequality.Rmd new file mode 100644 index 0000000..d291bd2 --- /dev/null +++ b/vignettes/genderequality.Rmd @@ -0,0 +1,94 @@ +--- +title: "genderequality" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{genderequality} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(hateimparlament) +library(dplyr) +library(ggplot2) +library(stringr) +library(tidyr) +library(rvest) +``` + +## Preparation of data + +First, you need to download all records of the current legislative period. +```r +fetch_all("../records/") # path to directory where records should be stored +``` +Second, those `.xml` files, need to be parsed into `R` `tibbles`. This is accomplished by: +```r +read_all("../records/") %>% repair() -> res +``` +We also used `repair` to fix a bunch of formatting issues in the records and unpacked +the result into more descriptive variables. + +For development purposes, we load the tables from csv files. +```{r} +res <- read_from_csv('../csv/') +``` +and unpack our tibbles +```{r} +comments <- res$comments +speeches <- res$speeches +speaker <- res$speaker +talks <- res$talks +``` + +Bevor we can do our analysis, we have to assign a gender to our politicans. + +```{r} +extract_href <- function(sel, html) { + html %>% + html_node(sel) %>% + html_attr("href") +} + +first_content_p_text <- function(url) { + res <- NA + i <- 1 + while(is.na(res)) { + read_html(url) %>% + html_node(str_glue("#mw-content-text > div.mw-parser-output > p:nth-child({i})")) %>% + html_text() -> res + i <- i + 1 + } + res +} + +abgeordneten_list_html <- read_html( + "https://de.wikipedia.org/wiki/Liste_der_Mitglieder_des_Deutschen_Bundestages_(19._Wahlperiode)") + +selectors <- str_glue("#mw-content-text > div.mw-parser-output > table:nth-child(20) > tbody > tr:nth-child({2:709}) > td:nth-child(2) > a") +link_part2 <- sapply(selectors, extract_href, abgeordneten_list_html) +link <- str_c("https://de.wikipedia.org", link_part2) + +text <- sapply(link, first_content_p_text) +text %>% + str_extract(" ist ein.") %>% + str_replace(" ist eine", "female") %>% + str_replace(" ist ein ", "male") -> + gender + +text %>% + str_extract("^([:upper:]?[:lower:]+[\\s\\-]?)*") %>% + str_trim() -> + names + +gender <- tibble(name = names, + gender = gender) +``` + From d5da63602036ff7eed42ad96caf4889f20855489 Mon Sep 17 00:00:00 2001 From: Leon Burgard Date: Fri, 6 Aug 2021 19:07:35 +0200 Subject: [PATCH 3/3] add documentations --- R/analyze.R | 33 ++++++++++++++++++++++++-- R/parse.R | 2 +- man/bar_plot_fractions.Rd | 46 ++++++++++++++++++++++++++++++++++++ man/join_speaker.Rd | 18 ++++++++++++++ man/party_colors.Rd | 16 +++++++++++++ man/word_usage_by_date.Rd | 20 ++++++++++++++++ vignettes/genderequality.Rmd | 4 +++- 7 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 man/bar_plot_fractions.Rd create mode 100644 man/join_speaker.Rd create mode 100644 man/party_colors.Rd create mode 100644 man/word_usage_by_date.Rd diff --git a/R/analyze.R b/R/analyze.R index 43aa990..30f2774 100644 --- a/R/analyze.R +++ b/R/analyze.R @@ -5,6 +5,14 @@ find_word <- function(res, word) { nrow)) } +#'Assign speaker-name to speaker-ID +#' +#' Assign speaker-name to speaker-ID. +#' +#' @param tb tibble +#' @param res list of tibbles +#' @param fraction_only If TRUE select only from "fraction" +#' #' @export join_speaker <- function(tb, res, fraction_only = F) { joined <- left_join(tb, res$speaker, by=c("speaker" = "id")) @@ -30,6 +38,20 @@ party_order <- factor(c("Fraktionslos", "AfD&Fraktionslos", "DIE LINKE", "BÜNDNIS 90 / DIE GRÜNEN", "SPD", "CDU/CSU", "FDP", "AfD", NA_character_)) +#'Bar chart of the fractions +#' +#' @param tb The tibble to be analyzed. +#' @param x_variable Default is fraction. +#' @param y_variable Default is n. +#' @param fill Default is fraction. +#' @param title Title of the chart. +#' @param xlab Description of x-Lab. Default is Fraction. +#' @param ylab Description of y-Lab. Default is n. +#' @param filllab Default is Fraction. +#' @param flipped Default is TRUE. If TRUE draw bars horizontally, else vertically. +#' @param position Default is dodge. +#' @param reorder Either reorder fraction factor by variable value or reorder fraction factor by party seat order in parliament (default). +#' #' @export bar_plot_fractions <- function(tb, x_variable = NULL, # default is fraction @@ -73,8 +95,15 @@ bar_plot_fractions <- function(tb, if (flipped) plt + coord_flip() else plt } -# Counts how many talks do match a given pattern and summarises by date -# +#'Word usage summarised by date +#' +#' Counts how many talks do match a given pattern and summarises by date. +#' +#' @param res List of Tibbles to be analysed. +#' @param patterns Words to look up. +#' @param name ? +#' @param tidy Default is FALSE. +#' #' @export word_usage_by_date <- function(res, patterns, name, tidy=F) { tb <- res$talks diff --git a/R/parse.R b/R/parse.R index aa7ceb6..55f17dd 100644 --- a/R/parse.R +++ b/R/parse.R @@ -194,7 +194,7 @@ parse_speechlist <- function(speechlist_xml, date) { } # create a tibble of speaker from a list of xml nodes representing speaker -parse_speakerliste <- function(speakerliste_xml) { +parse_speakerlist <- function(speakerliste_xml) { d <- sapply(speakerliste_xml, parse_speaker) tibble(id = d["id",], vorname = d["vorname",], diff --git a/man/bar_plot_fractions.Rd b/man/bar_plot_fractions.Rd new file mode 100644 index 0000000..e308374 --- /dev/null +++ b/man/bar_plot_fractions.Rd @@ -0,0 +1,46 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/analyze.R +\name{bar_plot_fractions} +\alias{bar_plot_fractions} +\title{Bar chart of the fractions} +\usage{ +bar_plot_fractions( + tb, + x_variable = NULL, + y_variable = NULL, + fill = NULL, + title = NULL, + xlab = "Fraction", + ylab = "n", + filllab = "Fraction", + flipped = TRUE, + position = "dodge", + reorder = FALSE +) +} +\arguments{ +\item{tb}{The tibble to be analyzed.} + +\item{x_variable}{Default is fraction.} + +\item{y_variable}{Default is n.} + +\item{fill}{Default is fraction.} + +\item{title}{Title of the chart.} + +\item{xlab}{Description of x-Lab. Default is Fraction.} + +\item{ylab}{Description of y-Lab. Default is n.} + +\item{filllab}{Default is Fraction.} + +\item{flipped}{Default is TRUE. If TRUE draw bars horizontally, else vertically.} + +\item{position}{Default is dodge.} + +\item{reorder}{Either reorder fraction factor by variable value or reorder fraction factor by party seat order in parliament (default).} +} +\description{ +Bar chart of the fractions +} diff --git a/man/join_speaker.Rd b/man/join_speaker.Rd new file mode 100644 index 0000000..a30fb22 --- /dev/null +++ b/man/join_speaker.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/analyze.R +\name{join_speaker} +\alias{join_speaker} +\title{Assign speaker-name to speaker-ID} +\usage{ +join_speaker(tb, res, fraction_only = F) +} +\arguments{ +\item{tb}{tibble} + +\item{res}{list of tibbles} + +\item{fraction_only}{If TRUE select only from "fraction"} +} +\description{ +Assign speaker-name to speaker-ID. +} diff --git a/man/party_colors.Rd b/man/party_colors.Rd new file mode 100644 index 0000000..e4417b6 --- /dev/null +++ b/man/party_colors.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/analyze.R +\docType{data} +\name{party_colors} +\alias{party_colors} +\title{Assignment of the official colors to the parties} +\format{ +An object of class \code{character} of length 8. +} +\usage{ +party_colors +} +\description{ +Assignment of the official colors to the parties +} +\keyword{datasets} diff --git a/man/word_usage_by_date.Rd b/man/word_usage_by_date.Rd new file mode 100644 index 0000000..4d897a8 --- /dev/null +++ b/man/word_usage_by_date.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/analyze.R +\name{word_usage_by_date} +\alias{word_usage_by_date} +\title{Word usage summarised by date} +\usage{ +word_usage_by_date(res, patterns, name, tidy = F) +} +\arguments{ +\item{res}{List of Tibbles to be analysed.} + +\item{patterns}{Words to look up.} + +\item{name}{?} + +\item{tidy}{Default is FALSE.} +} +\description{ +Counts how many talks do match a given pattern and summarises by date. +} diff --git a/vignettes/genderequality.Rmd b/vignettes/genderequality.Rmd index d291bd2..d4aa64d 100644 --- a/vignettes/genderequality.Rmd +++ b/vignettes/genderequality.Rmd @@ -88,7 +88,9 @@ text %>% str_trim() -> names -gender <- tibble(name = names, +gender <- tibble(speaker = names, gender = gender) + + ```