add documentation for read_from_csv_or_fetch

NAMESPACE

@@ -7,6 +7,7 @@ export(join_speaker)
 export(party_colors)
 export(read_all)
 export(read_from_csv)
+export(read_from_csv_or_fetch)
 export(repair)
 export(word_usage_by_date)
 export(write_to_csv)

R/parse.R

@@ -271,3 +271,22 @@ read_from_csv <- function(path="inst/csv/") {
     is_valid_res(res)
     res
 }
+
+#' @param path directory of csv files to read
+#' read data from csv files if they exist already
+#' otherwise fetch protocols and then write the data into csv files
+#' 
+#' @export
+read_from_csv_or_fetch <- function(path="inst/") {
+    path <- make_directory_path(path)
+    res <- tryCatch(read_from_csv(str_c(path, "csv/")),
+                    error = function(c) NULL)
+    if (!is.null(res)) return(res)
+    
+    fetch_all(str_c(path, "records/"), create=T)
+    read_all(str_c(path, "records/")) %>%
+        repair() ->
+        res
+    write_to_csv(res, str_c(path, "csv/"), create=T)
+    res
+}

README.md

@@ -9,26 +9,7 @@ Using the `remotes` package, this is easily installed via:
 ```r
 remotes::install_url("https://git.flavigny.de/christian/hateimparlament/archive/master.zip")
 ```
-Since the fetching and reading is very slow and depends on an internet connection, all vignettes
-use `read_from_csv` to read already parsed tibbles from `.csv` files.
-
-That's why, if you want to build the vignettes yourself, you need to
-download the source code, e.g. on Linux
-```
-git clone https://git.flavigny.de/christian/hateimparlament
-cd hateimparlament
-```
-then start `R` and do
-```r
-devtools::load_all()
-fetch_all(create = TRUE)
-read_all() %>% repair() -> res
-write_to_csv(res, create = TRUE)
-```
-Then finally, do:
-```r
-devtools::install(build_vignettes = TRUE)
-```
+If you want to build the vignettes, pass `build_vignettes = TRUE`.
 
 # Features
 

vignettes/explicittopic.Rmd

@@ -34,9 +34,10 @@ read_all("../inst/records/") %>% repair() -> res
 ```
 We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
 
 ## Analysis

vignettes/genderequality.Rmd

@@ -1,8 +1,8 @@
 ---
-title: "genderequality"
+title: "Differences in gender"
 output: rmarkdown::html_vignette
 vignette: >
-  %\VignetteIndexEntry{genderequality}
+  %\VignetteIndexEntry{Differences in gender}
   %\VignetteEngine{knitr::rmarkdown}
   %\VignetteEncoding{UTF-8}
 ---
@@ -20,7 +20,7 @@ library(dplyr)
 library(ggplot2)
 library(stringr)
 library(tidyr)
-library(rvest)
+library(xml2)
 ```
 
 ## Preparation of data
@@ -33,13 +33,15 @@ Second, those `.xml` files, need to be parsed into `R` `tibbles`. This is accomp
 ```r
 read_all("../records/") %>% repair() -> res
 ```
-We also used `repair` to fix a bunch of formatting issues in the records and unpacked
-the result into more descriptive variables.
+We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
+
+
 and unpack our tibbles
 ```{r}
 comments <- res$comments
@@ -48,53 +50,33 @@ speaker <- res$speaker
 talks <- res$talks
 ```
 
-Bevor we can do our analysis, we have to assign a gender to our politicans.
+Bevor we can do our analysis, we have to assign a gender to our politicans. We do this
+by reading the gender from the master data of all members of parliament, which is
+fetched from bundestag.de.
 
 ```{r}
-extract_href <- function(sel, html) {
-  html %>%
-    html_node(sel) %>%
-    html_attr("href")
+xml_get <- function(node, name) {
+    res <- xml_text(xml_find_all(node, name))
+    if (length(res) == 0) NA_character_
+    else res
 }
 
-first_content_p_text <- function(url) {
-  res <- NA
-  i <- 1
-  while(is.na(res)) {
-    read_html(url) %>% 
-      html_node(str_glue("#mw-content-text > div.mw-parser-output > p:nth-child({i})"))  %>% 
-      html_text() -> res
-    i <- i + 1
-  }
-  res
+x <- read_xml("../inst/masterdata.xml")
+mdbs <- xml_find_all(x, "MDB")
+
+ids <- c()
+genders <- c()
+for (mdb in mdbs) {
+    xml_get(mdb, "ID") -> mdb_id
+    xml_find_first(mdb, "BIOGRAFISCHE_ANGABEN") %>%
+        xml_get("GESCHLECHT") ->
+        mdb_gender
+    ids <- c(ids, mdb_id)
+    genders <- c(genders, if (mdb_gender == "männlich") "male" else "female")
 }
 
-abgeordneten_list_html <- read_html(
-  "https://de.wikipedia.org/wiki/Liste_der_Mitglieder_des_Deutschen_Bundestages_(19._Wahlperiode)")
-
-selectors <- str_glue("#mw-content-text > div.mw-parser-output > table:nth-child(20) > tbody > tr:nth-child({2:709}) > td:nth-child(2) > a")
-link_part2 <- sapply(selectors, extract_href, abgeordneten_list_html)
-link <- str_c("https://de.wikipedia.org", link_part2)
-
-text <- sapply(link, first_content_p_text)
-text %>% 
-  str_extract(" ist ein.") %>% 
-  str_replace(" ist eine", "female") %>% 
-  str_replace(" ist ein ", "male") ->
-  gender
-
-text %>% 
-  str_extract("^([:upper:]?[:lower:]+[\\s\\-]?)*") %>% 
-  str_trim() -> 
-  names
-
-gender <- tibble(speaker = names,
-                 gender = gender)
-
-speaker %>% 
-  unite("speaker", vorname, nachname, sep = " ") %>% 
-  right_join(gender, by = "speaker") -> 
-  speaker_with_gender
+gender <- tibble(id = ids, gender = genders)
+speaker_with_gender <- left_join(res$speaker, gender)
 ```
 
 ## Analyse
@@ -161,7 +143,7 @@ speeches %>%
 
 
 party_order <- factor(c("Fraktionslos", "AfD&Fraktionslos",
-                        "DIE LINKE", "BÜNDNIS 90 / DIE GRÜNEN", "SPD", "CDU/CSU",
+                        "DIE LINKE", "BÜNDNIS 90/DIE GRÜNEN", "SPD", "CDU/CSU",
                         "FDP", "AfD", NA_character_))
 
 speech_distribution %>% 
@@ -179,9 +161,8 @@ speeches %>%
   summarize(n = n()) %>%
   ungroup() %>% 
   arrange(-n) %>%
-  left_join(speaker, by=c("speaker" = "id")) %>% 
-  unite(name, vorname, nachname, sep = " ") %>% 
-  inner_join(gender, by=c("name"= "speaker")) %>% 
+  join_speaker(res) %>%
+  left_join(gender, by=c("speaker"="id")) %>% 
   group_by(gender) %>% 
   summarise(absolute=sum(n)) %>%
   filter(gender %in% c("female", "male")) %>%

vignettes/generalquestions.Rmd

@@ -34,9 +34,10 @@ read_all("../inst/records/") %>% repair() -> res
 ```
 We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
 
 ## Analysis

vignettes/hitlercomparison.Rmd

@@ -38,9 +38,10 @@ talks <- res$talks
 We also used `repair` to fix a bunch of formatting issues in the records and unpacked
 the result into more descriptive variables.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-tables <- read_from_csv('../inst/csv/')
+tables <- read_from_csv_or_fetch('../inst/')
 
 comments <- tables$comments
 speeches <- tables$speeches

vignettes/interaction.Rmd

@@ -34,9 +34,10 @@ read_all("../inst/records/") %>% repair() -> res
 ```
 We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
 
 ## Analysis