add some example analysing

analysis/funwithdata.R

@@ -0,0 +1,19 @@
+library(tidyverse)
+source("../scraping/fetch.R")
+source("../scraping/parse.R")
+source("../scraping/repair.R")
+
+# fetch_all()
+read_all() %>% repair() -> res
+
+reden <- res$reden
+redner <- res$redner
+talks <- res$talks
+
+# first tries
+
+left_join(reden, redner, by=c("redner" = "id")) %>%
+    group_by(fraktion) %>%
+    summarize(n = n()) %>%
+    ggplot(aes(x = fraktion, y = n)) +
+    geom_bar(stat = "identity")

scraping/parse.R

@@ -2,13 +2,40 @@ source("config.R")
 source("../utils/helpers.R")
 library("xml2")
 library(tibble)
+library(dplyr)
 library(magrittr)
+library(pbapply)
 
 # for usage see the example at the end
 
+read_all <- function() {
+    cat("Reading all protocols from", DOWNLOAD_DIR, "\n")
+    available_protocols <- list.files(DOWNLOAD_DIR)
+    res <- pblapply(available_protocols, read_one)
+
+    lapply(res, `[[`, "redner") %>%
+        bind_rows() %>%
+        distinct() ->
+        redner
+    
+    lapply(res, `[[`, "reden") %>%
+        bind_rows() %>%
+        distinct() ->
+        reden
+
+    lapply(res, `[[`, "talks") %>%
+        bind_rows() %>%
+        distinct() ->
+        talks
+
+    list(redner = redner, reden = reden, talks = talks)
+}
+
 # this reads all currently parseable data from one xml
 read_one <- function(name) {
-    x <- read_xml(paste0(DOWNLOAD_DIR, name))
+    x <- tryCatch(read_xml(paste0(DOWNLOAD_DIR, name)),
+                  error = function(c) NULL)
+    if (is.null(x)) return(NULL)
     cs <- xml_children(x)
 
     verlauf <- xml_find_first(x, "sitzungsverlauf")
@@ -22,6 +49,7 @@ read_one <- function(name) {
         xml_find_all("rede") %>%
         parse_redenliste() ->
         res
+
     list(redner = redner, reden = res$reden, talks = res$talks)
 }
 
@@ -60,7 +88,8 @@ parse_rede <- function(rede_xml) {
     reden <- list()
     for (node in cs) {
         if (xml_name(node) == "p") {
-            if (xml_attr(node, "klasse") == "redner") {
+            klasse <- xml_attr(node, "klasse")
+            if (!is.na(klasse) && klasse == "redner") {
                 if (!is.na(cur_redner)) {
                     rede <- c(rede_id = rede_id,
                               redner = cur_redner,
@@ -111,10 +140,10 @@ parse_rednerliste <- function(rednerliste_xml) {
 # EXAMPLE USE
 
 # make sure data ist downloaded via fetch.R
-res <- read_one("19038-data.xml")
-
-res$redner
-res$reden
-res$talks
+# res <- read_one("19126-data.xml")
+# 
+# res$redner
+# res$reden
+# res$talks
 
 # -------------------------------

scraping/repair.R

@@ -0,0 +1,38 @@
+source("../utils/helpers.R")
+fraktionen <- c("AFD" = "AfD",
+                "BÜNDNIS90/" = "BÜNDNIS 90 / DIE GRÜNEN",
+                "BÜNDNIS90/DIEGRÜNEN" = "BÜNDNIS 90 / DIE GRÜNEN",
+                "FRAKTIONSLOS" = "Fraktionslos",
+                "DIELINKE" = "DIE LINKE",
+                "SPD" = "SPD",
+                "CDU/CSU" = "CDU/CSU",
+                "FDP" = "FDP")
+
+
+# expects a tibble of redner and repairs
+repair_fraktion <- function(fraktion) {
+    cleaned <- str_to_upper %$% str_replace_all(fraktion, "\\s", "")
+    fraktionen[cleaned]
+}
+
+repair_redner <- function(redner) {
+    # fix fraktionsnames
+    redner %>% mutate(fraktion = Vectorize(repair_fraktion)(fraktion))
+}
+
+repair_reden <- function(reden) {
+    # TODO: fill with content
+    reden
+}
+
+repair_talks <- function(talks) {
+    # TODO: fill with content
+    talks
+}
+
+# repairs all tables
+repair <- function(parse_output) {
+    list(redner = repair_redner(parse_output$redner),
+         reden = repair_reden(parse_output$reden),
+         talks = repair_talks(parse_output$talks))
+}