Merge branch 'master' of gitea:christian/hateimparlament

.gitignore

@@ -8,3 +8,5 @@
 /parlament_49_53_texts/
 .Rproj.user
 *.Rproj
+*.RData
+*.Rhistory

hitler_texts/parse.py

@@ -1,8 +1,52 @@
-import os
-words = []
+import re
+
+german_words = []
+with open('/home/josua/deu_mixed-typical_2011_1M/deu_mixed-typical_2011_1M-words.txt') as f:
+    lines = f.readlines()
+    for line in lines:
+        #print(line.split(sep="\t"))
+        index, word, count = line.split(sep="\t")
+        if int(index) > 100:
+            german_words.append(word.lower())
+
+
+def get_words_from_line(line):
+    words = line.split(sep=" ")
+    ret_list = []
+    for word in words:
+        word = re.sub("[^a-zA-ZüöäÜÖÄßẞ]", "", word)
+        ret_list.append(word.lower())
+    return ret_list
+
+
+hitler_words = []
 for i in range(1, 7):
     with open(f'hitler_rede_{i}') as f:
         lines = f.readlines()
         for line in lines:
-            words.extend(line.split(sep=" "))
-            
+            hitler_words.extend(get_words_from_line(line))
+        
+with open(f'goebbels_sportpalast') as f:
+    lines = f.readlines()
+    for line in lines:
+        hitler_words.extend(get_words_from_line(line))
+
+with open(f'mein_kampf') as f:
+    lines = f.readlines()
+    for line in lines:
+        hitler_words.extend(get_words_from_line(line))
+
+hitler_words = set(hitler_words) #unique
+#filter_words = hitler_words.intersection(set(german_words))
+
+only_hitler_words = list(hitler_words.difference(german_words))
+
+with open("german_words", "w") as f:
+    for word in german_words:
+        word += "\n"
+        f.write(word)
+
+with open("hitler_words", "w") as f:
+    for word in only_hitler_words:
+        word += "\n"
+        f.write(word)

vignettes/hitlercomparison.Rmd

@@ -0,0 +1,185 @@
+---
+title: "funwithdata"
+output: rmarkdown::html_vignette
+vignette: >
+  %\VignetteIndexEntry{funwithdata}
+  %\VignetteEngine{knitr::rmarkdown}
+  %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+  collapse = TRUE,
+  comment = "#>"
+)
+```
+
+```{r setup}
+library(hateimparlament)
+library(dplyr)
+library(stringr)
+library(ggplot2)
+```
+
+## Preparation of data
+
+First, you need to download all records of the current legislative period.
+```r
+fetch_all("../records/") # path to directory where records should be stored
+```
+Second, those `.xml` files, need to be parsed into `R` `tibbles`. This is accomplished by:
+```r
+read_all("../records/") %>% repair() -> res
+
+reden <- res$reden
+redner <- res$redner
+talks <- res$talks
+```
+We also used `repair` to fix a bunch of formatting issues in the records and unpacked
+the result into more descriptive variables.
+
+For development purposes, we load the tables from csv files.
+```{r}
+tables <- read_from_csv('../csv/')
+
+comments <- tables$comments
+reden <- tables$reden
+redner <- tables$redner
+talks <- tables$talks
+```
+
+Further, we need to load a list of words that were used by Hitler but not by standard German texts.
+```{r}
+fil <- file('../hitler_texts/hitler_words')
+Worte <- readLines(fil)
+hitlerwords <- tibble(Worte)
+```
+
+## Analysis
+
+Now we extract the words that were used with higher frequency by one party and compare them with `hitlerwords`.
+```{r}
+talks %>% 
+  left_join(redner, by=c(redner='id')) %>% 
+  group_by(fraktion) %>%
+  summarize(full_text=str_c(content, collapse="\n")) -> talks_by_fraktion
+talks_by_fraktion
+```
+For each party, we want to get a tibble of words with frequency.
+```{r}
+#AfD
+Worte <- str_extract_all(talks_by_fraktion$full_text[[1]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+afdtotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/afdtotal) -> afd_words
+
+#AfD&Fraktionslos
+Worte <- str_extract_all(talks_by_fraktion$full_text[[2]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+afdundfraktionslostotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/afdundfraktionslostotal) -> afdundfraktionslos_words
+
+#BÜNDNIS 90 / DIE GRÜNEN
+Worte <- str_extract_all(talks_by_fraktion$full_text[[3]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+grünetotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/grünetotal) -> grüne_words
+
+#CDU/CSU
+Worte <- str_extract_all(talks_by_fraktion$full_text[[4]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+cdutotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/cdutotal) -> cdu_words
+
+#DIE LINKE
+Worte <- str_extract_all(talks_by_fraktion$full_text[[5]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+linketotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/linketotal) -> linke_words
+
+#FDP
+Worte <- str_extract_all(talks_by_fraktion$full_text[[6]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+fdptotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/fdptotal) -> fdp_words
+
+#Fraktionslos
+Worte <- str_extract_all(talks_by_fraktion$full_text[[7]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+fraktionslostotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/fraktionslostotal) -> fraktionslos_words
+
+#SPD
+Worte <- str_extract_all(talks_by_fraktion$full_text[[8]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+spdtotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/spdtotal) -> spd_words
+
+#NA
+Worte <- str_extract_all(talks_by_fraktion$full_text[[9]], "\\b[a-zA-ZäöüÄÖÜß]+\\b")[[1]]
+natotal = length(Worte)
+tibble(Worte) %>% group_by(Worte) %>% count() %>% mutate(freq =n/natotal) -> na_words
+
+#alle
+all_words <- bind_rows(afd_words, afdundfraktionslos_words, grüne_words, cdu_words, linke_words, fdp_words, fraktionslos_words, spd_words, na_words)
+total <- sum(all_words$n)
+all_words %>% group_by(Worte) %>% summarize(n = sum(n), part= sum(n)/total) -> all_words
+```
+
+Now we want to extract the words that are more frequently used by a specific `fraktion`.
+```{r}
+afd_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> afd_high_frequent
+
+afdundfraktionslos_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> afdundfraktionslos_high_frequent
+
+grüne_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> grüne_high_frequent
+
+cdu_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> cdu_high_frequent
+
+linke_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> linke_high_frequent
+
+fdp_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> fdp_high_frequent
+
+fraktionslos_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> fraktionslos_high_frequent
+
+spd_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> spd_high_frequent
+
+na_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> na_high_frequent
+
+```
+
+We compare these words with `hitlerwords`.
+
+```{r}
+afd_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> afd_hitler_comparison
+afdundfraktionslos_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> afdundfraktionslos_hitler_comparison
+grüne_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> grüne_hitler_comparison
+cdu_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> cdu_hitler_comparison
+linke_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> linke_hitler_comparison
+fdp_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> fdp_hitler_comparison
+fraktionslos_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> fraktionslos_hitler_comparison
+spd_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> spd_hitler_comparison
+na_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> na_hitler_comparison
+
+#not unique
+tibble(fraktion = c("AfD", "AfD und Fraktionslose", "Grüne", "CDU", "Linke", "FDP", "Fraktionslos", "SPD", "NA"),
+       absolute = c(nrow(afd_hitler_comparison), nrow(afdundfraktionslos_hitler_comparison), nrow(grüne_hitler_comparison), nrow(cdu_hitler_comparison), nrow(linke_hitler_comparison), nrow(fdp_hitler_comparison), nrow(fraktionslos_hitler_comparison), nrow(spd_hitler_comparison), nrow(na_hitler_comparison)),
+       total = c(nrow(afd_words), nrow(afdundfraktionslos_words), nrow(grüne_words), nrow(cdu_words), nrow(linke_words), nrow(fdp_words), nrow(fraktionslos_words), nrow(spd_words), nrow(na_words))
+      ) %>% mutate(relative = absolute/total) -> hitler_compare
+```
+
+Dead code:
+```r
+1000*nrow(afd_hitler_comparison) / nrow(afd_words)
+1000*nrow(afdundfraktionslos_hitler_comparison) / nrow(afdundfraktionslos_words)
+1000*nrow(grüne_hitler_comparison) / nrow(grüne_words)
+1000*nrow(cdu_hitler_comparison) / nrow(cdu_words)
+1000*nrow(linke_hitler_comparison) / nrow(linke_words)
+1000*nrow(fdp_hitler_comparison) / nrow(fdp_words)
+1000*nrow(fraktionslos_hitler_comparison) / nrow(fraktionslos_words)
+1000*nrow(spd_hitler_comparison) / nrow(spd_words)
+1000*nrow(na_hitler_comparison) / nrow(na_words)
+
+
+1000*sum(afd_hitler_comparison$fraktion_n) / afdtotal
+1000*sum(afdundfraktionslos_hitler_comparison$fraktion_n) / afdundfraktionslostotal
+1000*sum(grüne_hitler_comparison$fraktion_n) / grünetotal
+1000*sum(cdu_hitler_comparison$fraktion_n) / cdutotal
+1000*sum(linke_hitler_comparison$fraktion_n) / linketotal
+1000*sum(fdp_hitler_comparison$fraktion_n) / fdptotal
+1000*sum(fraktionslos_hitler_comparison$fraktion_n) / fraktionslostotal
+1000*sum(spd_hitler_comparison$fraktion_n) / spdtotal
+1000*sum(na_hitler_comparison$fraktion_n) / natotal
+```