Merge branch 'master' of https://git.flavigny.de/christian/hateimparlament

hitler_texts/parse.py

@@ -6,7 +6,15 @@ with open('/home/josua/deu_mixed-typical_2011_1M/deu_mixed-typical_2011_1M-words
     for line in lines:
         #print(line.split(sep="\t"))
         index, word, count = line.split(sep="\t")
-        if int(index) > 100:
+        if int(index) > 100 and int(count) > 5:
+            german_words.append(word.lower())
+
+with open('/home/josua/deu_mixed-typical_2011_1M/deu_news_1995_1M-words.txt') as f:
+    lines = f.readlines()
+    for line in lines:
+        #print(line.split(sep="\t"))
+        index, word, count = line.split(sep="\t")
+        if int(index) > 100 and int(count) > 5:# only words that are used more than 5 times
             german_words.append(word.lower())
 
 
@@ -36,11 +44,14 @@ with open(f'mein_kampf') as f:
     for line in lines:
         hitler_words.extend(get_words_from_line(line))
 
+german_words = set(german_words)
 hitler_words = set(hitler_words) #unique
 #filter_words = hitler_words.intersection(set(german_words))
 
 only_hitler_words = list(hitler_words.difference(german_words))
 
+print(only_hitler_words)
+
 with open("german_words", "w") as f:
     for word in german_words:
         word += "\n"

vignettes/hitlercomparison.Rmd

@@ -1,5 +1,5 @@
 ---
-title: "funwithdata"
+title: "Analysis of vocabulary"
 output: rmarkdown::html_vignette
 vignette: >
   %\VignetteIndexEntry{funwithdata}
@@ -63,7 +63,6 @@ talks %>%
   left_join(redner, by=c(redner='id')) %>% 
   group_by(fraktion) %>%
   summarize(full_text=str_c(content, collapse="\n")) -> talks_by_fraktion
-talks_by_fraktion
 ```
 For each party, we want to get a tibble of words with frequency.
 ```{r}
@@ -121,23 +120,31 @@ all_words %>% group_by(Worte) %>% summarize(n = sum(n), part= sum(n)/total) -> a
 Now we want to extract the words that are more frequently used by a specific `fraktion`.
 ```{r}
 afd_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> afd_high_frequent
+select(afd_high_frequent, fraktion_n, total_n)
 
 afdundfraktionslos_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> afdundfraktionslos_high_frequent
+select(afdundfraktionslos_high_frequent, fraktion_n, total_n)
 
 grüne_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> grüne_high_frequent
+select(grüne_high_frequent, fraktion_n, total_n)
 
 cdu_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> cdu_high_frequent
+select(cdu_high_frequent, fraktion_n, total_n)
 
 linke_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> linke_high_frequent
+select(linke_high_frequent, fraktion_n, total_n)
 
 fdp_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> fdp_high_frequent
+select(fdp_high_frequent, fraktion_n, total_n)
 
 fraktionslos_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> fraktionslos_high_frequent
+select(fraktionslos_high_frequent, fraktion_n, total_n)
 
 spd_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> spd_high_frequent
+select(spd_high_frequent, fraktion_n, total_n)
 
 na_words %>% transmute(freq, fraktion_n = n) %>% left_join(all_words) %>% transmute(fraktion_freq = freq, total_freq = part, fraktion_n, total_n = n, rel_quotient = fraktion_freq/total_freq, abs_quotient = fraktion_n/total_n) %>% arrange(-abs_quotient, -fraktion_n) %>% filter(rel_quotient > 1) -> na_high_frequent
-
+select(na_high_frequent, fraktion_n, total_n)
 ```
 
 We compare these words with `hitlerwords`.
@@ -154,32 +161,13 @@ spd_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerw
 na_high_frequent %>% mutate(Worte = str_to_lower(Worte)) %>% inner_join(hitlerwords) -> na_hitler_comparison
 
 #not unique
-tibble(fraktion = c("AfD", "AfD und Fraktionslose", "Grüne", "CDU", "Linke", "FDP", "Fraktionslos", "SPD", "NA"),
-       absolute = c(nrow(afd_hitler_comparison), nrow(afdundfraktionslos_hitler_comparison), nrow(grüne_hitler_comparison), nrow(cdu_hitler_comparison), nrow(linke_hitler_comparison), nrow(fdp_hitler_comparison), nrow(fraktionslos_hitler_comparison), nrow(spd_hitler_comparison), nrow(na_hitler_comparison)),
-       total = c(nrow(afd_words), nrow(afdundfraktionslos_words), nrow(grüne_words), nrow(cdu_words), nrow(linke_words), nrow(fdp_words), nrow(fraktionslos_words), nrow(spd_words), nrow(na_words))
-      ) %>% mutate(relative = absolute/total) -> hitler_compare
+tibble(fraktion = c("AfD", "AfD&Fraktionslos", "BÜNDNIS 90 / DIE GRÜNEN", "CDU/CSU", "DIE LINKE", "FDP", "Fraktionslos", "SPD"),
+       absolute = c(nrow(afd_hitler_comparison), nrow(afdundfraktionslos_hitler_comparison), nrow(grüne_hitler_comparison), nrow(cdu_hitler_comparison), nrow(linke_hitler_comparison), nrow(fdp_hitler_comparison), nrow(fraktionslos_hitler_comparison), nrow(spd_hitler_comparison)),
+       total = c(nrow(afd_words), nrow(afdundfraktionslos_words), nrow(grüne_words), nrow(cdu_words), nrow(linke_words), nrow(fdp_words), nrow(fraktionslos_words), nrow(spd_words))
+      ) %>% mutate(n = absolute/total) -> hitler_comparison
+hitler_comparison
 ```
-
-Dead code:
-```r
-1000*nrow(afd_hitler_comparison) / nrow(afd_words)
-1000*nrow(afdundfraktionslos_hitler_comparison) / nrow(afdundfraktionslos_words)
-1000*nrow(grüne_hitler_comparison) / nrow(grüne_words)
-1000*nrow(cdu_hitler_comparison) / nrow(cdu_words)
-1000*nrow(linke_hitler_comparison) / nrow(linke_words)
-1000*nrow(fdp_hitler_comparison) / nrow(fdp_words)
-1000*nrow(fraktionslos_hitler_comparison) / nrow(fraktionslos_words)
-1000*nrow(spd_hitler_comparison) / nrow(spd_words)
-1000*nrow(na_hitler_comparison) / nrow(na_words)
-
-
-1000*sum(afd_hitler_comparison$fraktion_n) / afdtotal
-1000*sum(afdundfraktionslos_hitler_comparison$fraktion_n) / afdundfraktionslostotal
-1000*sum(grüne_hitler_comparison$fraktion_n) / grünetotal
-1000*sum(cdu_hitler_comparison$fraktion_n) / cdutotal
-1000*sum(linke_hitler_comparison$fraktion_n) / linketotal
-1000*sum(fdp_hitler_comparison$fraktion_n) / fdptotal
-1000*sum(fraktionslos_hitler_comparison$fraktion_n) / fraktionslostotal
-1000*sum(spd_hitler_comparison$fraktion_n) / spdtotal
-1000*sum(na_hitler_comparison$fraktion_n) / natotal
+Finally, we want to plot our results:
+```{r, fig.width=7}
+bar_plot_fraktionen(hitler_comparison)
 ```