explain where vignettes are available

DESCRIPTION

@@ -1,13 +1,21 @@
 Package: hateimparlament
-Title: Protocolanalysis of German Bundestag
+Title: Recordanalysis Of Bundestag
 Version: 0.0.0.9000
-Authors@R: 
+Authors@R: c(
-    person(given = "First",
+    person(given = "Leon",
-           family = "Last",
+           family = "Burgard",
+           role = c("aut")),
+    person(given = "Josua",
+           family = "Kugler",
+           role = c("aut")),
+    person(given = "Christian",
+           family = "Merten",
            role = c("aut", "cre"),
-           email = "first.last@example.com",
+           email = "christian@merten.dev"))
-           comment = c(ORCID = "YOUR-ORCID-ID"))
+Description: Downloads, parses and analyses parliamentary records of the 19th legislative
-Description: Downloads, parses and analyses protocols of the current German parliament (Bundestag).
+    period of the German parliament (Bundestag).
+URL: https://git.flavigny.de/christian/hateimparlament
+BugReports: https://git.flavigny.de/christian/hateimparlament/issues
 License: GPL (>= 3)
 Encoding: UTF-8
 LazyData: true

NAMESPACE

@@ -7,6 +7,7 @@ export(join_speaker)
 export(party_colors)
 export(read_all)
 export(read_from_csv)
+export(read_from_csv_or_fetch)
 export(repair)
 export(word_usage_by_date)
 export(write_to_csv)

R/fetch.R

@@ -61,10 +61,3 @@ fetch_all <- function(download_dir="inst/records/", create=FALSE) {
     # if successful, set progressbar to 100%
     setTimerProgressBar(pb, 250)
 }
-
-stop_dir_not_creatable <- function(cond) {
-    # currently this has call: dir.create(download_dir)
-    # do we want to change this to fetch_all(...) ?
-    cond$message <- "Directory does not exist and can't be created. Probably because the path is not writeable."
-    stop(cond)
-}

R/helpers.R

@@ -19,6 +19,13 @@ check_directory <- function(path, create=F) {
     }
 }
 
+stop_dir_not_creatable <- function(cond) {
+    # currently this has call: dir.create(download_dir)
+    # do we want to change this to fetch_all(...) ?
+    cond$message <- "Directory does not exist and can't be created. Probably because the path is not writeable."
+    stop(cond)
+}
+
 # appends a file seperator at end of path if needed
 make_directory_path <- function(path) {
     if (!str_ends(path, .Platform$file.sep)) str_c(path, .Platform$file.sep)

R/parse.R

@@ -233,6 +233,11 @@ parse_speakerlist <- function(speakerliste_xml) {
 #' 
 #' @export
 write_to_csv <- function(tables, path="inst/csv/", create=F) {
+    is_valid_res(tables)
+    stopifnot("path must be of type character" = is.character(path))
+    stopifnot("create must be of type logical" = is.logical(create))
+
+    path <- make_directory_path(path)
     check_directory(path, create)
     write.table(tables$speaker, str_c(path, "speaker.csv"))
     write.table(tables$speeches, str_c(path, "speeches.csv"))
@@ -250,6 +255,9 @@ write_to_csv <- function(tables, path="inst/csv/", create=F) {
 #' 
 #' @export
 read_from_csv <- function(path="inst/csv/") {
+    stopifnot("path must be of type character" = is.character(path))
+
+    path <- make_directory_path(path)
     list(speaker = read.table(str_c(path, "speaker.csv")) %>%
              tibble() %>%
              mutate(id = as.character(id)),
@@ -259,5 +267,27 @@ read_from_csv <- function(path="inst/csv/") {
                     date = as.Date(date)),
          talks = tibble %$% read.table(str_c(path, "talks.csv")),
          comments = tibble %$% read.table(str_c(path, "comments.csv")),
-         applause = tibble %$% read.table(str_c(path, "applause.csv")))
+         applause = tibble %$% read.table(str_c(path, "applause.csv"))) -> res
+    is_valid_res(res)
+    res
+}
+
+#' Read records from csv or fetch
+#'
+#' @param path base directory where csv files are expected under path/csv
+#' and possibly records fetched and stored under path/records
+#' 
+#' @export
+read_from_csv_or_fetch <- function(path="inst/") {
+    path <- make_directory_path(path)
+    res <- tryCatch(read_from_csv(str_c(path, "csv/")),
+                    error = function(c) NULL)
+    if (!is.null(res)) return(res)
+    
+    fetch_all(str_c(path, "records/"), create=T)
+    read_all(str_c(path, "records/")) %>%
+        repair() ->
+        res
+    write_to_csv(res, str_c(path, "csv/"), create=T)
+    res
 }

README.md

@@ -1,19 +1,174 @@
-# How to develop
+# Description
 
+R package to analyze parliamentary records of the 19th legislative period of the Bundestag,
+the German parliament.
+
+# Installation
+
+Using the `remotes` package, this is easily installed via:
+```r
+remotes::install_url("https://git.flavigny.de/christian/hateimparlament/archive/master.zip")
+```
+If you want to build the vignettes, pass `build_vignettes = TRUE`. This takes a long time and
+fails sometimes, if bundestag.de times out, since
+in the beginning the necessary records are neither fetched nor parsed.
+
+## Install with vignettes
+
+An alternative for building
+the vignettes is to clone the repository and build the vignettes manually, e.g. on Linux
+```
+git clone https://git.flavigny.de/christian/hateimparlament
+cd hateimparlament
+```
+Then open a `R` shell and do
+```r
+devtools::load_all() # load package
+devtools::wd() # set working directory
+```
+Then fetch all records, read them and write the parsed tibbles to csv files.
+```r
+fetch_all(create = TRUE)
+read_all() %>% repair() -> res
+write_to_csv(res, create = TRUE)
+```
+Now you can install the package with vignettes by using
+```r
+devtools::install(build_vignettes = TRUE)
+```
+This makes all vignettes available via `browseVignettes()`.
+
+# Features
+
+The package mainly supplies 4 functionalities:
+
+## Download records
+
+To analyze records, they need to be downloaded. This is done with `fetch_all`:
+```r
+fetch_all("records/", create = TRUE) # path to directory where records should be stored
+```
+This downloads all parliamentary records and stores them as `.xml` files in the given directory.
+
+## Parse records
+
+To use the records in R, they are converted to `tibble`s with
+```r
+res_raw <- read_all("records/") # path to directory where records are stored
+```
+
+`res_raw` is a named list with 5 `tibble`s:
+
+### Speaker
+
+Table of all speakers of this legislative period.
+
+Fields:
+- `id`: Unique speaker id
+- `prename`: Prename
+- `lastname`: Surname
+- `fraction`: Name of fraction if the speaker is member of parliament.
+- `title`: Title, e.g. ,,Prof''
+- `role_short`: Short name of role, e.g. ,,Bundeskanzlerin''
+- `role_long`: Long name of role
+
+### Speeches
+
+Table of all speeches given during this legislative period.
+
+Fields:
+- `id`: Unique speech id
+- `speaker`: Principal speaker (the person standing behind the lectern during the speech).
+- `date`: Date of session
+
+### Talks
+
+Within a speech, there can be multiple talks by different people. Mostly this is the main speech
+by the principal speaker, but usually there are questions by other members of parliament or
+order calls by the president of the Bundestag.
+
+Fields:
+- `speech_id`: Speech in which this talk has been given
+- `speaker`: Person that actually talks
+- `content`: Spoken content
+
+### Comments
+
+These are the interjections that appear during the speeches.
+
+Fields:
+- `speech_id`: The speech that was interrupted
+- `on_speaker`: The speaker who was interrupted
+- `fraction`: The fraction of the commenter
+- `commenter`: The person who interrupted the speech
+- `comment`: The content of the comment
+
+### Applause
+
+Table containing all the rounds of applause that happened during this legislative period.
+
+Fields:
+- `speech_id`: Speech during which was applauded
+- `on_speaker`: Speaker who was applauded
+
+And then logical fields `CDU_CSU`, `SPD`, `FDP`, `DIE_LINKE`, `BUENDNIS_90_DIE_GRUENEN`, `AfD`
+for every fraction in the Bundestag, signifying whether this fraction applauded.
+
+## Repair records
+
+The parliamentary records usually contain some major and minor formatting issues. These are
+mostly resolved by using
+```
+res <- repair(res_raw)
+```
+By passing `lookup_speaker = TRUE`, even commenters in
+`res_raw$comments` are matched with their respective speaker id.
+
+## Analysis
+
+Also some functions are provided to analyze the parliamentary records and draw some plots:
+
+- `bar_plot_fractions`
+- `find_word`
+- `join_speaker`
+- `word_usage_by_date`
+
+See their usage with the `?` operator.
+
+In the vignettes you can find different analyses of the protocols, for example:
+
+- "Who talks the most?"
+- "Which party gives the most speeches?"
+- "Which party comments the most on which parties?"
+- "When are which topics discussed the most?"
+- ...
+
+# Contributing
+
+Developing works the easiest with `devtools`:
 ```r
-# everything works with devtools (loads some other packages too)
 library(devtools)
-
+```
-# reload all package functions
+When you changed something or added some functionality, you can reload all package functions with
+```r
 load_all()
+```
+If you want to avoid reading all records every time you start a new R session, you can
+write your parsed tibbles to CSV files:
 
-#write to CSV files to speed up loading
+```
 tables <- read_all()
 tables <- repair(tables)
-write_to_csv(tables)
+write_to_csv(tables, "path/to/csv/")
 ```
-We NEVER use source(...), etc.! Also NEVER use library(...). 
+Then later you can use
-But to add new packages (as dependency), use:
+```r
+res <- read_from_csv("path/to/csv/")
+```
+to load your stored tibbles very fast.
+
+NEVER use source(...), etc.! Also NEVER use library(...). 
+To add new packages (as dependency), use:
 ```r
 use_package("my-good-old-package")
 ```
@@ -27,87 +182,5 @@ document()
 
 Build vignettes
 ```r
-rmarkdown::render("vignettes/bla.Rmd")
+rmarkdown::render("vignettes/test.Rmd")
 ```
-
-# Download
-
-Before parsing, fetch.R must be run to download all protocols.
-```r
-fetch_all("../inst/records/") # path to directory where records should be stored
-```
-
-# Parsing
-
-## tables
-
-parse.R parses all downloaded logs and creates 5 tibbles.
-repair.R then cleans up the errors in these tibbles.
-```r
-read_all("../inst/records/") %>% repair()
-```
-
-
-
-### Speaker
-
-structure: `id` , `first_name` , `last_name` , `fraction` , `title` , `role_short`, `role_long`.
-
-
-
-Obtained from the `<speaker list>` entry at the end of the transcripts.
-
-### Speeches
-
-Structure: `id` , `speaker`
-
-The speeches `id` is specified in the protocol and is unique.A speech is a `<speech>` entry in the session history. A speech always has a main speaker (the one standing at the front of the lectern).
-
-Within a speech, there can be different speech entries:
-
-- Comments: Applause, interjections, etc.
-- Speeches: Typically mainly the main speaker, but also interjections. 
-These are stored in the talks, comments and applause tables when parsing.
-
-
-### Talks
-
-Structure: `speech_id` , `speaker` , `content`.
-
-These are the actual talk entries that appear within _speeches_.
-
-- `speech_id`: the speech in which the contribution appears.
-- `speaker`: The speaker of the speech entry.
-- `content`: The content of the speech.
-
-###comments
-
-These are the interjections that appear during the speeches.
-
-They have the following structure:
-- `speech_id`: The speech that was interrupted.
-- `on_speaker`: The speaker who was interrupted.
-- `fraction`
-- `commenter`: The person who interrupted the speech.
-- `comment`: The content of the comment.
-
-###applause
-
-The logical table shows which party applauded for which speaker with explicit speech and which did not.
-
-structure: `speech_id`, `on_speaker`, `CDU_CSU`, `SPD`, `FDP`, `DIE_LINKE`, `BUENDNIS_90_DIE_GRUENEN`, `AfD`
-
-
-# Analysis
-
-analysis.R provides some functions to analyze the "Plenarprotokolle" and to create plots.
-
-In the vignettes you can find different analyses of the protocols, for example:
-
-- "Who talks the most?"
-- "Which party gives the most speeches?"
-- "Which party comments the most on which parties?"
-- "When are which topics discussed the most?"
-- ...
-
-

inst/reports/abschlussbericht.tex

@@ -1,16 +1,28 @@
 \documentclass{article}
+\usepackage[ngerman]{babel}
 \usepackage[top=2.5cm, bottom=2.5cm]{geometry}
-
+\title{Abschlussbericht}
+\author{Leon Burgard, Josua Kugler, Christian Merten}
 \begin{document}
+    \maketitle
     \section*{Projektbeschreibung}
-    Wir haben zunächst die Plenarprotokolle der 19. Wahlperiode von der Website automatisiert herunterladen lassen.
+    Unser Projekt \glqq Plenarprotokolle \grqq stellt mittels dem Paket \verb|hateimparlament| Funktionen zur Analyse der Plenarprotokolle der 19. Wahlperiode des deutschen Bundestages zur Verfügung. Diese Funktionen können in vier Bereiche unterteilt werden:
-    Als nächstes haben wir die Daten in ein für die Analyse sinnvolles Format gebracht, d.h. 5 Tibbles  und Fehler ausgebessert.
+    \begin{enumerate}
-    Daraufhin konnten wir mit der Analyse beginnen.
+        \item Herunterladen der Protokolle
-    Insbesondere 
+        \item Konvertierung der XML-Dateien in Tibbles
-    \section*{Werkzeuge aus der Vorlesung}
+        \item Reparieren von Fehlern
-    Wir haben, da es hauptsächlich um Datenanalyse ging, sehr viel mit tidyverse gearbeitet.
+        \item Analyse
-    Ganz zu Beginn haben wir fürs fetchen der Protokolle rvest verwendet.
+    \end{enumerate}
-    Für die Visualisierung haben wir ggplot2 sowie vignettes genutzt.
+    Das Herunterladen der Protokolle gelingt über die Funktion 
+    \verb|fetch_all()|, welche auf die Website des deutschen Bundestages zugreift und die XML-Dateien einzeln herunterlädt. Hierzu haben wir das Paket rvest verwendet, welches wir bereits in der Vorlesung kennengelernt haben.
+    Durch \verb|read_all()| werden diese heruntergeladenen XML-Dateien in eine benannte Liste mit fünf Tibbles (speaker, speeches, talks, comments und applause) geschrieben. Allerdings benötigt man diese Tibbles immer wieder und es ist ziemlich zeitaufwändig die XML-Dateien immer wieder neu in Tibbles einzulesen, deshalb haben wir zusätzlich eine Funkion \verb|write_to_csv()| geschrieben, die die fertigen Tibbles als CSV-Dateien speichert. Diese können dann sehr schnell durch \verb|read_from_csv()| eingelesen werden, wodurch viel Zeit gesparrt wird.
+    Da diese Protokolle kleine Fehler enthalten, müssen diese noch im nächsten Schritt bereinigt werden, was mit \verb|repair()| funktioniert. Hierbei wird das Paket tidyverse viel benutzt, welches insgesamt sehr viel in unserem Projekt beansprucht wird, da wir uns mit der Datenanalyse beschäftigen.
+    In \verb|analyse.R| stellen wir noch einige Hilfsfunktionen bereit, die es dem Nutzer vereinfachen die Daten auszuwerten. Beispielsweise steht schon eine Funktion zur Verfügung, die ein Balkendiagramm erstellt, bei dem jede Partei des Bundestages sperat ausgewertet wird. Hierbei wird das Paket \verb|ggplot2| verwendet.
+    Im letzten Schritt unseres Projekts haben wir Fragestellungen festgelegt, die wir mithilfe von unserem Paket beantworten wollten. Die Daten und unsere Ergebnisse visualisierten wir mithilfe von \verb|ggplot2| und \verb|tidyverse| in Vignetten. 
     \section*{Organisation des Teams}
+    Während der ersten Projektphase wurden hauptsächlich die Funktionen zum Herunterladen der Dateien und Konvertieren und Reparieren der Tibbles geschrieben. Dies geschah größtenteils in Einzelarbeit, wobei hierbei die gegenseitige Kontrolle und Nachfragen die Funktionen optimiert haben. Zwischendurch wurde immer mal wieder zu einer HeiConf-Konferenz einberufen, um sich selbst den Zwischenstand klar zu machen und die Herausforderungen für die nächsten Wochen zu besprechen.
+    In der zweiten Hälfte des Projekts kümmerten wir uns dann um die Analyse der Daten und stellten unsere Ergebnisse in Vignetten da und erzeugten Dokumentationen für alle Funktionen, die für den Nutzer wichtig sind.
+    \newpage
     \section*{Meine Beteiligung}
+    \input{meine_beteiligung.tex}
 \end{document}

man/hateimparlament-package.Rd

@@ -4,15 +4,30 @@
 \name{hateimparlament-package}
 \alias{hateimparlament}
 \alias{hateimparlament-package}
-\title{hateimparlament: Protocolanalysis of German Bundestag}
+\title{hateimparlament: Recordanalysis Of Bundestag}
 \description{
-Downloads, parses and analyses protocols of the current German parliament (Bundestag).
+Downloads, parses and analyses parliamentary records of the 19th legislative
+    period of the German parliament (Bundestag).
 }
 \details{
 hateimparlament ist ein großartiges Paket!
+}
+\seealso{
+Useful links:
+\itemize{
+  \item \url{https://git.flavigny.de/christian/hateimparlament}
+  \item Report bugs at \url{https://git.flavigny.de/christian/hateimparlament/issues}
+}
+
 }
 \author{
-\strong{Maintainer}: First Last \email{first.last@example.com} (\href{https://orcid.org/YOUR-ORCID-ID}{ORCID})
+\strong{Maintainer}: Christian Merten \email{christian@merten.dev}
+
+Authors:
+\itemize{
+  \item Leon Burgard
+  \item Josua Kugler
+}
 
 }
 \keyword{internal}

man/read_from_csv_or_fetch.Rd

@@ -0,0 +1,15 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/parse.R
+\name{read_from_csv_or_fetch}
+\alias{read_from_csv_or_fetch}
+\title{Read records from csv or fetch}
+\usage{
+read_from_csv_or_fetch(path = "inst/")
+}
+\arguments{
+\item{path}{base directory where csv files are expected under path/csv
+and possibly records fetched and stored under path/records}
+}
+\description{
+Read records from csv or fetch
+}

vignettes/explicittopic.Rmd

@@ -1,8 +1,8 @@
 ---
-title: "explicittopic"
+title: "Analysis of covered topics"
 output: rmarkdown::html_vignette
 vignette: >
-  %\VignetteIndexEntry{explicittopic}
+  %\VignetteIndexEntry{Analysis of covered topics}
   %\VignetteEngine{knitr::rmarkdown}
   %\VignetteEncoding{UTF-8}
 ---
@@ -34,9 +34,10 @@ read_all("../inst/records/") %>% repair() -> res
 ```
 We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
 
 ## Analysis

vignettes/genderequality.Rmd

@@ -1,8 +1,8 @@
 ---
-title: "genderequality"
+title: "Differences in gender"
 output: rmarkdown::html_vignette
 vignette: >
-  %\VignetteIndexEntry{genderequality}
+  %\VignetteIndexEntry{Differences in gender}
   %\VignetteEngine{knitr::rmarkdown}
   %\VignetteEncoding{UTF-8}
 ---
@@ -35,10 +35,13 @@ read_all("../records/") %>% repair() -> res
 ```
 We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
+
+
 and unpack our tibbles
 ```{r}
 comments <- res$comments

vignettes/generalquestions.Rmd

@@ -1,8 +1,8 @@
 ---
-title: "generalquestions"
+title: "General questions"
 output: rmarkdown::html_vignette
 vignette: >
-  %\VignetteIndexEntry{generalquestions}
+  %\VignetteIndexEntry{General questions}
   %\VignetteEngine{knitr::rmarkdown}
   %\VignetteEncoding{UTF-8}
 ---
@@ -34,9 +34,10 @@ read_all("../inst/records/") %>% repair() -> res
 ```
 We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
 
 ## Analysis

vignettes/hitlercomparison.Rmd

@@ -38,9 +38,10 @@ talks <- res$talks
 We also used `repair` to fix a bunch of formatting issues in the records and unpacked
 the result into more descriptive variables.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-tables <- read_from_csv('../inst/csv/')
+tables <- read_from_csv_or_fetch('../inst/')
 
 comments <- tables$comments
 speeches <- tables$speeches

vignettes/interaction.Rmd

@@ -1,8 +1,8 @@
 ---
-title: "interaction"
+title: "Interaction between fractions"
 output: rmarkdown::html_vignette
 vignette: >
-  %\VignetteIndexEntry{interaction}
+  %\VignetteIndexEntry{Interaction between fractions}
   %\VignetteEngine{knitr::rmarkdown}
   %\VignetteEncoding{UTF-8}
 ---
@@ -34,9 +34,10 @@ read_all("../inst/records/") %>% repair() -> res
 ```
 We also used `repair` to fix a bunch of formatting issues in the records.
 
-For development purposes, we load the tables from csv files.
+For development purposes, we only fetch records if they are not already
+stored as csv files:
 ```{r}
-res <- read_from_csv('../inst/csv/')
+res <- read_from_csv_or_fetch('../inst/')
 ```
 
 ## Analysis