add error handling if directories are empty or do not exist
This commit is contained in:
@@ -30,17 +30,50 @@ fetch_batch <- function(offset, download_dir) {
|
|||||||
# TODO: error handling
|
# TODO: error handling
|
||||||
# - what if: page not reachable
|
# - what if: page not reachable
|
||||||
# - wrong format, etc.
|
# - wrong format, etc.
|
||||||
fetch_all <- function(download_dir="records/") {
|
|
||||||
|
#' Download available records
|
||||||
|
#'
|
||||||
|
#' This fetches all available records of the 19th legislative period of the german Bundestag.
|
||||||
|
#'
|
||||||
|
#' @param download_dir character
|
||||||
|
#'
|
||||||
|
#' @export
|
||||||
|
fetch_all <- function(download_dir="records/", create=FALSE) {
|
||||||
|
# check if download_dir path is a directory path
|
||||||
|
if (str_sub(download_dir, -1) != .Platform$file.sep)
|
||||||
|
download_dir <- str_c(download_dir, .Platform$file.sep)
|
||||||
|
|
||||||
|
# check if download_dir exists
|
||||||
|
if(file.access(download_dir, mode=0) == -1) {
|
||||||
|
if (create) {
|
||||||
|
tryCatch(dir.create(download_dir),
|
||||||
|
error = stop_dir_not_creatable,
|
||||||
|
warning = stop_dir_not_creatable)
|
||||||
|
} else {
|
||||||
|
stop("Directory does not exist. Use create = TRUE if you wish to create the directory.")
|
||||||
|
}
|
||||||
|
} else if (file.access(download_dir, mode=2) == -1) {
|
||||||
|
stop("Directory exists, but is not writeable.")
|
||||||
|
}
|
||||||
cat("Fetching all available records from bundestag.de. This may take a while ...\n")
|
cat("Fetching all available records from bundestag.de. This may take a while ...\n")
|
||||||
|
|
||||||
# create progress bar
|
# create progress bar
|
||||||
pb <<- timerProgressBar(min=0, max=250, width=40, char="+")
|
pb <<- timerProgressBar(min=0, max=250, width=40, char="+")
|
||||||
progress <<- 0
|
progress <<- 0
|
||||||
# close progress bar on exit (also on error)
|
# close progress bar on exit (also on error)
|
||||||
on.exit({close(pb); cat("Done.\n")})
|
on.exit({close(pb); cat("Done.\n")})
|
||||||
|
|
||||||
# fetch batch by batch
|
# fetch batch by batch
|
||||||
offset <- 0
|
offset <- 0
|
||||||
while(fetch_batch(offset, download_dir)) offset <- offset + 10
|
while(fetch_batch(offset, download_dir)) offset <- offset + 10
|
||||||
|
|
||||||
# if successful, set progressbar to 100%
|
# if successful, set progressbar to 100%
|
||||||
setTimerProgressBar(pb, 250)
|
setTimerProgressBar(pb, 250)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
stop_dir_not_creatable <- function(cond) {
|
||||||
|
# currently this has call: dir.create(download_dir)
|
||||||
|
# do we want to change this to fetch_all(...) ?
|
||||||
|
cond$message <- "Directory does not exist and can't be created. Probably because the path is not writeable."
|
||||||
|
stop(cond)
|
||||||
|
}
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ read_all <- function(path="records/") {
|
|||||||
distinct() ->
|
distinct() ->
|
||||||
talks
|
talks
|
||||||
|
|
||||||
|
if (length(available_protocols) == 0)
|
||||||
|
warning("The given directory is empty or does not exist.")
|
||||||
list(redner = redner, reden = reden, talks = talks)
|
list(redner = redner, reden = reden, talks = talks)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+4
-1
@@ -23,6 +23,7 @@ collect_unique <- function(xs) xs %>% clear_na() %>% unique() %>% str_c(collapse
|
|||||||
|
|
||||||
# expects a tibble of redner and repairs
|
# expects a tibble of redner and repairs
|
||||||
repair_redner <- function(redner) {
|
repair_redner <- function(redner) {
|
||||||
|
if (nrow(redner) == 0) return(redner)
|
||||||
redner %>% mutate(fraktion = Vectorize(repair_fraktion)(fraktion)) %>% # fix fraktion
|
redner %>% mutate(fraktion = Vectorize(repair_fraktion)(fraktion)) %>% # fix fraktion
|
||||||
group_by(id, vorname, nachname) %>%
|
group_by(id, vorname, nachname) %>%
|
||||||
summarize(fraktion = collect_unique(fraktion),
|
summarize(fraktion = collect_unique(fraktion),
|
||||||
@@ -32,16 +33,18 @@ repair_redner <- function(redner) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
repair_reden <- function(reden) {
|
repair_reden <- function(reden) {
|
||||||
|
if (nrow(reden) == 0) return(reden)
|
||||||
# TODO: fill with content
|
# TODO: fill with content
|
||||||
reden
|
reden
|
||||||
}
|
}
|
||||||
|
|
||||||
repair_talks <- function(talks) {
|
repair_talks <- function(talks) {
|
||||||
|
if (nrow(talks) == 0) return(talks)
|
||||||
# TODO: fill with content
|
# TODO: fill with content
|
||||||
talks
|
talks
|
||||||
}
|
}
|
||||||
|
|
||||||
#' Repairs parsed tables
|
#' Repair parsed tables
|
||||||
#'
|
#'
|
||||||
#' @export
|
#' @export
|
||||||
repair <- function(parse_output) {
|
repair <- function(parse_output) {
|
||||||
|
|||||||
Reference in New Issue
Block a user