|
|
|
@@ -30,17 +30,50 @@ fetch_batch <- function(offset, download_dir) { |
|
|
|
# TODO: error handling |
|
|
|
# - what if: page not reachable |
|
|
|
# - wrong format, etc. |
|
|
|
fetch_all <- function(download_dir="records/") { |
|
|
|
|
|
|
|
#' Download available records |
|
|
|
#' |
|
|
|
#' This fetches all available records of the 19th legislative period of the german Bundestag. |
|
|
|
#' |
|
|
|
#' @param download_dir character |
|
|
|
#' |
|
|
|
#' @export |
|
|
|
fetch_all <- function(download_dir="records/", create=FALSE) { |
|
|
|
# check if download_dir path is a directory path |
|
|
|
if (str_sub(download_dir, -1) != .Platform$file.sep) |
|
|
|
download_dir <- str_c(download_dir, .Platform$file.sep) |
|
|
|
|
|
|
|
# check if download_dir exists |
|
|
|
if(file.access(download_dir, mode=0) == -1) { |
|
|
|
if (create) { |
|
|
|
tryCatch(dir.create(download_dir), |
|
|
|
error = stop_dir_not_creatable, |
|
|
|
warning = stop_dir_not_creatable) |
|
|
|
} else { |
|
|
|
stop("Directory does not exist. Use create = TRUE if you wish to create the directory.") |
|
|
|
} |
|
|
|
} else if (file.access(download_dir, mode=2) == -1) { |
|
|
|
stop("Directory exists, but is not writeable.") |
|
|
|
} |
|
|
|
cat("Fetching all available records from bundestag.de. This may take a while ...\n") |
|
|
|
|
|
|
|
|
|
|
|
# create progress bar |
|
|
|
pb <<- timerProgressBar(min=0, max=250, width=40, char="+") |
|
|
|
progress <<- 0 |
|
|
|
# close progress bar on exit (also on error) |
|
|
|
on.exit({close(pb); cat("Done.\n")}) |
|
|
|
|
|
|
|
# fetch batch by batch |
|
|
|
offset <- 0 |
|
|
|
while(fetch_batch(offset, download_dir)) offset <- offset + 10 |
|
|
|
|
|
|
|
# if successful, set progressbar to 100% |
|
|
|
setTimerProgressBar(pb, 250) |
|
|
|
} |
|
|
|
|
|
|
|
stop_dir_not_creatable <- function(cond) { |
|
|
|
# currently this has call: dir.create(download_dir) |
|
|
|
# do we want to change this to fetch_all(...) ? |
|
|
|
cond$message <- "Directory does not exist and can't be created. Probably because the path is not writeable." |
|
|
|
stop(cond) |
|
|
|
} |