diff --git a/.Rbuildignore b/.Rbuildignore index 481c3ac..786f647 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -16,3 +16,4 @@ ^vignettes/precompile\.R$ ^reviews$ ^\.claude$ +\.qmd$ diff --git a/R/get_preliminary_damage_assessments.R b/R/get_preliminary_damage_assessments.R index a12b256..79e1b94 100644 --- a/R/get_preliminary_damage_assessments.R +++ b/R/get_preliminary_damage_assessments.R @@ -1,25 +1,7 @@ #' Derive collision-free local filenames for a set of PDA report URLs #' -#' The last path segment of each URL is percent-decoded and stripped of -#' characters that cannot appear in a Windows filename, which produces a -#' readable local name. That transformation is lossy, so two different URLs can -#' in principle reduce to the same name; where that happens, a short hash of the -#' full URL is appended to each of the colliding names so that both reports are -#' kept rather than one silently overwriting or being skipped past the other. -#' -#' The suffix is a hash of the URL rather than a running `_1`, `_2` counter for -#' two reasons. First, it depends only on the URL, so a given report always -#' resolves to the same filename and the `file.exists()` check in `save_pdf()` -#' still recognizes it as already downloaded on a later run -- a counter -#' assigned in encounter order would change between runs and re-download the -#' whole archive. Second, FEMA's own filenames already use a `_0`/`_1` suffix to -#' distinguish their duplicates (72 of the currently cached files do), so a -#' counter would be indistinguishable from the source naming. -#' -#' Collisions are resolved across the complete set of URLs passed in, so -#' `scrape_pda_pdfs()` must collect every listing page before calling this; -#' resolving against a partial set could assign a name that a later page's URL -#' also claims. +#' File names can have tricky characters, including escaped/special characters +#' derived from non-ASCI text. This handles that complexity. #' #' @param urls A character vector of PDF URLs. #' @@ -31,20 +13,15 @@ resolve_pdf_destinations = function(urls) { destinations1 = tibble::tibble(url = unique(urls)) %>% dplyr::mutate( base_name = url %>% - ## drop any query string or fragment, which would otherwise survive into - ## the filename ("report.pdf?v=2" -> "report?v=2.pdf", which is not a - ## legal Windows filename and fails to download at all) + ## a query string or fragment left in place produces an illegal Windows + ## filename ("report.pdf?v=2" -> "report?v=2.pdf") that cannot be written stringr::str_remove("[?#].*$") %>% stringr::str_extract("[^/]+$") %>% curl::curl_unescape() %>% - ## anchored so only a trailing extension is removed, not every - ## occurrence of ".pdf" in the name stringr::str_remove(stringr::regex("\\.pdf$", ignore_case = TRUE)) %>% stringr::str_replace_all("[<>:\"/\\\\|?*]", "_") %>% stringr::str_remove_all("[[:cntrl:]]") %>% stringr::str_squish() %>% - ## decoded "%20" would otherwise leave literal spaces in the path; no - ## currently cached file contains one, so this does not rename anything stringr::str_replace_all("\\s+", "_")) %>% dplyr::add_count(base_name, name = "base_name_count") %>% dplyr::mutate( @@ -58,12 +35,17 @@ resolve_pdf_destinations = function(urls) { stringr::str_c(base_name, ".pdf"))) if (any(destinations1$needs_hash)) { + is_empty_name = is.na(destinations1$base_name) | destinations1$base_name == "" + collided = destinations1$needs_hash & !is_empty_name + message( stringr::str_c( sum(destinations1$needs_hash), - " report URL(s) reduced to a filename already claimed by a different ", - "URL; a URL hash was appended so that every report is retained. ", - "Affected: ", + " report URL(s) were given a URL hash so that every report is retained: ", + sum(collided), + " reduced to a filename already claimed by a different URL, and ", + sum(is_empty_name), + " reduced to no filename at all. Affected: ", stringr::str_c( destinations1$destination_file[destinations1$needs_hash], collapse = ", "))) } @@ -71,15 +53,22 @@ resolve_pdf_destinations = function(urls) { destinations1 %>% dplyr::select(url, destination_file) } -#' Download a PDF from a URL and verify that a PDF is what arrived +#' The browser identity FEMA's site requires +#' +#' @return A length-one character `User-Agent` string. +#' @noRd +browser_user_agent = function() { + stringr::str_c( + "Mozilla/5.0 (Linux; Android 11; SAMSUNG SM-G973U) AppleWebKit/537.36 ", + "(KHTML, like Gecko) SamsungBrowser/14.2 Chrome/87.0.4280.141 Mobile ", + "Safari/537.36") +} + +#' Download a PDF from a URL and verify #' #' A download is kept only if the resulting file begins with the `%PDF` magic #' number and exceeds `minimum_bytes`. Anything else -- an HTML error page served -#' with a 200 status, a truncated transfer -- is deleted rather than left on -#' disk. This matters because the cache check is the file's existence: a bad file -#' that stayed would never be retried on a later run, turning one transient -#' failure into a permanent hole in the archive that looks no different from a -#' complete one. Deleting it means the next run downloads it again. +#' with a 200 status, a truncated transfer -- is deleted. #' #' @param url URL to the pdf. #' @param destfile The full local path the PDF should be written to, as @@ -93,9 +82,8 @@ save_pdf = function(url, destfile, minimum_bytes = 2048) { if (file.exists(destfile)) { return("cached") } - ## otherwise the FEMA domain blocks our requests headers = c( - "User-Agent" = "Mozilla/5.0 (Linux; Android 11; SAMSUNG SM-G973U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/14.2 Chrome/87.0.4280.141 Mobile Safari/537.36", + "User-Agent" = browser_user_agent(), "Accept" = "application/pdf", "Accept-Language" = "en-US", "Connection" = "keep-alive") @@ -103,12 +91,8 @@ save_pdf = function(url, destfile, minimum_bytes = 2048) { ## Some listing links are percent-encoded twice: FEMA publishes ## "...Luise%25C3%25B1oIndians.pdf", where "%25" is an encoded "%", so the ## request asks for a file literally named "Luise%C3%B1o..." and returns 404. - ## The real filename contains "n" with a tilde, correctly encoded as "%C3%B1". - ## Undoing one layer of encoding recovers it. Tried only after the URL as - ## published fails, so a filename that genuinely contains a percent sign is - ## still fetched correctly on the first attempt. + ## Undoing one layer of encoding recovers it. candidate_urls = unique(c(url, stringr::str_replace_all(url, "%25", "%"))) - downloaded = FALSE for (candidate_url in candidate_urls) { downloaded = tryCatch({ @@ -119,22 +103,19 @@ save_pdf = function(url, destfile, minimum_bytes = 2048) { mode = "wb") TRUE}, error = function(e) FALSE, - warning = function(w) FALSE) + warning = function(w) NA) - if (downloaded) { break } } + if (isTRUE(downloaded)) { break } } is_pdf = FALSE if (file.exists(destfile) && file.size(destfile) >= minimum_bytes) { connection = file(destfile, "rb") - ## closed immediately rather than via on.exit(): the file.remove() below runs - ## before function exit, and Windows refuses to delete a file that still has - ## an open handle, which would leave the bad download in place file_header = tryCatch( rawToChar(readBin(connection, "raw", 4)), finally = close(connection)) is_pdf = identical(file_header, "%PDF") } - if (!downloaded || !is_pdf) { + if (!is_pdf) { ## remove the partial or wrong-content file so this URL is retried next run if (file.exists(destfile)) { file.remove(destfile) } warning( @@ -147,38 +128,23 @@ save_pdf = function(url, destfile, minimum_bytes = 2048) { #' @title Download Preliminary Damage Assessment (PDA) Reports to Disk #' -#' @description Downloads every PDA report PDF that FEMA publishes into a local +#' @description Downloads every PDA report that FEMA publishes into a local #' directory, so that `get_preliminary_damage_assessments()` has a complete and #' current set of source documents to parse. Run this before regenerating the #' dataset; `get_preliminary_damage_assessments()` parses whatever is already on #' disk and never fetches anything itself. #' #' @details Walks every page of FEMA's PDA report listing at -#' https://www.fema.gov/disaster/how-declared/preliminary-damage-assessments/reports, -#' advancing until a page returns no PDF links, and downloads any report not -#' already present in `cache_directory`. Because the full listing is traversed on -#' every run, the set of files on disk after a successful run is the complete set -#' of reports FEMA publishes -- coverage does not depend on the caller working -#' out which page numbers hold new reports. +#' https://www.fema.gov/disaster/how-declared/preliminary-damage-assessments/reports +#' and downloads any report not +#' already present in `cache_directory`. #' #' A page that errors is retried, and exhausting the retries raises an error -#' rather than ending the walk. This matters because a transient failure and a -#' genuinely empty final page are otherwise indistinguishable, and treating the -#' former as the end of the listing would silently truncate the archive. -#' -#' The listing is server-rendered, so the PDF links are present in the HTML that -#' `httr::GET()` returns and no headless browser is needed. An earlier version -#' used `rvest::read_html_live()`, which requires `chromote`; that dependency is -#' not declared by this package and is frequently absent, which made the -#' function fail outright rather than merely run slowly. -#' -#' Note that FEMA's bot protection rejects requests that do not carry a -#' browser-like `User-Agent`, and rejects them from some networks regardless, so -#' a failure here is not necessarily a change to the listing. +#' rather than ending the walk. #' #' @param cache_directory The folder where scraped PDFs are written. #' @param max_pages A guard against an unbounded walk if the listing ever stops -#' returning empty pages. Raises an error if reached. +#' returning empty pages. #' @param attempts_per_page How many times to try a listing page before treating #' it as a failure. #' @param pages Which listing pages to read, as a numeric vector. The default @@ -228,10 +194,8 @@ scrape_pda_pdfs = function( on.exit(options(timeout = original_timeout), add = TRUE) options(timeout = 200) - ## the same browser-like header the PDF downloads use; without it FEMA - ## returns 403 rather than the listing listing_headers = httr::add_headers( - "User-Agent" = "Mozilla/5.0 (Linux; Android 11; SAMSUNG SM-G973U) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/14.2 Chrome/87.0.4280.141 Mobile Safari/537.36", + "User-Agent" = browser_user_agent(), "Accept" = "text/html", "Accept-Language" = "en-US") @@ -247,6 +211,7 @@ scrape_pda_pdfs = function( urls1 = character(0) page_number = if (walk_whole_listing) 0 else page_queue[1] page_index = 1 + pages_read = 0 repeat { page_urls = NULL @@ -263,14 +228,10 @@ scrape_pda_pdfs = function( rvest::html_elements("a") %>% rvest::html_attr("href") %>% purrr::discard(~ is.na(.x)) %>% - ## anchored to the extension: an unanchored "pdf" match also picks up - ## links such as "/help-with-pdf-files", which are not reports purrr::keep(~ stringr::str_detect(.x, stringr::regex("\\.pdf$", ignore_case = TRUE)))}, error = function(e) NULL) if (!is.null(page_urls)) { break } - ## exponential rather than linear: FEMA rate-limits a sustained walk and - ## the cooldown outlasts a few short pauses Sys.sleep(min(120, delay_seconds * 2 ^ attempt)) } @@ -285,17 +246,15 @@ scrape_pda_pdfs = function( "Reports from pages already read were not downloaded. To resume from ", "where this stopped, wait a few minutes and call:\n", " scrape_pda_pdfs(pages = ", page_number, ":", - max(page_number + 49, page_number), ")\n", + page_number + 49, ")\n", "or raise delay_seconds / attempts_per_page for a slower, more ", "patient walk."), call. = FALSE) } - ## an empty page that loaded successfully is the end of the listing; only - ## meaningful when walking, since an explicit page list may legitimately - ## include a page beyond the end if (length(page_urls) == 0 && walk_whole_listing) { break } urls1 = c(urls1, page_urls) + pages_read = pages_read + 1 if (walk_whole_listing) { page_number = page_number + 1 @@ -325,7 +284,7 @@ scrape_pda_pdfs = function( destinations = resolve_pdf_destinations(urls2) notify( - "Found ", nrow(destinations), " report(s) across ", page_number, + "Found ", nrow(destinations), " report(s) across ", pages_read, " listing page(s).") destinations2 = destinations %>% @@ -349,11 +308,7 @@ scrape_pda_pdfs = function( "Do not treat the archive as complete until this count is zero."), call. = FALSE) } - ## Files present locally but no longer listed on the site. FEMA has not been - ## observed to withdraw a report (as of the 2026-07-30 archive, all 1,356 - ## cached files were still listed), so this is a monitor rather than a known - ## problem: a withdrawn report would otherwise keep flowing into the dataset - ## with nothing to indicate it is no longer published. + ## Files present locally but not listed on the site. orphans = setdiff( list.files(cache_directory, recursive = TRUE, pattern = "(?i)pdf$"), destinations2$destination_file) @@ -370,58 +325,64 @@ scrape_pda_pdfs = function( invisible(destinations2) } -#' Helper function to extract values from the PDAs -#' @param text The inputted string of text -#' @param term1 Where to begin matching -#' @param term2 Where to finish matching +#' Extract the value a report prints between two field labels #' -#' @return All the text between term1 and term2, but not including either of the terms themselves +#' @param text The report text to search. +#' @param term1 The label the value follows. +#' @param term2 The label the value runs up to. +#' +#' @return The text between `term1` and `term2`, excluding both terms. #' @noRd extract_value = function(text, term1, term2) { - ## Colons are optional. The reports render these fields as - ## "