From 164299ee8db661c6dda6bd5cbb0eecfa5f438d69 Mon Sep 17 00:00:00 2001
From: Matt Strimas-Mackey
Date: Sat, 1 Aug 2026 03:40:33 -0700
Subject: [PATCH 1/4] refactor all download-related functions into fetch.R
---
DESCRIPTION | 2 +-
NEWS.md | 7 +
R/download.R | 278 ++--------
R/fetch.R | 359 +++++++++++++
R/load.R | 479 +++++-------------
cran-comments.md | 15 +-
docs/404.html | 2 +-
docs/CODE_OF_CONDUCT.html | 2 +-
docs/CONTRIBUTING.html | 2 +-
docs/LICENSE.html | 2 +-
docs/articles/api.html | 2 +-
docs/articles/applications.html | 2 +-
docs/articles/index.html | 2 +-
docs/articles/product-changelog.html | 2 +-
docs/articles/status.html | 150 +-----
docs/articles/status.md | 148 +-----
docs/articles/trends.html | 2 +-
docs/authors.html | 6 +-
docs/authors.md | 4 +-
docs/index.html | 2 +-
docs/news/index.html | 12 +-
docs/news/index.md | 13 +
docs/pkgdown.yml | 2 +-
.../abundance_palette-deprecated.html | 2 +-
docs/reference/assign_to_grid.html | 2 +-
docs/reference/calculate_mcc_f1.html | 2 +-
docs/reference/convert_ppy_to_cumulative.html | 2 +-
docs/reference/date_to_st_week.html | 2 +-
docs/reference/ebirdst-defunct.html | 2 +-
docs/reference/ebirdst-deprecated.html | 2 +-
docs/reference/ebirdst-package.html | 2 +-
docs/reference/ebirdst_data_dir.html | 4 +-
docs/reference/ebirdst_data_dir.md | 2 +-
docs/reference/ebirdst_data_inventory.html | 2 +-
docs/reference/ebirdst_delete.html | 2 +-
.../ebirdst_download_data_coverage.html | 2 +-
docs/reference/ebirdst_download_status.html | 2 +-
docs/reference/ebirdst_download_trends.html | 2 +-
docs/reference/ebirdst_palettes.html | 2 +-
.../ebirdst_predictor_descriptions.html | 2 +-
docs/reference/ebirdst_predictors.html | 2 +-
docs/reference/ebirdst_regional_stats.html | 2 +-
docs/reference/ebirdst_runs.html | 2 +-
docs/reference/ebirdst_version.html | 2 +-
docs/reference/get_species.html | 2 +-
docs/reference/get_species_path.html | 2 +-
docs/reference/grid_sample.html | 2 +-
docs/reference/index.html | 2 +-
docs/reference/load_config.html | 2 +-
docs/reference/load_data_coverage.html | 2 +-
docs/reference/load_fac_map_parameters.html | 2 +-
docs/reference/load_pi.html | 42 +-
docs/reference/load_pi.md | 13 +-
docs/reference/load_ppm.html | 24 +-
docs/reference/load_ppm.md | 11 +-
docs/reference/load_ranges.html | 2 +-
docs/reference/load_raster.html | 2 +-
docs/reference/load_regional_stats.html | 2 +-
docs/reference/load_trends.html | 2 +-
docs/reference/pipe.html | 2 +-
docs/reference/rasterize_trends.html | 2 +-
docs/reference/set_ebirdst_access_key.html | 2 +-
docs/reference/vectorize_trends.html | 2 +-
docs/search.json | 2 +-
man/load_pi.Rd | 16 +-
man/load_ppm.Rd | 10 +-
tests/testthat/test_fetch.R | 129 +++++
tests/testthat/test_loading.R | 53 ++
68 files changed, 888 insertions(+), 981 deletions(-)
create mode 100644 R/fetch.R
create mode 100644 tests/testthat/test_fetch.R
diff --git a/DESCRIPTION b/DESCRIPTION
index 37e8173..c337d7d 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,7 +1,7 @@
Type: Package
Package: ebirdst
Title: Access and Analyze eBird Status and Trends Data Products
-Version: 4.2023.0
+Version: 4.2023.1
Authors@R: c(
person("Matthew", "Strimas-Mackey", , "mes335@cornell.edu", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-8929-7776")),
diff --git a/NEWS.md b/NEWS.md
index 6bc186e..f2dc48a 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,10 @@
+# ebirdst 4.2023.1
+
+- Backend approach to file download has been refactored to an on-demand first approach
+- `list_available_pis()` no longer downloads every predictor importance raster to determine availability, only `pi_rangewide.csv`
+- The http fallback for VPNs that block https now also applies to file downloads, not just file listings
+- Errors for data that can't be found on-demand now include function-specific guidance, e.g. pointing to `list_available_pis()`
+
# ebirdst 4.2023.0
- Transition to having all the `load_*()` functions download directly rather than having to call `ebirdst_download_status()`
diff --git a/R/download.R b/R/download.R
index 161081a..3234ae7 100644
--- a/R/download.R
+++ b/R/download.R
@@ -118,11 +118,22 @@ ebirdst_download_status <- function(
}
# complete list of all available files for this species
- files <- get_download_file_list(
- species_code = species,
- path = path,
- dataset = "status"
+ keys <- list_object_keys(species_code = species, dataset = "status")
+
+ # decide which files to download
+ keys <- select_status_keys(
+ keys,
+ download_abundance = download_abundance,
+ download_occurrence = download_occurrence,
+ download_count = download_count,
+ download_ranges = download_ranges,
+ download_regional = download_regional,
+ download_pis = download_pis,
+ download_ppms = download_ppms,
+ download_all = download_all,
+ pattern = pattern
)
+
# path to data package
run_path <- file.path(
path,
@@ -130,69 +141,24 @@ ebirdst_download_status <- function(
species
)
- # decide which files to download
- # always download config file
- dl <- stringr::str_detect(files$file, pattern = "config.json$")
- if (download_abundance || download_all) {
- # add abundance
- dl <- stringr::str_detect(files$file, "\\_abundance\\_") | dl
- # add proportion of population
- dl <- stringr::str_detect(files$file, "\\_proportion-population\\_") | dl
- }
- if (download_occurrence || download_all) {
- # add occurrence
- dl <- stringr::str_detect(files$file, "\\_occurrence\\_") | dl
- }
- if (download_count || download_all) {
- # add count
- dl <- stringr::str_detect(files$file, "\\_count\\_") | dl
- }
- if (download_ranges || download_all) {
- # add ranges
- dl <- stringr::str_detect(files$file, "/ranges/") | dl
- }
- if (download_regional || download_all) {
- # add regional summary stats
- dl <- stringr::str_ends(files$file, "regional_stats.csv") | dl
- }
- if (download_pis || download_all) {
- # add pis
- dl <- stringr::str_detect(files$file, "/pis/") | dl
- }
- if (download_ppms || download_all) {
- # add ppms
- dl <- stringr::str_detect(files$file, "/ppms/") | dl
- }
- files <- files[dl, ]
-
- # apply pattern
- if (!is.null(pattern)) {
- stopifnot(is.character(pattern), length(pattern) == 1, !is.na(pattern))
- pat_match <- stringr::str_detect(basename(files$file), pattern = pattern)
- if (!any(pat_match)) {
- stop("No files matched pattern")
- }
-
- # always download config file
- is_config <- stringr::str_detect(
- basename(files$file),
- pattern = "config.json$"
- )
- files <- files[pat_match | is_config, ]
- }
-
# print files to download for dry run
if (dry_run) {
message("Downloading Status Data Products for ", species, " to:\n ", path)
- message(paste(c("File list:", files$file), collapse = "\n "))
- return(invisible(files$file))
+ message(paste(c("File list:", keys), collapse = "\n "))
+ return(invisible(keys))
}
if (show_progress) {
message(stringr::str_glue("Downloading Status Data Products for {species}"))
}
- download_files(files, force = force, show_progress = show_progress)
+ fetch_data(
+ keys,
+ path = path,
+ force = force,
+ show_progress = show_progress,
+ report_existing = TRUE
+ )
return(invisible(normalizePath(run_path)))
}
@@ -267,22 +233,24 @@ ebirdst_download_trends <- function(
run_paths <- character()
for (s in species_code) {
# complete list of all available files for this species
- files <- get_download_file_list(
- species_code = s,
- path = path,
- dataset = "trends"
- )
+ keys <- list_object_keys(species_code = s, dataset = "trends")
+ # only trends files
+ keys <- keys[stringr::str_detect(keys, "/trends/")]
+
# path to data package
run_path <- file.path(path, ebirdst_version()[["trends_version_year"]], s)
- # only trends files
- files <- files[stringr::str_detect(files$file, "/trends/"), ]
-
# download
if (show_progress) {
message(stringr::str_glue("Downloading Trends Data Products for {s}"))
}
- download_files(files, force = force, show_progress = show_progress)
+ fetch_data(
+ keys,
+ path = path,
+ force = force,
+ show_progress = show_progress,
+ report_existing = TRUE
+ )
run_paths <- c(run_paths, run_path)
}
@@ -329,7 +297,7 @@ ebirdst_download_data_coverage <- function(
stopifnot(is_flag(show_progress))
# complete list of all available files for this species
- files <- get_download_file_list(species_code = "data_coverage", path = path)
+ keys <- list_object_keys(species_code = "data_coverage", dataset = "status")
# path to data package
run_path <- file.path(
path,
@@ -340,25 +308,31 @@ ebirdst_download_data_coverage <- function(
# apply pattern
if (!is.null(pattern)) {
stopifnot(is.character(pattern), length(pattern) == 1, !is.na(pattern))
- pat_match <- stringr::str_detect(basename(files$file), pattern = pattern)
+ pat_match <- stringr::str_detect(basename(keys), pattern = pattern)
if (!any(pat_match)) {
stop("No files matched pattern")
}
- files <- files[pat_match, ]
+ keys <- keys[pat_match]
}
# print files to download for dry run
if (dry_run) {
message("Downloading Data Coverage Products to:\n ", path)
- message(paste(c("File list:", files$file), collapse = "\n "))
- return(invisible(files$file))
+ message(paste(c("File list:", keys), collapse = "\n "))
+ return(invisible(keys))
}
if (show_progress) {
message(stringr::str_glue("Downloading Data Coverage Products"))
}
- download_files(files, force = force, show_progress = show_progress)
+ fetch_data(
+ keys,
+ path = path,
+ force = force,
+ show_progress = show_progress,
+ report_existing = TRUE
+ )
return(invisible(normalizePath(run_path)))
}
@@ -459,161 +433,3 @@ ebirdst_version <- function() {
release_year = 2025
)
}
-
-
-# internal ----
-
-get_download_file_list <- function(
- species_code,
- path,
- dataset = c("status", "trends")
-) {
- stopifnot(
- is.character(species_code),
- length(species_code) == 1,
- !is.na(species_code)
- )
- dataset <- match.arg(dataset)
-
- # version of the data products that this package version corresponds to
- version_year <- ebirdst_version()[[paste0(dataset, "_version_year")]]
- # example data or a full data package
- is_example <- (species_code == "yebsap-example")
-
- # path to data package
- run_path <- file.path(path, version_year, species_code)
-
- if (is_example) {
- api_url <- paste0(
- "https://raw.githubusercontent.com/",
- "ebird/ebirdst_example-data/main/",
- "example-data/"
- )
- # file list
- fl <- system.file(
- "extdata",
- paste0("example-data_file-list_", dataset, ".txt"),
- package = "ebirdst"
- )
- files <- readLines(fl)
- } else {
- # api url and key
- key <- get_ebirdst_access_key()
- api_url <- "https://st-download.ebird.org/v1"
-
- # get file list for this species
- list_obj_url <- stringr::str_glue(
- "{api_url}/list-obj/{version_year}/",
- "{species_code}?key={key}"
- )
- files <- tryCatch(
- suppressWarnings({
- jsonlite::read_json(list_obj_url, simplifyVector = TRUE)
- }),
- error = function(e) NULL
- )
- if (is.null(files)) {
- # try http instead in case of ssl issues on vpn
- api_url <- "http://st-download.ebird.org/v1"
- # get file list for this species
- list_obj_url <- stringr::str_glue(
- "{api_url}/list-obj/{version_year}/",
- "{species_code}?key={key}"
- )
- files <- tryCatch(
- suppressWarnings({
- jsonlite::read_json(list_obj_url, simplifyVector = TRUE)
- }),
- error = function(e) NULL
- )
- if (is.null(files)) {
- stop(
- "Cannot access Status and Trends data URL. Ensure that you have ",
- "a working internet connection and a valid API key for the ",
- "Status and Trends data. Note that the API keys expire after ",
- "6 month, so may need to update your key. ",
- "Visit https://ebird.org/st/request"
- )
- }
- }
-
- # remove web_download folder
- web_down <- stringr::str_detect(dirname(files), pattern = "web_download")
- files <- files[!web_down]
-
- # remove additional species cause by bug in API
- # e.g. leafly will also return leafly2
- only_target <- stringr::str_detect(
- files,
- pattern = paste0("/", species_code, "/")
- )
- files <- files[only_target]
- }
-
- if (length(files) == 0) {
- stop("No data found for species ", species_code)
- }
-
- # prepare download paths
- files <- data.frame(file = files)
- if (is_example) {
- files$src_path <- paste0(api_url, files$file)
- } else {
- files$src_path <- stringr::str_glue(
- "{api_url}/fetch?objKey={files$file}",
- "&key={key}"
- )
- }
- files$dest_path <- file.path(path, files$file)
- files$exists <- file.exists(files$dest_path)
-
- return(files)
-}
-
-download_files <- function(files, force, show_progress) {
- # create necessary directories
- dirs <- unique(dirname(files$dest_path))
- for (d in dirs) {
- dir.create(d, showWarnings = FALSE, recursive = TRUE)
- }
-
- # check if already exists
- if (all(files$exists)) {
- if (!isTRUE(force)) {
- message("Data already exists, use force = TRUE to re-download.")
- return(invisible(0L))
- }
- } else if (any(files$exists)) {
- if (!isTRUE(force)) {
- message(paste(
- "Some files already exist, only downloading new files.",
- " Use force = TRUE to re-download all files."
- ))
- files <- files[!files$exists, ]
- }
- }
-
- # download
- n_files <- nrow(files)
- old_timeout <- getOption("timeout")
- options(timeout = max(3000, old_timeout))
- for (i in seq_len(n_files)) {
- if (show_progress) {
- message(stringr::str_glue(
- " Downloading file {i} of {n_files}: ",
- "{basename(files$file[i])}"
- ))
- }
- dl_response <- utils::download.file(
- files$src_path[i],
- files$dest_path[i],
- quiet = TRUE,
- mode = "wb"
- )
- if (dl_response != 0) {
- stop("Error downloading file: ", files$file[i])
- }
- }
- options(timeout = old_timeout)
- return(invisible(n_files))
-}
diff --git a/R/fetch.R b/R/fetch.R
new file mode 100644
index 0000000..4d11795
--- /dev/null
+++ b/R/fetch.R
@@ -0,0 +1,359 @@
+# this file contains the internal machinery shared by the ebirdst_download_*()
+# functions in download.R and the on-demand downloads performed by the
+# load_*() functions in load.R. the local path for a downloaded file is always
+# its object key (e.g. "2023/woothr/config.json") appended to the data
+# directory, and the API can fetch a single object directly by key, so
+# fetch_data() is the one function anything in the package needs to call to
+# make sure a set of files exist locally
+
+# internal ----
+
+# session-cached API base url; some VPNs block https to the download API, so
+# a fallback to http is cached here once discovered so it isn't re-probed on
+# every request
+ebirdst_env <- new.env(parent = emptyenv())
+ebirdst_env$api_base_url <- "https://st-download.ebird.org/v1"
+
+api_base_url <- function() {
+ return(ebirdst_env$api_base_url)
+}
+
+use_http_fallback <- function() {
+ ebirdst_env$api_base_url <- sub(
+ "^https://",
+ "http://",
+ ebirdst_env$api_base_url
+ )
+ return(invisible(ebirdst_env$api_base_url))
+}
+
+
+# resolve a species name/code to its eBird species code; mirrors the
+# validation in get_species_path() but doesn't require path to already exist
+resolve_species <- function(species) {
+ species_code <- get_species(species)
+ if (anyNA(species_code)) {
+ stop(
+ paste(species[is.na(species_code)], collapse = ", "),
+ " does not correspond to a valid Status and Trends species."
+ )
+ }
+ return(species_code)
+}
+
+
+# create the data directory if it doesn't already exist
+ensure_data_dir <- function(path) {
+ if (!dir.exists(path)) {
+ created <- dir.create(path, recursive = TRUE, showWarnings = FALSE)
+ if (!isTRUE(created)) {
+ stop("Unable to create data directory: ", path)
+ }
+ }
+ return(invisible(path))
+}
+
+
+# build object keys within the status or trends data package for a species;
+# vectorizes over the last argument, e.g. status_key("woothr", "weekly", files)
+status_key <- function(species_code, ...) {
+ version_year <- ebirdst_version()[["status_version_year"]]
+ return(paste(version_year, species_code, ..., sep = "/"))
+}
+
+trends_key <- function(species_code, ...) {
+ version_year <- ebirdst_version()[["trends_version_year"]]
+ return(paste(version_year, species_code, ..., sep = "/"))
+}
+
+
+# list all object keys available for a species, for callers that don't
+# already know the exact key(s) they want: flag/pattern-based selection in
+# ebirdst_download_status()/ebirdst_download_trends(), and PI availability in
+# list_available_pis()
+list_object_keys <- function(species_code, dataset = c("status", "trends")) {
+ stopifnot(
+ is.character(species_code),
+ length(species_code) == 1,
+ !is.na(species_code)
+ )
+ dataset <- match.arg(dataset)
+
+ version_year <- ebirdst_version()[[paste0(dataset, "_version_year")]]
+ is_example <- (species_code == "yebsap-example")
+
+ if (is_example) {
+ fl <- system.file(
+ "extdata",
+ paste0("example-data_file-list_", dataset, ".txt"),
+ package = "ebirdst"
+ )
+ keys <- readLines(fl)
+ } else {
+ key <- get_ebirdst_access_key()
+ list_obj_url <- stringr::str_glue(
+ "{api_base_url()}/list-obj/{version_year}/",
+ "{species_code}?key={key}"
+ )
+ keys <- tryCatch(
+ suppressWarnings({
+ jsonlite::read_json(list_obj_url, simplifyVector = TRUE)
+ }),
+ error = function(e) NULL
+ )
+ if (is.null(keys)) {
+ # try http instead in case of ssl issues on vpn
+ use_http_fallback()
+ list_obj_url <- stringr::str_glue(
+ "{api_base_url()}/list-obj/{version_year}/",
+ "{species_code}?key={key}"
+ )
+ keys <- tryCatch(
+ suppressWarnings({
+ jsonlite::read_json(list_obj_url, simplifyVector = TRUE)
+ }),
+ error = function(e) NULL
+ )
+ if (is.null(keys)) {
+ stop(
+ "Cannot access Status and Trends data URL. Ensure that you have ",
+ "a working internet connection and a valid API key for the ",
+ "Status and Trends data. Note that the API keys expire after ",
+ "6 month, so may need to update your key. ",
+ "Visit https://ebird.org/st/request"
+ )
+ }
+ }
+
+ # remove web_download folder
+ web_down <- stringr::str_detect(dirname(keys), pattern = "web_download")
+ keys <- keys[!web_down]
+
+ # remove additional species caused by bug in API, e.g. leafly will also
+ # return leafly2
+ only_target <- stringr::str_detect(
+ keys,
+ pattern = paste0("/", species_code, "/")
+ )
+ keys <- keys[only_target]
+ }
+
+ if (length(keys) == 0) {
+ stop("No data found for species ", species_code)
+ }
+
+ return(keys)
+}
+
+
+# select which object keys should be downloaded based on the download_* flags
+# and an optional filename pattern; the selection logic used by
+# ebirdst_download_status()
+select_status_keys <- function(
+ keys,
+ download_abundance = TRUE,
+ download_occurrence = FALSE,
+ download_count = FALSE,
+ download_ranges = FALSE,
+ download_regional = FALSE,
+ download_pis = FALSE,
+ download_ppms = FALSE,
+ download_all = FALSE,
+ pattern = NULL
+) {
+ # always download config file
+ dl <- stringr::str_detect(keys, pattern = "config.json$")
+ if (download_abundance || download_all) {
+ # add abundance
+ dl <- stringr::str_detect(keys, "\\_abundance\\_") | dl
+ # add proportion of population
+ dl <- stringr::str_detect(keys, "\\_proportion-population\\_") | dl
+ }
+ if (download_occurrence || download_all) {
+ # add occurrence
+ dl <- stringr::str_detect(keys, "\\_occurrence\\_") | dl
+ }
+ if (download_count || download_all) {
+ # add count
+ dl <- stringr::str_detect(keys, "\\_count\\_") | dl
+ }
+ if (download_ranges || download_all) {
+ # add ranges
+ dl <- stringr::str_detect(keys, "/ranges/") | dl
+ }
+ if (download_regional || download_all) {
+ # add regional summary stats
+ dl <- stringr::str_ends(keys, "regional_stats.csv") | dl
+ }
+ if (download_pis || download_all) {
+ # add pis
+ dl <- stringr::str_detect(keys, "/pis/") | dl
+ }
+ if (download_ppms || download_all) {
+ # add ppms
+ dl <- stringr::str_detect(keys, "/ppms/") | dl
+ }
+ keys <- keys[dl]
+
+ # apply pattern
+ if (!is.null(pattern)) {
+ stopifnot(is.character(pattern), length(pattern) == 1, !is.na(pattern))
+ pat_match <- stringr::str_detect(basename(keys), pattern = pattern)
+ if (!any(pat_match)) {
+ stop("No files matched pattern")
+ }
+
+ # always download config file
+ is_config <- stringr::str_detect(basename(keys), pattern = "config.json$")
+ keys <- keys[pat_match | is_config]
+ }
+
+ return(keys)
+}
+
+
+# build the source download url for a set of object keys
+object_key_url <- function(keys) {
+ is_example <- stringr::str_detect(keys, "yebsap-example")
+ urls <- character(length(keys))
+
+ if (any(is_example)) {
+ example_url <- paste0(
+ "https://raw.githubusercontent.com/",
+ "ebird/ebirdst_example-data/main/",
+ "example-data/"
+ )
+ urls[is_example] <- paste0(example_url, keys[is_example])
+ }
+ if (!all(is_example)) {
+ key <- get_ebirdst_access_key()
+ urls[!is_example] <- stringr::str_glue(
+ "{api_base_url()}/fetch?objKey={keys[!is_example]}",
+ "&key={key}"
+ )
+ }
+
+ return(urls)
+}
+
+
+# ensure the local files for a set of object keys exist, downloading any that
+# are missing (or all of them, if force = TRUE); returns the normalized local
+# paths. every download in the package funnels through here. `hint` is
+# appended to the error raised if a requested key can't be found, and
+# `report_existing` controls whether "already downloaded" messages are shown
+# (used by the ebirdst_download_*() functions, but not by on-demand loads,
+# which should stay silent when the requested data is already cached)
+fetch_data <- function(
+ keys,
+ path,
+ force = FALSE,
+ show_progress = interactive(),
+ hint = NULL,
+ report_existing = FALSE
+) {
+ ensure_data_dir(path)
+ dest_paths <- file.path(path, keys)
+ exists <- file.exists(dest_paths)
+
+ if (!isTRUE(force) && all(exists)) {
+ if (report_existing) {
+ message("Data already exists, use force = TRUE to re-download.")
+ }
+ return(invisible(normalizePath(dest_paths)))
+ }
+ if (!isTRUE(force) && any(exists) && report_existing) {
+ message(
+ "Some files already exist, only downloading new files. ",
+ "Use force = TRUE to re-download all files."
+ )
+ }
+
+ to_fetch <- if (isTRUE(force)) keys else keys[!exists]
+ fetch_dest <- file.path(path, to_fetch)
+
+ # create necessary directories
+ dirs <- unique(dirname(fetch_dest))
+ for (d in dirs) {
+ dir.create(d, showWarnings = FALSE, recursive = TRUE)
+ }
+
+ download_files(
+ object_key_url(to_fetch),
+ fetch_dest,
+ to_fetch,
+ show_progress = show_progress
+ )
+
+ missing <- keys[!file.exists(dest_paths)]
+ if (length(missing) > 0) {
+ msg <- paste0(
+ "The requested data could not be found:\n ",
+ paste(missing, collapse = "\n ")
+ )
+ if (!is.null(hint)) {
+ stop(msg, "\n", hint)
+ }
+ stop(msg)
+ }
+
+ return(invisible(normalizePath(dest_paths)))
+}
+
+
+# download files from src urls to local destination paths; on failure, retry
+# once over http in case https is being blocked (e.g. by a VPN), caching the
+# fallback for the rest of the session if it succeeds. a file that still
+# can't be downloaded after the retry is simply left missing on disk, so
+# fetch_data() can report it (with its caller-specific hint) rather than
+# failing here with a generic message. `keys` is used only to report progress
+download_files <- function(src, dest, keys, show_progress) {
+ n_files <- length(src)
+ old_timeout <- getOption("timeout")
+ options(timeout = max(3000, old_timeout))
+ on.exit(options(timeout = old_timeout), add = TRUE)
+
+ for (i in seq_len(n_files)) {
+ if (show_progress) {
+ message(stringr::str_glue(
+ " Downloading file {i} of {n_files}: ",
+ "{basename(keys[i])}"
+ ))
+ }
+ dl_response <- tryCatch(
+ suppressWarnings(
+ utils::download.file(src[i], dest[i], quiet = TRUE, mode = "wb")
+ ),
+ error = function(e) 1L
+ )
+ if (
+ dl_response != 0 && stringr::str_starts(src[i], "https://st-download")
+ ) {
+ use_http_fallback()
+ src[i] <- sub("^https://", "http://", src[i])
+ tryCatch(
+ suppressWarnings(
+ utils::download.file(src[i], dest[i], quiet = TRUE, mode = "wb")
+ ),
+ error = function(e) 1L
+ )
+ }
+ }
+
+ return(invisible(n_files))
+}
+
+
+# check that the geotiff driver is installed; required to load any of the
+# raster data products
+check_gtiff_support <- function() {
+ drv <- terra::gdal(drivers = TRUE)
+ drv <- drv$name[stringr::str_detect(drv$can, "read")]
+ if (!"GTiff" %in% drv) {
+ stop(
+ "GDAL does not have GeoTIFF support. GeoTIFF support is required to ",
+ "load Status and Trends raster data."
+ )
+ }
+ return(invisible(TRUE))
+}
diff --git a/R/load.R b/R/load.R
index 1b93299..a88d25a 100644
--- a/R/load.R
+++ b/R/load.R
@@ -110,28 +110,9 @@ load_raster <- function(
period <- match.arg(period)
resolution <- match.arg(resolution)
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
- species_code <- get_species(species)
- species_path <- get_species_path(
- species,
- path = path,
- dataset = "status",
- check_downloaded = FALSE
- )
+ check_gtiff_support()
- # check that the geotiff driver is installed
- drv <- terra::gdal(drivers = TRUE)
- drv <- drv$name[stringr::str_detect(drv$can, "read")]
- if (!"GTiff" %in% drv) {
- stop(
- "GDAL does not have GeoTIFF support. GeoTIFF support is required to ",
- "load Status and Trends raster data."
- )
- }
+ species_code <- resolve_species(species)
# load config file, downloading it on demand if necessary
p <- load_config(
@@ -156,7 +137,7 @@ load_raster <- function(
)
}
- # construct file name and path
+ # construct file name and key
if (period == "weekly") {
# assess which metric is being requested
if (is.null(metric)) {
@@ -180,7 +161,7 @@ load_raster <- function(
"{species_code}_{product}_{metric}",
"_{resolution}_{v}.tif"
)
- file <- file.path(species_path, "weekly", file)
+ key <- status_key(species_code, "weekly", file)
} else {
# assess which metric is being requested
if (is.null(metric)) {
@@ -195,38 +176,19 @@ load_raster <- function(
"{species_code}_{product}_{period}_{metric}",
"_{resolution}_{v}.tif"
)
- file <- file.path(species_path, "seasonal", file)
+ key <- status_key(species_code, "seasonal", file)
}
# download the requested product on demand if it isn't already present
- status_dl_flag <- switch(
- product,
- "abundance" = "download_abundance",
- "proportion-population" = "download_abundance",
- "count" = "download_count",
- "occurrence" = "download_occurrence"
- )
- fetch_if_missing(
- target = file,
+ local_file <- fetch_data(
+ key,
+ path = path,
force = force,
- downloader = function() {
- dl_args <- list(
- species = species_code,
- path = path,
- pattern = stringr::str_escape(basename(file)),
- force = force,
- show_progress = show_progress
- )
- dl_args[[status_dl_flag]] <- TRUE
- do.call(ebirdst_download_status, dl_args)
- }
+ show_progress = show_progress
)
- if (!file.exists(file)) {
- stop("The file for the requested product does not exist: \n ", file)
- }
# load and return raster stack
- return(terra::rast(file))
+ return(terra::rast(local_file))
}
@@ -322,11 +284,6 @@ load_trends <- function(
stopifnot(is_flag(fold_estimates))
stopifnot(is_flag(force), is_flag(show_progress))
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
v <- ebirdst_version()[["trends_version_year"]]
# trends species and seaons
@@ -343,15 +300,9 @@ load_trends <- function(
)
}
- # get paths to trends parquet files
+ # construct keys for trends parquet files
trends_paths <- character()
for (i in seq_along(species_code)) {
- p <- get_species_path(
- species_code[i],
- path = path,
- dataset = "trends",
- check_downloaded = FALSE
- )
if (fold_estimates) {
f <- stringr::str_glue(
"{species_code[i]}_{season[i]}_ebird-trends_",
@@ -363,31 +314,25 @@ load_trends <- function(
"{v}.parquet"
)
}
- trends_paths <- c(trends_paths, file.path(p, "trends", f))
+ trends_paths <- c(
+ trends_paths,
+ file.path(path, trends_key(species_code[i], "trends", f))
+ )
}
# download trends data on demand for any species not already present
- fetch_if_missing(
- target = trends_paths,
- force = force,
- downloader = function() {
- to_download <- if (isTRUE(force)) {
- species_code
- } else {
- species_code[!file.exists(trends_paths)]
- }
- ebirdst_download_trends(
- to_download,
- path = path,
- force = force,
- show_progress = show_progress
- )
- }
- )
- if (!all(file.exists(trends_paths))) {
- stop(
- "Trends data could not be found for the following species:\n ",
- paste(species[!file.exists(trends_paths)], collapse = ", ")
+ ensure_data_dir(path)
+ if (isTRUE(force)) {
+ missing <- species_code
+ } else {
+ missing <- species_code[!file.exists(trends_paths)]
+ }
+ if (length(missing) > 0) {
+ ebirdst_download_trends(
+ missing,
+ path = path,
+ force = force,
+ show_progress = show_progress
)
}
@@ -455,27 +400,7 @@ load_data_coverage <- function(
stopifnot(is.character(path), length(path) == 1)
stopifnot(is_flag(force), is_flag(show_progress))
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
- dc_path <- get_species_path(
- "data_coverage",
- path = path,
- dataset = "status",
- check_downloaded = FALSE
- )
-
- # check that the geotiff driver is installed
- drv <- terra::gdal(drivers = TRUE)
- drv <- drv$name[stringr::str_detect(drv$can, "read")]
- if (!"GTiff" %in% drv) {
- stop(
- "GDAL does not have GeoTIFF support. GeoTIFF support is required to ",
- "load Status and Trends raster data."
- )
- }
+ check_gtiff_support()
# generate vector of valid weeks
valid_weeks <- as.Date(paste(2018, seq(4, 366, 7)), format = "%Y %j")
@@ -502,35 +427,17 @@ load_data_coverage <- function(
# construct filenames
product <- paste0(product, "_mean")
files <- stringr::str_glue("{product}_{valid_weeks}.tif")
- files <- file.path(dc_path, product, files)
# download the requested weeks on demand if they aren't already present
- fetch_if_missing(
- target = files,
+ local_files <- fetch_data(
+ status_key("data_coverage", product, files),
+ path = path,
force = force,
- downloader = function() {
- to_download <- if (isTRUE(force)) files else files[!file.exists(files)]
- pattern <- paste(
- stringr::str_escape(basename(to_download)),
- collapse = "|"
- )
- ebirdst_download_data_coverage(
- path = path,
- pattern = pattern,
- force = force,
- show_progress = show_progress
- )
- }
+ show_progress = show_progress
)
- if (!all(file.exists(files))) {
- stop(
- "The files for the requested product could not be found:\n ",
- paste(basename(files[!file.exists(files)]), collapse = "\n ")
- )
- }
# load and return raster stack
- return(stats::setNames(terra::rast(files), valid_weeks))
+ return(stats::setNames(terra::rast(local_files), valid_weeks))
}
@@ -573,18 +480,7 @@ load_ranges <- function(
stopifnot(is_flag(force), is_flag(show_progress))
resolution <- match.arg(resolution)
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
- species_code <- get_species(species)
- species_path <- get_species_path(
- species,
- path = path,
- dataset = "status",
- check_downloaded = FALSE
- )
+ species_code <- resolve_species(species)
# load config file, downloading it on demand if necessary
p <- load_config(
@@ -607,29 +503,17 @@ load_ranges <- function(
"{species_code}_range_{label}",
"_{resolution}_{v}.gpkg"
)
- file <- file.path(species_path, "ranges", file)
# download the ranges on demand if they aren't already present
- fetch_if_missing(
- target = file,
+ local_file <- fetch_data(
+ status_key(species_code, "ranges", file),
+ path = path,
force = force,
- downloader = function() {
- ebirdst_download_status(
- species_code,
- path = path,
- download_ranges = TRUE,
- pattern = stringr::str_escape(basename(file)),
- force = force,
- show_progress = show_progress
- )
- }
+ show_progress = show_progress
)
- if (!file.exists(file)) {
- stop("The file for the requested product does not exist: \n ", file)
- }
# load polygons
- p <- sf::read_sf(dsn = file, layer = "range")
+ p <- sf::read_sf(dsn = local_file, layer = "range")
return(p)
}
@@ -689,38 +573,16 @@ load_regional_stats <- function(
stopifnot(is.character(path), length(path) == 1)
stopifnot(is_flag(force), is_flag(show_progress))
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
- species_code <- get_species(species)
- species_path <- get_species_path(
- species,
- path = path,
- dataset = "status",
- check_downloaded = FALSE
- )
+ species_code <- resolve_species(species)
# download the regional stats on demand if they aren't already present
- file <- file.path(species_path, "regional_stats.csv")
- fetch_if_missing(
- target = file,
+ file <- fetch_data(
+ status_key(species_code, "regional_stats.csv"),
+ path = path,
force = force,
- downloader = function() {
- ebirdst_download_status(
- species_code,
- path = path,
- download_regional = TRUE,
- pattern = "regional_stats.csv",
- force = force,
- show_progress = show_progress
- )
- }
+ show_progress = show_progress
)
- if (!file.exists(file)) {
- stop("The regional summary stats file could not be found for this species.")
- }
+
# load stats
stats <- dplyr::as_tibble(utils::read.csv(file, na = "", row.names = NULL))
stats[["region_area_km2"]] <- NULL
@@ -763,38 +625,21 @@ ebirdst_regional_stats <- function(
stopifnot(is_flag(force))
stopifnot(is_flag(show_progress))
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
# the regional stats file is stored at the annual results level, named for
# the status data version year
version_year <- ebirdst_version()[["status_version_year"]]
- obj_key <- file.path(
+ key <- file.path(
version_year,
sprintf("regional-stats_%s.parquet", version_year)
)
- dest_path <- file.path(path, obj_key)
# download the file on demand if it isn't already present
- if (!file.exists(dest_path) || force) {
- if (show_progress) {
- message("Downloading regional stats for all species")
- }
-
- # build the fetch url and download using the shared download machinery
- key <- get_ebirdst_access_key()
- api_url <- "https://st-download.ebird.org/v1"
- files <- data.frame(file = obj_key)
- files$src_path <- stringr::str_glue(
- "{api_url}/fetch?objKey={obj_key}",
- "&key={key}"
- )
- files$dest_path <- dest_path
- files$exists <- file.exists(dest_path)
- download_files(files, force = force, show_progress = show_progress)
- }
+ dest_path <- fetch_data(
+ key,
+ path = path,
+ force = force,
+ show_progress = show_progress
+ )
# load stats
stats <- dplyr::as_tibble(arrow::read_parquet(dest_path))
@@ -831,39 +676,16 @@ load_config <- function(
stopifnot(is.character(path), length(path) == 1)
stopifnot(is_flag(force), is_flag(show_progress))
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
+ species_code <- resolve_species(species)
- species_code <- get_species(species)
- species_path <- get_species_path(
- species,
+ # download the config file on demand if it isn't already present
+ cfg_file <- fetch_data(
+ status_key(species_code, "config.json"),
path = path,
- dataset = "status",
- check_downloaded = FALSE
- )
-
- # download the config file on demand if it isn't already present; passing
- # download_abundance = FALSE with no other product selected downloads only
- # config.json
- cfg_file <- file.path(species_path, "config.json")
- fetch_if_missing(
- target = cfg_file,
force = force,
- downloader = function() {
- ebirdst_download_status(
- species_code,
- path = path,
- download_abundance = FALSE,
- force = force,
- show_progress = show_progress
- )
- }
+ show_progress = show_progress
)
- if (!file.exists(cfg_file)) {
- stop("The file 'config.json' does not exist in: ", species_path)
- }
+
# load configuration file
p <- jsonlite::read_json(cfg_file, simplifyVector = TRUE)
names(p) <- tolower(names(p))
@@ -946,12 +768,12 @@ load_fac_map_parameters <- function(
#' a rank of 1 being the most important) relative to the full suite of
#' environmental predictors. The ranks are summarized to a 27 km resolution
#' raster grid for each predictor, where the cell values are the average across
-#' all models in the ensemble contributing to that cell. These data are
-#' available in raster format provided `download_pis = TRUE` was used when
-#' calling [ebirdst_download_status()]. PI estimates are available separately
-#' for both the occurrence and count sub-model and only the 30 most important
-#' predictors are distributed. Use [list_available_pis()] to see which
-#' predictors have PI data.
+#' all models in the ensemble contributing to that cell. If the requested data
+#' have not already been downloaded, they will be downloaded automatically on
+#' first use. PI estimates are available separately for both the occurrence
+#' and count sub-model and only the 30 most important predictors are
+#' distributed. Use [list_available_pis()] to see which predictors have PI
+#' data.
#'
#' @inheritParams load_raster
#' @param predictor character; the predictor that the PI data should be loaded
@@ -975,10 +797,8 @@ load_fac_map_parameters <- function(
#'
#' @examples
#' \dontrun{
-#' # download example data if hasn't already been downloaded
-#' ebirdst_download_status("yebsap-example", download_pis = TRUE)
-#'
#' # identify the top predictor
+#' # data will be downloaded automatically if not already present
#' top_preds <- list_available_pis("yebsap-example")
#' print(top_preds[1, ])
#'
@@ -998,55 +818,28 @@ load_pi <- function(
stopifnot(is_flag(force), is_flag(show_progress))
response <- match.arg(response)
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
- species_code <- get_species(species)
- species_path <- get_species_path(
- species,
- path = path,
- dataset = "status",
- check_downloaded = FALSE
- )
+ species_code <- resolve_species(species)
# construct file name; load_config() downloads config on demand and provides
# the data version year
year <- load_config(
- species = species,
+ species = species_code,
path = path,
force = force,
show_progress = show_progress
)[["srd_pred_year"]]
p <- stringr::str_replace_all(predictor, "_", "-")
tif <- stringr::str_glue("{species_code}_pi_{response}_{p}_27km_{year}.tif")
- tif <- file.path(species_path, "pis", tif)
# download the requested PI raster on demand if it isn't already present
- fetch_if_missing(
- target = tif,
+ local_tif <- fetch_data(
+ status_key(species_code, "pis", tif),
+ path = path,
force = force,
- downloader = function() {
- ebirdst_download_status(
- species_code,
- path = path,
- download_pis = TRUE,
- pattern = stringr::str_escape(basename(tif)),
- force = force,
- show_progress = show_progress
- )
- }
+ show_progress = show_progress,
+ hint = "To list predictors that have PI data use list_available_pis()."
)
- if (!file.exists(tif)) {
- stop(
- "GeoTIFF for ",
- predictor,
- " PI could not be found. To list predictors that have PI data use ",
- "list_available_pis()."
- )
- }
- return(terra::rast(tif))
+ return(terra::rast(local_tif))
}
@@ -1063,47 +856,20 @@ list_available_pis <- function(
stopifnot(is.character(path), length(path) == 1)
stopifnot(is_flag(force), is_flag(show_progress))
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
+ species_code <- resolve_species(species)
- species_code <- get_species(species)
- species_path <- get_species_path(
- species,
+ # download the pi rank csv on demand if it isn't already present; this does
+ # not require downloading any of the pi rasters themselves
+ csv_file <- fetch_data(
+ status_key(species_code, "pis", "pi_rangewide.csv"),
path = path,
- check_downloaded = FALSE
- )
-
- # download the PI data on demand if not already present; the full set of PI
- # files is needed to list the available predictors
- csv_file <- file.path(species_path, "pis", "pi_rangewide.csv")
- fetch_if_missing(
- target = csv_file,
force = force,
- downloader = function() {
- ebirdst_download_status(
- species_code,
- path = path,
- download_abundance = FALSE,
- download_pis = TRUE,
- force = force,
- show_progress = show_progress
- )
- }
+ show_progress = show_progress
)
- if (!file.exists(csv_file)) {
- stop("The PI data could not be found for this species.")
- }
- # load ranks
ranks <- utils::read.csv(csv_file, row.names = NULL, na = "")
- # available pis
- tifs <- list.files(file.path(species_path, "pis"), pattern = "*.tif")
- tifs <- tifs[!stringr::str_detect(tifs, "n-folds")]
- preds <- stringr::str_remove(tifs, "^[^_]+_pi_(occurrence|count)_")
- preds <- stringr::str_extract(preds, "[-a-z0-9]+")
- preds <- unique(stringr::str_replace_all(preds, "-", "_"))
+ # identify which of the ranked predictors have pi rasters available
+ preds <- available_pi_predictors(species_code, path = path)
preds <- preds[preds %in% ranks$predictor]
# return ranks
@@ -1120,9 +886,9 @@ list_available_pis <- function(
#' during model training and a suite of predictive performance metrics (PPMs)
#' are calculated. The PPMs for each base model are summarized to a 27 km
#' resolution raster grid, where the cell values are the average across all
-#' models in the ensemble contributing to that cell. These data are available in
-#' raster format provided `download_ppms = TRUE` was used when calling
-#' [ebirdst_download_status()].
+#' models in the ensemble contributing to that cell. If the requested data have
+#' not already been downloaded, they will be downloaded automatically on first
+#' use.
#'
#' @inheritParams load_raster
#' @param ppm character; the name of a single metric to load data for. See
@@ -1194,10 +960,8 @@ list_available_pis <- function(
#'
#' @examples
#' \dontrun{
-#' # download example data if hasn't already been downloaded
-#' ebirdst_download_status("yebsap-example", download_ppms = TRUE)
-#'
#' # load area under the precision-recall curve PPM raster
+#' # data will be downloaded automatically if not already present
#' load_ppm("yebsap-example", ppm = "binary_pr_auc")
#' }
load_ppm <- function(
@@ -1232,64 +996,57 @@ load_ppm <- function(
stopifnot(is_flag(force), is_flag(show_progress))
ppm <- match.arg(ppm)
- # create the data directory if needed so data can be downloaded on demand
- if (!dir.exists(path)) {
- dir.create(path, recursive = TRUE, showWarnings = FALSE)
- }
-
- species_code <- get_species(species)
- species_path <- get_species_path(
- species,
- path = path,
- dataset = "status",
- check_downloaded = FALSE
- )
+ species_code <- resolve_species(species)
# construct file name; load_config() downloads config on demand and provides
# the data version year
year <- load_config(
- species = species,
+ species = species_code,
path = path,
force = force,
show_progress = show_progress
)[["srd_pred_year"]]
p <- stringr::str_replace_all(ppm, "_", "-")
tif <- stringr::str_glue("{species_code}_ppm_{p}_mean_27km_{year}.tif")
- tif <- file.path(species_path, "ppms", tif)
# download on demand if the file isn't already present
- fetch_if_missing(
- target = tif,
+ local_tif <- fetch_data(
+ status_key(species_code, "ppms", tif),
+ path = path,
force = force,
- downloader = function() {
- ebirdst_download_status(
- species_code,
- path = path,
- download_ppms = TRUE,
- pattern = stringr::str_escape(basename(tif)),
- force = force,
- show_progress = show_progress
- )
- }
+ show_progress = show_progress,
+ hint = "GeoTIFF for this PPM could not be found for this species."
)
- if (!file.exists(tif)) {
- stop("GeoTIFF for ", ppm, " PPM could not be found for this species.")
- }
- return(terra::rast(tif))
+ return(terra::rast(local_tif))
}
# internal ----
-# download a data product on demand when its file(s) are not already present,
-# so that load_*() functions fetch missing data transparently instead of
-# erroring. `target` is one or more file paths, `downloader` is a zero-argument
-# function that downloads the missing data. returns TRUE if a download was
-# attempted
-fetch_if_missing <- function(target, downloader, force = FALSE) {
- if (!isTRUE(force) && all(file.exists(target))) {
- return(invisible(FALSE))
+# identify which predictors have pi rasters available for a species. prefers
+# a single remote listing call, which requires no downloads, and falls back
+# to globbing any pi tifs already downloaded locally if the listing can't be
+# reached (e.g. offline). filtering on "_pi_(occurrence|count)_" excludes the
+# other tifs that live alongside the pi rasters in the pis/ directory, e.g.
+# n-folds-modeled, start_day_of_year, end_day_of_year
+available_pi_predictors <- function(species_code, path) {
+ pi_pattern <- "_pi_(occurrence|count)_"
+
+ tifs <- tryCatch(
+ {
+ keys <- list_object_keys(species_code, dataset = "status")
+ keys <- keys[stringr::str_detect(keys, "/pis/")]
+ basename(keys[stringr::str_detect(basename(keys), pi_pattern)])
+ },
+ error = function(e) NULL
+ )
+ if (is.null(tifs)) {
+ pis_path <- file.path(path, status_key(species_code, "pis"))
+ tifs <- list.files(pis_path, pattern = paste0(pi_pattern, ".*\\.tif$"))
}
- downloader()
- return(invisible(TRUE))
+
+ preds <- stringr::str_remove(tifs, paste0("^[^_]+", pi_pattern))
+ preds <- stringr::str_extract(preds, "[-a-z0-9]+")
+ preds <- unique(stringr::str_replace_all(preds, "-", "_"))
+ return(preds)
}
diff --git a/cran-comments.md b/cran-comments.md
index 9e3b824..ebcb79a 100644
--- a/cran-comments.md
+++ b/cran-comments.md
@@ -1,12 +1,9 @@
-# ebirdst 4.2023.0
+# ebirdst 4.2023.1
-- Transition to having all the `load_*()` functions download directly rather than having to call `ebirdst_download_status()`
-- Converted vignettes to Quarto and moved them to website-only pkgdown articles; the package no longer ships built-in vignettes to CRAN (documentation lives at )
-- Add `ebirdst_regional_stats()` to load regional summary statistics for all species
-- Add `ebirdst_data_inventory()` and `ebirdst_delete()` to manage files downloaded by `ebirdst`
-- Move to air auto-formatting and jarl linting
-- Efficiency improvements for `grid_sample()`
-- `grid_sample_stratified()` gains a `cell_quantile_cap` argument to limit how many observations a single chronically over-sampled site (e.g. a bird feeder) can contribute
+- Backend approach to file download has been refactored to an on-demand first approach
+- `list_available_pis()` no longer downloads every predictor importance raster to determine availability, only `pi_rangewide.csv`
+- The http fallback for VPNs that block https now also applies to file downloads, not just file listings
+- Errors for data that can't be found on-demand now include function-specific guidance, e.g. pointing to `list_available_pis()`
## Test environments
@@ -20,7 +17,7 @@
0 errors | 0 warnings | 1 notes
-- NOTE: Version contains large components (4.2023.0). We've aligned our version numbers with the version numbers for the API that this package interacts with. The eBird Status and Trends data products are given a version corresponding to a year, with the current version being 2022, so we've included that year in our version number to indicate that this package only works with the 2023 version of the data.
+- NOTE: Version contains large components (4.2023.1). We've aligned our version numbers with the version numbers for the API that this package interacts with. The eBird Status and Trends data products are given a version corresponding to a year, with the current version being 2022, so we've included that year in our version number to indicate that this package only works with the 2023 version of the data.
## revdepcheck results
diff --git a/docs/404.html b/docs/404.html
index ea3321f..00c49b4 100644
--- a/docs/404.html
+++ b/docs/404.html
@@ -20,7 +20,7 @@
ebirdst
- 4.2023.0
+ 4.2023.1
+all models in the ensemble contributing to that cell. If the requested data
+have not already been downloaded, they will be downloaded automatically on
+first use. PI estimates are available separately for both the occurrence
+and count sub-model and only the 30 most important predictors are
+distributed. Use list_available_pis() to see which predictors have PI
+data.
if(FALSE){# \dontrun{
-# download example data if hasn't already been downloaded
-ebirdst_download_status("yebsap-example", download_pis =TRUE)
-# identify the top predictor
+# data will be downloaded automatically if not already presenttop_preds<-list_available_pis("yebsap-example")print(top_preds[1, ])
diff --git a/docs/reference/load_pi.md b/docs/reference/load_pi.md
index 13a6d0a..0de88f8 100644
--- a/docs/reference/load_pi.md
+++ b/docs/reference/load_pi.md
@@ -7,11 +7,10 @@ converted to ranks (with a rank of 1 being the most important) relative
to the full suite of environmental predictors. The ranks are summarized
to a 27 km resolution raster grid for each predictor, where the cell
values are the average across all models in the ensemble contributing to
-that cell. These data are available in raster format provided
-`download_pis = TRUE` was used when calling
-[`ebirdst_download_status()`](https://ebird.github.io/ebirdst/reference/ebirdst_download_status.md).
-PI estimates are available separately for both the occurrence and count
-sub-model and only the 30 most important predictors are distributed. Use
+that cell. If the requested data have not already been downloaded, they
+will be downloaded automatically on first use. PI estimates are
+available separately for both the occurrence and count sub-model and
+only the 30 most important predictors are distributed. Use
`list_available_pis()` to see which predictors have PI data.
## Usage
@@ -102,10 +101,8 @@ as the integer rank (`rank`) relative to the full suite of predictors
``` r
if (FALSE) { # \dontrun{
-# download example data if hasn't already been downloaded
-ebirdst_download_status("yebsap-example", download_pis = TRUE)
-
# identify the top predictor
+# data will be downloaded automatically if not already present
top_preds <- list_available_pis("yebsap-example")
print(top_preds[1, ])
diff --git a/docs/reference/load_ppm.html b/docs/reference/load_ppm.html
index 4677c64..14a5759 100644
--- a/docs/reference/load_ppm.html
+++ b/docs/reference/load_ppm.html
@@ -3,15 +3,15 @@
during model training and a suite of predictive performance metrics (PPMs)
are calculated. The PPMs for each base model are summarized to a 27 km
resolution raster grid, where the cell values are the average across all
-models in the ensemble contributing to that cell. These data are available in
-raster format provided download_ppms = TRUE was used when calling
-ebirdst_download_status().">
+models in the ensemble contributing to that cell. If the requested data have
+not already been downloaded, they will be downloaded automatically on first
+use.">
Skip to contents
@@ -19,7 +19,7 @@
ebirdst
- 4.2023.0
+ 4.2023.1
@@ -57,9 +57,9 @@
Load predictive performance metric (PPM) rasters
during model training and a suite of predictive performance metrics (PPMs)
are calculated. The PPMs for each base model are summarized to a 27 km
resolution raster grid, where the cell values are the average across all
-models in the ensemble contributing to that cell. These data are available in
-raster format provided download_ppms = TRUE was used when calling
-ebirdst_download_status().
+models in the ensemble contributing to that cell. If the requested data have
+not already been downloaded, they will be downloaded automatically on first
+use.
if(FALSE){# \dontrun{
-# download example data if hasn't already been downloaded
-ebirdst_download_status("yebsap-example", download_ppms =TRUE)
-# load area under the precision-recall curve PPM raster
+# data will be downloaded automatically if not already presentload_ppm("yebsap-example", ppm ="binary_pr_auc")}# }