From 1a36b405d636db3306d07259fc2590bd3d63b7ca Mon Sep 17 00:00:00 2001 From: lshep Date: Mon, 9 Dec 2024 13:27:04 -0500 Subject: [PATCH 01/10] First pass of replacing httr::HEAD with httr2 - httr cache_info automatically took into account cache-control when evaluating expires, httr2 does not - httr cache_info also formatted dates differently with parse_http_date - transfered/modified httr functions cache_info, parse_http_date, parse_cache-control that are no longer available in httr2 - modified .httr_get_cache_info that previously used httr:HEAD to httr2 --- DESCRIPTION | 7 +-- NAMESPACE | 1 + R/BiocFileCache-class.R | 1 + R/httr.R | 94 +++++++++++++++++++++++++++++++++++------ 4 files changed, 87 insertions(+), 16 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 35e98eb..c5c8ccd 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: BiocFileCache Title: Manage Files Across Sessions -Version: 2.15.0 +Version: 2.15.1 Authors@R: c(person("Lori", "Shepherd", email = "lori.shepherd@roswellpark.org", role = c("aut", "cre")), @@ -24,12 +24,13 @@ Imports: DBI, filelock, curl, - httr + httr, + httr2 BugReports: https://github.com/Bioconductor/BiocFileCache/issues DevelopmentURL: https://github.com/Bioconductor/BiocFileCache License: Artistic-2.0 Encoding: UTF-8 -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.2 biocViews: DataImport VignetteBuilder: knitr Suggests: diff --git a/NAMESPACE b/NAMESPACE index 307cd63..c168c9f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -60,6 +60,7 @@ exportMethods(show) import(RSQLite) import(dbplyr) import(httr) +import(httr2) import(methods) importFrom(DBI,dbExecute) importFrom(DBI,dbSendStatement) diff --git a/R/BiocFileCache-class.R b/R/BiocFileCache-class.R index eb6b81f..c257392 100644 --- a/R/BiocFileCache-class.R +++ b/R/BiocFileCache-class.R @@ -1,4 +1,5 @@ #' @import methods +#' @import httr2 #' @import httr #' @importFrom utils tar zip untar unzip #' @importFrom dplyr mutate diff --git a/R/httr.R b/R/httr.R index 234c6de..63bebbb 100644 --- a/R/httr.R +++ b/R/httr.R @@ -1,23 +1,74 @@ +# taken/based on httr::parse_http_date +.fmt_http_date <- function(x, failure = structure(NA_real_, class = "Date")) { + if (length(x) == 0) + return(NULL) + fmts <- c("%a, %d %b %Y %H:%M:%S", "%A, %d-%b-%y %H:%M:%S", + "%a %b %d %H:%M:%S %Y") + for (fmt in fmts) { + parsed <- as.POSIXct(strptime(x, fmt, tz = "GMT")) + if (all(!is.na(parsed))) + return(parsed) + } + rep(failure, length(x)) +} + +# taken/based on httr::parse_cache_control +.fmt_cache_control <- function(cachecontrol){ + if (is.null(cachecontrol)) + return(list()) + pieces <- strsplit(cachecontrol, ",")[[1]] + pieces <- gsub("^\\s+|\\s+$", "", pieces) + pieces <- tolower(pieces) + is_value <- grepl("=", pieces) + flags <- pieces[!is_value] + keyvalues <- strsplit(pieces[is_value], "\\s*=\\s*") + keys <- c(rep("flags", length(flags)), lapply(keyvalues, + "[[", 1)) + values <- c(flags, lapply(keyvalues, "[[", 2)) + stats::setNames(values, keys) +} + +# taken/based on httr::cache_info +.get_http_expires <- function(response){ + expires <- .fmt_http_date(resp_header(response, "expires"), Inf) %||% NULL + control <- .fmt_cache_control(resp_header(response, "cache-control")) + max_age <- as.integer(control$`max-age`) %||% NULL + if (!is.null(max_age)) { + expires <- .fmt_http_date(resp_header(response, "date")) + max_age + } else { + if (!is.null(resp_header(response, "expires"))) { + expires <- .fmt_http_date(resp_header(response, "expires"), -Inf) + } else { + expires <- NULL + } + } + expires +} + .httr_get_cache_info <- function(link) { response = withCallingHandlers({ - HEAD(link) + request(link) %>% + req_method("HEAD") %>% + req_perform() + }, warning = function(w) { invokeRestart("muffleWarning") + }, error = function(e){ + stop("Error while performing HEAD request") }) tryCatch({ - ci <- cache_info(response) - last_mod <- as.character(ci$modified) - etag <- gsub("\"", "",as.character(ci$etag)) - expires <- as.character(ci$expires) - if (length(last_mod) == 0L) - last_mod <- NA_character_ - if (length(etag) == 0L) - etag <- NA_character_ - if (length(expires) == 0L) - expires <- NA_character_ + etag <- ifelse(resp_header_exists(response, "etag"), + gsub("\"", "",as.character(resp_header(response, "etag"))), + NA_character_) + last_mod <- ifelse(resp_header_exists(response, "last-modified"), + as.character(.fmt_http_date(resp_header(response, "last-modified"))), + NA_character_) + expires <- .get_http_expires(response) + expires <- ifelse(is.null(expires), NA_character_, as.character(expires)) + c(etag = etag, modified = last_mod, expires = expires) }, error = function(err) { @@ -31,11 +82,23 @@ } else { modified = NA_character_ } - if ("expires" %in% names(response$headers)){ - expires = as.character(response[["headers"]]["expires"]) + if ("cache-control" %in% names(response$headers)){ + cachecontrol = + .fmt_cache_control(as.character(response[["headers"]]["cache-control"])) + max_age = as.integer(control$`max-age`) %||% NULL + if (!is.null(max_age)) { + expires <- .fmt_http_date(as.character(response[["headers"]]["date"])) + max_age + }else{ + if ("expires" %in% names(response$headers)){ + expires = as.character(response[["headers"]]["expires"]) + } else { + expires = NA_character_ + } + } } else { expires = NA_character_ } + c(etag = etag, modified = modified, expires = expires) }) } @@ -44,6 +107,8 @@ .httr_download <- function(websource, localfile, proxy, config, ...) { + + ## retrieve file from hub to cache tryCatch({ @@ -84,4 +149,7 @@ call.=FALSE) FALSE }) + + + } From 5e8e5ae53654feeb68667ec488829191cdc30e39 Mon Sep 17 00:00:00 2001 From: lshep Date: Thu, 12 Dec 2024 15:20:05 -0500 Subject: [PATCH 02/10] First pass at replacing httr::GET with httr2 --- DESCRIPTION | 2 +- R/httr.R | 59 ++++++++++++++++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index c5c8ccd..2f81a74 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: BiocFileCache Title: Manage Files Across Sessions -Version: 2.15.1 +Version: 2.15.2 Authors@R: c(person("Lori", "Shepherd", email = "lori.shepherd@roswellpark.org", role = c("aut", "cre")), diff --git a/R/httr.R b/R/httr.R index 63bebbb..f8ce4cd 100644 --- a/R/httr.R +++ b/R/httr.R @@ -108,38 +108,53 @@ function(websource, localfile, proxy, config, ...) { - ## retrieve file from hub to cache tryCatch({ if(!is(proxy, "request")){ if (proxy == ""){ - proxy = NULL - }else{ - proxy = use_proxy(proxy) + proxy <- NULL + } else { + proxy <- request() %>% + req_proxy(proxy) # req_proxy to set the proxy in httr2 } } - if (!all(file.exists(dirname(localfile)))) - dir.create(dirname(localfile), recursive=TRUE) - - ## Download the resource in a way that supports https - if (interactive() && (packageVersion("httr") > "1.0.0")) { - response <- suppressWarnings({ - GET(websource, progress(con = stderr()), config=config, - write_disk(localfile, overwrite=TRUE), proxy, ...) - }) - cat("\n") ## line break after progress bar + if((length(config)==0)){ + config <- NULL } else { - response <- suppressWarnings({ - GET(websource, write_disk(localfile, overwrite=TRUE), - proxy, config=config, ...) - }) + stopifnot(is.list(config)) + } + + if (!all(file.exists(dirname(localfile)))) dir.create(dirname(localfile), recursive=TRUE) + + # set up request using httr2 + req <- request(websource) + # This enables progress bars in httr2 + if(interactive()){ + req <- req %>% req_progress() } - if (length(status_code(response))) { - if (status_code(response) != 302L) - stop_for_status(response) + # Apply the proxy if it's not NULL + if (!is.null(proxy)) { + req <- req %>% req_proxy(proxy) } + # Apply the config if it's not NULL + if (!is.null(config)) { + req <- req %>% req_options(!!!config) + } + ## httr2 req_perform does not have an overwrite + ## assume overwrite will occur automatically + ## Perform the request and capture the response + response <- req_perform(req, path=localfile, ...) + + cat("\n") ## line break after progress bar + +# No longer needed? +# if (length(status_code(response))) { +# if (status_code(response) != 302L) +# stop_for_status(response) +# } + TRUE }, error = function(err) { warning("download failed", @@ -150,6 +165,4 @@ FALSE }) - - } From 5b50ae1773d8bbf6893d4d95f84fbf4457d16784 Mon Sep 17 00:00:00 2001 From: lshep Date: Fri, 13 Dec 2024 08:50:43 -0500 Subject: [PATCH 03/10] Fix R CMD check Notes - global variables - describe instead of itemize --- DESCRIPTION | 2 +- R/BiocFileCache-class.R | 4 ++-- R/zzz.R | 2 ++ man/BiocFileCache-class.Rd | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 2f81a74..7b32644 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: BiocFileCache Title: Manage Files Across Sessions -Version: 2.15.2 +Version: 2.15.3 Authors@R: c(person("Lori", "Shepherd", email = "lori.shepherd@roswellpark.org", role = c("aut", "cre")), diff --git a/R/BiocFileCache-class.R b/R/BiocFileCache-class.R index c257392..29a93fa 100644 --- a/R/BiocFileCache-class.R +++ b/R/BiocFileCache-class.R @@ -31,7 +31,7 @@ #' 'BiocFileCacheReadOnly' classes. #' #' Slots unique to 'BiocFileCache' and related classes: -#' \itemize{ +#' \describe{ #' \item{'cache': }{character(1) on-disk location (directory path) of the #' cache} #' \item{'rid': }{character() of unique rids in the cache. } @@ -40,7 +40,7 @@ #' The cache creates an RSQLite database to keep track of local and remote #' resources. Each item located in the database will have the following #' information: -#' \itemize{ +#' \describe{ #' \item{'rid': }{resource id. Autogenerated. This is a unique identifier #' automatically generated when a resource is added to the cache} #' \item{'rname': }{resource name. This is given by the user when a diff --git a/R/zzz.R b/R/zzz.R index ed0093b..4ee58df 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,3 +1,5 @@ +globalVariables(c("rid", "id", "control", "access_time")) + .onLoad <- function(libname, pkgname, ...) { ## options from getOption or Sys.env or default, in that order if (is.null(getBFCOption("CACHE"))) { diff --git a/man/BiocFileCache-class.Rd b/man/BiocFileCache-class.Rd index 3d52dba..e3b82af 100644 --- a/man/BiocFileCache-class.Rd +++ b/man/BiocFileCache-class.Rd @@ -482,7 +482,7 @@ The package defines 'BiocFileCache', 'BiocFileCacheBase' and 'BiocFileCacheReadOnly' classes. Slots unique to 'BiocFileCache' and related classes: -\itemize{ +\describe{ \item{'cache': }{character(1) on-disk location (directory path) of the cache} \item{'rid': }{character() of unique rids in the cache. } @@ -491,7 +491,7 @@ Slots unique to 'BiocFileCache' and related classes: The cache creates an RSQLite database to keep track of local and remote resources. Each item located in the database will have the following information: -\itemize{ +\describe{ \item{'rid': }{resource id. Autogenerated. This is a unique identifier automatically generated when a resource is added to the cache} \item{'rname': }{resource name. This is given by the user when a From 1c231095d7b3993e0f303a58909ea5671226114b Mon Sep 17 00:00:00 2001 From: lshep Date: Fri, 13 Dec 2024 09:48:05 -0500 Subject: [PATCH 04/10] Update how to use behind proxy information --- DESCRIPTION | 2 +- R/BiocFileCache-class.R | 24 +++++++++++---------- R/httr.R | 13 +++++------ tests/testthat/test_httr.R | 2 +- vignettes/BiocFileCache.Rmd | 17 +++++++++------ vignettes/BiocFileCache_Troubleshooting.Rmd | 22 ++++++++++--------- vignettes/BiocFileCache_UseCases.Rmd | 20 +++++++++-------- 7 files changed, 53 insertions(+), 47 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7b32644..265a135 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: BiocFileCache Title: Manage Files Across Sessions -Version: 2.15.3 +Version: 2.15.4 Authors@R: c(person("Lori", "Shepherd", email = "lori.shepherd@roswellpark.org", role = c("aut", "cre")), diff --git a/R/BiocFileCache-class.R b/R/BiocFileCache-class.R index 29a93fa..bbecaa2 100644 --- a/R/BiocFileCache-class.R +++ b/R/BiocFileCache-class.R @@ -1,6 +1,5 @@ #' @import methods #' @import httr2 -#' @import httr #' @importFrom utils tar zip untar unzip #' @importFrom dplyr mutate #' @importFrom tools R_user_dir @@ -303,7 +302,7 @@ setMethod("bfcnew", "BiocFileCache", setGeneric("bfcadd", function( x, rname, fpath = rname, rtype=c("auto", "relative", "local", "web"), - action=c("copy", "move", "asis"), proxy="", + action=c("copy", "move", "asis"), proxy=list(), download=TRUE, config=list(), ext=NA_character_, fname=c("unique", "exact"),... ) standardGeneric("bfcadd"), @@ -316,7 +315,7 @@ setGeneric("bfcadd", setMethod("bfcadd", "missing", function( x, rname, fpath = rname, rtype=c("auto", "relative", "local", "web"), - action=c("copy", "move", "asis"), proxy="", + action=c("copy", "move", "asis"), proxy=list(), download=TRUE, config=list(), ext=NA_character_, fname=c("unique", "exact"), ... ) @@ -341,13 +340,16 @@ setMethod("bfcadd", "missing", #' the file to the cache directory; or \code{asis} leave the file #' in current location but save the path in the cache. If 'rtype #' == "relative"', action can not be "asis". -#' @param proxy character(1) (Optional) proxy server. +#' @param proxy list() (Optional) options to be passed to +#' \code{httr2::req_proxy}. #' @param download logical(1) If \code{rtype=web}, should remote #' resource be downloaded locally immediately. -#' @param config list() passed as config argument in \code{httr::GET} +#' @param config list() passed as argument to \code{httr2::req_options}. The +#' names of items shoudl be valid curl options as defined in +#' \code{curl::curl_options}. #' @param ... For 'bfcadd', 'bfcupdate' and 'bfcdownload': Additional #' arguments passed to internal download functions for use with -#' \code{httr::GET}. For 'bfcrpaths': Additional arguments passed +#' \code{httr2::req_perform}. For 'bfcrpaths': Additional arguments passed #' to 'bfcadd', or \code{exact} passed to 'bfcquery'. For #' 'bfcquery': Additional arguments passed to \code{grepl}. For #' 'exportbfc': Additional arguments to the selected outputMethod @@ -382,7 +384,7 @@ setMethod("bfcadd", "BiocFileCache", x, rname, fpath = rname, rtype = c("auto", "relative", "local", "web"), action = c("copy", "move", "asis"), - proxy = "", download = TRUE, config = list(), ext=NA_character_, + proxy = list(), download = TRUE, config = list(), ext=NA_character_, fname=c("unique", "exact"),...) { stopifnot( @@ -620,7 +622,7 @@ setMethod("bfcupdate", "missing", #' @exportMethod bfcupdate setMethod("bfcupdate", "BiocFileCache", function(x, rids, ..., rname=NULL, rpath=NULL, fpath=NULL, - proxy="", config=list(), ask=TRUE) + proxy=list(), config=list(), ask=TRUE) { stopifnot(!missing(rids), all(rids %in% bfcrid(x))) stopifnot( @@ -1051,7 +1053,7 @@ setMethod("bfcneedsupdate", "BiocFileCacheBase", #' @export setGeneric("bfcdownload", - function(x, rid, proxy="", config=list(), ask=TRUE, FUN, ...) + function(x, rid, proxy=list(), config=list(), ask=TRUE, FUN, ...) standardGeneric("bfcdownload"), signature = "x" ) @@ -1061,7 +1063,7 @@ setGeneric("bfcdownload", #' @param rid character(1) Unique resource id. #' @exportMethod bfcdownload setMethod("bfcdownload", "missing", - function(x, rid, proxy="", config=list(), ask=TRUE, FUN, ...) + function(x, rid, proxy=list(), config=list(), ask=TRUE, FUN, ...) { bfcdownload(x=BiocFileCache(), rid=rid, proxy=proxy, config=config, ask=ask, FUN=FUN, ...) @@ -1082,7 +1084,7 @@ setMethod("bfcdownload", "missing", #' @aliases bfcdownload #' @exportMethod bfcdownload setMethod("bfcdownload", "BiocFileCache", - function(x, rid, proxy="", config=list(), ask=TRUE, FUN, ...) + function(x, rid, proxy=list(), config=list(), ask=TRUE, FUN, ...) { stopifnot( !missing(rid), length(rid) > 0L, diff --git a/R/httr.R b/R/httr.R index f8ce4cd..99f2a5b 100644 --- a/R/httr.R +++ b/R/httr.R @@ -111,13 +111,10 @@ ## retrieve file from hub to cache tryCatch({ - if(!is(proxy, "request")){ - if (proxy == ""){ - proxy <- NULL - } else { - proxy <- request() %>% - req_proxy(proxy) # req_proxy to set the proxy in httr2 - } + if ((length(proxy)==0)){ + proxy <- NULL + } else { + stopifnot(is.list(proxy)) } if((length(config)==0)){ @@ -136,7 +133,7 @@ } # Apply the proxy if it's not NULL if (!is.null(proxy)) { - req <- req %>% req_proxy(proxy) + req <- req %>% req_proxy(!!!proxy) } # Apply the config if it's not NULL if (!is.null(config)) { diff --git a/tests/testthat/test_httr.R b/tests/testthat/test_httr.R index a08eeab..e42fad5 100644 --- a/tests/testthat/test_httr.R +++ b/tests/testthat/test_httr.R @@ -1,4 +1,4 @@ -context("httr") +context("httr2") test_that("internal .httr_get_cache_info works", { diff --git a/vignettes/BiocFileCache.Rmd b/vignettes/BiocFileCache.Rmd index 64e5cbd..12c110d 100644 --- a/vignettes/BiocFileCache.Rmd +++ b/vignettes/BiocFileCache.Rmd @@ -653,15 +653,18 @@ removebfc(bfc) # Access Behind a Proxy -BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing -web resources. This can be problematic if operating behind a proxy. The easiest -solution is to set the `httr::set_config` with the proxy information. +BiocFileCache uses CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. You can pass proxy +information into the bfcadd functions ```{r eval=FALSE} -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) +proxy=list(proxy_url = "http://myproxy", # Proxy host URL + proxy_port = 8080, # Proxy port + proxy_username = "my_user", # Proxy username + proxy_password = "my_password") +bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) ``` diff --git a/vignettes/BiocFileCache_Troubleshooting.Rmd b/vignettes/BiocFileCache_Troubleshooting.Rmd index 27fd5c4..69fe369 100644 --- a/vignettes/BiocFileCache_Troubleshooting.Rmd +++ b/vignettes/BiocFileCache_Troubleshooting.Rmd @@ -58,18 +58,20 @@ bfc <- BiocFileCache(path, ask = FALSE) # Access Behind a Proxy -BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing -web resources. This can be problematic if operating behind a proxy. The easiest -solution is to set the `httr::set_config` with the proxy information. - -``` -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) +BiocFileCache uses CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. You can pass proxy +information into the bfcadd functions + +```{r eval=FALSE} +proxy=list(proxy_url = "http://myproxy", # Proxy host URL + proxy_port = 8080, # Proxy port + proxy_username = "my_user", # Proxy username + proxy_password = "my_password") +bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) ``` - # Group Cache Access The situation may occur where a cache is desired to be shared across multiple diff --git a/vignettes/BiocFileCache_UseCases.Rmd b/vignettes/BiocFileCache_UseCases.Rmd index 66474e7..9fbd920 100644 --- a/vignettes/BiocFileCache_UseCases.Rmd +++ b/vignettes/BiocFileCache_UseCases.Rmd @@ -354,17 +354,19 @@ bfcadd("myfile", url, download=FALSE, ext=".Rdata") # Access Behind a Proxy -BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing -web resources. This can be problematic if operating behind a proxy. The easiest -solution is to set the `httr::set_config` with the proxy information. +BiocFileCache uses CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. You can pass proxy +information into the bfcadd functions +```{r eval=FALSE} +proxy=list(proxy_url = "http://myproxy", # Proxy host URL + proxy_port = 8080, # Proxy port + proxy_username = "my_user", # Proxy username + proxy_password = "my_password") +bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) ``` -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) -``` - # Group Cache Access From a65f61472a353e1273b8e7ad018da80259b43467 Mon Sep 17 00:00:00 2001 From: lshep Date: Fri, 13 Dec 2024 10:04:45 -0500 Subject: [PATCH 05/10] Clean up --- DESCRIPTION | 1 - NAMESPACE | 1 - man/BiocFileCache-class.Rd | 19 +++++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 265a135..6d64d17 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -24,7 +24,6 @@ Imports: DBI, filelock, curl, - httr, httr2 BugReports: https://github.com/Bioconductor/BiocFileCache/issues DevelopmentURL: https://github.com/Bioconductor/BiocFileCache diff --git a/NAMESPACE b/NAMESPACE index c168c9f..f46e15f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -59,7 +59,6 @@ exportMethods(removebfc) exportMethods(show) import(RSQLite) import(dbplyr) -import(httr) import(httr2) import(methods) importFrom(DBI,dbExecute) diff --git a/man/BiocFileCache-class.Rd b/man/BiocFileCache-class.Rd index e3b82af..a3e7bee 100644 --- a/man/BiocFileCache-class.Rd +++ b/man/BiocFileCache-class.Rd @@ -133,7 +133,7 @@ bfcrid(x) fpath = rname, rtype = c("auto", "relative", "local", "web"), action = c("copy", "move", "asis"), - proxy = "", + proxy = list(), download = TRUE, config = list(), ext = NA_character_, @@ -147,7 +147,7 @@ bfcrid(x) fpath = rname, rtype = c("auto", "relative", "local", "web"), action = c("copy", "move", "asis"), - proxy = "", + proxy = list(), download = TRUE, config = list(), ext = NA_character_, @@ -178,7 +178,7 @@ bfcrid(x) rname = NULL, rpath = NULL, fpath = NULL, - proxy = "", + proxy = list(), config = list(), ask = TRUE ) @@ -217,9 +217,9 @@ bfcmeta(x, name, ...) <- value \S4method{bfcneedsupdate}{BiocFileCacheBase}(x, rids) -\S4method{bfcdownload}{missing}(x, rid, proxy = "", config = list(), ask = TRUE, FUN, ...) +\S4method{bfcdownload}{missing}(x, rid, proxy = list(), config = list(), ask = TRUE, FUN, ...) -\S4method{bfcdownload}{BiocFileCache}(x, rid, proxy = "", config = list(), ask = TRUE, FUN, ...) +\S4method{bfcdownload}{BiocFileCache}(x, rid, proxy = list(), config = list(), ask = TRUE, FUN, ...) \S4method{bfcremove}{missing}(x, rids) @@ -276,7 +276,7 @@ of \code{BiocFileCache()}.} \item{...}{For 'bfcadd', 'bfcupdate' and 'bfcdownload': Additional arguments passed to internal download functions for use with -\code{httr::GET}. For 'bfcrpaths': Additional arguments passed +\code{httr2::req_perform}. For 'bfcrpaths': Additional arguments passed to 'bfcadd', or \code{exact} passed to 'bfcquery'. For 'bfcquery': Additional arguments passed to \code{grepl}. For 'exportbfc': Additional arguments to the selected outputMethod @@ -320,12 +320,15 @@ the file to the cache directory; or \code{asis} leave the file in current location but save the path in the cache. If 'rtype == "relative"', action can not be "asis".} -\item{proxy}{character(1) (Optional) proxy server.} +\item{proxy}{list() (Optional) options to be passed to +\code{httr2::req_proxy}.} \item{download}{logical(1) If \code{rtype=web}, should remote resource be downloaded locally immediately.} -\item{config}{list() passed as config argument in \code{httr::GET}} +\item{config}{list() passed as argument to \code{httr2::req_options}. The +names of items shoudl be valid curl options as defined in +\code{curl::curl_options}.} \item{rids}{character() Vector of rids.} From 5fb80df4ec8f5e94a27d150b1ab154dcf61bca99 Mon Sep 17 00:00:00 2001 From: lshep Date: Fri, 20 Dec 2024 11:17:15 -0500 Subject: [PATCH 06/10] Revert "Update how to use behind proxy information" This reverts commit 1c231095d7b3993e0f303a58909ea5671226114b. --- DESCRIPTION | 2 +- R/BiocFileCache-class.R | 24 ++++++++++----------- R/httr.R | 13 ++++++----- tests/testthat/test_httr.R | 2 +- vignettes/BiocFileCache.Rmd | 17 ++++++--------- vignettes/BiocFileCache_Troubleshooting.Rmd | 22 +++++++++---------- vignettes/BiocFileCache_UseCases.Rmd | 20 ++++++++--------- 7 files changed, 47 insertions(+), 53 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6d64d17..3249b14 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: BiocFileCache Title: Manage Files Across Sessions -Version: 2.15.4 +Version: 2.15.3 Authors@R: c(person("Lori", "Shepherd", email = "lori.shepherd@roswellpark.org", role = c("aut", "cre")), diff --git a/R/BiocFileCache-class.R b/R/BiocFileCache-class.R index bbecaa2..29a93fa 100644 --- a/R/BiocFileCache-class.R +++ b/R/BiocFileCache-class.R @@ -1,5 +1,6 @@ #' @import methods #' @import httr2 +#' @import httr #' @importFrom utils tar zip untar unzip #' @importFrom dplyr mutate #' @importFrom tools R_user_dir @@ -302,7 +303,7 @@ setMethod("bfcnew", "BiocFileCache", setGeneric("bfcadd", function( x, rname, fpath = rname, rtype=c("auto", "relative", "local", "web"), - action=c("copy", "move", "asis"), proxy=list(), + action=c("copy", "move", "asis"), proxy="", download=TRUE, config=list(), ext=NA_character_, fname=c("unique", "exact"),... ) standardGeneric("bfcadd"), @@ -315,7 +316,7 @@ setGeneric("bfcadd", setMethod("bfcadd", "missing", function( x, rname, fpath = rname, rtype=c("auto", "relative", "local", "web"), - action=c("copy", "move", "asis"), proxy=list(), + action=c("copy", "move", "asis"), proxy="", download=TRUE, config=list(), ext=NA_character_, fname=c("unique", "exact"), ... ) @@ -340,16 +341,13 @@ setMethod("bfcadd", "missing", #' the file to the cache directory; or \code{asis} leave the file #' in current location but save the path in the cache. If 'rtype #' == "relative"', action can not be "asis". -#' @param proxy list() (Optional) options to be passed to -#' \code{httr2::req_proxy}. +#' @param proxy character(1) (Optional) proxy server. #' @param download logical(1) If \code{rtype=web}, should remote #' resource be downloaded locally immediately. -#' @param config list() passed as argument to \code{httr2::req_options}. The -#' names of items shoudl be valid curl options as defined in -#' \code{curl::curl_options}. +#' @param config list() passed as config argument in \code{httr::GET} #' @param ... For 'bfcadd', 'bfcupdate' and 'bfcdownload': Additional #' arguments passed to internal download functions for use with -#' \code{httr2::req_perform}. For 'bfcrpaths': Additional arguments passed +#' \code{httr::GET}. For 'bfcrpaths': Additional arguments passed #' to 'bfcadd', or \code{exact} passed to 'bfcquery'. For #' 'bfcquery': Additional arguments passed to \code{grepl}. For #' 'exportbfc': Additional arguments to the selected outputMethod @@ -384,7 +382,7 @@ setMethod("bfcadd", "BiocFileCache", x, rname, fpath = rname, rtype = c("auto", "relative", "local", "web"), action = c("copy", "move", "asis"), - proxy = list(), download = TRUE, config = list(), ext=NA_character_, + proxy = "", download = TRUE, config = list(), ext=NA_character_, fname=c("unique", "exact"),...) { stopifnot( @@ -622,7 +620,7 @@ setMethod("bfcupdate", "missing", #' @exportMethod bfcupdate setMethod("bfcupdate", "BiocFileCache", function(x, rids, ..., rname=NULL, rpath=NULL, fpath=NULL, - proxy=list(), config=list(), ask=TRUE) + proxy="", config=list(), ask=TRUE) { stopifnot(!missing(rids), all(rids %in% bfcrid(x))) stopifnot( @@ -1053,7 +1051,7 @@ setMethod("bfcneedsupdate", "BiocFileCacheBase", #' @export setGeneric("bfcdownload", - function(x, rid, proxy=list(), config=list(), ask=TRUE, FUN, ...) + function(x, rid, proxy="", config=list(), ask=TRUE, FUN, ...) standardGeneric("bfcdownload"), signature = "x" ) @@ -1063,7 +1061,7 @@ setGeneric("bfcdownload", #' @param rid character(1) Unique resource id. #' @exportMethod bfcdownload setMethod("bfcdownload", "missing", - function(x, rid, proxy=list(), config=list(), ask=TRUE, FUN, ...) + function(x, rid, proxy="", config=list(), ask=TRUE, FUN, ...) { bfcdownload(x=BiocFileCache(), rid=rid, proxy=proxy, config=config, ask=ask, FUN=FUN, ...) @@ -1084,7 +1082,7 @@ setMethod("bfcdownload", "missing", #' @aliases bfcdownload #' @exportMethod bfcdownload setMethod("bfcdownload", "BiocFileCache", - function(x, rid, proxy=list(), config=list(), ask=TRUE, FUN, ...) + function(x, rid, proxy="", config=list(), ask=TRUE, FUN, ...) { stopifnot( !missing(rid), length(rid) > 0L, diff --git a/R/httr.R b/R/httr.R index 99f2a5b..f8ce4cd 100644 --- a/R/httr.R +++ b/R/httr.R @@ -111,10 +111,13 @@ ## retrieve file from hub to cache tryCatch({ - if ((length(proxy)==0)){ - proxy <- NULL - } else { - stopifnot(is.list(proxy)) + if(!is(proxy, "request")){ + if (proxy == ""){ + proxy <- NULL + } else { + proxy <- request() %>% + req_proxy(proxy) # req_proxy to set the proxy in httr2 + } } if((length(config)==0)){ @@ -133,7 +136,7 @@ } # Apply the proxy if it's not NULL if (!is.null(proxy)) { - req <- req %>% req_proxy(!!!proxy) + req <- req %>% req_proxy(proxy) } # Apply the config if it's not NULL if (!is.null(config)) { diff --git a/tests/testthat/test_httr.R b/tests/testthat/test_httr.R index e42fad5..a08eeab 100644 --- a/tests/testthat/test_httr.R +++ b/tests/testthat/test_httr.R @@ -1,4 +1,4 @@ -context("httr2") +context("httr") test_that("internal .httr_get_cache_info works", { diff --git a/vignettes/BiocFileCache.Rmd b/vignettes/BiocFileCache.Rmd index 12c110d..64e5cbd 100644 --- a/vignettes/BiocFileCache.Rmd +++ b/vignettes/BiocFileCache.Rmd @@ -653,18 +653,15 @@ removebfc(bfc) # Access Behind a Proxy -BiocFileCache uses CRAN package `httr2` functions for accessing and downloading -web resources. This can be problematic if operating behind a -proxy. Unfortunately unlike the previously used `httr::set_config`, there is no -option to globally set the proxy for httr2 requests. You can pass proxy -information into the bfcadd functions +BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing +web resources. This can be problematic if operating behind a proxy. The easiest +solution is to set the `httr::set_config` with the proxy information. ```{r eval=FALSE} -proxy=list(proxy_url = "http://myproxy", # Proxy host URL - proxy_port = 8080, # Proxy port - proxy_username = "my_user", # Proxy username - proxy_password = "my_password") -bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) +proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") +## or +proxy <- httr::use_proxy(Sys.getenv('http_proxy')) +httr::set_config(proxy) ``` diff --git a/vignettes/BiocFileCache_Troubleshooting.Rmd b/vignettes/BiocFileCache_Troubleshooting.Rmd index 69fe369..27fd5c4 100644 --- a/vignettes/BiocFileCache_Troubleshooting.Rmd +++ b/vignettes/BiocFileCache_Troubleshooting.Rmd @@ -58,20 +58,18 @@ bfc <- BiocFileCache(path, ask = FALSE) # Access Behind a Proxy -BiocFileCache uses CRAN package `httr2` functions for accessing and downloading -web resources. This can be problematic if operating behind a -proxy. Unfortunately unlike the previously used `httr::set_config`, there is no -option to globally set the proxy for httr2 requests. You can pass proxy -information into the bfcadd functions - -```{r eval=FALSE} -proxy=list(proxy_url = "http://myproxy", # Proxy host URL - proxy_port = 8080, # Proxy port - proxy_username = "my_user", # Proxy username - proxy_password = "my_password") -bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) +BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing +web resources. This can be problematic if operating behind a proxy. The easiest +solution is to set the `httr::set_config` with the proxy information. + +``` +proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") +## or +proxy <- httr::use_proxy(Sys.getenv('http_proxy')) +httr::set_config(proxy) ``` + # Group Cache Access The situation may occur where a cache is desired to be shared across multiple diff --git a/vignettes/BiocFileCache_UseCases.Rmd b/vignettes/BiocFileCache_UseCases.Rmd index 9fbd920..66474e7 100644 --- a/vignettes/BiocFileCache_UseCases.Rmd +++ b/vignettes/BiocFileCache_UseCases.Rmd @@ -354,19 +354,17 @@ bfcadd("myfile", url, download=FALSE, ext=".Rdata") # Access Behind a Proxy -BiocFileCache uses CRAN package `httr2` functions for accessing and downloading -web resources. This can be problematic if operating behind a -proxy. Unfortunately unlike the previously used `httr::set_config`, there is no -option to globally set the proxy for httr2 requests. You can pass proxy -information into the bfcadd functions +BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing +web resources. This can be problematic if operating behind a proxy. The easiest +solution is to set the `httr::set_config` with the proxy information. -```{r eval=FALSE} -proxy=list(proxy_url = "http://myproxy", # Proxy host URL - proxy_port = 8080, # Proxy port - proxy_username = "my_user", # Proxy username - proxy_password = "my_password") -bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) ``` +proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") +## or +proxy <- httr::use_proxy(Sys.getenv('http_proxy')) +httr::set_config(proxy) +``` + # Group Cache Access From 8f19fd0754d4c3509b08e1b99e0bfd8182cf8a6f Mon Sep 17 00:00:00 2001 From: lshep Date: Fri, 20 Dec 2024 11:18:18 -0500 Subject: [PATCH 07/10] Revert "Clean up" This reverts commit a65f61472a353e1273b8e7ad018da80259b43467. --- DESCRIPTION | 1 + NAMESPACE | 1 + man/BiocFileCache-class.Rd | 19 ++++++++----------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3249b14..7b32644 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -24,6 +24,7 @@ Imports: DBI, filelock, curl, + httr, httr2 BugReports: https://github.com/Bioconductor/BiocFileCache/issues DevelopmentURL: https://github.com/Bioconductor/BiocFileCache diff --git a/NAMESPACE b/NAMESPACE index f46e15f..c168c9f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -59,6 +59,7 @@ exportMethods(removebfc) exportMethods(show) import(RSQLite) import(dbplyr) +import(httr) import(httr2) import(methods) importFrom(DBI,dbExecute) diff --git a/man/BiocFileCache-class.Rd b/man/BiocFileCache-class.Rd index a3e7bee..e3b82af 100644 --- a/man/BiocFileCache-class.Rd +++ b/man/BiocFileCache-class.Rd @@ -133,7 +133,7 @@ bfcrid(x) fpath = rname, rtype = c("auto", "relative", "local", "web"), action = c("copy", "move", "asis"), - proxy = list(), + proxy = "", download = TRUE, config = list(), ext = NA_character_, @@ -147,7 +147,7 @@ bfcrid(x) fpath = rname, rtype = c("auto", "relative", "local", "web"), action = c("copy", "move", "asis"), - proxy = list(), + proxy = "", download = TRUE, config = list(), ext = NA_character_, @@ -178,7 +178,7 @@ bfcrid(x) rname = NULL, rpath = NULL, fpath = NULL, - proxy = list(), + proxy = "", config = list(), ask = TRUE ) @@ -217,9 +217,9 @@ bfcmeta(x, name, ...) <- value \S4method{bfcneedsupdate}{BiocFileCacheBase}(x, rids) -\S4method{bfcdownload}{missing}(x, rid, proxy = list(), config = list(), ask = TRUE, FUN, ...) +\S4method{bfcdownload}{missing}(x, rid, proxy = "", config = list(), ask = TRUE, FUN, ...) -\S4method{bfcdownload}{BiocFileCache}(x, rid, proxy = list(), config = list(), ask = TRUE, FUN, ...) +\S4method{bfcdownload}{BiocFileCache}(x, rid, proxy = "", config = list(), ask = TRUE, FUN, ...) \S4method{bfcremove}{missing}(x, rids) @@ -276,7 +276,7 @@ of \code{BiocFileCache()}.} \item{...}{For 'bfcadd', 'bfcupdate' and 'bfcdownload': Additional arguments passed to internal download functions for use with -\code{httr2::req_perform}. For 'bfcrpaths': Additional arguments passed +\code{httr::GET}. For 'bfcrpaths': Additional arguments passed to 'bfcadd', or \code{exact} passed to 'bfcquery'. For 'bfcquery': Additional arguments passed to \code{grepl}. For 'exportbfc': Additional arguments to the selected outputMethod @@ -320,15 +320,12 @@ the file to the cache directory; or \code{asis} leave the file in current location but save the path in the cache. If 'rtype == "relative"', action can not be "asis".} -\item{proxy}{list() (Optional) options to be passed to -\code{httr2::req_proxy}.} +\item{proxy}{character(1) (Optional) proxy server.} \item{download}{logical(1) If \code{rtype=web}, should remote resource be downloaded locally immediately.} -\item{config}{list() passed as argument to \code{httr2::req_options}. The -names of items shoudl be valid curl options as defined in -\code{curl::curl_options}.} +\item{config}{list() passed as config argument in \code{httr::GET}} \item{rids}{character() Vector of rids.} From 8262875938100e74e1c3fe5efa39b278afe39298 Mon Sep 17 00:00:00 2001 From: lshep Date: Fri, 20 Dec 2024 12:00:31 -0500 Subject: [PATCH 08/10] Update proxy information and documentation --- DESCRIPTION | 3 +-- NAMESPACE | 1 - R/BiocFileCache-class.R | 10 ++++++---- R/httr.R | 11 +++-------- man/BiocFileCache-class.Rd | 9 ++++++--- tests/testthat/test_httr.R | 2 +- vignettes/BiocFileCache.Rmd | 14 +++++++------- vignettes/BiocFileCache_Troubleshooting.Rmd | 17 +++++++++-------- vignettes/BiocFileCache_UseCases.Rmd | 18 +++++++++--------- 9 files changed, 42 insertions(+), 43 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7b32644..6d64d17 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: BiocFileCache Title: Manage Files Across Sessions -Version: 2.15.3 +Version: 2.15.4 Authors@R: c(person("Lori", "Shepherd", email = "lori.shepherd@roswellpark.org", role = c("aut", "cre")), @@ -24,7 +24,6 @@ Imports: DBI, filelock, curl, - httr, httr2 BugReports: https://github.com/Bioconductor/BiocFileCache/issues DevelopmentURL: https://github.com/Bioconductor/BiocFileCache diff --git a/NAMESPACE b/NAMESPACE index c168c9f..f46e15f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -59,7 +59,6 @@ exportMethods(removebfc) exportMethods(show) import(RSQLite) import(dbplyr) -import(httr) import(httr2) import(methods) importFrom(DBI,dbExecute) diff --git a/R/BiocFileCache-class.R b/R/BiocFileCache-class.R index 29a93fa..891355a 100644 --- a/R/BiocFileCache-class.R +++ b/R/BiocFileCache-class.R @@ -1,6 +1,5 @@ #' @import methods #' @import httr2 -#' @import httr #' @importFrom utils tar zip untar unzip #' @importFrom dplyr mutate #' @importFrom tools R_user_dir @@ -341,13 +340,16 @@ setMethod("bfcadd", "missing", #' the file to the cache directory; or \code{asis} leave the file #' in current location but save the path in the cache. If 'rtype #' == "relative"', action can not be "asis". -#' @param proxy character(1) (Optional) proxy server. +#' @param proxy character(1) (Optional) proxy server passed to +#' \code{httr2::req_proxy} #' @param download logical(1) If \code{rtype=web}, should remote #' resource be downloaded locally immediately. -#' @param config list() passed as config argument in \code{httr::GET} +#' @param config list() passed as argument to \code{httr2::req_options}. The +#' names of items shoudl be valid curl options as defined in +#' \code{curl::curl_options}. #' @param ... For 'bfcadd', 'bfcupdate' and 'bfcdownload': Additional #' arguments passed to internal download functions for use with -#' \code{httr::GET}. For 'bfcrpaths': Additional arguments passed +#' \code{httr2::req_perform}. For 'bfcrpaths': Additional arguments passed #' to 'bfcadd', or \code{exact} passed to 'bfcquery'. For #' 'bfcquery': Additional arguments passed to \code{grepl}. For #' 'exportbfc': Additional arguments to the selected outputMethod diff --git a/R/httr.R b/R/httr.R index f8ce4cd..0bd7c85 100644 --- a/R/httr.R +++ b/R/httr.R @@ -111,14 +111,9 @@ ## retrieve file from hub to cache tryCatch({ - if(!is(proxy, "request")){ - if (proxy == ""){ - proxy <- NULL - } else { - proxy <- request() %>% - req_proxy(proxy) # req_proxy to set the proxy in httr2 - } - } + if (proxy == ""){ + proxy <- NULL + } if((length(config)==0)){ config <- NULL diff --git a/man/BiocFileCache-class.Rd b/man/BiocFileCache-class.Rd index e3b82af..cba6053 100644 --- a/man/BiocFileCache-class.Rd +++ b/man/BiocFileCache-class.Rd @@ -276,7 +276,7 @@ of \code{BiocFileCache()}.} \item{...}{For 'bfcadd', 'bfcupdate' and 'bfcdownload': Additional arguments passed to internal download functions for use with -\code{httr::GET}. For 'bfcrpaths': Additional arguments passed +\code{httr2::req_perform}. For 'bfcrpaths': Additional arguments passed to 'bfcadd', or \code{exact} passed to 'bfcquery'. For 'bfcquery': Additional arguments passed to \code{grepl}. For 'exportbfc': Additional arguments to the selected outputMethod @@ -320,12 +320,15 @@ the file to the cache directory; or \code{asis} leave the file in current location but save the path in the cache. If 'rtype == "relative"', action can not be "asis".} -\item{proxy}{character(1) (Optional) proxy server.} +\item{proxy}{character(1) (Optional) proxy server passed to +\code{httr2::req_proxy}} \item{download}{logical(1) If \code{rtype=web}, should remote resource be downloaded locally immediately.} -\item{config}{list() passed as config argument in \code{httr::GET}} +\item{config}{list() passed as argument to \code{httr2::req_options}. The +names of items shoudl be valid curl options as defined in +\code{curl::curl_options}.} \item{rids}{character() Vector of rids.} diff --git a/tests/testthat/test_httr.R b/tests/testthat/test_httr.R index a08eeab..e42fad5 100644 --- a/tests/testthat/test_httr.R +++ b/tests/testthat/test_httr.R @@ -1,4 +1,4 @@ -context("httr") +context("httr2") test_that("internal .httr_get_cache_info works", { diff --git a/vignettes/BiocFileCache.Rmd b/vignettes/BiocFileCache.Rmd index 64e5cbd..1632dce 100644 --- a/vignettes/BiocFileCache.Rmd +++ b/vignettes/BiocFileCache.Rmd @@ -653,15 +653,15 @@ removebfc(bfc) # Access Behind a Proxy -BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing -web resources. This can be problematic if operating behind a proxy. The easiest -solution is to set the `httr::set_config` with the proxy information. +BiocFileCache uses CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. You can pass proxy +information into the bfcadd functions. ```{r eval=FALSE} -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) +proxy <- "http://my_user:my_password@myproxy:8080" +bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) ``` diff --git a/vignettes/BiocFileCache_Troubleshooting.Rmd b/vignettes/BiocFileCache_Troubleshooting.Rmd index 27fd5c4..928ad56 100644 --- a/vignettes/BiocFileCache_Troubleshooting.Rmd +++ b/vignettes/BiocFileCache_Troubleshooting.Rmd @@ -58,15 +58,16 @@ bfc <- BiocFileCache(path, ask = FALSE) # Access Behind a Proxy -BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing -web resources. This can be problematic if operating behind a proxy. The easiest -solution is to set the `httr::set_config` with the proxy information. -``` -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) +BiocFileCache uses CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. You can pass proxy +information into the bfcadd functions. + +```{r eval=FALSE} +proxy <- "http://my_user:my_password@myproxy:8080" +bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) ``` diff --git a/vignettes/BiocFileCache_UseCases.Rmd b/vignettes/BiocFileCache_UseCases.Rmd index 66474e7..2e0f713 100644 --- a/vignettes/BiocFileCache_UseCases.Rmd +++ b/vignettes/BiocFileCache_UseCases.Rmd @@ -354,17 +354,17 @@ bfcadd("myfile", url, download=FALSE, ext=".Rdata") # Access Behind a Proxy -BiocFileCache uses CRAN package `httr` functions `HEAD` and `GET` for accessing -web resources. This can be problematic if operating behind a proxy. The easiest -solution is to set the `httr::set_config` with the proxy information. -``` -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) -``` +BiocFileCache uses CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. You can pass proxy +information into the bfcadd functions. +```{r eval=FALSE} +proxy <- "http://my_user:my_password@myproxy:8080" +bfcadd(bfc, rname="uniquename", fpath="https://remoteresource", proxy=proxy) +``` # Group Cache Access From 5493c9435ff7e34d34ce6bded596b18243087c49 Mon Sep 17 00:00:00 2001 From: lshep Date: Tue, 22 Apr 2025 11:25:34 -0400 Subject: [PATCH 09/10] Update to https instead of http --- R/BiocFileCache-class.R | 2 +- man/BiocFileCache-class.Rd | 2 +- tests/testthat/test_BiocFileCache_class.R | 18 +++++++++--------- tests/testthat/test_httr.R | 2 +- .../test_makeBiocFileCacheFromDataFrame.R | 2 +- vignettes/BiocFileCache.Rmd | 6 +++--- vignettes/BiocFileCache_UseCases.Rmd | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/R/BiocFileCache-class.R b/R/BiocFileCache-class.R index be94101..e0c67f9 100644 --- a/R/BiocFileCache-class.R +++ b/R/BiocFileCache-class.R @@ -377,7 +377,7 @@ setMethod("bfcadd", "missing", #' file.exists(fl3) # TRUE #' #' # add a remote resource -#' url <- "http://httpbin.org/get" +#' url <- "https://httpbin.org/get" #' bfcadd(bfc0, "TestWeb", fpath=url) #' @aliases bfcadd #' @exportMethod bfcadd diff --git a/man/BiocFileCache-class.Rd b/man/BiocFileCache-class.Rd index cba6053..bb7faa9 100644 --- a/man/BiocFileCache-class.Rd +++ b/man/BiocFileCache-class.Rd @@ -623,7 +623,7 @@ file.exists(fl2) # FALSE file.exists(fl3) # TRUE # add a remote resource -url <- "http://httpbin.org/get" +url <- "https://httpbin.org/get" bfcadd(bfc0, "TestWeb", fpath=url) bfcinfo(bfc0) bfcpath(bfc0, rid3) diff --git a/tests/testthat/test_BiocFileCache_class.R b/tests/testthat/test_BiocFileCache_class.R index f811aca..4aac3eb 100644 --- a/tests/testthat/test_BiocFileCache_class.R +++ b/tests/testthat/test_BiocFileCache_class.R @@ -40,7 +40,7 @@ test_that("bfcadd and bfcnew works", { expect_true(!file.exists(fl)) # test add web resource - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" path <- bfcadd(bfc, 'test-4', url, rtype="web") rid <- names(path) expect_identical(length(bfc), 4L) @@ -61,13 +61,13 @@ test_that("bfcadd and bfcnew works", { expect_error(bfcadd(bfc, 'test-2', fl, rtype='local', action='asis')) # test no fpath given - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" path <- bfcadd(bfc, url) expect_identical(.sql_get_fpath(bfc,names(path)), .sql_get_rname(bfc,names(path))) # test web resource not download - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" path <- bfcadd(bfc, 'test-noDownload', url, rtype="web", download=FALSE) rid <- names(path) expect_identical(length(bfc), 7L) @@ -178,12 +178,12 @@ add1 <- bfcadd(bfc, 'test-1', fl) rid1 <- names(add1) add2 <- bfcadd(bfc, 'test-2', fl, rtype='local', action='asis') rid2 <- names(add2) -url <- "http://httpbin.org/get" +url <- "https://httpbin.org/get" add3 <- bfcadd(bfc, 'test-3', url, rtype="web") rid3 <- names(add3) path <- bfcnew(bfc, 'test-4') rid4 <- names(path) -url <- "http://httpbin.org/get" +url <- "https://httpbin.org/get" add5 <- bfcadd(bfc, 'test-5', url, rtype="web", download=FALSE) rid5 <- names(add5) @@ -461,7 +461,7 @@ test_that("bfcneedsupdate works", { expect_error(bfcneedsupdate(bfc, 7)) # test expires and last modified not available - link = "http://httpbin.org/get" + link = "https://httpbin.org/get" bfcupdate(bfc, rid3, fpath=link, ask=FALSE) expect_true(is.na(bfcneedsupdate(bfc, rid3))) expect_true(is.na(as.data.frame(bfcinfo(bfc,rid3))$last_modified_time)) @@ -550,12 +550,12 @@ test_that("exportbfc and importbfc works",{ rid1 <- names(add1) add2 <- bfcadd(bfc, 'local', fl, rtype='local', action='asis') rid2 <- names(add2) - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" add3 <- bfcadd(bfc, 'web', url, rtype="web") rid3 <- names(add3) path <- bfcnew(bfc, 'notfound') rid4 <- names(path) - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" add5 <- bfcadd(bfc, 'webno', url, rtype="web", download=FALSE) rid5 <- names(add5) @@ -602,7 +602,7 @@ test_that("bfcsync and bfcremove works", { rid1 <- names(add1) add2 <- bfcadd(bfc2, 'test-2', fl, rtype='local', action='asis') rid2 <- names(add2) - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" add3 <- bfcadd(bfc2, 'test-3', url, rtype="web") rid3 <- names(add3) path <- bfcnew(bfc2, 'test-4') diff --git a/tests/testthat/test_httr.R b/tests/testthat/test_httr.R index e42fad5..e5d1954 100644 --- a/tests/testthat/test_httr.R +++ b/tests/testthat/test_httr.R @@ -3,7 +3,7 @@ context("httr2") test_that("internal .httr_get_cache_info works", { # example neither - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" info <- .httr_get_cache_info(url) expect_true(all(is.na(info))) expect_identical(length(info), 3L) diff --git a/tests/testthat/test_makeBiocFileCacheFromDataFrame.R b/tests/testthat/test_makeBiocFileCacheFromDataFrame.R index 5d3e28c..461cc75 100644 --- a/tests/testthat/test_makeBiocFileCacheFromDataFrame.R +++ b/tests/testthat/test_makeBiocFileCacheFromDataFrame.R @@ -6,7 +6,7 @@ test_that("makeBiocFileCacheFromDataFrame works",{ fl <- tempfile(); file.create(fl) add1 <- bfcadd(bfc2, 'relative', fl) add2 <- bfcadd(bfc2, 'local', fl, rtype='local', action='asis') - url <- "http://httpbin.org/get" + url <- "https://httpbin.org/get" add3 <- bfcadd(bfc2, 'noDown', url, rtype="web", download=FALSE) url <- "https://www.wikipedia.org/" add4 <- bfcadd(bfc2, 'web', url, rtype="web") diff --git a/vignettes/BiocFileCache.Rmd b/vignettes/BiocFileCache.Rmd index 1632dce..b15727e 100644 --- a/vignettes/BiocFileCache.Rmd +++ b/vignettes/BiocFileCache.Rmd @@ -204,7 +204,7 @@ file.exists(fl3) # TRUE - left asis, original location tracked Now let's add some examples with remote sources: ```{r, bfcaddremote} -url <- "http://httpbin.org/get" +url <- "https://httpbin.org/get" add5 <- bfcadd(bfc, "TestWeb", fpath=url) rid5 <- names(add5) @@ -305,7 +305,7 @@ bfcpath(bfc, "BFC2") bfcpath(bfc, "BFC5") bfcrpath(bfc, rids="BFC5") bfcrpath(bfc) -bfcrpath(bfc, c("http://httpbin.org/get","Test3_addAsis")) +bfcrpath(bfc, c("https://httpbin.org/get","Test3_addAsis")) ``` Managing remote resources locally involves knowing when to update the local copy @@ -604,7 +604,7 @@ been downloaded yet. ```{r, mock} tbl <- data.frame(rtype=c("web","web"), rpath=c(NA_character_,NA_character_), - fpath=c("http://httpbin.org/get", + fpath=c("https://httpbin.org/get", "https://en.wikipedia.org/wiki/Bioconductor"), keywords = c("httpbin", "wiki"), stringsAsFactors=FALSE) tbl diff --git a/vignettes/BiocFileCache_UseCases.Rmd b/vignettes/BiocFileCache_UseCases.Rmd index 2e0f713..e4fc271 100644 --- a/vignettes/BiocFileCache_UseCases.Rmd +++ b/vignettes/BiocFileCache_UseCases.Rmd @@ -345,7 +345,7 @@ bfcadd argument `ext` to assign an extension to identify the type of file that was saved. For example ``` -url = "http://httpbin.org/get" +url = "https://httpbin.org/get" bfcadd("myfile", url, download=FALSE) # would save a file `_get` in the cache bfcadd("myfile", url, download=FALSE, ext=".Rdata") From df1d8b7c4c4002d303efa4a1aae32d07aa9f5af2 Mon Sep 17 00:00:00 2001 From: lshep Date: Tue, 22 Apr 2025 12:23:07 -0400 Subject: [PATCH 10/10] Version bump for changing httr to httr2 --- DESCRIPTION | 2 +- NEWS | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index bf00a2e..c3017bb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: BiocFileCache Title: Manage Files Across Sessions -Version: 2.17.1 +Version: 2.99.0 Authors@R: c(person("Lori", "Shepherd", email = "lori.shepherd@roswellpark.org", role = c("aut", "cre")), diff --git a/NEWS b/NEWS index e7f213d..bb39de7 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +CHANGES IN VERSION 2.99.0 +---------------------- + +MAJOR UPDATES + + o Update downloading method from httr to httr2 + CHANGES IN VERSION 2.15 ----------------------