diff --git a/DESCRIPTION b/DESCRIPTION index d5d790d..89282b9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: AnnotationHub Type: Package Title: Client to access AnnotationHub resources -Version: 3.17.0 +Version: 3.99.0 Authors@R: c(person("Bioconductor Package", "Maintainer", email="maintainer@bioconductor.org", role="cre"), @@ -23,9 +23,9 @@ Description: This package provides a client for the Bioconductor files retrieved by the user, helping with quick and reproducible access. License: Artistic-2.0 -Depends: BiocGenerics (>= 0.15.10), BiocFileCache (>= 1.5.1) +Depends: BiocGenerics (>= 0.15.10), BiocFileCache (>= 2.99.0) Imports: utils, methods, grDevices, RSQLite, BiocManager, BiocVersion, curl, rappdirs, - AnnotationDbi (>= 1.31.19), S4Vectors, httr, yaml, dplyr + AnnotationDbi (>= 1.31.19), S4Vectors, httr2, yaml, dplyr Suggests: IRanges, GenomicRanges, GenomeInfoDb, VariantAnnotation, Rsamtools, rtracklayer, BiocStyle, knitr, AnnotationForge, rBiopaxParser, RUnit, txdbmaker, MSnbase, mzR, Biostrings, CompoundDb, keras, diff --git a/NAMESPACE b/NAMESPACE index eb651b9..7987832 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -13,11 +13,8 @@ importFrom(grDevices, rgb) importFrom(utils, packageVersion, read.csv, read.delim, unzip, capture.output, read.table, .DollarNames ) -importFrom(httr, - GET, content, parse_url, progress, write_disk, - stop_for_status, status_code, use_proxy, HEAD, headers -) importFrom(yaml, yaml.load) +import(httr2) importMethodsFrom(AnnotationDbi, dbconn, dbfile) importFrom(AnnotationDbi, dbFileConnect) @@ -71,8 +68,8 @@ export( .Hub, AnnotationHub, removeCache, getAnnotationHubOption, setAnnotationHubOption, - possibleDates, mcols, - .httr_proxy, .hub_option_key, .db_close, + possibleDates, mcols, .getProxyValue, + .hub_option_key, .db_close, recordStatus, DispatchClassList, refreshHub ) diff --git a/NEWS b/NEWS index 537551d..b554005 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +CHANGES IN VERSION 3.99.0 +---------------------- + +MAJOR UPDATES + + o Update downloading method from httr to httr2 + CHANGES IN VERSION 3.13.0 ------------------------- diff --git a/R/AnnotationHub-class.R b/R/AnnotationHub-class.R index 053adfd..fc7b875 100644 --- a/R/AnnotationHub-class.R +++ b/R/AnnotationHub-class.R @@ -30,9 +30,7 @@ AnnotationHub <- } if (!is.null(proxy)) { message("Assuming valid proxy connection through '", - ifelse(is(proxy,"request"), - paste(unlist(proxy), collapse=":"), - proxy), + proxy, "'", "\n If you experience connection issues consider ", "using 'localHub=TRUE'") diff --git a/R/AnnotationHubOption.R b/R/AnnotationHubOption.R index fae36e8..24ebef8 100644 --- a/R/AnnotationHubOption.R +++ b/R/AnnotationHubOption.R @@ -29,13 +29,11 @@ setAnnotationHubOption <- function(arg, value) stopifnot(isSingleInteger(value)) value }, PROXY={ - if (is.null(value) || inherits(value, "request")) + if (is.null(value) || (isSingleString(value))) value - else if (isSingleString(value)) { - .httr_proxy(value) - } else { - txt <- "'value' must be an httr proxy request (use_proxy()), - character(1), or NULL" + else { + txt <- + "'value' must be an single string value character(1) in embedded format (e.g. http://my_user:my_password@myproxy:8080), or NULL" stop(paste(strwrap(txt, exdent=2), collapse="\n")) } }, LOCAL={ @@ -47,13 +45,14 @@ setAnnotationHubOption <- function(arg, value) }) } -.httr_proxy <- function(value) -{ - rm <- parse_url(value) - if (is.null(rm$scheme)) - stop("PROXY 'value' does not include scheme (e.g., 'http://')") - rm$url <- paste0(rm$scheme, "://", rm$hostname) - if (!is.null(rm$port)) - rm$port <- as.integer(rm$port) - do.call(use_proxy, rm[c("url", "port", "username", "password")]) -} +## .httr_proxy <- function(value) +## { +## rm <- parse_url(value) +## if (is.null(rm$scheme)) +## stop("PROXY 'value' does not include scheme (e.g., 'http://')") +## rm$url <- paste0(rm$scheme, "://", rm$hostname) +## if (!is.null(rm$port)) +## rm$port <- as.integer(rm$port) +## do.call(use_proxy, rm[c("url", "port", "username", "password")]) +## } + diff --git a/R/Hub-class.R b/R/Hub-class.R index 9313503..4ac95ca 100644 --- a/R/Hub-class.R +++ b/R/Hub-class.R @@ -29,7 +29,7 @@ setClass("Hub", # find snapshot date if (!localHub){ tryCatch({ - db_date <- .restrictDateByVersion(db_path) + db_date <- .restrictDateByVersion(db_path, proxy) }, error=function(err) { stop("failed to connect", "\n reason: ", conditionMessage(err), @@ -127,6 +127,7 @@ setReplaceMethod("isLocalHub", "Hub", function(x, value) { stopifnot(value %in% c(TRUE, FALSE)) + proxy <- .getProxyValue(x) # not a no op if (isLocalHub(x) != value){ # switch from FALSE to TRUE @@ -136,7 +137,7 @@ setReplaceMethod("isLocalHub", "Hub", # switch from TRUE to FALSE }else{ db_path <- x@.db_path - db_date <- .restrictDateByVersion(db_path) + db_date <- .restrictDateByVersion(db_path, proxy) db_uid <- .db_uid0(db_path, db_date, value) x <- new(as.character(class(x)), cache=hubCache(x), hub=hubUrl(x), date=db_date, .db_path=db_path, .db_uid=db_uid, @@ -159,9 +160,10 @@ setReplaceMethod("snapshotDate", "Hub", }, error=function(err) { stop("'value' must be a single date or character string") }) + proxy <- .getProxyValue(x) ## 'value' must be < BiocManager::version() release date - restrict <- .restrictDateByVersion(dbfile(x)) + restrict <- .restrictDateByVersion(dbfile(x), proxy) dates <- .possibleDates(dbfile(x)) valid_range <- range(dates[as.POSIXlt(dates) <= as.POSIXlt(restrict)]) if (as.POSIXlt(value) > max(valid_range) || @@ -345,7 +347,8 @@ setReplaceMethod("cache", "Hub", localHub <- isLocalHub(x) db_path <- x@.db_path if (!localHub){ - db_date <- .restrictDateByVersion(db_path) + proxy <- .getProxyValue(x) + db_date <- .restrictDateByVersion(db_path, proxy) } else { dates <-.possibleDates(db_path) db_date <- dates[length(dates)] @@ -419,7 +422,8 @@ setMethod("removeResources", "character", localHub <- isLocalHub(hub) db_path <- hub@.db_path if (!localHub){ - db_date <- .restrictDateByVersion(db_path) + proxy <- .getProxyValue(x) + db_date <- .restrictDateByVersion(db_path, proxy) } else { dates <-.possibleDates(db_path) db_date <- dates[length(dates)] @@ -499,7 +503,10 @@ getSize <- function(hub, tbl) vapply(urls, FUN=function(url){ tryCatch({ - headers(HEAD(url))$`content-length` + response = request(url) %>% req_method("HEAD") %>% req_perform() + ifelse(resp_header_exists(response, "content-length"), + resp_header(response, "content-length"), + NA_character_) }, error = function(err){ NA_character_ }) diff --git a/R/Hub-utils.R b/R/Hub-utils.R index c43e3eb..24baa5c 100644 --- a/R/Hub-utils.R +++ b/R/Hub-utils.R @@ -155,12 +155,20 @@ refreshHub <- function(..., hub, cache, proxy, ### ## returns the release date for BiocManager::version() -.biocVersionDate <- function(biocversion) { +.biocVersionDate <- function(biocversion, proxy) { if (length(biocversion) > 1L) stop("length(biocversion) must == 1") - yaml <- httr::content(GET("https://bioconductor.org/config.yaml"), - encoding="UTF-8", as="text") + if ((proxy == "") || is.null(proxy)){ + proxy <- NULL + } + + req <- request("https://bioconductor.org/config.yaml") + # Apply the proxy if it's not NULL + if (!is.null(proxy)) { + req <- req %>% req_proxy(proxy) + } + yaml <- req %>% req_perform() %>% resp_body_string(encoding = "UTF-8") obj <- yaml.load(yaml) release_dates <- obj$release_dates version_date <- release_dates[biocversion == names(release_dates)] @@ -172,9 +180,9 @@ refreshHub <- function(..., hub, cache, proxy, } ## single date closest to the release date for BiocManager::version() -.restrictDateByVersion <- function(path) { +.restrictDateByVersion <- function(path, proxy) { dates <- as.POSIXlt(.possibleDates(path), format='%Y-%m-%d') - restrict <- as.POSIXlt(.biocVersionDate(BiocManager::version()), + restrict <- as.POSIXlt(.biocVersionDate(BiocManager::version(), proxy), format='%Y-%m-%d') if (length(restrict)) ## release as.character(max(dates[dates <= restrict])) @@ -197,7 +205,8 @@ refreshHub <- function(..., hub, cache, proxy, possibleDates <- function(x) { path <- dbfile(x) dates <- .possibleDates(path) - restrict <- .restrictDateByVersion(path) + proxy <- .getProxyValue(x) + restrict <- .restrictDateByVersion(path, proxy) dates[as.POSIXlt(dates) <= as.POSIXlt(restrict)] } @@ -412,3 +421,18 @@ convertHub <- function(oldcachepath=NULL, newcachepath=NULL, hubCache(hub) } + +.getProxyValue <- function(x){ + + proxy <- NULL + if(is(x, "AnnotationHub")){ + proxy <- getAnnotationHubOption("PROXY") + } + if(is(x, "ExperimentHub")){ + proxy <- getExperimentHubOption("PROXY") + } + if(is.null(proxy) && is(x, "Hub")){ + proxy <- Sys.getenv("HUB_PROXY") + } + proxy +} diff --git a/man/Hub-utils.Rd b/man/Hub-utils.Rd index 92d3996..e5dec32 100644 --- a/man/Hub-utils.Rd +++ b/man/Hub-utils.Rd @@ -82,8 +82,9 @@ getInfoOnIds(hub, 69303) "https://annotationhub.bioconductor.org/fetch/69303" # or in R -httr::GET("https://annotationhub.bioconductor.org/fetch/69303", - write_disk(, overwrite=FALSE)) +library(httr2) +req <- request("https://annotationhub.bioconductor.org/fetch/69303") +response <- req_perform(req, path=, ...) # 3. To add to a hub cache (not recommended) hub <- AnnotationHub() diff --git a/man/getAnnotationHubOption.Rd b/man/getAnnotationHubOption.Rd index 7cec548..78ded35 100644 --- a/man/getAnnotationHubOption.Rd +++ b/man/getAnnotationHubOption.Rd @@ -38,19 +38,24 @@ setAnnotationHubOption(arg, value) avoid accidental download of a large number of AnnotationHub members.} - \item{\dQuote{PROXY}:}{\code{request} object returned by - \code{httr::use_proxy()}. The \code{request} object describes a proxy + \item{\dQuote{PROXY}:}{Describes a proxy connection allowing Internet access, usually through a restrictive firewall. Setting this option sends all AnnotationHub requests through - the proxy. Default: NULL. + the proxy. Default: NULL. - In \code{setAnnotationHubOption("PROXY", value)}, \code{value} can be one of NULL, - a \code{request} object returned by \code{httr::use_proxy()}, or a - well-formed URL as character(1). The URL can be completely + In \code{setAnnotationHubOption("PROXY", value)}, \code{value} can + be NULL or a well-formed URL as character(1). The URL can be completely specified by \code{http://username:password@proxy.dom.com:8080}; \code{username:password} and port (e.g. \code{:8080}) are - optional.If behind a proxy it will also be useful to set the - \code{httr::set_config(proxy)} with the proxy information.} + optional. + + Unfortunately unlike the previously used + `httr::set_config`, there is no option to globally set the proxy + for httr2 requests. To get around this you can also set a system + wide environment variable "ANNOTATION_HUB_PROXY" for AnnotationHub + proxy or "HUB_PROXY" that will work across all Hub classes + (e.g. AnnotationHub and ExperimentHub) + } \item{\dQuote{LOCAL}:}{logical(1). TRUE/FALSE should the AnnotationHub create a hub consisting only of previously downloaded resourcesd. diff --git a/vignettes/AnnotationHub.Rmd b/vignettes/AnnotationHub.Rmd index 3edf1b8..f0212c9 100644 --- a/vignettes/AnnotationHub.Rmd +++ b/vignettes/AnnotationHub.Rmd @@ -247,18 +247,24 @@ vignette("TroubleshootingTheHubs", package="AnnotationHub") # Accessing Behind A Proxy -The AnnotationHub uses CRAN package `httr` functions `HEAD` and `GET` for accessing -web resources. This can be problematic if operating behind a proxy. The easiest -solution is to set the `httr::set_config` with the proxy information. + +The ExperimentHub and AnnotationHub use CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. You can pass proxy information for Hubs by setting the options like the following: ``` -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) +proxy <- "http://my_user:my_password@myproxy:8080" AnnotationHub::setAnnotationHubOption("PROXY", proxy) +## or +ExperimentHub::setExperimentHubOption("PROXY", proxy) ``` +Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. To get around this you can +also set a system wide environment variable "ANNOTATION_HUB_PROXY" for +AnnotationHub, "EXPERIMENT_HUB_PROXY" for ExperimentHub, or "HUB_PROXY" that +will work across all Hub classes (e.g. AnnotationHub and ExperimentHub)} + # Session info diff --git a/vignettes/TroubleshootingTheHubs.Rmd b/vignettes/TroubleshootingTheHubs.Rmd index 0791468..a2b3cc2 100644 --- a/vignettes/TroubleshootingTheHubs.Rmd +++ b/vignettes/TroubleshootingTheHubs.Rmd @@ -30,21 +30,23 @@ asked issues. If the question or answer cannot be found here please ask on the ## Accessing Behind A Proxy -The ExperimentHub and AnnotationHub use CRAN package `httr` functions `HEAD` and -`GET` for accessing web resources. This can be problematic if operating behind a -proxy. The easiest solution is to set the `httr::set_config` with the proxy -information. +The ExperimentHub and AnnotationHub use CRAN package `httr2` functions for accessing and downloading +web resources. This can be problematic if operating behind a +proxy. You can pass proxy information for Hubs by setting the options like the following: ``` -proxy <- httr::use_proxy("http://my_user:my_password@myproxy:8080") -## or -proxy <- httr::use_proxy(Sys.getenv('http_proxy')) -httr::set_config(proxy) +proxy <- "http://my_user:my_password@myproxy:8080" AnnotationHub::setAnnotationHubOption("PROXY", proxy) ## or ExperimentHub::setExperimentHubOption("PROXY", proxy) ``` +Unfortunately unlike the previously used `httr::set_config`, there is no +option to globally set the proxy for httr2 requests. To get around this you can +also set a system wide environment variable "ANNOTATION_HUB_PROXY" for +AnnotationHub, "EXPERIMENT_HUB_PROXY" for ExperimentHub, or "HUB_PROXY" that +will work across all Hub classes (e.g. AnnotationHub and ExperimentHub)} + ## Invalid Cache An invalid cache ERROR results from a missing sqlite or index file in the Hub's