diff --git a/coderbuild/broad_sanger/01-broadSangerSamples.R b/coderbuild/broad_sanger/01-broadSangerSamples.R index b3c0965e..c853b4e2 100755 --- a/coderbuild/broad_sanger/01-broadSangerSamples.R +++ b/coderbuild/broad_sanger/01-broadSangerSamples.R @@ -10,6 +10,9 @@ library(dplyr) library(readr) library(XML) +## Retry helpers for flaky third-party downloads +source("retry_utils.R") + ##the only thing that Priyanka has here is TRP identifiers, so collecting those # tab<-read.table('DepMap_Argonne_Mapping.csv',sep=',',header=T)%>% # dplyr::select(Argonne_ID,DepMap_ID)%>% @@ -22,18 +25,83 @@ library(XML) ###[1] "Panc-05-04" "L3-3" "Panc-02-03" "Panc-10-05" "G-292-clone-A141B1" "Hs-729" "Panc-03-27" "Panc-04-03" "Hs-688-A-T" ###[10] "Panc-08-13" "PE-CA-PJ34-clone-C12" "Hep-3B2-1-7" "PE-CA-PJ41-clone-D2" "PE-CA-PJ49" "Ishikawa-Heraklio-02-ER" -##here are all the models in depmap 23Q2, downloded on 10/11/2023 -depmap_models<-readr::read_csv('https://figshare.com/ndownloader/files/40448834')#|> +## --------------------------------------------------------------------------- +## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE. +## +## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in +## front of https://depmap.org/portal/api/download/files. That endpoint now +## returns an HTML challenge page with HTTP 200, so read_csv() silently parses +## the HTML and every downstream column lookup fails. It cannot be scripted +## around, and DepMap explicitly asks that the portal not be scraped. +## +## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in +## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +## +## build_samples.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script, +## which downloads them from Synapse into $DEPMAP_DIR. This script only reads +## from disk -- it never contacts the DepMap portal. +## +## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE: +## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData +## (accept the terms, then use the download button on each file) +## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities +## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py +## 4. Re-run the build with --depmap-ready +## +## Currently pinned to: DepMap Public 26Q1 +## Files required by coderdata (4 total, ~1.29 GB): +## Model.csv (samples - this script) +## OmicsSomaticMutations.csv (omics) +## OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv (omics) +## PortalOmicsCNGeneLog2.csv (omics) +## --------------------------------------------------------------------------- +## DEPMAP_DIR is exported by build_samples.sh from fetch_depmap.py, so the +## release name is defined in exactly one place. The fallback keeps this script +## runnable standalone for debugging. +DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data") +depmap_model_file <- file.path(DEPMAP_DIR, "Model.csv") + +if (!file.exists(depmap_model_file)) { + stop(sprintf(paste0( + "Required DepMap file not found: %s\n", + "It should have been downloaded from Synapse by fetch_depmap.py before\n", + "this script ran. DepMap files can no longer be downloaded from the DepMap\n", + "portal (Cloudflare challenge), so they are read from the Synapse folder\n", + "syn75028495. To refresh them for a new release, download from\n", + "https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."), + depmap_model_file)) +} + +message("Using DepMap file from Synapse copy: ", depmap_model_file) +depmap_models<-readr::read_csv(depmap_model_file)#|> # dplyr::rename(DepMap_ID='ModelID') +## Transient container-local input -- drop it once it is in memory. +unlink(depmap_model_file) -sanger_models<-readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv") +## cog.sanger.ac.uk intermittently drops the connection mid-transfer +## (OpenSSL "unexpected eof"); this failed twice before succeeding on the +## third attempt on 2026-08-30. +sanger_models<-with_retries( + function() readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv", + show_col_types = FALSE), + what = "download Sanger model_list") print(paste("Downloaded",nrow(depmap_models),'dep map identifiers and',nrow(sanger_models),'sanger models')) ##query for cellosaurus automagically to get loadest version url='https://ftp.expasy.org/databases/cellosaurus/cellosaurus.xml' -if(!file.exists('/tmp/cell.xml')) - curl_download(url,'/tmp/cell.xml',quiet=TRUE)#curl(url, "r", h) +## 672 MB download; a partial file would poison every later run because the +## existence check below would treat it as complete. Download to a temp path +## and only move it into place once it has fully arrived. +if(!file.exists('/tmp/cell.xml')) { + with_retries(function() { + tmp_cello <- '/tmp/cell.xml.partial' + curl_download(url, tmp_cello, quiet=TRUE) + if (file.info(tmp_cello)$size < 1e8) + stop(sprintf('cellosaurus download truncated (%s bytes)', file.info(tmp_cello)$size)) + file.rename(tmp_cello, '/tmp/cell.xml') + }, what = "download cellosaurus.xml") +} cello<-XML::xmlParse('/tmp/cell.xml') cdf<-XML::xmlToList(cello) @@ -72,14 +140,14 @@ print(paste('Got',nrow(full.res),'human cellosaurus samples')) allmod<-sanger_models|> dplyr::rename(ModelID='BROAD_ID')|> left_join(depmap_models)|> - dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type)|> + dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type,ModelType)|> subset(!is.na(RRID))|> ##only take those we can map to cellosaurus distinct() ##we missed a handful here, so let's add back missedmod<-subset(depmap_models,ModelID%in%setdiff(depmap_models$ModelID,allmod$Depmap_ID))|> subset(!is.na(RRID))|> - dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype')|> + dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype',ModelType)|> mutate(PatientID=NA) allmod<-rbind(allmod,missedmod) @@ -122,7 +190,11 @@ long.df<-full.df%>% dplyr::select(-c(StrippedCellLineName))|> tidyr::pivot_longer(cols=c(PatientID,DepMap,Sanger,CCLE,COSMIC,Cellosaurus),names_to='other_id_source', values_to='other_id')%>% - mutate(model_type='cell line')%>% + mutate(model_type=dplyr::case_when( + ModelType=='Organoid' ~ 'patient derived organoid', + TRUE ~ 'cell line' + ))%>% + dplyr::select(-ModelType)%>% subset(!is.na(other_id))%>% subset(other_id!="") diff --git a/coderbuild/broad_sanger/02-broadSangerOmics.R b/coderbuild/broad_sanger/02-broadSangerOmics.R index bfd5780d..1a77a9de 100755 --- a/coderbuild/broad_sanger/02-broadSangerOmics.R +++ b/coderbuild/broad_sanger/02-broadSangerOmics.R @@ -6,56 +6,34 @@ library(tidyr) library(dplyr) library(rio) library(httr2) +library(data.table) + +## Shared retry/resume helpers (download_file_resumable, with_retries). +source("retry_utils.R") Sys.setenv(VROOM_CONNECTION_SIZE=100000000) +## Resumable, size-verified download. +## +## Kept under the original name so callers are unchanged, but the transport is +## no longer httr2. req_retry() restarts the whole request instead of resuming +## it, and the files pulled here are the largest in the pipeline -- Sanger ships +## rnaseq_all at ~897MB and WES_pureCN_CNV_genes at ~935MB. On a link that drops +## part-way, restarting a 935MB file just fails again at a similar point: build +## v24 died exactly that way on a 112MB file, breaking 12MB in, three times. +## +## download_file_resumable() (coderbuild/utils/retry_utils.R) uses wget -c, so +## each attempt continues from what is already on disk, and only promotes the +## file once its size matches the server's. +## +## req_verbose() is also gone: full curl logging across ~2GB of transfers buried +## the real errors in the build log. robust_download_httr2 <- function(url, dest, - max_tries = 5, - timeout_secs = 1500) { - # browser-style User-Agent - message("Downloading: ", url) - ua <- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" - - req <- request(url) |> - req_headers( - `User-Agent` = ua, - Accept = "application/octet-stream" - ) |> - req_timeout(timeout_secs) |> - req_retry(max_tries = max_tries, retry_on_failure = TRUE) |> - req_verbose() # ← turn on full curl logging - - resp <- tryCatch( - { - req |> req_perform(path = dest) - }, - error = function(e) { - message("🚨 Download failed for: ", url) - message(" • curl error: ", e$message) - if (file.exists(dest)) { - message(" • partial file size: ", - file.info(dest)$size, " bytes") - } - stop(e) # re-throw so your script still aborts - } - ) - - # if we get here, the download succeeded; sanity-check length - hdrs <- resp |> resp_headers() - if (!is.null(hdrs$`content-length`)) { - expected <- as.numeric(hdrs$`content-length`) - actual <- file.info(dest)$size - if (actual != expected) { - stop(sprintf( - "Incomplete download for %s: expected %d bytes but got %d", - url, expected, actual - )) - } - } - - invisible(dest) + max_tries = 5, + timeout_secs = 1500) { + download_file_resumable(url, dest, what = basename(dest)) } @@ -77,10 +55,69 @@ download_and_extract_zip_httr2 <- function(url, dest_zip, extract_dir, max_tries ##### DEPMAP FILES - -depmap_filenames=list(copy_number='https://figshare.com/ndownloader/files/40448840', - transcriptomics='https://figshare.com/ndownloader/files/40449128', - mutations='https://figshare.com/ndownloader/files/40449638') +## --------------------------------------------------------------------------- +## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE. +## +## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in +## front of https://depmap.org/portal/api/download/files. That endpoint now +## returns an HTML challenge page with HTTP 200, so read_csv() silently parses +## the HTML and every downstream column lookup fails. It cannot be scripted +## around, and DepMap explicitly asks that the portal not be scraped. +## +## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in +## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +## +## build_omics.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script, +## which downloads them from Synapse into $DEPMAP_DIR. This script only reads +## from disk -- it never contacts the DepMap portal. +## +## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE: +## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData +## (accept the terms, then use the download button on each file) +## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities +## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py +## 4. Re-run the build with --depmap-ready +## +## Currently pinned to: DepMap Public 26Q1 +## --------------------------------------------------------------------------- +## DEPMAP_DIR is exported by build_omics.sh from fetch_depmap.py, so the release +## name is defined in exactly one place. The fallback keeps this script runnable +## standalone for debugging. +DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data") + +depmap_filenames = list( + copy_number = file.path(DEPMAP_DIR, "PortalOmicsCNGeneLog2.csv"), + transcriptomics = file.path(DEPMAP_DIR, "OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv"), + mutations = file.path(DEPMAP_DIR, "OmicsSomaticMutations.csv") +) + +missing_depmap <- unlist(depmap_filenames)[!file.exists(unlist(depmap_filenames))] +if (length(missing_depmap) > 0) { + stop(sprintf(paste0( + "Required DepMap file(s) not found:\n %s\n", + "They should have been downloaded from Synapse by fetch_depmap.py before\n", + "this script ran. DepMap files can no longer be downloaded from the DepMap\n", + "portal (Cloudflare challenge), so they are read from the Synapse folder\n", + "syn75028495. To refresh them for a new release, download from\n", + "https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."), + paste(missing_depmap, collapse = "\n "))) +} +message("Using DepMap files from Synapse copy in ", DEPMAP_DIR) + +## fetch_depmap.py has already downloaded these into a container-local scratch +## directory, so there is nothing to download here -- we read them in place. +## +## Each file is deleted as soon as it has been read: they are transient build +## inputs totalling ~1.3 GB, and this step is memory- and disk-sensitive (the +## copy_number melt alone is large). Freeing each input before processing the +## next keeps the container's footprint down. The container is `--rm` anyway, +## so this is belt-and-braces, but it matters while the step is running. +depmap_consume <- function(path) { + if (!is.null(path) && file.exists(path)) { + unlink(path) + message("Removed consumed DepMap input: ", path) + } +} ##### SANGER FILES sanger_filenames=list(transcriptomics='https://cog.sanger.ac.uk/cmp/download/rnaseq_all_20220624.zip', copy_number='https://cog.sanger.ac.uk/cmp/download/WES_pureCN_CNV_genes_latest.csv.gz', @@ -88,29 +125,29 @@ sanger_filenames=list(transcriptomics='https://cog.sanger.ac.uk/cmp/download/rna ###### VARIANT SCHEMA HARMONIZATION -variant_schema =list(`3'UTR`=c("3'UTR",'THREE_PRIME_UTR','3prime_UTR_variant','3prime_UTR_ess_splice'), - `5'Flank`=c("FIVE_PRIME_FLANK","5'Flank",'upstream'), - `5'UTR`=c("5'UTR",'5prime_UTR_variant','5prime_UTR_variant','5prime_UTR_ess_splice'), - Undetermined=c('COULD_NOT_DETERMINE'), +variant_schema =list(`3'UTR`=c("3'UTR",'THREE_PRIME_UTR','3prime_UTR_variant','3prime_UTR_ess_splice','3_prime_UTR_variant'), + `5'Flank`=c("FIVE_PRIME_FLANK","5'Flank",'upstream','upstream_gene_variant'), + `5'UTR`=c("5'UTR",'5prime_UTR_variant','5prime_UTR_variant','5prime_UTR_ess_splice','5_prime_UTR_variant'), + Undetermined=c('COULD_NOT_DETERMINE','protein_altering_variant'), De_novo_Start_InFrame=c('DE_NOVO_START_IN_FRAME','De_novo_Start_InFrame'), De_novo_Start_OutOfFrame=c('DE_NOVO_START_OUT_FRAME','De_novo_Start_OutOfFrame'), - Frame_Shift_Del=c('FRAME_SHIFT_DEL','Frame_Shift_Del','frameshift'), + Frame_Shift_Del=c('FRAME_SHIFT_DEL','Frame_Shift_Del','frameshift','frameshift_variant'), Frame_Shift_Ins=c('FRAME_SHIFT_INS','Frame_Shift_Ins'), - IGR=c('IGR','nc_variant'), - In_Frame_Del=c('IN_FRAME_DEL','In_Frame_Del','inframe'), - In_Frame_Ins=c('IN_FRAME_INS','In_Frame_Ins'), - Intron=c('INTRON','Intron','intronic','intron'), - Missense_Mutation=c('Missense_Mutation','MISSENSE','missense'), - Nonsense_Mutation=c('Nonsense_Mutation','NONSENSE','nonsense'), - Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP'), - RNA=c('RNA'), + IGR=c('IGR','nc_variant','intergenic_variant','downstream_gene_variant'), + In_Frame_Del=c('IN_FRAME_DEL','In_Frame_Del','inframe','inframe_deletion'), + In_Frame_Ins=c('IN_FRAME_INS','In_Frame_Ins','inframe_insertion'), + Intron=c('INTRON','Intron','intronic','intron','intron_variant'), + Missense_Mutation=c('Missense_Mutation','MISSENSE','missense','missense_variant'), + Nonsense_Mutation=c('Nonsense_Mutation','NONSENSE','nonsense','stop_gained'), + Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP','stop_lost'), + RNA=c('RNA','non_coding_transcript_exon_variant','non_coding_transcript_variant'), Start_Codon_SNP=c('START_CODON_SNP','Start_Codon_SNP'), Start_Codon_Del=c('Start_Codon_Del','START_CODON_DEL','start_lost'), Start_Codon_Ins=c('Start_Codon_Ins','START_CODON_INS'), Stop_Codon_Del=c('Stop_Codon_Del','stop_lost'), Stop_Codon_Ins=c('Stop_Codon_Ins'), - Silent=c('Silent','SILENT','silent'), - Splice_Site=c('Splice_Site','SPLICE_SITE','splice_region'), + Silent=c('Silent','SILENT','silent','synonymous_variant'), + Splice_Site=c('Splice_Site','SPLICE_SITE','splice_region','splice_donor_variant','splice_acceptor_variant','splice_region_variant','splice_polypyrimidine_tract_variant','splice_donor_region_variant','splice_donor_5th_base_variant'), Translation_Start_Site=c('Translation_Start_Site','start_lost')) depmap_vtab<-do.call('rbind',sapply(names(variant_schema),function(x) cbind(rep(x,length(variant_schema[[x]])),unlist(variant_schema[[x]])))) @@ -149,7 +186,9 @@ sanger_files<-function(fi,value){ #read in file local_cn <- file.path(tempdir(), "sanger_copy_number.csv.gz") robust_download_httr2(fi, local_cn) - exp_file <- readr::read_csv(local_cn) + exp_file <- data.table::fread(local_cn, + select = c("model_id","symbol","gatk_mean_log2_copy_ratio","source","data_type","cn_category"), + data.table = FALSE) # exp_file <- readr::read_csv(fi) ##already in long form <3 <3 <3 # file.remove(fi) smap<-sanger_samples|> @@ -173,7 +212,8 @@ sanger_files<-function(fi,value){ dplyr::select(other_id,copy_number,entrez_id,Sanger='cn_category')|> left_join(smap)|> distinct() - rm(exp_file) + rm(exp_file); gc() + if (file.exists(local_cn)) file.remove(local_cn) print('copy call') @@ -198,7 +238,7 @@ sanger_files<-function(fi,value){ tidyr::pivot_longer(cols=c(IMPROVE,Sanger), names_to='source', values_to='copy_call') - rm(res) + rm(res); gc() # full<-lres }else if(value=='methylation'){ ###IF DATA REPRESENT RRBS### @@ -294,7 +334,7 @@ sanger_files<-function(fi,value){ dat<-exp_file[-c(1:4),-1] colnames(dat)[1]<-'gene_symbol' - rm(exp_file) + rm(exp_file); gc() ddat<-apply(dat,2,unlist)|> as.data.frame()|> @@ -302,14 +342,19 @@ sanger_files<-function(fi,value){ left_join(genes)|> dplyr::select(-gene_symbol) - rm(dat) + ## gc() here matters: without it R still holds dat when pivot_longer + ## allocates the long frame, which is the peak of the whole step. + ## Measured on a 20k gene x 800 sample matrix (16M output rows): + ## peak RSS 1,351MB -> 1,219MB, run time 8.2s -> 8.1s. The other + ## branches in this function already pair rm() with gc(). + rm(dat); gc() ddat$entrez_id<-as.numeric(ddat$entrez_id) res = tidyr::pivot_longer(data=as.data.frame(ddat),cols=c(1:(ncol(ddat)-1)), names_to='other_id',values_to='transcriptomics', values_transform=list(expression=as.numeric))|> distinct() - rm(ddat) + rm(ddat); gc() smap<-samps|> dplyr::select(improve_sample_id,other_id,study,source)|>distinct() @@ -428,25 +473,38 @@ depmap_files<-function(fi,value){ ##now every data type is parsed slightly differently, so we need to change our formatting ##and mapping to get it into a unified 3 column schema if(value=='copy_number'){ - # exp_file <- readr::read_csv(fi) - local_path <- "/tmp/depmap_copy_number.csv.gz" - robust_download_httr2(fi, local_path) - exp_file <- readr::read_csv(local_path) - - - print('Long to wide') - res = exp_file|> - tidyr::pivot_longer(cols=c(2:ncol(exp_file)), - names_to='gene_entrez',values_to='copy_number', - values_transform=list(copy_number=as.numeric))|> + local_path <- fi + ## Use data.table::melt for memory-efficient wide->long conversion + exp_dt <- data.table::fread(local_path, data.table = TRUE) + depmap_consume(local_path) # read into memory; drop the on-disk copy + id_col <- colnames(exp_dt)[1] + data.table::setnames(exp_dt, id_col, "other_id") + gene_cols <- setdiff(colnames(exp_dt), "other_id") + + print('Long to wide (data.table melt)') + res <- data.table::melt(exp_dt, id.vars = "other_id", + measure.vars = gene_cols, + variable.name = "gene_entrez", + value.name = "copy_number", + variable.factor = FALSE) + rm(exp_dt, gene_cols); gc() + ## (the DepMap input was already removed by depmap_consume above) + + res <- as.data.frame(res) |> + dplyr::mutate( + copy_number = suppressWarnings(as.numeric(copy_number)), + ## PortalOmicsCNGeneLog2 stores log2(absolute copies); convert to ratio + copy_number = 2^copy_number / 2 + ) |> dplyr::distinct() - rm(exp_file) - - colnames(res)[1]<-'other_id' print('String manipulations') - res<-res|> - tidyr::separate_wider_delim(gene_entrez,' ',names=c('gene_symbol','entrez_id')) + res <- res |> + dplyr::mutate( + gene_symbol = trimws(stringr::str_extract(gene_entrez, "^[^(]+")), + entrez_id = stringr::str_extract(gene_entrez, "(?<=\\()\\d+(?=\\))") + ) |> + dplyr::select(-gene_entrez) print('join with gene') res<-res|> @@ -493,12 +551,14 @@ depmap_files<-function(fi,value){ }else if(value=='mutations'){ ####IF DATA REPRESENTS MUTATIONS##### - local_mut <- file.path(tempdir(), "depmap_mutations.csv.gz") - robust_download_httr2(fi, local_mut) + local_mut <- fi exp_file <- readr::read_csv(local_mut)|> + dplyr::filter(IsDefaultEntryForModel %in% c(TRUE, "Yes"))|> dplyr::select(EntrezGeneID,HgncName,other_id='ModelID',VariantInfo,mutation='DNAChange')|> - distinct() + distinct()|> + dplyr::mutate(VariantInfo = sub("&.*", "", VariantInfo)) + depmap_consume(local_mut) # read into memory; drop the on-disk copy res<-exp_file|> mutate(entrez_id=as.numeric(EntrezGeneID))|> @@ -537,9 +597,15 @@ depmap_files<-function(fi,value){ return(full) }else if(value=='transcriptomics'){ #if gene expression # exp_file <- readr::read_csv(fi) - local_tx <- file.path(tempdir(), "depmap_transcriptomics.csv.gz") - robust_download_httr2(fi, local_tx) - exp_file <- readr::read_csv(local_tx) + local_tx <- fi + ## 26Q1 format has metadata columns (SequencingID, ModelConditionID, + ## IsDefaultEntryForMC, IsDefaultEntryForModel) before the gene columns. + ## Filter to one canonical run per model, then keep only ModelID + gene cols. + exp_file <- readr::read_csv(local_tx, show_col_types = FALSE)|> + dplyr::filter(IsDefaultEntryForModel %in% c(TRUE, "Yes"))|> + dplyr::rename(other_id = ModelID)|> + dplyr::select(other_id, dplyr::matches("\\(\\d+\\)")) + depmap_consume(local_tx) # read into memory; drop the on-disk copy print("wide to long") res = tidyr::pivot_longer(data=exp_file,cols=c(2:ncol(exp_file)), @@ -549,8 +615,12 @@ depmap_files<-function(fi,value){ colnames(res)[1]<-'other_id' print('fixing gene names') - res<-res|> - tidyr::separate_wider_delim(gene_entrez,' ',names=c('gene_symbol','entrez_par')) + res <- res |> + dplyr::mutate( + gene_symbol = trimws(stringr::str_extract(gene_entrez, "^[^(]+")), + entrez_par = stringr::str_extract(gene_entrez, "(?<=\\()\\d+(?=\\))") + ) |> + dplyr::select(-gene_entrez) print('join with gene') res<-res|> @@ -670,12 +740,10 @@ main<-function(){ print(dt) temps<-sanger_files(sanger_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct() readr::write_csv(temps,file=paste0('/tmp/sanger_',dt,'.csv.gz')) + rm(temps); gc() tempd<-depmap_files(depmap_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct() readr::write_csv(tempd,file=paste0('/tmp/broad_',dt,'.csv.gz')) - -# readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz')) - rm(tempd) - rm(temps) + rm(tempd); gc() }) } diff --git a/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py b/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py index c57da5b7..59be0e0b 100644 --- a/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py +++ b/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py @@ -1,3 +1,5 @@ +import os +import time import pandas as pd import argparse from zipfile import ZipFile @@ -5,28 +7,98 @@ from requests.adapters import HTTPAdapter from urllib3.util import Retry -def robust_download(url, dest_path, max_retries=5): - session = requests.Session() +# Waits between whole-download attempts: 1, 3, 10 and 15 minutes, matching +# coderbuild/utils/retry_utils.R. Long enough to ride out a real incident +# rather than only a momentary blip; if all attempts fail the build fails. +RETRY_SLEEPS = (60, 180, 600, 900) + + +def robust_download(url, dest_path, sleeps=RETRY_SLEEPS): + """Download `url` to `dest_path`, resuming an interrupted transfer. + + The previous implementation mounted a urllib3 Retry adapter and assumed that + made it robust. It did not: Retry only covers connection setup and HTTP + status codes. Once the response body is streaming, a truncated body raises + ChunkedEncodingError/ProtocolError out of iter_content(), which went straight + to the except clause and became a fatal RuntimeError -- no retry, no resume. + + That is exactly how build v24 died, 12MB into a 112MB file: + + Failed to download .../Proteomics_20221214.zip: + ('Connection broken: IncompleteRead(12353536 bytes read, + 100318054 more expected)') + + So: retry the whole transfer, and RESUME it with a Range request rather than + starting over, since restarting a large file on a flaky link tends to fail + again at a similar point. Verified that this host sends accept-ranges: bytes + and answers a Range request with 206. + + Bytes accumulate in ".part", which is renamed onto dest_path only once + the finished size matches the server's Content-Length. An unverified or + partial file therefore never appears at the destination. + """ + part = dest_path + ".part" + retry_strategy = Retry( - total=max_retries, + total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504], allowed_methods={"GET", "HEAD"}, raise_on_status=False, ) adapter = HTTPAdapter(max_retries=retry_strategy) + session = requests.Session() session.mount("https://", adapter) session.mount("http://", adapter) + expected = None try: - with session.get(url, stream=True, timeout=(5, 60)) as r: - r.raise_for_status() - with open(dest_path, "wb") as f: - for chunk in r.iter_content(chunk_size=1024 * 1024): - if chunk: # filter out keep-alive chunks - f.write(chunk) - except requests.exceptions.RequestException as e: - raise RuntimeError(f"Failed to download {url}: {e}") from e + head = session.head(url, timeout=(5, 30), allow_redirects=True) + if head.ok and head.headers.get("Content-Length"): + expected = int(head.headers["Content-Length"]) + except requests.exceptions.RequestException: + pass # size unknown; completeness is then judged by a clean stream end + + last_error = None + for attempt in range(len(sleeps) + 1): + have = os.path.getsize(part) if os.path.exists(part) else 0 + try: + headers = {"Range": f"bytes={have}-"} if have else {} + with session.get(url, stream=True, timeout=(5, 60), headers=headers) as r: + if have and r.status_code == 206: + mode = "ab" # server honoured the resume + else: + if have: + print(f" server ignored Range for {url}; restarting", flush=True) + r.raise_for_status() + mode, have = "wb", 0 + with open(part, mode) as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + if chunk: # filter out keep-alive chunks + f.write(chunk) + + got = os.path.getsize(part) + if expected is not None and got != expected: + raise IOError(f"incomplete: {got} of {expected} bytes") + + os.replace(part, dest_path) + return + + except (requests.exceptions.RequestException, IOError) as e: + last_error = e + got = os.path.getsize(part) if os.path.exists(part) else 0 + # Keep the .part file: the next attempt resumes from here. Deleting + # it would mean a flaky link could never accumulate a large file. + if attempt < len(sleeps): + wait = sleeps[attempt] + print(f" download of {url} failed at {got} bytes " + f"({e}); attempt {attempt + 1}/{len(sleeps) + 1}, " + f"resuming in {wait // 60} min", flush=True) + time.sleep(wait) + + raise RuntimeError( + f"Failed to download {url} after {len(sleeps) + 1} attempts: {last_error}" + ) def main(): diff --git a/coderbuild/broad_sanger/03-createDrugFile.R b/coderbuild/broad_sanger/03-createDrugFile.R index b1b04615..c4375066 100644 --- a/coderbuild/broad_sanger/03-createDrugFile.R +++ b/coderbuild/broad_sanger/03-createDrugFile.R @@ -9,6 +9,10 @@ source_python("pubchem_retrieval.py") library('PharmacoGx') +## Pinned PSet list shared with 04a-drugResponseData.R +source("pinned_psets.R") +## Retry/caching helpers for flaky third-party downloads +source("retry_utils.R") all.dsets<-PharmacoGx::availablePSets() @@ -26,9 +30,10 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' ignore_file_path <- '/tmp/ignore_chems.txt' for(cel in cell.lines){ - files<-subset(all.dsets,`Dataset Name`==cel)%>% - dplyr::select(`PSet Name`)%>% - unlist() + ## PSets are pinned (see coderbuild/utils/pinned_psets.R) rather than + ## taken straight from availablePSets(), so a new upstream PSet cannot + ## silently change or break the release. + files<-psets_for_dataset(all.dsets, cel) for(f in files){ print(f) @@ -36,9 +41,19 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' cel='GDSCv2' if(f=='GDSC_2020(v1-8.2)') cel='GDSCv1' - #tmpfile<-paste0(cel,'doseResponse') - dset<<-downloadPSet(f,saveDir='.',timeout=10000) + ## Skip PSets already merged into the cumulative output on an + ## earlier attempt, so a failure late in the list does not force a + ## re-run of everything before it. The marker is written only after + ## the merge below succeeds. + done_tag <- paste0("drugs-", f) + if (pset_is_done(done_tag)) { + message(sprintf( + "[03-createDrugFile] PSet '%s' already processed in a previous attempt; skipping.", f)) + next + } + + dset<<-download_pset_retry(f) mapping <- sensitivityInfo(dset)##get the dataset dose response data @@ -72,6 +87,44 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' print(paste('Found',length(chem_list),'chemicals for dataset',cel)) + ## Guard: never hand an empty/NULL chem_list to the Python writer. + ## update_dataframe_and_write_tsv() iterates it and fails with the + ## opaque "TypeError: 'NoneType' object is not iterable", which is + ## how the Aug 2026 CTRPv2.1_2016 schema change surfaced. + ## + ## Zero is also a legitimate outcome -- e.g. every compound in this + ## PSet was already covered by a previous PSet or by efile -- so + ## this skips rather than errors. A PSet whose treatment column is + ## missing entirely is reported loudly, since that means an + ## unrecognised schema rather than "nothing new to do". + ## An unrecognised schema is a HARD FAILURE, not a skip. PSets are + ## pinned (coderbuild/utils/pinned_psets.R), so if a pinned PSet no + ## longer exposes a treatment column it means upstream changed its + ## shape. Continuing would silently omit that PSet's drugs from the + ## release, which is exactly the kind of quiet data loss the pinning + ## exists to prevent -- so stop and make it visible. + if (is.null(mapping$treatmentid)) { + stop(sprintf( + paste0("PSet '%s': sensitivityInfo() has no 'treatmentid' or ", + "'drugid' column (columns: %s).\n", + "A pinned PSet has changed schema. Refusing to continue, ", + "because skipping it would drop its drugs from the release.\n", + "Either add an explicit column mapping for this PSet or ", + "remove it from PINNED_PSETS after confirming that is correct."), + f, paste(colnames(mapping), collapse = ", "))) + } + + ## Zero new chemicals is a legitimate outcome and NOT data loss: + ## every compound in this PSet was already fetched via an earlier + ## PSet in this run or via efile, so its drugs are already in the + ## cumulative output. Nothing is dropped by moving on. + if (length(chem_list) == 0) { + message(sprintf( + "[03-createDrugFile] PSet '%s': all chemicals already present; nothing new to fetch.", f)) + mark_pset_done(done_tag) + next + } + #build prev_drug_filepaths for this iteration (include original and accumulated) --- prev_list <- c() @@ -102,12 +155,19 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' } fwrite(combined, output_file_path, sep="\t") file.remove(temp_out) + ## Merge succeeded -- record this PSet as done so a later + ## attempt skips it entirely. + mark_pset_done(done_tag) } else { warning("Expected temporary output not found: ", temp_out) } - ##clean up file when done - file.remove(paste0(f,'.rds')) + ## The downloaded PSet is deliberately NOT deleted. It lives in + ## PSET_CACHE_DIR under /tmp (the bind mount to the host's local/), + ## so a retried attempt reuses it instead of re-downloading. These + ## are large -- CTRPv2 is 228 MB and one PSet is 1.5 GB -- and + ## re-fetching them is what made a single transient Zenodo 504 + ## expensive. Clear local/ between releases to reclaim the space. } } diff --git a/coderbuild/broad_sanger/03a-nci60Drugs.py b/coderbuild/broad_sanger/03a-nci60Drugs.py index 21a188ff..568c397f 100644 --- a/coderbuild/broad_sanger/03a-nci60Drugs.py +++ b/coderbuild/broad_sanger/03a-nci60Drugs.py @@ -10,6 +10,31 @@ import pubchem_retrieval as pr import random as rand from urllib import request +import download_utils +import io + +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + +def _fetch(url: str) -> bytes: + """Download url with a browser User-Agent (NCI wiki blocks Python's default).""" + req = request.Request(url, headers={"User-Agent": _BROWSER_UA}) + with request.urlopen(req) as resp: + return resp.read() + +def _read_csv_url(url: str, **kwargs) -> pl.DataFrame: + return pl.read_csv(io.BytesIO(_fetch(url)), **kwargs) + +def _retrieve_url(url: str, dest: str) -> None: + """Download url to dest, resuming an interrupted transfer. + + This also fetches the ~346MB DOSERESP.zip, and _fetch() reads the whole + body into memory in one call -- the same shape that truncated with + IncompleteRead in build v32. Small CSVs still go through _fetch(). + """ + download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA}) ##drug files smi_strings='https://wiki.nci.nih.gov/download/attachments/155844992/nsc_smiles.csv?version=1&modificationDate=1710381820000&api=v2&download=true' @@ -41,11 +66,12 @@ #jan 2025 cas = 'https://wiki.nci.nih.gov/download/attachments/155844992/nsc_cas.csv?version=4&modificationDate=1735931080720&api=v2' -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=11&modificationDate=1712351454136&api=v2' -##OCT 2024 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' +##oct 2024 +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' #jan 2025 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#may 2025 +conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=19&modificationDate=1775183341937&api=v2' def main(): parser = argparse.ArgumentParser() @@ -58,15 +84,20 @@ def main(): 'InChIKey':[],'formula':[],'weight':[],'pubchem_id':[]} print('Downloading NSC identifiers for nci60 data') - names = pl.read_csv(chemnames,ignore_errors=True) + names = _read_csv_url(chemnames, ignore_errors=True) #castab = pd.read_csv(cas) - pubchems = pl.read_csv(pc_ids) - smiles = pl.read_csv(smi_strings) + pubchems = _read_csv_url(pc_ids) + smiles = _read_csv_url(smi_strings) print('Getting experimental data to filter drugs') if not os.path.exists('DOSERESP.csv'): - resp = request.urlretrieve(conc_data,'doseresp.zip') - os.system('unzip doseresp.zip') + _retrieve_url(conc_data, 'doseresp.zip') + # -o overwrites without prompting; the exit status is checked so a + # corrupt or truncated archive fails here rather than surfacing later + # as a confusing parse error on a stale DOSERESP.csv. + _rc = os.system('unzip -o doseresp.zip') + if _rc != 0: + raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).") dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True) pubchems = pubchems.filter(pl.col('NSC').is_in(dose_resp['NSC'])) smiles = smiles.filter(pl.col("NSC").is_in(dose_resp['NSC']) @@ -81,13 +112,10 @@ def main(): ##first filter to see if there are structures/drugs in teh data already. i dont think this does much. if os.path.exists(opts.output): curdrugs = pl.read_csv(opts.output,separator='\t') - # cs = set(curdrugs['isoSMILES']) - smiles = smiles.filter(pl.col('SMILES').is_not_null()) - upper=[a.upper() for a in smiles['SMILES']] - smiles= pl.DataFrame({'NSC':smiles['NSC'],'upper':upper})#smiles.with_columns(upper=upper) - ##reduce to smiels only in current drugs - # ssmiles = smiles.filter(~pl.col('upper').is_in(curdrugs['isoSMILES'])) - ssmiles = smiles.filter(~pl.col('upper').is_in(curdrugs['canSMILES'])) + smiles_notnull = smiles.filter(pl.col('SMILES').is_not_null()) + upper=[a.upper() for a in smiles_notnull['SMILES']] + smiles_upper = pl.DataFrame({'NSC':smiles_notnull['NSC'],'upper':upper}) + ssmiles = smiles_upper.filter(~pl.col('upper').is_in(curdrugs['canSMILES'])) pubchems = pubchems.filter(pl.col('NSC').is_in(ssmiles['NSC'])) arr = set(pubchems['CID']) diff --git a/coderbuild/broad_sanger/04-drug_dosage_and_curves.py b/coderbuild/broad_sanger/04-drug_dosage_and_curves.py index 3b6f7245..e5bc7005 100755 --- a/coderbuild/broad_sanger/04-drug_dosage_and_curves.py +++ b/coderbuild/broad_sanger/04-drug_dosage_and_curves.py @@ -14,29 +14,47 @@ import argparse import pandas as pd import subprocess +import glob +import multiprocessing +from datetime import datetime + +def _log(msg): + print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] {msg}', flush=True) parser = argparse.ArgumentParser() parser.add_argument('--curSampleFile',dest='samplefile',default=None,help='DepMap sample file') - parser.add_argument('--drugfile',dest='dfile',default=None,help='Drug database') +parser.add_argument('--workers', type=int, default=max(1, round(multiprocessing.cpu_count() * 3 / 4)), help='Number of parallel worker processes for curve fitting (default: 3/4 of available CPUs)') opts = parser.parse_args() samplefile = opts.samplefile drugfile = opts.dfile -cmd = ['/opt/venv/bin/python','04b-nci60-updated.py','--sampleFile='+samplefile,'--drugFile='+drugfile] -print(cmd) -subprocess.run(cmd) +def run(cmd): + _log('running: ' + ' '.join(cmd)) + res = subprocess.run(cmd) + if res.returncode != 0: + _log(f'WARNING: command exited {res.returncode}: {" ".join(cmd)}') + return res.returncode + +# A failure here is FATAL, not a warning. +# +# In build v32 this exited 1 after DOSERESP.zip truncated, run() merely logged +# "WARNING: command exited 1", and the build carried on. nci60DoseResponse was +# never written, so the fitting below saw 7 studies instead of 8 and nci60 was +# published as eight empty files -- which pass schema validation, because a +# file with no rows cannot violate a schema. +_rc = run(['/opt/venv/bin/python','04b-nci60-updated.py','--sampleFile='+samplefile,'--drugFile='+drugfile]) +if _rc != 0: + raise RuntimeError( + f'04b-nci60-updated.py failed (exit {_rc}). It produces the NCI60 ' + f'dose-response input, and continuing without it publishes nci60 as ' + f'empty files that still validate.') ####step 4a - get dose response data -cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'CTRPv2,FIMM,GDSC'] -print(cmd) -subprocess.run(cmd) - -cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'gCSI,PRISM,CCLE'] -print(cmd) -subprocess.run(cmd) +run(['Rscript','04a-drugResponseData.R',samplefile,drugfile,'CTRPv2,FIMM,GDSC']) +run(['Rscript','04a-drugResponseData.R',samplefile,drugfile,'gCSI,PRISM,CCLE']) #cmd = 'Rscript 04a-drugResponseData.R '+samplefile+' '+drugfile+' NCI60' @@ -44,18 +62,108 @@ #os.system(cmd) ########Step 4b fit curves -allfiles=[a for a in os.listdir('./') if 'DoseResponse' in a] -print(allfiles) -for a in allfiles: - subprocess.run(['/opt/venv/bin/python','fit_curve.py','--input='+a,'--output='+a]) +# Collect dose-response inputs from BOTH places they are written. +# +# 04b-nci60-updated.py writes ./nci60DoseResponse into the working directory, +# but 04a-drugResponseData.R writes through dose_response_cache_path(), i.e. +# $DOSE_RESPONSE_CACHE_DIR/_doseResponse.tsv, so that a failed +# attempt can resume instead of regenerating every study. This scan only looked +# in './' for 'DoseResponse', which matches neither the directory nor the +# lowercase 'd', so it silently found NCI60 alone. +# +# The consequence was severe and completely silent: broad_sanger_experiments.tsv +# came out NCI60-only, and 05b_separate_datasets.py derives each split dataset's +# sample and drug ids FROM the experiments rows, so ccle, ctrpv2, fimm, gcsi, +# gdscv1, gdscv2 and prism were published with every file present but empty -- +# samples, omics, drugs and experiments alike. Header-only files are schema +# valid, so validate passed. Build v27 shipped exactly that. +# +# Excluding '.0' matters too: fit_curve writes '.0', which itself +# contains 'DoseResponse', so a retry would re-fit its own output. +CACHE_DIR = os.environ.get('DOSE_RESPONSE_CACHE_DIR', '/tmp/dose_response_cache') +jobs = [] # (input path, output stem -- kept local so '.0' lands in ./) +for a in sorted(os.listdir('./')): + if 'DoseResponse' in a and not a.endswith('.0'): + jobs.append((a, a)) +for pth in sorted(glob.glob(os.path.join(CACHE_DIR, '*_doseResponse.tsv'))): + stem = os.path.basename(pth)[:-len('.tsv')] + if any(stem == j[1] for j in jobs): + continue + jobs.append((pth, stem)) + +if not jobs: + raise RuntimeError('No DoseResponse files found — data generation step likely failed') +_log(f'fitting {len(jobs)} DoseResponse files with {opts.workers} workers: ' + f'{[j[1] for j in jobs]}') +fit_failures = [] +for src, stem in jobs: + _log(f'starting fit_curve: {stem} (from {src})') + cmd = ['/opt/venv/bin/python', 'fit_curve.py', + '--input=' + src, '--output=' + stem, '--workers=' + str(opts.workers)] + if 'nci60' in stem.lower(): + cmd.append('--chunk_size=25000') + rc = run(cmd) + if rc != 0: + _log(f'WARNING: fit_curve failed for {stem} (exit {rc}) — will be excluded from output') + fit_failures.append(stem) + _log(f'finished fit_curve: {stem}') ###step 4c concatenate all files -outfiles = [a for a in os.listdir("./") if ".0" in a] +outfiles = [a for a in os.listdir("./") if a.endswith('.0')] +if not outfiles: + raise RuntimeError('No .0 output files found after curve fitting — all fit_curve steps failed') final_file = [] for of in outfiles: - final_file.append(pd.read_csv(of,sep='\t')) - -pd.concat(final_file).drop_duplicates().to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t') + df = pd.read_csv(of, sep='\t') + if df.empty: + _log(f'WARNING: {of} is empty, skipping') + continue + _log(f'loaded {of}: {len(df)} rows') + final_file.append(df) + +if not final_file: + raise RuntimeError('All .0 output files were empty — cannot produce experiments output') +if fit_failures: + _log(f'WARNING: {len(fit_failures)} dataset(s) missing from output: {fit_failures}') + +combined = pd.concat(final_file).drop_duplicates() + +# Every study that was fitted must appear in the output. +# +# 05b_separate_datasets.py derives each split dataset's sample and drug ids from +# these rows, so a study missing here is not a partial result -- it publishes +# that whole dataset as header-only files: samples, omics, drugs, experiments. +# Those pass schema validation (no rows, nothing to violate), which is why v27 +# shipped seven empty datasets and still reported "validate successful". +# Check against the studies this step is SUPPOSED to produce, not merely the +# ones that happened to yield an input file. +# +# The previous version derived the expectation from `jobs`, so a study whose +# dose-response file was never generated simply was not expected, and its +# absence passed silently. That is precisely how v32 shipped an empty nci60: +# the input never existed, so nothing noticed it was gone. +REQUIRED_STUDIES = {'nci60', 'ccle', 'ctrpv2', 'fimm', 'gcsi', + 'gdscv1', 'gdscv2', 'prism'} +fitted_studies = {s.replace('_doseResponse', '').replace('DoseResponse', '').lower() + for s in (j[1] for j in jobs)} - {''} +not_fitted = sorted(REQUIRED_STUDIES - fitted_studies) +if not_fitted: + raise RuntimeError( + f"No dose-response input was produced for: {', '.join(not_fitted)}. " + f"Fitted only {sorted(fitted_studies)}. Each missing study becomes an " + f"entirely empty published dataset, so the build is stopping here.") +expected_studies = fitted_studies | REQUIRED_STUDIES +got_studies = {str(v).lower() for v in combined['study'].unique()} if 'study' in combined.columns else set() +missing_studies = sorted(st for st in expected_studies if st not in got_studies) +if missing_studies: + raise RuntimeError( + f"broad_sanger_experiments is missing every row for: {', '.join(missing_studies)}. " + f"Present: {sorted(got_studies)}. Each missing study becomes an entirely empty " + f"published dataset, so the build is stopping rather than shipping header-only files.") +_log(f'studies in output: {sorted(got_studies)} ({len(combined):,} rows)') + +combined.to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t') +_log(f'wrote /tmp/broad_sanger_experiments.tsv') #os.system('cat *.0 > /tmp/broad_sanger_experiments.tsv') #os.system('gzip -f /tmp/experiments.tsv') diff --git a/coderbuild/broad_sanger/04a-drugResponseData.R b/coderbuild/broad_sanger/04a-drugResponseData.R index 144ce9e9..a5d943bb 100755 --- a/coderbuild/broad_sanger/04a-drugResponseData.R +++ b/coderbuild/broad_sanger/04a-drugResponseData.R @@ -9,6 +9,11 @@ library(readr) library(dplyr) #} +## Pinned PSet list shared with 03-createDrugFile.R +source("pinned_psets.R") +## Retry/caching helpers for flaky third-party downloads +source("retry_utils.R") + all.dsets<-PharmacoGx::availablePSets() ##first define a generic dose response function @@ -137,7 +142,7 @@ getDoseRespData<-function(dset,studyName,improve_samples,drug.map){ ##now map drugs, samples, genes to ids in database files - write.table(doseRep,file=paste0(tolower(studyName),'DoseResponse'),sep='\t',row.names=F,quote=F) + write.table(doseRep,file=dose_response_cache_path(studyName),sep='\t',row.names=F,quote=F) doseRep } @@ -154,9 +159,10 @@ getCellLineDoseData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC' all.dose.rep<-do.call(rbind,lapply(cell.lines,function(cel){ # print(cel) - files<-subset(all.dsets,`Dataset Name`==cel)%>% - dplyr::select(`PSet Name`)%>% - unlist() + ## PSets are pinned (see coderbuild/utils/pinned_psets.R) and resolved + ## through the same helper the drugs step uses, so drugs and experiments + ## can never disagree about which PSets are in the release. + files<-psets_for_dataset(all.dsets, cel) res<-do.call(rbind,lapply(files,function(f){ @@ -165,12 +171,24 @@ getCellLineDoseData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC' cel='GDSCv2' if(f=='GDSC_2020(v1-8.2)') cel='GDSCv1' - tmpfile<-paste0(cel,'doseResponse') + ## Resume support: if this PSet's dose-response was already computed on + ## an earlier attempt, read it back instead of re-downloading the PSet + ## and recomputing. + ## + ## This check was previously broken and silently never matched: it looked + ## for paste0(cel,'doseResponse') while getDoseRespData() wrote + ## paste0(tolower(studyName),'DoseResponse') -- different case on both + ## halves, so e.g. CTRPv2 wrote 'ctrpv2DoseResponse' and this looked for + ## 'CTRPv2doseResponse'. Every attempt therefore redid all the work. + ## Both sides now go through dose_response_cache_path(), and the file + ## lives under /tmp so it survives between container attempts. + tmpfile<-dose_response_cache_path(cel) if(file.exists(tmpfile)){ + message(sprintf("[04a] Reusing cached dose-response for '%s' (%s)", cel, tmpfile)) dres<-read.table(tmpfile,sep='\t',header=T) }else{ - dset<<-downloadPSet(f,saveDir='.',timeout=10000) + dset<<-download_pset_retry(f) url=subset(all.dsets,`PSet Name`==f)$Download #print(url) diff --git a/coderbuild/broad_sanger/04b-nci60-updated.py b/coderbuild/broad_sanger/04b-nci60-updated.py index 3f1e8080..4c8022d1 100644 --- a/coderbuild/broad_sanger/04b-nci60-updated.py +++ b/coderbuild/broad_sanger/04b-nci60-updated.py @@ -5,38 +5,59 @@ import polars as pl import argparse -#from zipfile import ZipFile import os -#from io import BytesIO import re from urllib import request +import download_utils -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=11&modificationDate=1712351454136&api=v2' -##OCT 2024 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + +def _retrieve_url(url: str, dest: str) -> None: + """Download url, resuming an interrupted transfer. + + DOSERESP.zip is ~346MB. The previous single urlopen().read() truncated at + 32MB in build v32 with IncompleteRead, which cost the entire nci60 dataset: + this script exited 1, its caller only warned, nci60DoseResponse was never + written, and nci60 shipped as eight empty files that still validated. + """ + download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA}) + +##oct 2024 +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' #jan 2025 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#may 2025 +conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=19&modificationDate=1775183341937&api=v2' -cancelled = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP_Cancelled.csv?version=1&modificationDate=1660871847000&api=v2&download=true' +#may 2025 +oneconc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/ONECONC.zip?version=19&modificationDate=1775183401634&api=v2' def main(): - + parser = argparse.ArgumentParser() - parser.add_argument('--sampleFile',dest='samplefile',default=None,help='DepMap sample file') + parser.add_argument('--sampleFile',dest='samplefile',default=None,help='DepMap sample file') parser.add_argument('--drugFile',dest='dfile',default=None,help='Drug database') - + opts = parser.parse_args() - + samplefile = opts.samplefile drugfile = opts.dfile if not os.path.exists('DOSERESP.csv'): - resp = request.urlretrieve(conc_data,'doseresp.zip') - os.system('unzip doseresp.zip') - + _retrieve_url(conc_data, 'doseresp.zip') + # -o overwrites without prompting; the exit status is checked so a + # corrupt or truncated archive fails here rather than surfacing later + # as a confusing parse error on a stale DOSERESP.csv. + _rc = os.system('unzip -o doseresp.zip') + if _rc != 0: + raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).") + samples = pl.read_csv(samplefile,quote_char='"') drugs = pl.read_csv(drugfile,separator='\t',quote_char='"') - + dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True) ##update drug mapping @@ -51,16 +72,14 @@ def main(): drugmapping = drugmapping.unique() ###update sample mapping - on = samples[['other_names','improve_sample_id']] - on.columns=['common_name','improve_sample_id'] + on = samples[['other_names','improve_sample_id']].rename({'other_names': 'common_name'}) #there should be 71 cell lines, but there are 163. # 82 map to the 'other_names' # 81 map to neither sampmapping = pl.concat([on[['common_name','improve_sample_id']],samples[['common_name','improve_sample_id']]]) - - sampmapping = sampmapping.unique() - sampmapping.columns = ['CELL_NAME','improve_sample_id'] + + sampmapping = sampmapping.unique().rename({'common_name': 'CELL_NAME'}) ###create a time mapping tabel timemapping = pl.DataFrame( @@ -75,30 +94,15 @@ def main(): ##now we can merge all the data into the dose response data frame merged = dose_resp[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC']].join(sampmapping,on='CELL_NAME',how='left') merged = merged.join(timemapping,on='EXPID',how='left') - + ##clean up mssing samples nonulls = merged.filter(pl.col('improve_sample_id').is_not_null()) nulls = merged.filter(pl.col('improve_sample_id').is_null()) - - # newnames = pl.DataFrame( - # { - # 'new_name':[re.split(' |\(|\/',a)[0] for a in nulls['CELL_NAME']], - # 'CELL_NAME':nulls['CELL_NAME'] - # } - # ) - # newnames = newnames.unique() - - - # fixed = nulls[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC','time','time_unit']].join(newnames,on='CELL_NAME',how='left') - # merged.columns = ['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','old_CELL_NAME','EXPID','NSC','time','time_unit','CELL_NAME'] - # fixed = merged.join(sampmapping,on='CELL_NAME',how='left')[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','old_CELL_NAME','EXPID','NSC','improve_sample_id','time','time_unit']] - # fixed.columns = ['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC','improve_sample_id','time','time_unit'] -# fixed = fixed.filter(pl.col('improve_sample_id').is_not_null()) - - merged = nonulls#pl.concat([nonulls,fixed]) - - ###we get a few more results added, but still missing a bunch + + merged = nonulls + + ###we get a few more results added, but still missing a bunch merged = merged.join(drugmapping,on='NSC',how='left') nulldrugs = merged.filter(pl.col('improve_drug_id').is_null()) nonulls = merged.filter(pl.col('improve_drug_id').is_not_null()) @@ -106,7 +110,7 @@ def main(): ###now update all the concentrations to be in Moles (some are in uM, all are log10) ##some are provided as molecular weights ('v') or other ('s') and we can't compare molar = merged.filter(pl.col('CONCENTRATION_UNIT')=='M') - + finaldf = pl.DataFrame( { 'source':['NCI60_24' for a in molar['improve_drug_id']], ##2024 build @@ -122,7 +126,7 @@ def main(): ) ##write to file finaldf.write_csv('nci60DoseResponse',separator='\t') - + if __name__=='__main__': main() diff --git a/coderbuild/broad_sanger/05b_separate_datasets.py b/coderbuild/broad_sanger/05b_separate_datasets.py index d00680fe..eb110684 100644 --- a/coderbuild/broad_sanger/05b_separate_datasets.py +++ b/coderbuild/broad_sanger/05b_separate_datasets.py @@ -37,9 +37,11 @@ def main(): exp = pl.read_csv(exp_in_filename, separator="\t") # Keeping memory down, so I will not be making copies. exp = exp.filter(pl.col("study") == dataset) - # Extract information to separate out datasets - exp_improve_sample_ids = exp["improve_sample_id"].unique().to_list() - exp_improve_drug_ids = exp["improve_drug_id"].unique().to_list() + # Extract information to separate out datasets. + # improve_sample_id is always an integer (cast via Float64 to handle "1.0" TSV notation). + # improve_drug_id is always a string ("SMI_XXXXX"); cast to Utf8 for consistent is_in matching. + exp_improve_sample_ids = exp["improve_sample_id"].cast(pl.Float64).cast(pl.Int64).unique().to_list() + exp_improve_drug_ids = exp["improve_drug_id"].cast(pl.Utf8).unique().to_list() #Ensure that the improve_sample_id column is in integer form. exp = exp.with_columns(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64)) @@ -65,7 +67,7 @@ def main(): samples_filename_out = f"/tmp/{dataset}_{samples}.csv".lower() samples_df = pl.read_csv(samples_filename_in) - samples_df = samples_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids)) + samples_df = samples_df.filter(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64).is_in(exp_improve_sample_ids)) samples_df.write_csv(samples_filename_out) #csv #Rewrite as gzipped if needed @@ -86,7 +88,7 @@ def main(): omics_filename_out = f"/tmp/{dataset}_{omics}.csv".lower() omics_df = pl.read_csv(omics_filename_in) - omics_df = omics_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids)) + omics_df = omics_df.filter(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64).is_in(exp_improve_sample_ids)) # omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset])) omics_df.write_csv(omics_filename_out) #csv @@ -116,7 +118,7 @@ def main(): else: drugs_df = pl.read_csv(drugs_filename_in,separator="\t") - drugs_df = drugs_df.filter(pl.col("improve_drug_id").is_in(exp_improve_drug_ids)) + drugs_df = drugs_df.filter(pl.col("improve_drug_id").cast(pl.Utf8).is_in(exp_improve_drug_ids)) drugs_df.write_csv(drugs_filename_out,separator="\t") #tsv if drugs_filename_in.endswith(".gz"): diff --git a/coderbuild/broad_sanger/build_omics.sh b/coderbuild/broad_sanger/build_omics.sh index 024e0d17..46e1b964 100644 --- a/coderbuild/broad_sanger/build_omics.sh +++ b/coderbuild/broad_sanger/build_omics.sh @@ -3,6 +3,19 @@ set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR +# DepMap files are no longer downloadable from the DepMap portal (Cloudflare +# challenge). Pull the static copy from Synapse first; the R script reads them +# from $DEPMAP_DIR and deletes each one once consumed. +# +# $DEPMAP_DIR is container-local (NOT under /tmp, which is the bind-mounted +# host local/ directory), so these transient inputs never touch host disk. +# The trap is a backstop in case the R script exits before consuming them. +echo "Fetching DepMap files from Synapse..." +DEPMAP_DIR="$(/opt/venv/bin/python fetch_depmap.py --print-dest)" +export DEPMAP_DIR +trap 'rm -rf "$DEPMAP_DIR"' EXIT +/opt/venv/bin/python fetch_depmap.py --dest "$DEPMAP_DIR" + echo "Running 02a-broad_sanger_proteomics.py with gene file $1 and sample file $2." /opt/venv/bin/python 02a-broad_sanger_proteomics.py --gene $1 --sample $2 diff --git a/coderbuild/broad_sanger/build_samples.sh b/coderbuild/broad_sanger/build_samples.sh index 4302ba44..49220052 100644 --- a/coderbuild/broad_sanger/build_samples.sh +++ b/coderbuild/broad_sanger/build_samples.sh @@ -3,5 +3,18 @@ set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR +# DepMap files are no longer downloadable from the DepMap portal (Cloudflare +# challenge). Pull the static copy from Synapse first; the R script reads them +# from $DEPMAP_DIR and deletes each one once consumed. +# +# $DEPMAP_DIR is container-local (NOT under /tmp, which is the bind-mounted +# host local/ directory), so these transient inputs never touch host disk. +# The trap is a backstop in case the R script exits before consuming them. +echo "Fetching DepMap files from Synapse..." +DEPMAP_DIR="$(/opt/venv/bin/python fetch_depmap.py --print-dest)" +export DEPMAP_DIR +trap 'rm -rf "$DEPMAP_DIR"' EXIT +/opt/venv/bin/python fetch_depmap.py --dest "$DEPMAP_DIR" + echo "Running 01-broadSangerSamples.R." Rscript 01-broadSangerSamples.R diff --git a/coderbuild/broad_sanger/exp_requirements.r b/coderbuild/broad_sanger/exp_requirements.r index 9cc16e17..7323b1f4 100755 --- a/coderbuild/broad_sanger/exp_requirements.r +++ b/coderbuild/broad_sanger/exp_requirements.r @@ -7,3 +7,12 @@ install.packages('tidyverse') install.packages("BiocManager") BiocManager::install("PharmacoGx",update=TRUE,ask=FALSE) + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("reticulate", "tidyverse", "BiocManager", "PharmacoGx") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/broad_sanger/omics_requirements.r b/coderbuild/broad_sanger/omics_requirements.r index 26d27844..83ce09b7 100755 --- a/coderbuild/broad_sanger/omics_requirements.r +++ b/coderbuild/broad_sanger/omics_requirements.r @@ -7,4 +7,14 @@ install.packages("dplyr") install.packages("XML") #install.packages('reticulate') install.packages('tidyr') -install.packages('httr2') \ No newline at end of file +install.packages('httr2') +install.packages('data.table') + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("curl", "rio", "readr", "stringr", "dplyr", "XML", "tidyr", "httr2", "data.table") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/broad_sanger/requirements.r b/coderbuild/broad_sanger/requirements.r index 629f4fe0..a1e60b9d 100755 --- a/coderbuild/broad_sanger/requirements.r +++ b/coderbuild/broad_sanger/requirements.r @@ -11,6 +11,7 @@ install.packages("XML") install.packages('reticulate') install.packages('tidyr') install.packages('httr2') +install.packages('data.table') #install.packages("BiocManager") BiocManager::install("PharmacoGx",update=TRUE,ask=FALSE) BiocManager::install("org.Hs.eg.db",update=TRUE,ask=FALSE) diff --git a/coderbuild/broad_sanger/requirements.txt b/coderbuild/broad_sanger/requirements.txt index a00c3bd6..9a5e7191 100755 --- a/coderbuild/broad_sanger/requirements.txt +++ b/coderbuild/broad_sanger/requirements.txt @@ -11,4 +11,5 @@ polars-lts-cpu==0.19.17 mordredcommunity rdkit coderdata==0.1.40 -psutil \ No newline at end of file +psutil +synapseclient>=4.0 \ No newline at end of file