diff --git a/.gitignore b/.gitignore index fe663199..bfd44a7a 100755 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,11 @@ build/lib build/local coderdata/_version.py local/ + +# Build logs and PID files. +# Build logs capture full `docker run` command lines. Those command lines pass +# credentials as `-e NAME=value`, so older logs contain a live +# SYNAPSE_AUTH_TOKEN in cleartext (build_all_v19.log had 47 copies). +# coderbuild/build_all.py now redacts them via _redact(), but never commit these. +*.log +build_v*.pid diff --git a/coderbuild/broad_sanger/01-broadSangerSamples.R b/coderbuild/broad_sanger/01-broadSangerSamples.R index b3c0965e..c853b4e2 100755 --- a/coderbuild/broad_sanger/01-broadSangerSamples.R +++ b/coderbuild/broad_sanger/01-broadSangerSamples.R @@ -10,6 +10,9 @@ library(dplyr) library(readr) library(XML) +## Retry helpers for flaky third-party downloads +source("retry_utils.R") + ##the only thing that Priyanka has here is TRP identifiers, so collecting those # tab<-read.table('DepMap_Argonne_Mapping.csv',sep=',',header=T)%>% # dplyr::select(Argonne_ID,DepMap_ID)%>% @@ -22,18 +25,83 @@ library(XML) ###[1] "Panc-05-04" "L3-3" "Panc-02-03" "Panc-10-05" "G-292-clone-A141B1" "Hs-729" "Panc-03-27" "Panc-04-03" "Hs-688-A-T" ###[10] "Panc-08-13" "PE-CA-PJ34-clone-C12" "Hep-3B2-1-7" "PE-CA-PJ41-clone-D2" "PE-CA-PJ49" "Ishikawa-Heraklio-02-ER" -##here are all the models in depmap 23Q2, downloded on 10/11/2023 -depmap_models<-readr::read_csv('https://figshare.com/ndownloader/files/40448834')#|> +## --------------------------------------------------------------------------- +## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE. +## +## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in +## front of https://depmap.org/portal/api/download/files. That endpoint now +## returns an HTML challenge page with HTTP 200, so read_csv() silently parses +## the HTML and every downstream column lookup fails. It cannot be scripted +## around, and DepMap explicitly asks that the portal not be scraped. +## +## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in +## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +## +## build_samples.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script, +## which downloads them from Synapse into $DEPMAP_DIR. This script only reads +## from disk -- it never contacts the DepMap portal. +## +## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE: +## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData +## (accept the terms, then use the download button on each file) +## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities +## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py +## 4. Re-run the build with --depmap-ready +## +## Currently pinned to: DepMap Public 26Q1 +## Files required by coderdata (4 total, ~1.29 GB): +## Model.csv (samples - this script) +## OmicsSomaticMutations.csv (omics) +## OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv (omics) +## PortalOmicsCNGeneLog2.csv (omics) +## --------------------------------------------------------------------------- +## DEPMAP_DIR is exported by build_samples.sh from fetch_depmap.py, so the +## release name is defined in exactly one place. The fallback keeps this script +## runnable standalone for debugging. +DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data") +depmap_model_file <- file.path(DEPMAP_DIR, "Model.csv") + +if (!file.exists(depmap_model_file)) { + stop(sprintf(paste0( + "Required DepMap file not found: %s\n", + "It should have been downloaded from Synapse by fetch_depmap.py before\n", + "this script ran. DepMap files can no longer be downloaded from the DepMap\n", + "portal (Cloudflare challenge), so they are read from the Synapse folder\n", + "syn75028495. To refresh them for a new release, download from\n", + "https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."), + depmap_model_file)) +} + +message("Using DepMap file from Synapse copy: ", depmap_model_file) +depmap_models<-readr::read_csv(depmap_model_file)#|> # dplyr::rename(DepMap_ID='ModelID') +## Transient container-local input -- drop it once it is in memory. +unlink(depmap_model_file) -sanger_models<-readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv") +## cog.sanger.ac.uk intermittently drops the connection mid-transfer +## (OpenSSL "unexpected eof"); this failed twice before succeeding on the +## third attempt on 2026-08-30. +sanger_models<-with_retries( + function() readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv", + show_col_types = FALSE), + what = "download Sanger model_list") print(paste("Downloaded",nrow(depmap_models),'dep map identifiers and',nrow(sanger_models),'sanger models')) ##query for cellosaurus automagically to get loadest version url='https://ftp.expasy.org/databases/cellosaurus/cellosaurus.xml' -if(!file.exists('/tmp/cell.xml')) - curl_download(url,'/tmp/cell.xml',quiet=TRUE)#curl(url, "r", h) +## 672 MB download; a partial file would poison every later run because the +## existence check below would treat it as complete. Download to a temp path +## and only move it into place once it has fully arrived. +if(!file.exists('/tmp/cell.xml')) { + with_retries(function() { + tmp_cello <- '/tmp/cell.xml.partial' + curl_download(url, tmp_cello, quiet=TRUE) + if (file.info(tmp_cello)$size < 1e8) + stop(sprintf('cellosaurus download truncated (%s bytes)', file.info(tmp_cello)$size)) + file.rename(tmp_cello, '/tmp/cell.xml') + }, what = "download cellosaurus.xml") +} cello<-XML::xmlParse('/tmp/cell.xml') cdf<-XML::xmlToList(cello) @@ -72,14 +140,14 @@ print(paste('Got',nrow(full.res),'human cellosaurus samples')) allmod<-sanger_models|> dplyr::rename(ModelID='BROAD_ID')|> left_join(depmap_models)|> - dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type)|> + dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type,ModelType)|> subset(!is.na(RRID))|> ##only take those we can map to cellosaurus distinct() ##we missed a handful here, so let's add back missedmod<-subset(depmap_models,ModelID%in%setdiff(depmap_models$ModelID,allmod$Depmap_ID))|> subset(!is.na(RRID))|> - dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype')|> + dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype',ModelType)|> mutate(PatientID=NA) allmod<-rbind(allmod,missedmod) @@ -122,7 +190,11 @@ long.df<-full.df%>% dplyr::select(-c(StrippedCellLineName))|> tidyr::pivot_longer(cols=c(PatientID,DepMap,Sanger,CCLE,COSMIC,Cellosaurus),names_to='other_id_source', values_to='other_id')%>% - mutate(model_type='cell line')%>% + mutate(model_type=dplyr::case_when( + ModelType=='Organoid' ~ 'patient derived organoid', + TRUE ~ 'cell line' + ))%>% + dplyr::select(-ModelType)%>% subset(!is.na(other_id))%>% subset(other_id!="") diff --git a/coderbuild/broad_sanger/02-broadSangerOmics.R b/coderbuild/broad_sanger/02-broadSangerOmics.R index 10f248e7..131876b9 100755 --- a/coderbuild/broad_sanger/02-broadSangerOmics.R +++ b/coderbuild/broad_sanger/02-broadSangerOmics.R @@ -6,56 +6,34 @@ library(tidyr) library(dplyr) library(rio) library(httr2) +library(data.table) + +## Shared retry/resume helpers (download_file_resumable, with_retries). +source("retry_utils.R") Sys.setenv(VROOM_CONNECTION_SIZE=100000000) +## Resumable, size-verified download. +## +## Kept under the original name so callers are unchanged, but the transport is +## no longer httr2. req_retry() restarts the whole request instead of resuming +## it, and the files pulled here are the largest in the pipeline -- Sanger ships +## rnaseq_all at ~897MB and WES_pureCN_CNV_genes at ~935MB. On a link that drops +## part-way, restarting a 935MB file just fails again at a similar point: build +## v24 died exactly that way on a 112MB file, breaking 12MB in, three times. +## +## download_file_resumable() (coderbuild/utils/retry_utils.R) uses wget -c, so +## each attempt continues from what is already on disk, and only promotes the +## file once its size matches the server's. +## +## req_verbose() is also gone: full curl logging across ~2GB of transfers buried +## the real errors in the build log. robust_download_httr2 <- function(url, dest, - max_tries = 5, - timeout_secs = 1500) { - # browser-style User-Agent - message("Downloading: ", url) - ua <- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" - - req <- request(url) |> - req_headers( - `User-Agent` = ua, - Accept = "application/octet-stream" - ) |> - req_timeout(timeout_secs) |> - req_retry(max_tries = max_tries, retry_on_failure = TRUE) |> - req_verbose() # ← turn on full curl logging - - resp <- tryCatch( - { - req |> req_perform(path = dest) - }, - error = function(e) { - message("🚨 Download failed for: ", url) - message(" • curl error: ", e$message) - if (file.exists(dest)) { - message(" • partial file size: ", - file.info(dest)$size, " bytes") - } - stop(e) # re-throw so your script still aborts - } - ) - - # if we get here, the download succeeded; sanity-check length - hdrs <- resp |> resp_headers() - if (!is.null(hdrs$`content-length`)) { - expected <- as.numeric(hdrs$`content-length`) - actual <- file.info(dest)$size - if (actual != expected) { - stop(sprintf( - "Incomplete download for %s: expected %d bytes but got %d", - url, expected, actual - )) - } - } - - invisible(dest) + max_tries = 5, + timeout_secs = 1500) { + download_file_resumable(url, dest, what = basename(dest)) } @@ -77,10 +55,69 @@ download_and_extract_zip_httr2 <- function(url, dest_zip, extract_dir, max_tries ##### DEPMAP FILES - -depmap_filenames=list(copy_number='https://figshare.com/ndownloader/files/40448840', - transcriptomics='https://figshare.com/ndownloader/files/40449128', - mutations='https://figshare.com/ndownloader/files/40449638') +## --------------------------------------------------------------------------- +## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE. +## +## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in +## front of https://depmap.org/portal/api/download/files. That endpoint now +## returns an HTML challenge page with HTTP 200, so read_csv() silently parses +## the HTML and every downstream column lookup fails. It cannot be scripted +## around, and DepMap explicitly asks that the portal not be scraped. +## +## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in +## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +## +## build_omics.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script, +## which downloads them from Synapse into $DEPMAP_DIR. This script only reads +## from disk -- it never contacts the DepMap portal. +## +## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE: +## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData +## (accept the terms, then use the download button on each file) +## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities +## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py +## 4. Re-run the build with --depmap-ready +## +## Currently pinned to: DepMap Public 26Q1 +## --------------------------------------------------------------------------- +## DEPMAP_DIR is exported by build_omics.sh from fetch_depmap.py, so the release +## name is defined in exactly one place. The fallback keeps this script runnable +## standalone for debugging. +DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data") + +depmap_filenames = list( + copy_number = file.path(DEPMAP_DIR, "PortalOmicsCNGeneLog2.csv"), + transcriptomics = file.path(DEPMAP_DIR, "OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv"), + mutations = file.path(DEPMAP_DIR, "OmicsSomaticMutations.csv") +) + +missing_depmap <- unlist(depmap_filenames)[!file.exists(unlist(depmap_filenames))] +if (length(missing_depmap) > 0) { + stop(sprintf(paste0( + "Required DepMap file(s) not found:\n %s\n", + "They should have been downloaded from Synapse by fetch_depmap.py before\n", + "this script ran. DepMap files can no longer be downloaded from the DepMap\n", + "portal (Cloudflare challenge), so they are read from the Synapse folder\n", + "syn75028495. To refresh them for a new release, download from\n", + "https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."), + paste(missing_depmap, collapse = "\n "))) +} +message("Using DepMap files from Synapse copy in ", DEPMAP_DIR) + +## fetch_depmap.py has already downloaded these into a container-local scratch +## directory, so there is nothing to download here -- we read them in place. +## +## Each file is deleted as soon as it has been read: they are transient build +## inputs totalling ~1.3 GB, and this step is memory- and disk-sensitive (the +## copy_number melt alone is large). Freeing each input before processing the +## next keeps the container's footprint down. The container is `--rm` anyway, +## so this is belt-and-braces, but it matters while the step is running. +depmap_consume <- function(path) { + if (!is.null(path) && file.exists(path)) { + unlink(path) + message("Removed consumed DepMap input: ", path) + } +} ##### SANGER FILES sanger_filenames=list(transcriptomics='https://cog.sanger.ac.uk/cmp/download/rnaseq_all_20220624.zip', copy_number='https://cog.sanger.ac.uk/cmp/download/WES_pureCN_CNV_genes_latest.csv.gz', @@ -88,29 +125,29 @@ sanger_filenames=list(transcriptomics='https://cog.sanger.ac.uk/cmp/download/rna ###### VARIANT SCHEMA HARMONIZATION -variant_schema =list(`3'UTR`=c("3'UTR",'THREE_PRIME_UTR','3prime_UTR_variant','3prime_UTR_ess_splice'), - `5'Flank`=c("FIVE_PRIME_FLANK","5'Flank",'upstream'), - `5'UTR`=c("5'UTR",'5prime_UTR_variant','5prime_UTR_variant','5prime_UTR_ess_splice'), - Undetermined=c('COULD_NOT_DETERMINE'), +variant_schema =list(`3'UTR`=c("3'UTR",'THREE_PRIME_UTR','3prime_UTR_variant','3prime_UTR_ess_splice','3_prime_UTR_variant'), + `5'Flank`=c("FIVE_PRIME_FLANK","5'Flank",'upstream','upstream_gene_variant'), + `5'UTR`=c("5'UTR",'5prime_UTR_variant','5prime_UTR_variant','5prime_UTR_ess_splice','5_prime_UTR_variant'), + Undetermined=c('COULD_NOT_DETERMINE','protein_altering_variant'), De_novo_Start_InFrame=c('DE_NOVO_START_IN_FRAME','De_novo_Start_InFrame'), De_novo_Start_OutOfFrame=c('DE_NOVO_START_OUT_FRAME','De_novo_Start_OutOfFrame'), - Frame_Shift_Del=c('FRAME_SHIFT_DEL','Frame_Shift_Del','frameshift'), + Frame_Shift_Del=c('FRAME_SHIFT_DEL','Frame_Shift_Del','frameshift','frameshift_variant'), Frame_Shift_Ins=c('FRAME_SHIFT_INS','Frame_Shift_Ins'), - IGR=c('IGR','nc_variant'), - In_Frame_Del=c('IN_FRAME_DEL','In_Frame_Del','inframe'), - In_Frame_Ins=c('IN_FRAME_INS','In_Frame_Ins'), - Intron=c('INTRON','Intron','intronic','intron'), - Missense_Mutation=c('Missense_Mutation','MISSENSE','missense'), - Nonsense_Mutation=c('Nonsense_Mutation','NONSENSE','nonsense'), - Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP'), - RNA=c('RNA'), + IGR=c('IGR','nc_variant','intergenic_variant','downstream_gene_variant'), + In_Frame_Del=c('IN_FRAME_DEL','In_Frame_Del','inframe','inframe_deletion'), + In_Frame_Ins=c('IN_FRAME_INS','In_Frame_Ins','inframe_insertion'), + Intron=c('INTRON','Intron','intronic','intron','intron_variant'), + Missense_Mutation=c('Missense_Mutation','MISSENSE','missense','missense_variant'), + Nonsense_Mutation=c('Nonsense_Mutation','NONSENSE','nonsense','stop_gained'), + Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP','stop_lost'), + RNA=c('RNA','non_coding_transcript_exon_variant','non_coding_transcript_variant'), Start_Codon_SNP=c('START_CODON_SNP','Start_Codon_SNP'), Start_Codon_Del=c('Start_Codon_Del','START_CODON_DEL','start_lost'), Start_Codon_Ins=c('Start_Codon_Ins','START_CODON_INS'), Stop_Codon_Del=c('Stop_Codon_Del','stop_lost'), Stop_Codon_Ins=c('Stop_Codon_Ins'), - Silent=c('Silent','SILENT','silent'), - Splice_Site=c('Splice_Site','SPLICE_SITE','splice_region'), + Silent=c('Silent','SILENT','silent','synonymous_variant'), + Splice_Site=c('Splice_Site','SPLICE_SITE','splice_region','splice_donor_variant','splice_acceptor_variant','splice_region_variant','splice_polypyrimidine_tract_variant','splice_donor_region_variant','splice_donor_5th_base_variant'), Translation_Start_Site=c('Translation_Start_Site','start_lost')) depmap_vtab<-do.call('rbind',sapply(names(variant_schema),function(x) cbind(rep(x,length(variant_schema[[x]])),unlist(variant_schema[[x]])))) @@ -149,7 +186,9 @@ sanger_files<-function(fi,value){ #read in file local_cn <- file.path(tempdir(), "sanger_copy_number.csv.gz") robust_download_httr2(fi, local_cn) - exp_file <- readr::read_csv(local_cn) + exp_file <- data.table::fread(local_cn, + select = c("model_id","symbol","gatk_mean_log2_copy_ratio","source","data_type","cn_category"), + data.table = FALSE) # exp_file <- readr::read_csv(fi) ##already in long form <3 <3 <3 # file.remove(fi) smap<-sanger_samples|> @@ -174,7 +213,8 @@ sanger_files<-function(fi,value){ dplyr::select(other_id,copy_number,entrez_id,Sanger='cn_category')|> left_join(smap)|> distinct() - rm(exp_file) + rm(exp_file); gc() + if (file.exists(local_cn)) file.remove(local_cn) print('copy call') @@ -199,7 +239,7 @@ sanger_files<-function(fi,value){ tidyr::pivot_longer(cols=c(IMPROVE,Sanger), names_to='source', values_to='copy_call') - rm(res) + rm(res); gc() # full<-lres }else if(value=='methylation'){ ###IF DATA REPRESENT RRBS### @@ -295,7 +335,7 @@ sanger_files<-function(fi,value){ dat<-exp_file[-c(1:4),-1] colnames(dat)[1]<-'gene_symbol' - rm(exp_file) + rm(exp_file); gc() ddat<-apply(dat,2,unlist)|> as.data.frame()|> @@ -303,14 +343,19 @@ sanger_files<-function(fi,value){ left_join(genes)|> dplyr::select(-gene_symbol) - rm(dat) + ## gc() here matters: without it R still holds dat when pivot_longer + ## allocates the long frame, which is the peak of the whole step. + ## Measured on a 20k gene x 800 sample matrix (16M output rows): + ## peak RSS 1,351MB -> 1,219MB, run time 8.2s -> 8.1s. The other + ## branches in this function already pair rm() with gc(). + rm(dat); gc() ddat$entrez_id<-as.numeric(ddat$entrez_id) res = tidyr::pivot_longer(data=as.data.frame(ddat),cols=c(1:(ncol(ddat)-1)), names_to='other_id',values_to='transcriptomics', values_transform=list(expression=as.numeric))|> distinct() - rm(ddat) + rm(ddat); gc() smap<-samps|> dplyr::select(improve_sample_id,other_id,study,source)|>distinct() @@ -429,25 +474,37 @@ depmap_files<-function(fi,value){ ##now every data type is parsed slightly differently, so we need to change our formatting ##and mapping to get it into a unified 3 column schema if(value=='copy_number'){ - # exp_file <- readr::read_csv(fi) - local_path <- "/tmp/depmap_copy_number.csv.gz" - robust_download_httr2(fi, local_path) - exp_file <- readr::read_csv(local_path) - - - print('Long to wide') - res = exp_file|> - tidyr::pivot_longer(cols=c(2:ncol(exp_file)), - names_to='gene_entrez',values_to='copy_number', - values_transform=list(copy_number=as.numeric))|> + local_path <- fi + ## Use data.table::melt for memory-efficient wide->long conversion + exp_dt <- data.table::fread(local_path, data.table = TRUE) + depmap_consume(local_path) # read into memory; drop the on-disk copy + id_col <- colnames(exp_dt)[1] + data.table::setnames(exp_dt, id_col, "other_id") + gene_cols <- setdiff(colnames(exp_dt), "other_id") + + print('Long to wide (data.table melt)') + res <- data.table::melt(exp_dt, id.vars = "other_id", + measure.vars = gene_cols, + variable.name = "gene_entrez", + value.name = "copy_number", + variable.factor = FALSE) + rm(exp_dt, gene_cols); gc() + ## (the DepMap input was already removed by depmap_consume above) + + res <- as.data.frame(res) |> + dplyr::mutate( + ## PortalOmicsCNGeneLog2 is already log2(relative CN + 1); keep as-is + copy_number = suppressWarnings(as.numeric(copy_number)) + ) |> dplyr::distinct() - rm(exp_file) - - colnames(res)[1]<-'other_id' print('String manipulations') - res<-res|> - tidyr::separate_wider_delim(gene_entrez,' ',names=c('gene_symbol','entrez_id')) + res <- res |> + dplyr::mutate( + gene_symbol = trimws(stringr::str_extract(gene_entrez, "^[^(]+")), + entrez_id = stringr::str_extract(gene_entrez, "(?<=\\()\\d+(?=\\))") + ) |> + dplyr::select(-gene_entrez) print('join with gene') res<-res|> @@ -494,12 +551,14 @@ depmap_files<-function(fi,value){ }else if(value=='mutations'){ ####IF DATA REPRESENTS MUTATIONS##### - local_mut <- file.path(tempdir(), "depmap_mutations.csv.gz") - robust_download_httr2(fi, local_mut) + local_mut <- fi exp_file <- readr::read_csv(local_mut)|> + dplyr::filter(IsDefaultEntryForModel %in% c(TRUE, "Yes"))|> dplyr::select(EntrezGeneID,HgncName,other_id='ModelID',VariantInfo,mutation='DNAChange')|> - distinct() + distinct()|> + dplyr::mutate(VariantInfo = sub("&.*", "", VariantInfo)) + depmap_consume(local_mut) # read into memory; drop the on-disk copy res<-exp_file|> mutate(entrez_id=as.numeric(EntrezGeneID))|> @@ -538,9 +597,15 @@ depmap_files<-function(fi,value){ return(full) }else if(value=='transcriptomics'){ #if gene expression # exp_file <- readr::read_csv(fi) - local_tx <- file.path(tempdir(), "depmap_transcriptomics.csv.gz") - robust_download_httr2(fi, local_tx) - exp_file <- readr::read_csv(local_tx) + local_tx <- fi + ## 26Q1 format has metadata columns (SequencingID, ModelConditionID, + ## IsDefaultEntryForMC, IsDefaultEntryForModel) before the gene columns. + ## Filter to one canonical run per model, then keep only ModelID + gene cols. + exp_file <- readr::read_csv(local_tx, show_col_types = FALSE)|> + dplyr::filter(IsDefaultEntryForModel %in% c(TRUE, "Yes"))|> + dplyr::rename(other_id = ModelID)|> + dplyr::select(other_id, dplyr::matches("\\(\\d+\\)")) + depmap_consume(local_tx) # read into memory; drop the on-disk copy print("wide to long") res = tidyr::pivot_longer(data=exp_file,cols=c(2:ncol(exp_file)), @@ -550,8 +615,12 @@ depmap_files<-function(fi,value){ colnames(res)[1]<-'other_id' print('fixing gene names') - res<-res|> - tidyr::separate_wider_delim(gene_entrez,' ',names=c('gene_symbol','entrez_par')) + res <- res |> + dplyr::mutate( + gene_symbol = trimws(stringr::str_extract(gene_entrez, "^[^(]+")), + entrez_par = stringr::str_extract(gene_entrez, "(?<=\\()\\d+(?=\\))") + ) |> + dplyr::select(-gene_entrez) print('join with gene') res<-res|> @@ -671,12 +740,10 @@ main<-function(){ print(dt) temps<-sanger_files(sanger_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct() readr::write_csv(temps,file=paste0('/tmp/sanger_',dt,'.csv.gz')) + rm(temps); gc() tempd<-depmap_files(depmap_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct() readr::write_csv(tempd,file=paste0('/tmp/broad_',dt,'.csv.gz')) - -# readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz')) - rm(tempd) - rm(temps) + rm(tempd); gc() }) } diff --git a/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py b/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py index c57da5b7..59be0e0b 100644 --- a/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py +++ b/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py @@ -1,3 +1,5 @@ +import os +import time import pandas as pd import argparse from zipfile import ZipFile @@ -5,28 +7,98 @@ from requests.adapters import HTTPAdapter from urllib3.util import Retry -def robust_download(url, dest_path, max_retries=5): - session = requests.Session() +# Waits between whole-download attempts: 1, 3, 10 and 15 minutes, matching +# coderbuild/utils/retry_utils.R. Long enough to ride out a real incident +# rather than only a momentary blip; if all attempts fail the build fails. +RETRY_SLEEPS = (60, 180, 600, 900) + + +def robust_download(url, dest_path, sleeps=RETRY_SLEEPS): + """Download `url` to `dest_path`, resuming an interrupted transfer. + + The previous implementation mounted a urllib3 Retry adapter and assumed that + made it robust. It did not: Retry only covers connection setup and HTTP + status codes. Once the response body is streaming, a truncated body raises + ChunkedEncodingError/ProtocolError out of iter_content(), which went straight + to the except clause and became a fatal RuntimeError -- no retry, no resume. + + That is exactly how build v24 died, 12MB into a 112MB file: + + Failed to download .../Proteomics_20221214.zip: + ('Connection broken: IncompleteRead(12353536 bytes read, + 100318054 more expected)') + + So: retry the whole transfer, and RESUME it with a Range request rather than + starting over, since restarting a large file on a flaky link tends to fail + again at a similar point. Verified that this host sends accept-ranges: bytes + and answers a Range request with 206. + + Bytes accumulate in ".part", which is renamed onto dest_path only once + the finished size matches the server's Content-Length. An unverified or + partial file therefore never appears at the destination. + """ + part = dest_path + ".part" + retry_strategy = Retry( - total=max_retries, + total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504], allowed_methods={"GET", "HEAD"}, raise_on_status=False, ) adapter = HTTPAdapter(max_retries=retry_strategy) + session = requests.Session() session.mount("https://", adapter) session.mount("http://", adapter) + expected = None try: - with session.get(url, stream=True, timeout=(5, 60)) as r: - r.raise_for_status() - with open(dest_path, "wb") as f: - for chunk in r.iter_content(chunk_size=1024 * 1024): - if chunk: # filter out keep-alive chunks - f.write(chunk) - except requests.exceptions.RequestException as e: - raise RuntimeError(f"Failed to download {url}: {e}") from e + head = session.head(url, timeout=(5, 30), allow_redirects=True) + if head.ok and head.headers.get("Content-Length"): + expected = int(head.headers["Content-Length"]) + except requests.exceptions.RequestException: + pass # size unknown; completeness is then judged by a clean stream end + + last_error = None + for attempt in range(len(sleeps) + 1): + have = os.path.getsize(part) if os.path.exists(part) else 0 + try: + headers = {"Range": f"bytes={have}-"} if have else {} + with session.get(url, stream=True, timeout=(5, 60), headers=headers) as r: + if have and r.status_code == 206: + mode = "ab" # server honoured the resume + else: + if have: + print(f" server ignored Range for {url}; restarting", flush=True) + r.raise_for_status() + mode, have = "wb", 0 + with open(part, mode) as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + if chunk: # filter out keep-alive chunks + f.write(chunk) + + got = os.path.getsize(part) + if expected is not None and got != expected: + raise IOError(f"incomplete: {got} of {expected} bytes") + + os.replace(part, dest_path) + return + + except (requests.exceptions.RequestException, IOError) as e: + last_error = e + got = os.path.getsize(part) if os.path.exists(part) else 0 + # Keep the .part file: the next attempt resumes from here. Deleting + # it would mean a flaky link could never accumulate a large file. + if attempt < len(sleeps): + wait = sleeps[attempt] + print(f" download of {url} failed at {got} bytes " + f"({e}); attempt {attempt + 1}/{len(sleeps) + 1}, " + f"resuming in {wait // 60} min", flush=True) + time.sleep(wait) + + raise RuntimeError( + f"Failed to download {url} after {len(sleeps) + 1} attempts: {last_error}" + ) def main(): diff --git a/coderbuild/broad_sanger/03-createDrugFile.R b/coderbuild/broad_sanger/03-createDrugFile.R index b1b04615..c4375066 100644 --- a/coderbuild/broad_sanger/03-createDrugFile.R +++ b/coderbuild/broad_sanger/03-createDrugFile.R @@ -9,6 +9,10 @@ source_python("pubchem_retrieval.py") library('PharmacoGx') +## Pinned PSet list shared with 04a-drugResponseData.R +source("pinned_psets.R") +## Retry/caching helpers for flaky third-party downloads +source("retry_utils.R") all.dsets<-PharmacoGx::availablePSets() @@ -26,9 +30,10 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' ignore_file_path <- '/tmp/ignore_chems.txt' for(cel in cell.lines){ - files<-subset(all.dsets,`Dataset Name`==cel)%>% - dplyr::select(`PSet Name`)%>% - unlist() + ## PSets are pinned (see coderbuild/utils/pinned_psets.R) rather than + ## taken straight from availablePSets(), so a new upstream PSet cannot + ## silently change or break the release. + files<-psets_for_dataset(all.dsets, cel) for(f in files){ print(f) @@ -36,9 +41,19 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' cel='GDSCv2' if(f=='GDSC_2020(v1-8.2)') cel='GDSCv1' - #tmpfile<-paste0(cel,'doseResponse') - dset<<-downloadPSet(f,saveDir='.',timeout=10000) + ## Skip PSets already merged into the cumulative output on an + ## earlier attempt, so a failure late in the list does not force a + ## re-run of everything before it. The marker is written only after + ## the merge below succeeds. + done_tag <- paste0("drugs-", f) + if (pset_is_done(done_tag)) { + message(sprintf( + "[03-createDrugFile] PSet '%s' already processed in a previous attempt; skipping.", f)) + next + } + + dset<<-download_pset_retry(f) mapping <- sensitivityInfo(dset)##get the dataset dose response data @@ -72,6 +87,44 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' print(paste('Found',length(chem_list),'chemicals for dataset',cel)) + ## Guard: never hand an empty/NULL chem_list to the Python writer. + ## update_dataframe_and_write_tsv() iterates it and fails with the + ## opaque "TypeError: 'NoneType' object is not iterable", which is + ## how the Aug 2026 CTRPv2.1_2016 schema change surfaced. + ## + ## Zero is also a legitimate outcome -- e.g. every compound in this + ## PSet was already covered by a previous PSet or by efile -- so + ## this skips rather than errors. A PSet whose treatment column is + ## missing entirely is reported loudly, since that means an + ## unrecognised schema rather than "nothing new to do". + ## An unrecognised schema is a HARD FAILURE, not a skip. PSets are + ## pinned (coderbuild/utils/pinned_psets.R), so if a pinned PSet no + ## longer exposes a treatment column it means upstream changed its + ## shape. Continuing would silently omit that PSet's drugs from the + ## release, which is exactly the kind of quiet data loss the pinning + ## exists to prevent -- so stop and make it visible. + if (is.null(mapping$treatmentid)) { + stop(sprintf( + paste0("PSet '%s': sensitivityInfo() has no 'treatmentid' or ", + "'drugid' column (columns: %s).\n", + "A pinned PSet has changed schema. Refusing to continue, ", + "because skipping it would drop its drugs from the release.\n", + "Either add an explicit column mapping for this PSet or ", + "remove it from PINNED_PSETS after confirming that is correct."), + f, paste(colnames(mapping), collapse = ", "))) + } + + ## Zero new chemicals is a legitimate outcome and NOT data loss: + ## every compound in this PSet was already fetched via an earlier + ## PSet in this run or via efile, so its drugs are already in the + ## cumulative output. Nothing is dropped by moving on. + if (length(chem_list) == 0) { + message(sprintf( + "[03-createDrugFile] PSet '%s': all chemicals already present; nothing new to fetch.", f)) + mark_pset_done(done_tag) + next + } + #build prev_drug_filepaths for this iteration (include original and accumulated) --- prev_list <- c() @@ -102,12 +155,19 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' } fwrite(combined, output_file_path, sep="\t") file.remove(temp_out) + ## Merge succeeded -- record this PSet as done so a later + ## attempt skips it entirely. + mark_pset_done(done_tag) } else { warning("Expected temporary output not found: ", temp_out) } - ##clean up file when done - file.remove(paste0(f,'.rds')) + ## The downloaded PSet is deliberately NOT deleted. It lives in + ## PSET_CACHE_DIR under /tmp (the bind mount to the host's local/), + ## so a retried attempt reuses it instead of re-downloading. These + ## are large -- CTRPv2 is 228 MB and one PSet is 1.5 GB -- and + ## re-fetching them is what made a single transient Zenodo 504 + ## expensive. Clear local/ between releases to reclaim the space. } } diff --git a/coderbuild/broad_sanger/03a-nci60Drugs.py b/coderbuild/broad_sanger/03a-nci60Drugs.py index 21a188ff..568c397f 100644 --- a/coderbuild/broad_sanger/03a-nci60Drugs.py +++ b/coderbuild/broad_sanger/03a-nci60Drugs.py @@ -10,6 +10,31 @@ import pubchem_retrieval as pr import random as rand from urllib import request +import download_utils +import io + +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + +def _fetch(url: str) -> bytes: + """Download url with a browser User-Agent (NCI wiki blocks Python's default).""" + req = request.Request(url, headers={"User-Agent": _BROWSER_UA}) + with request.urlopen(req) as resp: + return resp.read() + +def _read_csv_url(url: str, **kwargs) -> pl.DataFrame: + return pl.read_csv(io.BytesIO(_fetch(url)), **kwargs) + +def _retrieve_url(url: str, dest: str) -> None: + """Download url to dest, resuming an interrupted transfer. + + This also fetches the ~346MB DOSERESP.zip, and _fetch() reads the whole + body into memory in one call -- the same shape that truncated with + IncompleteRead in build v32. Small CSVs still go through _fetch(). + """ + download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA}) ##drug files smi_strings='https://wiki.nci.nih.gov/download/attachments/155844992/nsc_smiles.csv?version=1&modificationDate=1710381820000&api=v2&download=true' @@ -41,11 +66,12 @@ #jan 2025 cas = 'https://wiki.nci.nih.gov/download/attachments/155844992/nsc_cas.csv?version=4&modificationDate=1735931080720&api=v2' -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=11&modificationDate=1712351454136&api=v2' -##OCT 2024 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' +##oct 2024 +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' #jan 2025 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#may 2025 +conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=19&modificationDate=1775183341937&api=v2' def main(): parser = argparse.ArgumentParser() @@ -58,15 +84,20 @@ def main(): 'InChIKey':[],'formula':[],'weight':[],'pubchem_id':[]} print('Downloading NSC identifiers for nci60 data') - names = pl.read_csv(chemnames,ignore_errors=True) + names = _read_csv_url(chemnames, ignore_errors=True) #castab = pd.read_csv(cas) - pubchems = pl.read_csv(pc_ids) - smiles = pl.read_csv(smi_strings) + pubchems = _read_csv_url(pc_ids) + smiles = _read_csv_url(smi_strings) print('Getting experimental data to filter drugs') if not os.path.exists('DOSERESP.csv'): - resp = request.urlretrieve(conc_data,'doseresp.zip') - os.system('unzip doseresp.zip') + _retrieve_url(conc_data, 'doseresp.zip') + # -o overwrites without prompting; the exit status is checked so a + # corrupt or truncated archive fails here rather than surfacing later + # as a confusing parse error on a stale DOSERESP.csv. + _rc = os.system('unzip -o doseresp.zip') + if _rc != 0: + raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).") dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True) pubchems = pubchems.filter(pl.col('NSC').is_in(dose_resp['NSC'])) smiles = smiles.filter(pl.col("NSC").is_in(dose_resp['NSC']) @@ -81,13 +112,10 @@ def main(): ##first filter to see if there are structures/drugs in teh data already. i dont think this does much. if os.path.exists(opts.output): curdrugs = pl.read_csv(opts.output,separator='\t') - # cs = set(curdrugs['isoSMILES']) - smiles = smiles.filter(pl.col('SMILES').is_not_null()) - upper=[a.upper() for a in smiles['SMILES']] - smiles= pl.DataFrame({'NSC':smiles['NSC'],'upper':upper})#smiles.with_columns(upper=upper) - ##reduce to smiels only in current drugs - # ssmiles = smiles.filter(~pl.col('upper').is_in(curdrugs['isoSMILES'])) - ssmiles = smiles.filter(~pl.col('upper').is_in(curdrugs['canSMILES'])) + smiles_notnull = smiles.filter(pl.col('SMILES').is_not_null()) + upper=[a.upper() for a in smiles_notnull['SMILES']] + smiles_upper = pl.DataFrame({'NSC':smiles_notnull['NSC'],'upper':upper}) + ssmiles = smiles_upper.filter(~pl.col('upper').is_in(curdrugs['canSMILES'])) pubchems = pubchems.filter(pl.col('NSC').is_in(ssmiles['NSC'])) arr = set(pubchems['CID']) diff --git a/coderbuild/broad_sanger/04-drug_dosage_and_curves.py b/coderbuild/broad_sanger/04-drug_dosage_and_curves.py index 3b6f7245..e5bc7005 100755 --- a/coderbuild/broad_sanger/04-drug_dosage_and_curves.py +++ b/coderbuild/broad_sanger/04-drug_dosage_and_curves.py @@ -14,29 +14,47 @@ import argparse import pandas as pd import subprocess +import glob +import multiprocessing +from datetime import datetime + +def _log(msg): + print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] {msg}', flush=True) parser = argparse.ArgumentParser() parser.add_argument('--curSampleFile',dest='samplefile',default=None,help='DepMap sample file') - parser.add_argument('--drugfile',dest='dfile',default=None,help='Drug database') +parser.add_argument('--workers', type=int, default=max(1, round(multiprocessing.cpu_count() * 3 / 4)), help='Number of parallel worker processes for curve fitting (default: 3/4 of available CPUs)') opts = parser.parse_args() samplefile = opts.samplefile drugfile = opts.dfile -cmd = ['/opt/venv/bin/python','04b-nci60-updated.py','--sampleFile='+samplefile,'--drugFile='+drugfile] -print(cmd) -subprocess.run(cmd) +def run(cmd): + _log('running: ' + ' '.join(cmd)) + res = subprocess.run(cmd) + if res.returncode != 0: + _log(f'WARNING: command exited {res.returncode}: {" ".join(cmd)}') + return res.returncode + +# A failure here is FATAL, not a warning. +# +# In build v32 this exited 1 after DOSERESP.zip truncated, run() merely logged +# "WARNING: command exited 1", and the build carried on. nci60DoseResponse was +# never written, so the fitting below saw 7 studies instead of 8 and nci60 was +# published as eight empty files -- which pass schema validation, because a +# file with no rows cannot violate a schema. +_rc = run(['/opt/venv/bin/python','04b-nci60-updated.py','--sampleFile='+samplefile,'--drugFile='+drugfile]) +if _rc != 0: + raise RuntimeError( + f'04b-nci60-updated.py failed (exit {_rc}). It produces the NCI60 ' + f'dose-response input, and continuing without it publishes nci60 as ' + f'empty files that still validate.') ####step 4a - get dose response data -cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'CTRPv2,FIMM,GDSC'] -print(cmd) -subprocess.run(cmd) - -cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'gCSI,PRISM,CCLE'] -print(cmd) -subprocess.run(cmd) +run(['Rscript','04a-drugResponseData.R',samplefile,drugfile,'CTRPv2,FIMM,GDSC']) +run(['Rscript','04a-drugResponseData.R',samplefile,drugfile,'gCSI,PRISM,CCLE']) #cmd = 'Rscript 04a-drugResponseData.R '+samplefile+' '+drugfile+' NCI60' @@ -44,18 +62,108 @@ #os.system(cmd) ########Step 4b fit curves -allfiles=[a for a in os.listdir('./') if 'DoseResponse' in a] -print(allfiles) -for a in allfiles: - subprocess.run(['/opt/venv/bin/python','fit_curve.py','--input='+a,'--output='+a]) +# Collect dose-response inputs from BOTH places they are written. +# +# 04b-nci60-updated.py writes ./nci60DoseResponse into the working directory, +# but 04a-drugResponseData.R writes through dose_response_cache_path(), i.e. +# $DOSE_RESPONSE_CACHE_DIR/_doseResponse.tsv, so that a failed +# attempt can resume instead of regenerating every study. This scan only looked +# in './' for 'DoseResponse', which matches neither the directory nor the +# lowercase 'd', so it silently found NCI60 alone. +# +# The consequence was severe and completely silent: broad_sanger_experiments.tsv +# came out NCI60-only, and 05b_separate_datasets.py derives each split dataset's +# sample and drug ids FROM the experiments rows, so ccle, ctrpv2, fimm, gcsi, +# gdscv1, gdscv2 and prism were published with every file present but empty -- +# samples, omics, drugs and experiments alike. Header-only files are schema +# valid, so validate passed. Build v27 shipped exactly that. +# +# Excluding '.0' matters too: fit_curve writes '.0', which itself +# contains 'DoseResponse', so a retry would re-fit its own output. +CACHE_DIR = os.environ.get('DOSE_RESPONSE_CACHE_DIR', '/tmp/dose_response_cache') +jobs = [] # (input path, output stem -- kept local so '.0' lands in ./) +for a in sorted(os.listdir('./')): + if 'DoseResponse' in a and not a.endswith('.0'): + jobs.append((a, a)) +for pth in sorted(glob.glob(os.path.join(CACHE_DIR, '*_doseResponse.tsv'))): + stem = os.path.basename(pth)[:-len('.tsv')] + if any(stem == j[1] for j in jobs): + continue + jobs.append((pth, stem)) + +if not jobs: + raise RuntimeError('No DoseResponse files found — data generation step likely failed') +_log(f'fitting {len(jobs)} DoseResponse files with {opts.workers} workers: ' + f'{[j[1] for j in jobs]}') +fit_failures = [] +for src, stem in jobs: + _log(f'starting fit_curve: {stem} (from {src})') + cmd = ['/opt/venv/bin/python', 'fit_curve.py', + '--input=' + src, '--output=' + stem, '--workers=' + str(opts.workers)] + if 'nci60' in stem.lower(): + cmd.append('--chunk_size=25000') + rc = run(cmd) + if rc != 0: + _log(f'WARNING: fit_curve failed for {stem} (exit {rc}) — will be excluded from output') + fit_failures.append(stem) + _log(f'finished fit_curve: {stem}') ###step 4c concatenate all files -outfiles = [a for a in os.listdir("./") if ".0" in a] +outfiles = [a for a in os.listdir("./") if a.endswith('.0')] +if not outfiles: + raise RuntimeError('No .0 output files found after curve fitting — all fit_curve steps failed') final_file = [] for of in outfiles: - final_file.append(pd.read_csv(of,sep='\t')) - -pd.concat(final_file).drop_duplicates().to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t') + df = pd.read_csv(of, sep='\t') + if df.empty: + _log(f'WARNING: {of} is empty, skipping') + continue + _log(f'loaded {of}: {len(df)} rows') + final_file.append(df) + +if not final_file: + raise RuntimeError('All .0 output files were empty — cannot produce experiments output') +if fit_failures: + _log(f'WARNING: {len(fit_failures)} dataset(s) missing from output: {fit_failures}') + +combined = pd.concat(final_file).drop_duplicates() + +# Every study that was fitted must appear in the output. +# +# 05b_separate_datasets.py derives each split dataset's sample and drug ids from +# these rows, so a study missing here is not a partial result -- it publishes +# that whole dataset as header-only files: samples, omics, drugs, experiments. +# Those pass schema validation (no rows, nothing to violate), which is why v27 +# shipped seven empty datasets and still reported "validate successful". +# Check against the studies this step is SUPPOSED to produce, not merely the +# ones that happened to yield an input file. +# +# The previous version derived the expectation from `jobs`, so a study whose +# dose-response file was never generated simply was not expected, and its +# absence passed silently. That is precisely how v32 shipped an empty nci60: +# the input never existed, so nothing noticed it was gone. +REQUIRED_STUDIES = {'nci60', 'ccle', 'ctrpv2', 'fimm', 'gcsi', + 'gdscv1', 'gdscv2', 'prism'} +fitted_studies = {s.replace('_doseResponse', '').replace('DoseResponse', '').lower() + for s in (j[1] for j in jobs)} - {''} +not_fitted = sorted(REQUIRED_STUDIES - fitted_studies) +if not_fitted: + raise RuntimeError( + f"No dose-response input was produced for: {', '.join(not_fitted)}. " + f"Fitted only {sorted(fitted_studies)}. Each missing study becomes an " + f"entirely empty published dataset, so the build is stopping here.") +expected_studies = fitted_studies | REQUIRED_STUDIES +got_studies = {str(v).lower() for v in combined['study'].unique()} if 'study' in combined.columns else set() +missing_studies = sorted(st for st in expected_studies if st not in got_studies) +if missing_studies: + raise RuntimeError( + f"broad_sanger_experiments is missing every row for: {', '.join(missing_studies)}. " + f"Present: {sorted(got_studies)}. Each missing study becomes an entirely empty " + f"published dataset, so the build is stopping rather than shipping header-only files.") +_log(f'studies in output: {sorted(got_studies)} ({len(combined):,} rows)') + +combined.to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t') +_log(f'wrote /tmp/broad_sanger_experiments.tsv') #os.system('cat *.0 > /tmp/broad_sanger_experiments.tsv') #os.system('gzip -f /tmp/experiments.tsv') diff --git a/coderbuild/broad_sanger/04a-drugResponseData.R b/coderbuild/broad_sanger/04a-drugResponseData.R index 144ce9e9..a5d943bb 100755 --- a/coderbuild/broad_sanger/04a-drugResponseData.R +++ b/coderbuild/broad_sanger/04a-drugResponseData.R @@ -9,6 +9,11 @@ library(readr) library(dplyr) #} +## Pinned PSet list shared with 03-createDrugFile.R +source("pinned_psets.R") +## Retry/caching helpers for flaky third-party downloads +source("retry_utils.R") + all.dsets<-PharmacoGx::availablePSets() ##first define a generic dose response function @@ -137,7 +142,7 @@ getDoseRespData<-function(dset,studyName,improve_samples,drug.map){ ##now map drugs, samples, genes to ids in database files - write.table(doseRep,file=paste0(tolower(studyName),'DoseResponse'),sep='\t',row.names=F,quote=F) + write.table(doseRep,file=dose_response_cache_path(studyName),sep='\t',row.names=F,quote=F) doseRep } @@ -154,9 +159,10 @@ getCellLineDoseData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC' all.dose.rep<-do.call(rbind,lapply(cell.lines,function(cel){ # print(cel) - files<-subset(all.dsets,`Dataset Name`==cel)%>% - dplyr::select(`PSet Name`)%>% - unlist() + ## PSets are pinned (see coderbuild/utils/pinned_psets.R) and resolved + ## through the same helper the drugs step uses, so drugs and experiments + ## can never disagree about which PSets are in the release. + files<-psets_for_dataset(all.dsets, cel) res<-do.call(rbind,lapply(files,function(f){ @@ -165,12 +171,24 @@ getCellLineDoseData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC' cel='GDSCv2' if(f=='GDSC_2020(v1-8.2)') cel='GDSCv1' - tmpfile<-paste0(cel,'doseResponse') + ## Resume support: if this PSet's dose-response was already computed on + ## an earlier attempt, read it back instead of re-downloading the PSet + ## and recomputing. + ## + ## This check was previously broken and silently never matched: it looked + ## for paste0(cel,'doseResponse') while getDoseRespData() wrote + ## paste0(tolower(studyName),'DoseResponse') -- different case on both + ## halves, so e.g. CTRPv2 wrote 'ctrpv2DoseResponse' and this looked for + ## 'CTRPv2doseResponse'. Every attempt therefore redid all the work. + ## Both sides now go through dose_response_cache_path(), and the file + ## lives under /tmp so it survives between container attempts. + tmpfile<-dose_response_cache_path(cel) if(file.exists(tmpfile)){ + message(sprintf("[04a] Reusing cached dose-response for '%s' (%s)", cel, tmpfile)) dres<-read.table(tmpfile,sep='\t',header=T) }else{ - dset<<-downloadPSet(f,saveDir='.',timeout=10000) + dset<<-download_pset_retry(f) url=subset(all.dsets,`PSet Name`==f)$Download #print(url) diff --git a/coderbuild/broad_sanger/04b-nci60-updated.py b/coderbuild/broad_sanger/04b-nci60-updated.py index 3f1e8080..4c8022d1 100644 --- a/coderbuild/broad_sanger/04b-nci60-updated.py +++ b/coderbuild/broad_sanger/04b-nci60-updated.py @@ -5,38 +5,59 @@ import polars as pl import argparse -#from zipfile import ZipFile import os -#from io import BytesIO import re from urllib import request +import download_utils -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=11&modificationDate=1712351454136&api=v2' -##OCT 2024 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + +def _retrieve_url(url: str, dest: str) -> None: + """Download url, resuming an interrupted transfer. + + DOSERESP.zip is ~346MB. The previous single urlopen().read() truncated at + 32MB in build v32 with IncompleteRead, which cost the entire nci60 dataset: + this script exited 1, its caller only warned, nci60DoseResponse was never + written, and nci60 shipped as eight empty files that still validated. + """ + download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA}) + +##oct 2024 +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' #jan 2025 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#may 2025 +conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=19&modificationDate=1775183341937&api=v2' -cancelled = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP_Cancelled.csv?version=1&modificationDate=1660871847000&api=v2&download=true' +#may 2025 +oneconc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/ONECONC.zip?version=19&modificationDate=1775183401634&api=v2' def main(): - + parser = argparse.ArgumentParser() - parser.add_argument('--sampleFile',dest='samplefile',default=None,help='DepMap sample file') + parser.add_argument('--sampleFile',dest='samplefile',default=None,help='DepMap sample file') parser.add_argument('--drugFile',dest='dfile',default=None,help='Drug database') - + opts = parser.parse_args() - + samplefile = opts.samplefile drugfile = opts.dfile if not os.path.exists('DOSERESP.csv'): - resp = request.urlretrieve(conc_data,'doseresp.zip') - os.system('unzip doseresp.zip') - + _retrieve_url(conc_data, 'doseresp.zip') + # -o overwrites without prompting; the exit status is checked so a + # corrupt or truncated archive fails here rather than surfacing later + # as a confusing parse error on a stale DOSERESP.csv. + _rc = os.system('unzip -o doseresp.zip') + if _rc != 0: + raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).") + samples = pl.read_csv(samplefile,quote_char='"') drugs = pl.read_csv(drugfile,separator='\t',quote_char='"') - + dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True) ##update drug mapping @@ -51,16 +72,14 @@ def main(): drugmapping = drugmapping.unique() ###update sample mapping - on = samples[['other_names','improve_sample_id']] - on.columns=['common_name','improve_sample_id'] + on = samples[['other_names','improve_sample_id']].rename({'other_names': 'common_name'}) #there should be 71 cell lines, but there are 163. # 82 map to the 'other_names' # 81 map to neither sampmapping = pl.concat([on[['common_name','improve_sample_id']],samples[['common_name','improve_sample_id']]]) - - sampmapping = sampmapping.unique() - sampmapping.columns = ['CELL_NAME','improve_sample_id'] + + sampmapping = sampmapping.unique().rename({'common_name': 'CELL_NAME'}) ###create a time mapping tabel timemapping = pl.DataFrame( @@ -75,30 +94,15 @@ def main(): ##now we can merge all the data into the dose response data frame merged = dose_resp[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC']].join(sampmapping,on='CELL_NAME',how='left') merged = merged.join(timemapping,on='EXPID',how='left') - + ##clean up mssing samples nonulls = merged.filter(pl.col('improve_sample_id').is_not_null()) nulls = merged.filter(pl.col('improve_sample_id').is_null()) - - # newnames = pl.DataFrame( - # { - # 'new_name':[re.split(' |\(|\/',a)[0] for a in nulls['CELL_NAME']], - # 'CELL_NAME':nulls['CELL_NAME'] - # } - # ) - # newnames = newnames.unique() - - - # fixed = nulls[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC','time','time_unit']].join(newnames,on='CELL_NAME',how='left') - # merged.columns = ['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','old_CELL_NAME','EXPID','NSC','time','time_unit','CELL_NAME'] - # fixed = merged.join(sampmapping,on='CELL_NAME',how='left')[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','old_CELL_NAME','EXPID','NSC','improve_sample_id','time','time_unit']] - # fixed.columns = ['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC','improve_sample_id','time','time_unit'] -# fixed = fixed.filter(pl.col('improve_sample_id').is_not_null()) - - merged = nonulls#pl.concat([nonulls,fixed]) - - ###we get a few more results added, but still missing a bunch + + merged = nonulls + + ###we get a few more results added, but still missing a bunch merged = merged.join(drugmapping,on='NSC',how='left') nulldrugs = merged.filter(pl.col('improve_drug_id').is_null()) nonulls = merged.filter(pl.col('improve_drug_id').is_not_null()) @@ -106,7 +110,7 @@ def main(): ###now update all the concentrations to be in Moles (some are in uM, all are log10) ##some are provided as molecular weights ('v') or other ('s') and we can't compare molar = merged.filter(pl.col('CONCENTRATION_UNIT')=='M') - + finaldf = pl.DataFrame( { 'source':['NCI60_24' for a in molar['improve_drug_id']], ##2024 build @@ -122,7 +126,7 @@ def main(): ) ##write to file finaldf.write_csv('nci60DoseResponse',separator='\t') - + if __name__=='__main__': main() diff --git a/coderbuild/broad_sanger/05b_separate_datasets.py b/coderbuild/broad_sanger/05b_separate_datasets.py index d00680fe..eb110684 100644 --- a/coderbuild/broad_sanger/05b_separate_datasets.py +++ b/coderbuild/broad_sanger/05b_separate_datasets.py @@ -37,9 +37,11 @@ def main(): exp = pl.read_csv(exp_in_filename, separator="\t") # Keeping memory down, so I will not be making copies. exp = exp.filter(pl.col("study") == dataset) - # Extract information to separate out datasets - exp_improve_sample_ids = exp["improve_sample_id"].unique().to_list() - exp_improve_drug_ids = exp["improve_drug_id"].unique().to_list() + # Extract information to separate out datasets. + # improve_sample_id is always an integer (cast via Float64 to handle "1.0" TSV notation). + # improve_drug_id is always a string ("SMI_XXXXX"); cast to Utf8 for consistent is_in matching. + exp_improve_sample_ids = exp["improve_sample_id"].cast(pl.Float64).cast(pl.Int64).unique().to_list() + exp_improve_drug_ids = exp["improve_drug_id"].cast(pl.Utf8).unique().to_list() #Ensure that the improve_sample_id column is in integer form. exp = exp.with_columns(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64)) @@ -65,7 +67,7 @@ def main(): samples_filename_out = f"/tmp/{dataset}_{samples}.csv".lower() samples_df = pl.read_csv(samples_filename_in) - samples_df = samples_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids)) + samples_df = samples_df.filter(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64).is_in(exp_improve_sample_ids)) samples_df.write_csv(samples_filename_out) #csv #Rewrite as gzipped if needed @@ -86,7 +88,7 @@ def main(): omics_filename_out = f"/tmp/{dataset}_{omics}.csv".lower() omics_df = pl.read_csv(omics_filename_in) - omics_df = omics_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids)) + omics_df = omics_df.filter(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64).is_in(exp_improve_sample_ids)) # omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset])) omics_df.write_csv(omics_filename_out) #csv @@ -116,7 +118,7 @@ def main(): else: drugs_df = pl.read_csv(drugs_filename_in,separator="\t") - drugs_df = drugs_df.filter(pl.col("improve_drug_id").is_in(exp_improve_drug_ids)) + drugs_df = drugs_df.filter(pl.col("improve_drug_id").cast(pl.Utf8).is_in(exp_improve_drug_ids)) drugs_df.write_csv(drugs_filename_out,separator="\t") #tsv if drugs_filename_in.endswith(".gz"): diff --git a/coderbuild/broad_sanger/build_omics.sh b/coderbuild/broad_sanger/build_omics.sh index 024e0d17..46e1b964 100644 --- a/coderbuild/broad_sanger/build_omics.sh +++ b/coderbuild/broad_sanger/build_omics.sh @@ -3,6 +3,19 @@ set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR +# DepMap files are no longer downloadable from the DepMap portal (Cloudflare +# challenge). Pull the static copy from Synapse first; the R script reads them +# from $DEPMAP_DIR and deletes each one once consumed. +# +# $DEPMAP_DIR is container-local (NOT under /tmp, which is the bind-mounted +# host local/ directory), so these transient inputs never touch host disk. +# The trap is a backstop in case the R script exits before consuming them. +echo "Fetching DepMap files from Synapse..." +DEPMAP_DIR="$(/opt/venv/bin/python fetch_depmap.py --print-dest)" +export DEPMAP_DIR +trap 'rm -rf "$DEPMAP_DIR"' EXIT +/opt/venv/bin/python fetch_depmap.py --dest "$DEPMAP_DIR" + echo "Running 02a-broad_sanger_proteomics.py with gene file $1 and sample file $2." /opt/venv/bin/python 02a-broad_sanger_proteomics.py --gene $1 --sample $2 diff --git a/coderbuild/broad_sanger/build_samples.sh b/coderbuild/broad_sanger/build_samples.sh index 4302ba44..49220052 100644 --- a/coderbuild/broad_sanger/build_samples.sh +++ b/coderbuild/broad_sanger/build_samples.sh @@ -3,5 +3,18 @@ set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR +# DepMap files are no longer downloadable from the DepMap portal (Cloudflare +# challenge). Pull the static copy from Synapse first; the R script reads them +# from $DEPMAP_DIR and deletes each one once consumed. +# +# $DEPMAP_DIR is container-local (NOT under /tmp, which is the bind-mounted +# host local/ directory), so these transient inputs never touch host disk. +# The trap is a backstop in case the R script exits before consuming them. +echo "Fetching DepMap files from Synapse..." +DEPMAP_DIR="$(/opt/venv/bin/python fetch_depmap.py --print-dest)" +export DEPMAP_DIR +trap 'rm -rf "$DEPMAP_DIR"' EXIT +/opt/venv/bin/python fetch_depmap.py --dest "$DEPMAP_DIR" + echo "Running 01-broadSangerSamples.R." Rscript 01-broadSangerSamples.R diff --git a/coderbuild/broad_sanger/exp_requirements.r b/coderbuild/broad_sanger/exp_requirements.r index 9cc16e17..7323b1f4 100755 --- a/coderbuild/broad_sanger/exp_requirements.r +++ b/coderbuild/broad_sanger/exp_requirements.r @@ -7,3 +7,12 @@ install.packages('tidyverse') install.packages("BiocManager") BiocManager::install("PharmacoGx",update=TRUE,ask=FALSE) + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("reticulate", "tidyverse", "BiocManager", "PharmacoGx") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/broad_sanger/omics_requirements.r b/coderbuild/broad_sanger/omics_requirements.r index 26d27844..83ce09b7 100755 --- a/coderbuild/broad_sanger/omics_requirements.r +++ b/coderbuild/broad_sanger/omics_requirements.r @@ -7,4 +7,14 @@ install.packages("dplyr") install.packages("XML") #install.packages('reticulate') install.packages('tidyr') -install.packages('httr2') \ No newline at end of file +install.packages('httr2') +install.packages('data.table') + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("curl", "rio", "readr", "stringr", "dplyr", "XML", "tidyr", "httr2", "data.table") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/broad_sanger/requirements.r b/coderbuild/broad_sanger/requirements.r index 629f4fe0..a1e60b9d 100755 --- a/coderbuild/broad_sanger/requirements.r +++ b/coderbuild/broad_sanger/requirements.r @@ -11,6 +11,7 @@ install.packages("XML") install.packages('reticulate') install.packages('tidyr') install.packages('httr2') +install.packages('data.table') #install.packages("BiocManager") BiocManager::install("PharmacoGx",update=TRUE,ask=FALSE) BiocManager::install("org.Hs.eg.db",update=TRUE,ask=FALSE) diff --git a/coderbuild/broad_sanger/requirements.txt b/coderbuild/broad_sanger/requirements.txt index a00c3bd6..9a5e7191 100755 --- a/coderbuild/broad_sanger/requirements.txt +++ b/coderbuild/broad_sanger/requirements.txt @@ -11,4 +11,5 @@ polars-lts-cpu==0.19.17 mordredcommunity rdkit coderdata==0.1.40 -psutil \ No newline at end of file +psutil +synapseclient>=4.0 \ No newline at end of file diff --git a/coderbuild/build_all.py b/coderbuild/build_all.py index d98f338c..37290fd1 100644 --- a/coderbuild/build_all.py +++ b/coderbuild/build_all.py @@ -12,7 +12,34 @@ from glob import glob import sys import requests - +import threading +import atexit +import signal +import tempfile +import yaml +from datetime import datetime + +def _log(*args, **kwargs): + ts = datetime.now().strftime('[%Y-%m-%d %H:%M:%S]') + print(ts, *args, **kwargs) + +def _redact(cmd): + """Render a command for logging with secret values masked. + + Tokens are passed to containers as `-e NAME=value`, so logging the command + verbatim wrote them in cleartext. build_all_v19.log contained the live + SYNAPSE_AUTH_TOKEN 47 times, and the build logs sit untracked in the repo + root where a `git add -A` would commit them. + """ + out = [] + for part in cmd: + if '=' in part: + key, _, val = part.partition('=') + if val and any(m in key.upper() for m in ('TOKEN', 'SECRET', 'PASSWORD', 'KEY')): + part = f'{key}=***REDACTED***' + out.append(part) + return ' '.join(out) + def main(): parser=argparse.ArgumentParser( description="This script initializes all docker containers, builds datasets, validates them, and uploads to Figshare.", @@ -35,19 +62,253 @@ def main(): parser.add_argument('--samples',dest='samples',default=False,action='store_true', help="Build all sample files.") parser.add_argument('--omics',dest='omics',default=False,action='store_true', help="Build all omics files.") parser.add_argument('--drugs',dest='drugs',default=False,action='store_true', help="Build all drug files") + parser.add_argument('--prev_drugs', type=str, default='', help='Comma-separated list of previously built drug files (e.g. local_old/broad_sanger_drugs.tsv,local_old/beataml_drugs.tsv). Each matching dataset is still rebuilt, but its previous file seeds pubchem_retrieval so only genuinely new drugs are queried. Existing drugs are preserved; removed drugs fall off naturally.') parser.add_argument('--misc', action='store_true', help="Run the final misc post-build step (e.g., split broad_sanger datasets).") parser.add_argument('--exp',dest='exp',default=False,action='store_true', help="Build all experiment file.") parser.add_argument('--validate', action='store_true', help="Run schema checker on all local files. Note this will be run, whether specified or not, if figshare arguments are included.") parser.add_argument('--figshare', action='store_true', help="Upload all local data to Figshare. FIGSHARE_TOKEN must be set in local environment.") parser.add_argument('--all',dest='all',default=False,action='store_true', help="Run all data build commands. This includes docker, samples, omics, drugs, exp arguments. This does not run the validate or figshare commands") parser.add_argument('--high_mem',dest='high_mem',default=False,action='store_true',help = "If you have 32 or more CPUs, this option is recommended. It will run many code portions in parallel. If you don't have enough memory, this will cause a run failure.") - parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,hcmi',help='Datasets to process. Defaults to all available.') + parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,cptac,hcmi',help='Datasets to process. Defaults to all available.') parser.add_argument('--version', type=str, required=False, help='Version number for the Figshare upload title (e.g., "0.1.29"). This is required for Figshare upload. This must be a higher version than previously published versions.') parser.add_argument('--github-username', type=str, required=False, help='GitHub username for the repository.') parser.add_argument('--github-email', type=str, required=False, help='GitHub email for the repository.') - + parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', + help='Confirm that the static DepMap files on Synapse are up to date for ' + 'the current DepMap release. Required for any build that includes the ' + 'broad_sanger dataset, because DepMap files can no longer be downloaded ' + 'programmatically and must be refreshed by hand.') + parser.add_argument('--continue', dest='resume', default=False, action='store_true', + help='Resume a partial build: skip steps whose sentinel output already ' + 'exists in local/, rebuild Docker images only for datasets that ' + 'failed in the previous run, and log all activity to ' + 'local/build_progress.log.') + args = parser.parse_args() - + + # ------------------------------------------------------------------------- + # Resume / progress tracking helpers + # ------------------------------------------------------------------------- + PROGRESS_LOG = 'local/build_progress.log' + _progress_lock = threading.Lock() + + def _step_sentinel(step_name): + """Return the file/dir whose existence proves this step completed.""" + if step_name == 'genes file': + return 'local/genes.csv' + if step_name == 'phosphosites file': + return 'local/phosphosites.csv' + # All other steps are "{dataset} {type}" e.g. "colorectal omics" + da, _, stype = step_name.rpartition(' ') + if not stype: + return None + if stype == 'samples': + return f'local/{da}_samples.csv' + if stype == 'drugs': + return f'local/{da}_drugs.tsv' + if stype == 'omics': + # broad_sanger writes broad_transcriptomics, while all other datasets + # use their dataset name as the output prefix. + output_prefix = 'broad' if da == 'broad_sanger' else da + # transcriptomics is the last large file written — proves full completion + for ext in ('.csv.gz', '.csv'): + candidate = f'local/{output_prefix}_transcriptomics{ext}' + if os.path.exists(candidate): + return candidate + # default expectation for first-time check + gzip_ds = {'broad_sanger', 'beataml', 'pancreatic', 'hcmi'} + ext = '.csv.gz' if da in gzip_ds else '.csv' + return f'local/{output_prefix}_transcriptomics{ext}' + if stype == 'experiments': + for ext in ('.tsv', '.tsv.gz'): + candidate = f'local/{da}_experiments{ext}' + if os.path.exists(candidate): + return candidate + return f'local/{da}_experiments.tsv' + if stype == 'misc': + return 'local/all_files_dir' + return None + + def _sentinel_done(step_name): + """Return True if the sentinel for this step exists and is non-trivial.""" + s = _step_sentinel(step_name) + if s is None: + return False + if os.path.isdir(s): + return bool(os.listdir(s)) + return os.path.isfile(s) and os.path.getsize(s) > 100 + + def _step_cleanup_globs(step_name): + """Glob patterns (relative to cwd) to remove when a step fails.""" + if step_name == 'genes file': + return ['local/genes.csv'] + if step_name == 'phosphosites file': + return ['local/phosphosites.csv'] + da, _, stype = step_name.rpartition(' ') + if not stype: + return [] + if stype == 'samples': + return [f'local/{da}_samples.csv'] + if stype == 'drugs': + return [f'local/{da}_drugs.tsv', f'local/{da}_drug_descriptors.tsv.gz'] + if stype == 'omics': + if da == 'broad_sanger': + # broad_sanger omics produces both broad_* and sanger_* files + return [ + 'local/broad_mutations*', 'local/broad_transcriptomics*', + 'local/broad_copy_number*', 'local/broad_proteomics*', + 'local/sanger_mutations*', 'local/sanger_transcriptomics*', + 'local/sanger_copy_number*', 'local/sanger_proteomics*', + ] + return [ + f'local/{da}_mutations*', f'local/{da}_transcriptomics*', + f'local/{da}_copy_number*', f'local/{da}_proteomics*', + f'local/{da}_phosphoproteomics*', + ] + if stype == 'experiments': + return [f'local/{da}_experiments.tsv', f'local/{da}_experiments.tsv.gz'] + if stype == 'misc': + return ['local/all_files_dir'] + return [] + + def _cleanup_step(step_name): + """Remove partial outputs from a failed step; log what was removed.""" + removed = [] + for pattern in _step_cleanup_globs(step_name): + for f in glob(pattern): + try: + if os.path.isdir(f): + shutil.rmtree(f) + else: + os.remove(f) + removed.append(f) + except Exception as exc: + _log(f'[cleanup] could not remove {f}: {exc}') + if removed: + _log(f'[cleanup] {step_name}: removed {len(removed)} file(s): {", ".join(removed)}') + _write_progress(step_name, 'CLEANUP', f'removed: {", ".join(removed)}') + else: + _log(f'[cleanup] {step_name}: nothing to remove') + + def _clean_local_intermediates(reason='', protect_prefixes=()): + """Move leftover intermediate files in local/ into a quarantine + directory (local/unexpected_intermediates/) to keep the working set + small (raw downloads like rnaseq_all_data_*.csv, cell.xml, + proteomics_all_*.csv, *.xlsx, etc. are never needed after the step that + consumed them and otherwise accumulate until a later step hits ENOSPC). + + Files are MOVED, not deleted, so a mis-declared build output is never + silently lost -- if something you expected to be uploaded ends up in + local/unexpected_intermediates/, it means it was not declared in + schema/expected_files.yaml. NOTE: because the quarantine directory is on + the same filesystem, moving does NOT reclaim disk space during the + build; clear local/unexpected_intermediates/ (or point it at another + volume) if you hit disk pressure. + + Safety: a file is KEPT in place if it is a schema-declared output (per + schema/expected_files.yaml -- these may still be sitting in local/ before + the move to all_files_dir), a reference file, an incremental-build cache, + build infrastructure, or matches one of protect_prefixes (e.g. + broad*/sanger*, which build_misc.sh still needs to split broad_sanger). + Only regular files matching local/*.* are considered (mirrors the move + step's own glob), so directories like all_files_dir are never touched. + """ + try: + with open('schema/expected_files.yaml') as _f: + _schema = yaml.safe_load(_f) + except Exception as exc: + _log(f'[cleanup-intermediates] {reason}: skipped, cannot read schema: {exc}') + return + keep = set() + for _entries in _schema.get('datasets', {}).values(): + for _entry in _entries: + _bn = os.path.basename(_entry['file']) + keep.add(_bn) + keep.add(_bn + '.gz') + if _bn.endswith('.gz'): + keep.add(_bn[:-3]) + protected_exact = {'genes.csv', 'phosphosites.csv', 'prev_drug_files.txt'} + quarantine = os.path.join('local', 'unexpected_intermediates') + moved, moved_bytes = 0, 0 + for f in glob(os.path.join('local', '*.*')): + if os.path.isdir(f): + continue + bn = os.path.basename(f) + if bn in keep or bn in protected_exact: + continue + if '_prev.' in bn or bn.endswith(('.cid', '.log', '.jsonl')): + continue + if any(bn.startswith(p) for p in protect_prefixes): + continue + try: + os.makedirs(quarantine, exist_ok=True) + sz = os.path.getsize(f) + dest = os.path.join(quarantine, bn) + if os.path.exists(dest): + os.remove(dest) # replace a stale copy from an earlier pass + shutil.move(f, dest) + moved += 1 + moved_bytes += sz + except Exception as exc: + _log(f'[cleanup-intermediates] could not move {f}: {exc}') + if moved: + _log(f'[cleanup-intermediates] {reason}: moved {moved} file(s) ' + f'({moved_bytes / 1e9:.2f} GB) to {quarantine}/') + else: + _log(f'[cleanup-intermediates] {reason}: nothing to move') + + def _write_progress(step, status, note=''): + """Append one line to local/build_progress.log (thread-safe).""" + ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + line = f'{ts} | {status:<8} | {step}' + if note: + line += f' ({note})' + line += '\n' + with _progress_lock: + try: + with open(PROGRESS_LOG, 'a') as f: + f.write(line) + except Exception: + pass # never let logging crash the build + + def _step_to_docker_dataset(step_name): + """Map a step name to the dataset name used by process_docker.""" + if step_name in ('genes file', 'gene file'): + return 'genes' + if step_name == 'phosphosites file': + return 'phosphosites' + da, _, stype = step_name.rpartition(' ') + if not stype: + return None + return da # e.g. "colorectal", "broad_sanger", "liver" + + def _last_run_failures(): + """ + Parse build_progress.log and return the set of step names that + FAILED in the most recent run (since the last START line) and were + not subsequently fixed (no SUCCESS line after the FAILURE). + """ + if not os.path.exists(PROGRESS_LOG): + return set() + current_failures = set() + in_run = False + with open(PROGRESS_LOG) as f: + for line in f: + parts = [p.strip() for p in line.split('|', 2)] + if len(parts) < 3: + continue + _ts, status, rest = parts + step = rest.split('(')[0].strip() # strip any trailing note + if status == 'START': + in_run = True + current_failures = set() + elif in_run and status == 'FAILURE': + current_failures.add(step) + elif in_run and status == 'SUCCESS': + current_failures.discard(step) + return current_failures + + # ------------------------------------------------------------------------- # Simulation command for testing order of everything: # def run_docker_cmd(cmd_arr, filename): # # Simulate running the command by printing what would be run @@ -56,37 +317,91 @@ def main(): # time.sleep(2) # print(f'Completed: {filename}') - def run_docker_cmd(cmd_arr,filename): + # --- Container cleanup on exit/interrupt --- + _active_cid_files = [] + _cid_lock = threading.Lock() + + def _kill_all_containers(): + with _cid_lock: + cids = list(_active_cid_files) + for cid_file in cids: + try: + with open(cid_file) as f: + cid = f.read().strip() + if cid: + _log(f"Cleaning up container {cid[:12]}...") + subprocess.run(['docker', 'kill', cid], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except Exception: + pass + + def _signal_handler(_signum, _frame): + _log("\nBuild interrupted — killing all running containers...") + _kill_all_containers() + sys.exit(1) + + atexit.register(_kill_all_containers) + signal.signal(signal.SIGINT, _signal_handler) + signal.signal(signal.SIGTERM, _signal_handler) + + def run_docker_cmd(cmd_arr, filename): ''' Essentially a wrapper for 'docker run'. Also provides output. + Tracks each container via --cidfile so they can be killed on exit/interrupt. + With --continue: skips steps whose sentinel file already exists, and + cleans up partial outputs after all retries are exhausted. ''' - retries=3 + # --- Resume: skip if already done --- + if args.resume and _sentinel_done(filename): + sentinel = _step_sentinel(filename) + _log(f'[{filename}] skipping — already complete ({sentinel})') + _write_progress(filename, 'SKIPPED', f'sentinel: {sentinel}') + return + + retries = 3 delays = [3 * 60, 10 * 60] # 3 minutes, 10 minutes - print('running...'+filename) + _log('running...' + filename) env = os.environ.copy() if 'SYNAPSE_AUTH_TOKEN' not in env.keys(): - print('You need to set the SYNAPSE_AUTH_TOKEN to acess the MPNST, beatAML, bladder, pancreatic, liver, or sarcoma datasets') - docker_run = ['docker','run','--rm','-v',env['PWD']+'/local/:/tmp/','--platform=linux/amd64'] + _log('You need to set the SYNAPSE_AUTH_TOKEN to acess the MPNST, beatAML, bladder, pancreatic, liver, sarcoma, cnf datasets') + docker_run = ['docker', 'run', '--rm', '-v', env['PWD']+'/local/:/tmp/', '--platform=linux/amd64'] else: - docker_run = ['docker','run','--rm','-v',env['PWD']+'/local/:/tmp/','-e','SYNAPSE_AUTH_TOKEN='+env['SYNAPSE_AUTH_TOKEN'],'--platform=linux/amd64'] - cmd = docker_run+cmd_arr - print(cmd) - + docker_run = ['docker', 'run', '--rm', '-v', env['PWD']+'/local/:/tmp/', '-e', 'SYNAPSE_AUTH_TOKEN='+env['SYNAPSE_AUTH_TOKEN'], '--platform=linux/amd64'] + attempt = 1 while attempt <= retries: - print(f"[{filename}] Attempt {attempt}/{retries}: {' '.join(cmd)}") - res = subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr) - if res.returncode == 0: - print(f"[{filename}] succeeded on attempt {attempt}.") - return - else: - print(f"[{filename}] failed (exit {res.returncode}).") - if attempt < retries: - delay = delays[attempt - 1] - print(f"[{filename}] waiting {delay//60} minutes before retrying...") - print(cmd) - time.sleep(delay) + with tempfile.NamedTemporaryFile(suffix='.cid', delete=False) as _tf: + cid_file = _tf.name + os.remove(cid_file) # Docker requires the file to not exist before writing + cmd = docker_run + ['--cidfile', cid_file] + cmd_arr + with _cid_lock: + _active_cid_files.append(cid_file) + try: + _log(f"[{filename}] Attempt {attempt}/{retries}: {_redact(cmd)}") + res = subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr) + if res.returncode == 0: + _log(f"[{filename}] succeeded on attempt {attempt}.") + _write_progress(filename, 'SUCCESS') + return + else: + _log(f"[{filename}] failed (exit {res.returncode}).") + if attempt < retries: + delay = delays[attempt - 1] + _log(f"[{filename}] waiting {delay//60} minutes before retrying...") + time.sleep(delay) + finally: + with _cid_lock: + if cid_file in _active_cid_files: + _active_cid_files.remove(cid_file) + try: + os.remove(cid_file) + except OSError: + pass attempt += 1 + + # All retries exhausted — log failure and remove partial outputs + _write_progress(filename, 'FAILURE', f'all {retries} attempts failed') + _cleanup_step(filename) raise RuntimeError(f"{filename} failed after {retries} attempts") @@ -113,13 +428,17 @@ def process_docker(datasets): 'colorectal': ['colorectal'], 'cptac': ['cptac'], 'genes': ['genes'], + 'phosphosites': ['phosphosites'], 'upload': ['upload'], 'liver': ['liver'], - 'novartis': ['novartis'] + 'novartis': ['novartis'], + 'cnf': ['cnf'] } # Collect container names to build based on the datasets provided. Always build genes and upload. datasets_to_build = ['genes', 'upload'] + if 'cnf' in datasets: + datasets_to_build.append('phosphosites') for dataset in datasets: datasets_to_build.extend(dataset_map.get(dataset, [])) @@ -129,50 +448,153 @@ def process_docker(datasets): log_file_path = 'local/docker.log' env = os.environ.copy() - print(f"Docker-compose is building images for {', '.join(datasets_to_build)}. View output in {log_file_path}.") + _log(f"Docker-compose is building images for {', '.join(datasets_to_build)}. View output in {log_file_path}.") with open(log_file_path, 'w') as log_file: try: # Execute the docker-compose command res = subprocess.run(compose_command, env=env, stdout=log_file, stderr=log_file, text=True, check=True) log_file.write("Docker images built successfully.\n") - print(f"Docker images for {', '.join(datasets_to_build)} built successfully. Details logged in {log_file_path}.") + _log(f"Docker images for {', '.join(datasets_to_build)} built successfully. Details logged in {log_file_path}.") except subprocess.CalledProcessError as e: log_file.write(f"Docker compose build failed with error: {e}\n") - print(f"Docker compose build failed. See {log_file_path} for details.") + _log(f"Docker compose build failed. See {log_file_path} for details.") raise - def process_drugs(executor, datasets): + + # target_class values in schema/expected_files.yaml, grouped by the build + # step that produces them. + _STEP_CLASSES = { + 'samples': {'Sample'}, + 'omics': {'Transcriptomics', 'Proteomics', 'Mutations', 'Copy Number', + 'Phosphoproteomics'}, + 'drugs': {'Drug', 'Drug Descriptor'}, + 'experiments': {'Experiments', 'Combinations'}, + } + + def _verify_step_outputs(dataset, step): + """Fail a step that exited 0 without writing the files it declares. + + build_all only checks a container's exit status, so a script that + returns 0 after doing nothing counts as success. beataml omics did + exactly that on 2026-09-13: it ran for 88 seconds (against 4m24s in a + known-good build), printed "Starting Transcriptomics Data", never + reached proteomics, exited 0, and wrote none of its three files. The + build carried on through experiments, misc and three upload stages and + only fell over ~11 hours later in validate. + + schema/expected_files.yaml already declares what each dataset must + produce, so check it here and fail at the step that went wrong, while + the log still shows why. + """ + want = _STEP_CLASSES.get(step) + if not want: + return + try: + with open('schema/expected_files.yaml') as fh: + declared = yaml.safe_load(fh)['datasets'].get(dataset, []) + except Exception as exc: + _log(f'[verify] could not read expected_files.yaml ({exc}); skipping output check.') + return + + missing = [] + for entry in declared: + if entry.get('target_class') not in want: + continue + base = os.path.basename(entry['file']) + stem = base[:-3] if base.endswith('.gz') else base + # the build writes either the plain or the gzipped form + if not any(os.path.exists(os.path.join('local', c)) + for c in (stem, stem + '.gz')): + missing.append(stem) + + if missing: + raise RuntimeError( + f"{dataset} {step} reported success but produced none of: " + f"{', '.join(missing)}. The step exited 0 without writing the " + f"files schema/expected_files.yaml declares for it, so the build " + f"is stopping here rather than failing later in validate.") + + def _await_all(futures, step_label): + """Retrieve every future so no step failure can be silently dropped. + + This is load-bearing. Each process_* function used to await only the + PREVIOUS future before submitting the next, so the LAST dataset's + exception was never retrieved -- and an unretrieved exception in a + ThreadPoolExecutor is discarded, not raised. In --high_mem mode nothing + was awaited at all. + + On 2026-09-04 that let `hcmi omics` fail all three attempts while the + build logged "All omics files completed" and carried on to publish and + validate a release with hcmi transcriptomics missing entirely. + + Raises the first failure after collecting the rest, so one broken step + cannot hide behind a later success. + """ + errors = [] + for fut in futures: + try: + fut.result() + except Exception as exc: + errors.append(exc) + if errors: + if len(errors) > 1: + _log(f'{step_label}: {len(errors)} steps failed; raising the first.') + raise errors[0] + + def process_drugs(executor, datasets, prev_drugs=None): ''' - Build all drug files sequentially + Build all drug files sequentially. + + prev_drugs: list of local paths (e.g. ['local_old/broad_sanger_drugs.tsv', ...]) + Files are copied into local/ and listed in local/prev_drug_files.txt. + pubchem_retrieval.py auto-loads this file on import and uses it as a + local cache: known drugs skip HTTP requests entirely, but ID assignment + proceeds exactly as in a fresh run (dflist chain is unchanged). ''' last_drug_future = None - dflist = [] - - # WE NEED A METHOD TO CONFIRM THAT DRUG FILES ARE NOT INCOMPLETE - ##THIS IS BUILT IN- always rerun drug code to check - # Check for existing files and update dflist with processed files - for da in datasets: - if da not in ['cptac', 'hcmi']: - file_path = f'local/{da}_drugs.tsv' - desc_path = f'local/{da}_drug_descriptor.tsv.gz' - #if os.path.exists(file_path): ##always rerun drug process - # dflist.append(f'/tmp/{da}_drugs.tsv') # Add to dflist if already processed + _futures = [] + dflist = [] + + if prev_drugs: + docker_paths = [] + for p in prev_drugs: + base = os.path.basename(p) + if base.endswith('.tsv.gz'): + stem = base[:-len('.tsv.gz')] + prev_name = f'{stem}_prev.tsv.gz' + elif base.endswith('.tsv'): + stem = base[:-len('.tsv')] + prev_name = f'{stem}_prev.tsv' + else: + stem = os.path.splitext(base)[0] + prev_name = f'{stem}_prev.tsv' + shutil.copy(p, f'local/{prev_name}') + docker_paths.append(f'/tmp/{prev_name}') + _log(f'Copied prev drug cache: {p} → local/{prev_name}') + with open('local/prev_drug_files.txt', 'w') as f: + f.write('\n'.join(docker_paths) + '\n') + _log(f'Wrote local/prev_drug_files.txt with {len(docker_paths)} cache file(s).') for da in datasets: if da not in ['cptac', 'hcmi']: di = 'broad_sanger_exp' if da == 'broad_sanger' else da - #if not os.path.exists(f'local/{da}_drugs.tsv'): if last_drug_future: last_drug_future.result() # Ensure the last drug process is completed before starting the next - last_drug_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_drugs.sh', ','.join(dflist)], f'{da} drugs') + def _drugs_step(image=di, arg=','.join(dflist), ds=da): + run_docker_cmd([image, 'bash', 'build_drugs.sh', arg], f'{ds} drugs') + _verify_step_outputs(ds, 'drugs') + last_drug_future = executor.submit(_drugs_step) + _futures.append(last_drug_future) dflist.append(f'/tmp/{da}_drugs.tsv') - + _await_all(_futures, 'drugs') + def process_samples(executor, datasets): ''' Build all samples files sequentially ''' last_sample_future = None + _futures = [] sf = '' for da in datasets: file_path = f'local/{da}_samples.csv' @@ -185,49 +607,77 @@ def process_samples(executor, datasets): if not os.path.exists(f'local/{da}_samples.csv'): if last_sample_future: last_sample_future.result() - last_sample_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_samples.sh', sf], f'{da} samples') + def _samples_step(image=di, arg=sf, ds=da): + run_docker_cmd([image, 'bash', 'build_samples.sh', arg], f'{ds} samples') + _verify_step_outputs(ds, 'samples') + last_sample_future = executor.submit(_samples_step) + _futures.append(last_sample_future) sf = f'/tmp/{da}_samples.csv' - - def process_omics(executor, datasets,high_mem): + _await_all(_futures, 'samples') + + def process_phosphosites(executor): + ''' + Build the phosphosites reference file if it does not exist. + Caller must ensure genes.csv exists first. Returns a Future (or None). + ''' + if not os.path.exists('local/phosphosites.csv'): + return executor.submit( + run_docker_cmd, + ['phosphosites', 'bash', 'build_phosphosites.sh', '/tmp/genes.csv'], + 'phosphosites file', + ) + return None + + def process_omics(executor, datasets, high_mem): ''' Build all omics files concurrently ''' last_omics_future = None + _futures = [] for da in datasets: di = 'broad_sanger_omics' if da == 'broad_sanger' else da - #Run all at once: + omics_cmd = [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'] + if da == 'cnf': + omics_cmd.append('/tmp/phosphosites.csv') + def _omics_step(cmd=omics_cmd, ds=da): + run_docker_cmd(cmd, f'{ds} omics') + _verify_step_outputs(ds, 'omics') if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'], f'{da} omics') - #Run one at a time. + _futures.append(executor.submit(_omics_step)) else: if last_omics_future: - last_omics_future.result() - last_omics_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'], f'{da} omics') - + last_omics_future.result() + last_omics_future = executor.submit(_omics_step) + _futures.append(last_omics_future) + _await_all(_futures, 'omics') + def process_experiments(executor, datasets, high_mem): ''' Build all experiments files concurrently ''' last_experiments_future = None + _futures = [] for da in datasets: if da not in ['cptac', 'hcmi']: di = 'broad_sanger_exp' if da == 'broad_sanger' else da if not os.path.exists(f'local/{da}_experiments.tsv'): #Run all at once if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments') + _futures.append(executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments')) #Run one at a time else: if last_experiments_future: last_experiments_future.result() last_experiments_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments') - + _futures.append(last_experiments_future) + _await_all(_futures, 'experiments') def process_misc(executor, datasets, high_mem): ''' Run all misc scripts concurrently or one at a time. ''' last_misc_future = None + _futures = [] #Currently this only applies to broad_sanger. Add others here if they need a final step. if "broad_sanger" in datasets: datasets = ["broad_sanger"] @@ -238,17 +688,19 @@ def process_misc(executor, datasets, high_mem): di = 'broad_sanger_omics' if da == 'broad_sanger' else da #Run all at once: if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc') + _futures.append(executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc')) #Run one at a time. else: if last_misc_future: last_misc_future.result() last_misc_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc') - + _futures.append(last_misc_future) + _await_all(_futures, 'misc') def process_genes(executor): - if not os.path.exists('/tmp/genes.csv'): - executor.submit(run_docker_cmd,['genes','bash','build_genes.sh'],'gene file') + if not os.path.exists('local/genes.csv'): + return executor.submit(run_docker_cmd,['genes','bash','build_genes.sh'],'gene file') + return None def run_docker_upload_cmd(cmd_arr, all_files_dir, name, version): @@ -270,14 +722,20 @@ def run_docker_upload_cmd(cmd_arr, all_files_dir, name, version): # Full command to run including version update docker_run.extend(cmd_arr) - print('Executing:', ' '.join(docker_run)) + _log('Executing:', _redact(docker_run)) # res = subprocess.run(docker_run, stdout=subprocess.PIPE, stderr=subprocess.PIPE) res = subprocess.run(docker_run, stdout=sys.stdout, stderr=sys.stderr) if res.returncode != 0: - print(res.stderr) + # Do NOT log res.stderr here. stderr is passed through to sys.stderr + # above, so subprocess does not capture it and res.stderr is always + # None -- which is exactly what this line used to print, hiding the + # real cause of a `validate failed` behind the word "None". + # The container's own output is already interleaved above. + _log(f'{name} failed: exited with code {res.returncode}.' + f' See the container output immediately above for the cause.') exit(f'{name} failed') else: - print(f'{name} successful') + _log(f'{name} successful') def decompress_file(file_path): @@ -320,10 +778,78 @@ def get_latest_commit_hash(owner, repo, branch='main'): # Error handling for required tokens if args.figshare and not figshare_token: raise ValueError("FIGSHARE_TOKEN environment variable is not set.") - if any(dataset in args.datasets for dataset in ['beataml', 'mpnst', 'bladder', 'pancreatic','sarcoma','liver','novartis','colorectal']) and not synapse_auth_token: + # NOTE: broad_sanger now needs a Synapse token too -- its DepMap inputs are + # read from Synapse (syn75028495) because the DepMap portal added a + # Cloudflare challenge. See the DepMap gate below. + if any(dataset in args.datasets for dataset in ['beataml', 'mpnst', 'bladder', 'pancreatic','sarcoma','liver','novartis','colorectal','cnf','broad_sanger']) and not synapse_auth_token: if args.docker or args.samples or args.omics or args.drugs or args.exp or args.all: # Token only required if building data, not upload or validate. - raise ValueError("SYNAPSE_AUTH_TOKEN is required for accessing MPNST, beatAML, bladder, pancreatic, liver, novartis, colorectal or sarcoma datasets.") - + raise ValueError("SYNAPSE_AUTH_TOKEN is required for accessing MPNST, beatAML, bladder, pancreatic, liver, novartis, colorectal, sarcoma, cnf, broad_sanger (DepMap files) datasets.") + + # ------------------------------------------------------------------------- + # DepMap freshness gate. + # + # DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in + # front of https://depmap.org/portal/api/download/files, so the broad_sanger + # samples and omics steps can no longer fetch DepMap data from the portal. + # coderdata instead reads a static copy from the "DepMap Raw" Synapse folder + # (syn75028495); coderbuild/utils/fetch_depmap.py downloads it inside the + # container at the start of build_samples.sh and build_omics.sh. + # + # That copy must be refreshed BY HAND for each new DepMap release. Because a + # stale copy fails silently -- the build succeeds and quietly publishes the + # previous release's cell lines -- we require an explicit confirmation flag + # rather than trusting that someone remembered. + # + # Only the steps that actually read the DepMap files are gated (samples and + # omics); --docker on its own still works for image debugging. + # ------------------------------------------------------------------------- + DEPMAP_SYNAPSE_FOLDER = 'syn75028495' + DEPMAP_REQUIRED_FILES = [ + 'Model.csv', + 'OmicsSomaticMutations.csv', + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv', + 'PortalOmicsCNGeneLog2.csv', + ] + if 'broad_sanger' in args.datasets and (args.samples or args.omics or args.all): + if not args.depmap_ready: + raise SystemExit( + "\n" + "=====================================================================\n" + " STOP: DepMap files must be confirmed up to date before building\n" + "=====================================================================\n" + " This build includes broad_sanger, which requires DepMap data.\n" + "\n" + " DepMap files can NO LONGER be downloaded from the DepMap portal --\n" + " it now serves a Cloudflare 'verify you are a person' challenge.\n" + " coderdata reads a STATIC COPY from Synapse instead, which must be\n" + " updated MANUALLY for each new DepMap release.\n" + "\n" + f" Synapse folder: https://www.synapse.org/Synapse:{DEPMAP_SYNAPSE_FOLDER}\n" + "\n" + " To refresh the files for a new DepMap release:\n" + " 1. Download them from the DepMap download page:\n" + " https://depmap.org/portal/data_page/?tab=allData\n" + " (accept the terms, then use the download button on each file)\n" + f" 2. Upload them to {DEPMAP_SYNAPSE_FOLDER} as NEW VERSIONS of the\n" + " existing entities (this keeps the syn IDs stable)\n" + " 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py\n" + "\n" + " Files tracked (4 total, ~1.29 GB):\n" + + ''.join(f" - {f}\n" for f in DEPMAP_REQUIRED_FILES) + + "\n" + " Repository: https://github.com/PNNL-CompBio/coderdata\n" + "\n" + " Once you have confirmed the DepMap files on Synapse are current,\n" + " re-run this command with:\n" + "\n" + " --depmap-ready\n" + "\n" + "=====================================================================\n" + ) + _log(f"DepMap files confirmed current on Synapse ({DEPMAP_SYNAPSE_FOLDER}) " + f"via --depmap-ready; fetch_depmap.py will download them in-container.") + + ###### ### Begin Pipeline ##### @@ -332,6 +858,9 @@ def get_latest_commit_hash(owner, repo, branch='main'): if not os.path.exists('local'): os.mkdir('local') + # Record this run in the progress log + _write_progress('BUILD START', 'START', f'args={sys.argv[1:]}') + # Get dataset names - default is all. datasets = args.datasets.split(',') @@ -339,28 +868,56 @@ def get_latest_commit_hash(owner, repo, branch='main'): # Ouput is logged at local/docker.log if args.docker or args.all: process_docker(datasets) - print("Docker image generation completed") + _log("Docker image generation completed") + + # --continue: rebuild Docker images only for datasets that failed last run + if args.resume: + failed_steps = _last_run_failures() + if failed_steps: + failed_datasets = { + _step_to_docker_dataset(s) for s in failed_steps + if _step_to_docker_dataset(s) + } + # Only rebuild images for datasets we're actually processing + failed_datasets &= set(datasets) | {'genes', 'phosphosites'} + if failed_datasets: + _log(f'--continue: rebuilding Docker images for failed datasets: {sorted(failed_datasets)}') + process_docker(list(failed_datasets)) + else: + _log('--continue: no failed datasets from last run match current dataset list — skipping Docker rebuild') + else: + _log('--continue: no failures found in build_progress.log — skipping Docker rebuild') - ### Build Drugs files, Samples files, and Genes file. These two steps are run in Parallel. + ### Build Drugs files, Samples files, and Genes file. These two steps are run in Parallel. ### Within each step, sequential running is required. with ThreadPoolExecutor() as executor: if args.samples or args.all: sample_thread = executor.submit(process_samples,executor, datasets) if args.drugs or args.all: - drug_thread = executor.submit(process_drugs,executor, datasets) + prev_drugs = [p.strip() for p in args.prev_drugs.split(',') if p.strip()] + drug_thread = executor.submit(process_drugs, executor, datasets, prev_drugs or None) + + # Genes must finish before phosphosites can start (phosphosites reads genes.csv). + # Run genes now and wait; then submit phosphosites (which can overlap with samples/drugs). if args.samples or args.omics or args.exp or args.all: - gene_thread = executor.submit(process_genes,executor) - - # Wait for both processes to complete before proceeding to omics and experiments + genes_future = process_genes(executor) + if genes_future is not None: + genes_future.result() + + phosphosite_future = None + if (args.omics or args.all) and 'cnf' in datasets: + phosphosite_future = process_phosphosites(executor) + + # Wait for all remaining tasks to complete before proceeding to omics and experiments if args.drugs or args.all: drug_thread.result() - if args.samples or args.all:##need to wait for samples for all of these + if args.samples or args.all: sample_thread.result() - if args.samples or args.omics or args.exp or args.all: - gene_thread.result() - - print("All samples, drugs files, and genes file completed or skipped") + if phosphosite_future is not None: + phosphosite_future.result() + + _log("All samples, drugs files, genes, and phosphosites files completed or skipped") ### At this point in the pipeline, all samples and drugs files have been created. There are no blockers to proceed. @@ -375,10 +932,16 @@ def get_latest_commit_hash(owner, repo, branch='main'): if args.omics or args.all: omics_thread.result() - print("All omics files completed") + _log("All omics files completed") if args.exp or args.all: exp_thread.result() - print("All experiments files completed") + _log("All experiments files completed") + + # Omics + experiments are done. Drop their large raw intermediates now + # (e.g. rnaseq_all_data_*.csv ~5GB, cell.xml, proteomics_all_*.csv), but keep + # broad*/sanger* — build_misc.sh still needs them to split broad_sanger. + _clean_local_intermediates(reason='after omics/experiments', + protect_prefixes=('broad', 'sanger')) ### Final Step, some datasets may need an additional post build step. Add this here @@ -389,7 +952,13 @@ def get_latest_commit_hash(owner, repo, branch='main'): misc_thread = executor.submit(process_misc, executor, datasets, args.high_mem) if args.misc or args.all: misc_thread.result() - print("Final build step complete.") + _log("Final build step complete.") + + # misc has split broad_sanger into its per-dataset outputs, so the broad*/ + # sanger* intermediates are no longer needed. Clear all remaining + # intermediates from local/ before the disk-heavy map/align/upload steps + # (this is where the previous build hit "No space left on device"). + _clean_local_intermediates(reason='after misc (pre-upload)') ###### @@ -399,10 +968,8 @@ def get_latest_commit_hash(owner, repo, branch='main'): # if args.figshare or args.validate: # FigShare File Prefixes: - prefixes = ['beataml', 'hcmi', 'cptac', 'pancreatic', 'bladder','sarcoma', 'genes', 'drugs', 'liver','novartis','colorectal','mpnst'] broad_sanger_datasets = ["ccle","ctrpv2","fimm","gdscv1","gdscv2","gcsi","prism","nci60"] if "broad_sanger" in datasets: - prefixes.extend(broad_sanger_datasets) datasets.extend(broad_sanger_datasets) datasets.remove("broad_sanger") @@ -420,9 +987,21 @@ def get_latest_commit_hash(owner, repo, branch='main'): if args.figshare and not args.version: raise ValueError("Version must be specified when pushing to figshare") - # Move relevant files to a designated directory + # Build exact allowlist from schema/expected_files.yaml so only + # schema-declared output files (not _prev caches, docker.log, etc.) + # end up in all_files_dir. + with open('schema/expected_files.yaml') as _f: + _schema = yaml.safe_load(_f) + expected_basenames = set() + for _entries in _schema.get('datasets', {}).values(): + for _entry in _entries: + _bn = os.path.basename(_entry['file']) + expected_basenames.add(_bn) + expected_basenames.add(_bn + '.gz') + + # Move only expected files to a designated directory for file in glob(os.path.join("local", '*.*')): - if any(file.startswith(os.path.join("local", prefix)) for prefix in prefixes): + if os.path.basename(file) in expected_basenames: shutil.move(file, os.path.join(all_files_dir, os.path.basename(file))) # Decompress all compressed files in the directory for schema checking @@ -430,20 +1009,21 @@ def get_latest_commit_hash(owner, repo, branch='main'): decompress_file(file) ### These should be done before schema checking. - sample_mapping_command = ['python3', 'coderbuild/utils/map_improve_sample_ids.py', '--local_dir', "/tmp", '--version', args.version] + version_args = ['--version', args.version] if args.version is not None else [] + sample_mapping_command = ['python3', 'coderbuild/utils/map_improve_sample_ids.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(sample_mapping_command, 'all_files_dir', 'Map_Samples', args.version) - - drug_mapping_command = ['python3', 'coderbuild/utils/map_improve_drug_ids.py', '--local_dir', "/tmp", '--version', args.version] + + drug_mapping_command = ['python3', 'coderbuild/utils/map_improve_drug_ids.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command, 'all_files_dir', 'Map_Drugs', args.version) - - drug_mapping_command_2 = ['python3', 'coderbuild/utils/align_drug_descriptors.py', '--local_dir', "/tmp", '--version', args.version] + + drug_mapping_command_2 = ['python3', 'coderbuild/utils/align_drug_descriptors.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command_2, 'all_files_dir', 'Align_Drug_Descriptors', args.version) # Run schema checker - This will always run if uploading data. schema_check_command = ['python3', 'coderbuild/utils/check_schema.py', '--datasets'] + datasets run_docker_upload_cmd(schema_check_command, 'all_files_dir', 'validate', args.version) - print("Validation complete. Proceeding with file compression/decompression adjustments") + _log("Validation complete. Proceeding with file compression/decompression adjustments") # Compress or decompress files based on specific conditions after checking for file in glob(os.path.join(all_files_dir, '*')): @@ -453,7 +1033,7 @@ def get_latest_commit_hash(owner, repo, branch='main'): elif not ('samples' in file or 'figshare' in file) and not is_compressed: compress_file(file) - print("File compression and decompression adjustments are complete.") + _log("File compression and decompression adjustments are complete.") ### Upload to Figshare using Docker if args.figshare and args.version and figshare_token: diff --git a/coderbuild/build_dataset.py b/coderbuild/build_dataset.py index 930c0e2b..150d36c4 100644 --- a/coderbuild/build_dataset.py +++ b/coderbuild/build_dataset.py @@ -45,11 +45,13 @@ def process_docker(dataset,validate): 'cptac': ['cptac'], 'sarcoma': ['sarcoma'], 'genes': ['genes'], + 'phosphosites': ['phosphosites'], 'upload': ['upload'], - 'colorectal': ['colorectal'], + 'colorectal': ['colorectal'], 'bladder': ['bladder'], 'liver': ['liver'], - 'novartis': ['novartis'] + 'novartis': ['novartis'], + 'cnf': ['cnf'] } # Collect container names to build based on the dataset provided. Always build 'genes'. @@ -57,7 +59,10 @@ def process_docker(dataset,validate): # Append upload if validation step is included if validate is True: datasets_to_build.append('upload') - + # phosphosites container is required when building cnf + if dataset == 'cnf': + datasets_to_build.append('phosphosites') + datasets_to_build.extend(dataset_map.get(dataset, [])) compose_command = ['docker', 'compose', '-f', compose_file, 'build'] + datasets_to_build @@ -79,10 +84,26 @@ def process_docker(dataset,validate): def process_genes(executor): ''' - Build the genes file if it does not exist. + Build the genes file if it does not exist. Returns a Future (or None if already built). ''' if not os.path.exists('local/genes.csv'): - executor.submit(run_docker_cmd, ['genes', 'bash', 'build_genes.sh'], 'genes file') + return executor.submit(run_docker_cmd, ['genes', 'bash', 'build_genes.sh'], 'genes file') + return None + + +def process_phosphosites(executor): + ''' + Build the phosphosites reference file if it does not exist. + Only needed when the cnf dataset is being built. Returns a Future (or None). + Caller must ensure genes.csv exists before calling this. + ''' + if not os.path.exists('local/phosphosites.csv'): + return executor.submit( + run_docker_cmd, + ['phosphosites', 'bash', 'build_phosphosites.sh', '/tmp/genes.csv'], + 'phosphosites file', + ) + return None def process_samples(executor, dataset, use_prev_dataset, should_continue): ''' @@ -133,7 +154,8 @@ def process_omics(executor, dataset, should_continue): 'bladder': ['copy_number', 'mutations', 'transcriptomics'], 'colorectal':['copy_number', 'mutations', 'transcriptomics'], 'novartis':['copy_number', 'mutations', 'transcriptomics'], - 'liver':['copy_number', 'mutations', 'transcriptomics','proteomics'] + 'liver':['copy_number', 'mutations', 'transcriptomics','proteomics'], + 'cnf': ['transcriptomics', 'proteomics', 'phosphoproteomics'], } expected_omics = dataset_omics_files.get(dataset, []) @@ -167,7 +189,10 @@ def process_omics(executor, dataset, should_continue): di = 'broad_sanger_omics' if dataset == 'broad_sanger' else dataset filename = f'{dataset} omics' - executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{dataset}_samples.csv'], filename) + omics_cmd = [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{dataset}_samples.csv'] + if dataset == 'cnf': + omics_cmd.append('/tmp/phosphosites.csv') + executor.submit(run_docker_cmd, omics_cmd, filename) def process_experiments(executor, dataset, should_continue): @@ -245,6 +270,8 @@ def run_schema_checker(dataset): ''' # Prepare the directory with the built files prefixes = ['genes', dataset] + if dataset == 'cnf': + prefixes.append('phosphosites') datasets = [dataset] broad_sanger_datasets = ["ccle","ctrpv2","fimm","gdscv1","gdscv2","gcsi","prism","nci60"] all_files_dir = 'all_files_dir' @@ -280,9 +307,63 @@ def main(): parser.add_argument('--build', action='store_true', help='Run data build.') parser.add_argument('--validate', action='store_true', help='Run schema checker on the built files') parser.add_argument('--continue', dest='should_continue', action='store_true', help='Continue from where the build left off by skipping existing files') + parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', + help='Confirm that the static DepMap files on Synapse are up to date for the ' + 'current DepMap release. Required when building broad_sanger.') args = parser.parse_args() + # DepMap freshness gate -- see the matching block in build_all.py for the + # full rationale. DepMap files can no longer be fetched from the DepMap + # portal (Cloudflare challenge), so coderdata reads a static copy from + # Synapse (fetch_depmap.py downloads it in-container). That copy is + # hand-maintained and must be confirmed current before a broad_sanger build. + DEPMAP_SYNAPSE_FOLDER = 'syn75028495' + DEPMAP_REQUIRED_FILES = [ + 'Model.csv', + 'OmicsSomaticMutations.csv', + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv', + 'PortalOmicsCNGeneLog2.csv', + ] + if args.dataset == 'broad_sanger' and args.build: + if not args.depmap_ready: + raise SystemExit( + "\n" + "=====================================================================\n" + " STOP: DepMap files must be confirmed up to date before building\n" + "=====================================================================\n" + " broad_sanger requires DepMap data.\n" + "\n" + " DepMap files can NO LONGER be downloaded from the DepMap portal --\n" + " it now serves a Cloudflare 'verify you are a person' challenge.\n" + " coderdata reads a STATIC COPY from Synapse instead, which must be\n" + " updated MANUALLY for each new DepMap release.\n" + "\n" + f" Synapse folder: https://www.synapse.org/Synapse:{DEPMAP_SYNAPSE_FOLDER}\n" + "\n" + " To refresh the files for a new DepMap release:\n" + " 1. Download them from the DepMap download page:\n" + " https://depmap.org/portal/data_page/?tab=allData\n" + " (accept the terms, then use the download button on each file)\n" + f" 2. Upload them to {DEPMAP_SYNAPSE_FOLDER} as NEW VERSIONS of the\n" + " existing entities (this keeps the syn IDs stable)\n" + " 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py\n" + "\n" + " Files tracked (4 total, ~1.29 GB):\n" + + ''.join(f" - {f}\n" for f in DEPMAP_REQUIRED_FILES) + + "\n" + " Repository: https://github.com/PNNL-CompBio/coderdata\n" + "\n" + " Once you have confirmed the DepMap files on Synapse are current,\n" + " re-run this command with:\n" + "\n" + " --depmap-ready\n" + "\n" + "=====================================================================\n" + ) + print(f"DepMap files confirmed current on Synapse ({DEPMAP_SYNAPSE_FOLDER}) " + f"via --depmap-ready; fetch_depmap.py will download them in-container.") + if not os.path.exists('local'): os.mkdir('local') @@ -292,16 +373,25 @@ def main(): if args.build: # Use ThreadPoolExecutor for parallel execution with ThreadPoolExecutor() as executor: - # Always build genes file - process_genes(executor) + # Genes must finish before phosphosites can start (phosphosites reads genes.csv) + genes_future = process_genes(executor) + if genes_future is not None: + genes_future.result() + + # Now safe to start phosphosites (genes.csv is present) + phosphosites_future = None + if args.dataset == 'cnf': + phosphosites_future = process_phosphosites(executor) - # Build samples and drugs + # Build samples and drugs in parallel while phosphosites runs samples_future = executor.submit(process_samples, executor, args.dataset, args.use_prev_dataset, args.should_continue) drugs_future = executor.submit(process_drugs, executor, args.dataset, args.use_prev_dataset, args.should_continue) samples_future.result() drugs_future.result() - + if phosphosites_future is not None: + phosphosites_future.result() + print("Samples and Drugs Files Completed.") with ThreadPoolExecutor() as executor: diff --git a/coderbuild/cnf/01-samples-cnf.py b/coderbuild/cnf/01-samples-cnf.py new file mode 100644 index 00000000..41bb14ea --- /dev/null +++ b/coderbuild/cnf/01-samples-cnf.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +""" +01-samples-cnf.py: generate cnf_samples.csv for the cNF organoid drug screen. + +Specimens are gathered from four sources: + + 1. Drug-screen file index on Synapse (syn51301431, dataType='drug screen') + 2. Global proteomics matrix (env CNF_GLOBAL_PROT_SYN_ID, default syn74815895) + 3. RNA discovery file (env CNF_RNA_DISCOVERY_SYN_ID, default syn71333780). + This is the corrected-abundance file containing every RNA condition + processed for the project (organoid, tumor tissue, drug-treated + organoid, skin), used here for sample discovery only — NOT to be + confused with CNF_RNA_TPM_SYN_ID, which build_omics.py reads for the + actual TPM data and may be filtered to organoids only. + 4. Normal Skin Synapse folder (env CNF_NORMAL_SKIN_SYN_ID, default syn74284682) + +Each specimen is classified as one of: + - "organoid" → model_type = "patient derived organoid" + - "treated_organoid" → model_type = "patient derived organoid" + (drug-treated organoid; canonical ID preserves the + treatment label, e.g. NF0021_T1_Onalespid_1uM) + - "tumor_tissue" → model_type = "tumor" + - "normal_skin" → model_type = "normal_tissue" + +Tissue and organoid samples for the same tumor are kept as separate rows with +distinct improve_sample_ids — they have different biology (fresh tumor vs. +cultured organoid) and the schema model_types differ. Treated and untreated +organoids of the same tumor are also separate rows because their canonical +IDs differ (treated samples preserve the treatment label). + +Cohort assignment is intentionally omitted. The upstream omics data is +batch-corrected and per-modality cohort membership disagrees for some +specimens, so a single per-sample cohort label cannot be assigned cleanly. +The `other_id_source` field carries a uniform project label. + +Sample IDs continue from the previous samples file: max(improve_sample_id) + 1. + +Usage: + python 01-samples-cnf.py [--output /tmp/cnf_samples.csv] +""" + +import argparse +import logging +import os +import re +import sys + +import pandas as pd +import synapseclient + +from cnf_utils import ( + MODEL_TYPE_MAP, + classify_specimen, + patient_from_specimen, +) + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- +DRUG_SCREEN_TABLE = "syn51301431" +GLOBAL_PROT_SYN_ID = os.environ.get("CNF_GLOBAL_PROT_SYN_ID", "syn74815895") +# RNA file used for SPECIMEN DISCOVERY only — needs to contain all +# conditions (organoid + tissue + treated + skin). The corrected-abundance +# file at syn71333780 fits this. Also used by build_omics.py as the TPM +# data source (the column is named correctedAbundance but stores TPM). +RNA_DISCOVERY_SYN_ID = os.environ.get("CNF_RNA_DISCOVERY_SYN_ID", "syn71333780") +NORMAL_SKIN_SYN_ID = os.environ.get("CNF_NORMAL_SKIN_SYN_ID", "syn74284682") + +DEFAULT_OUTPUT = "/tmp/cnf_samples.csv" + +CANCER_TYPE = "Cutaneous Neurofibroma" +SPECIES = "Homo sapiens (Human)" +OTHER_ID_SOURCE = "NF1_cNF_Project" + + +# ---------------------------------------------------------------------------- +# Helpers +# ---------------------------------------------------------------------------- +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +def get_starting_id(prev_samples_path: str) -> int: + """Read previous samples file and return max(improve_sample_id) + 1.""" + if not os.path.exists(prev_samples_path): + logging.warning("Previous samples file %s not found; starting at 1.", + prev_samples_path) + return 1 + + df = pd.read_csv(prev_samples_path) + if "improve_sample_id" not in df.columns or df.empty: + logging.warning("No improve_sample_id column or empty file; starting at 1.") + return 1 + + max_id = int(pd.to_numeric(df["improve_sample_id"], errors="coerce").max()) + logging.info("Previous samples max improve_sample_id = %d", max_id) + return max_id + 1 + + +def merge_specimen_dicts(*dicts: dict[str, str]) -> dict[str, str]: + """Union per-source dicts; warn on type disagreements (first-seen wins).""" + merged: dict[str, str] = {} + for d in dicts: + for spec, stype in d.items(): + if spec in merged and merged[spec] != stype: + logging.warning( + "Type conflict for %s: %s (kept) vs %s (ignored)", + spec, merged[spec], stype, + ) + continue + merged.setdefault(spec, stype) + return merged + + +# ---------------------------------------------------------------------------- +# Source-specific specimen extraction +# ---------------------------------------------------------------------------- +def specimens_from_drug_screen(syn) -> dict[str, str]: + """Pull canonical_id → sample_type for drug-screen specimens.""" + query = ( + "select id, individualID, specimenID " + f"from {DRUG_SCREEN_TABLE} " + "where dataType='drug screen'" + ) + logging.info("Querying %s for drug-screen specimens", DRUG_SCREEN_TABLE) + table = syn.tableQuery(query).asDataFrame() + raw = table["specimenID"].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + by_type = _count_by_type(out) + logging.info(" Drug screen: %s", by_type) + return out + + +def _count_by_type(d: dict[str, str]) -> dict[str, int]: + out: dict[str, int] = {} + for stype in d.values(): + out[stype] = out.get(stype, 0) + 1 + return out + + +def _detect_specimen_column(df: pd.DataFrame) -> str: + """Return the name of the specimen column in an omics file.""" + candidates = ["Specimen", "specimen", "specimen_norm", "sample_id", + "specimenID", "Sample"] + for c in candidates: + if c in df.columns: + return c + raise KeyError( + f"No specimen column among {candidates} in columns {list(df.columns)[:15]}" + ) + + +def _read_synapse_table(syn, syn_id: str) -> pd.DataFrame: + """Fetch a Synapse file and read it as CSV/TSV.""" + logging.info(" Fetching %s", syn_id) + entity = syn.get(syn_id) + path = entity.path + if path.endswith(".tsv") or path.endswith(".tsv.gz"): + return pd.read_csv(path, sep="\t") + return pd.read_csv(path) + + +def specimens_from_proteomics(syn) -> dict[str, str]: + """Pull canonical_id → sample_type from the global proteomics matrix.""" + if not GLOBAL_PROT_SYN_ID: + logging.warning("CNF_GLOBAL_PROT_SYN_ID not set; skipping proteomics") + return {} + + logging.info("Reading proteomics specimens") + try: + df = _read_synapse_table(syn, GLOBAL_PROT_SYN_ID) + col = _detect_specimen_column(df) + raw = df[col].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + logging.info(" Proteomics: %s", _count_by_type(out)) + return out + except Exception as exc: + logging.warning(" Proteomics read failed: %s", exc) + return {} + + +def specimens_from_rna_discovery(syn) -> dict[str, str]: + """Discover RNA specimens from a comprehensive long-format RNA file. + + Reads CNF_RNA_DISCOVERY_SYN_ID (default syn71333780) — the corrected- + abundance RNA file that contains every condition processed for the + project: organoid + tumor tissue + drug-treated organoid + skin. + + This is intentionally a different file from the one read by + build_omics.py (CNF_RNA_TPM_SYN_ID, which is the gene-level TPM matrix + and may be filtered to organoids only). Sample discovery and data + ingestion have different requirements: discovery wants the maximum + sample list; ingestion wants TPM values. + + Specimens are passed through classify_specimen, so drug-treated + samples like NF0021_T1_Onalespid.1uM get caught regardless of their + condition_norm value. + """ + if not RNA_DISCOVERY_SYN_ID: + logging.warning("CNF_RNA_DISCOVERY_SYN_ID not set; skipping RNA discovery") + return {} + + logging.info("Reading RNA discovery file (%s)", RNA_DISCOVERY_SYN_ID) + try: + df = _read_synapse_table(syn, RNA_DISCOVERY_SYN_ID) + col = _detect_specimen_column(df) + + raw = df[col].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + logging.info(" RNA discovery: %s", _count_by_type(out)) + return out + except Exception as exc: + logging.warning(" RNA discovery read failed: %s", exc) + return {} + + +def specimens_from_normal_skin_tree(syn) -> dict[str, str]: + """Discover skin specimens by listing the Normal Skin Synapse folder. + + The "Normal Skin" tree contains one immediate child per patient (folders + named NF0017, NF0018, ...) plus, in some cases, top-level skin FASTQ + files like NF0035-SKIN-1-04-16-2025_*.fastq.gz. Both forms encode the + patient ID at the start, so we match `^NF\\d{4}` against each child name + and emit one canonical NFxxxx_skin entry per unique patient. + + This is intentionally one API call deep — it avoids walking the whole + WGS tree (which has hundreds of empty lane-stub folders). For most + cases this captures every patient with a matched normal. + """ + if not NORMAL_SKIN_SYN_ID: + logging.warning("CNF_NORMAL_SKIN_SYN_ID not set; skipping Normal Skin tree") + return {} + + logging.info("Listing Normal Skin tree (%s)", NORMAL_SKIN_SYN_ID) + out: dict[str, str] = {} + try: + for child in syn.getChildren(NORMAL_SKIN_SYN_ID): + name = str(child.get("name", "")) + m = re.match(r"^(NF\d{4})", name) + if m: + patient = m.group(1) + out[f"{patient}_skin"] = "normal_skin" + logging.info(" Normal Skin tree: %s", _count_by_type(out)) + except Exception as exc: + logging.warning(" Normal Skin tree listing failed: %s", exc) + return out + + +# ---------------------------------------------------------------------------- +# Sample table construction +# ---------------------------------------------------------------------------- +def build_samples_table(specimens: dict[str, str], start_id: int) -> pd.DataFrame: + """Build the schema-compliant Sample table from {canonical_id: sample_type}. + + Sort order: organoid → treated_organoid → tumor_tissue → normal_skin + (each by ID). Drug-modelable untreated organoids get the lowest + improve_sample_ids — useful for downstream filtering. + """ + sort_priority = { + "organoid": 0, + "treated_organoid": 1, + "tumor_tissue": 2, + "normal_skin": 3, + } + sorted_specimens = sorted( + specimens.items(), + key=lambda kv: (sort_priority.get(kv[1], 99), kv[0]), + ) + + rows = [] + next_id = start_id + for specimen, stype in sorted_specimens: + patient = patient_from_specimen(specimen) + rows.append({ + "improve_sample_id": next_id, + "other_id": specimen, + "other_id_source": OTHER_ID_SOURCE, + "common_name": specimen, + "cancer_type": CANCER_TYPE, + "other_names": patient, + "species": SPECIES, + "model_type": MODEL_TYPE_MAP[stype], + }) + next_id += 1 + + df = pd.DataFrame(rows) + logging.info("Built %d sample rows; ID range %d–%d", + len(df), start_id, next_id - 1) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "prev_samples", + help="Path to previous samples CSV (used to find max improve_sample_id)", + ) + parser.add_argument("--output", default=DEFAULT_OUTPUT, + help=f"Output CSV path (default: {DEFAULT_OUTPUT})") + args = parser.parse_args() + + configure_logging() + + syn = synapseclient.Synapse() + syn.login() + + # Pull specimens from each source as canonical_id → sample_type dicts + drug_d = specimens_from_drug_screen(syn) + prot_d = specimens_from_proteomics(syn) + rna_d = specimens_from_rna_discovery(syn) + skin_d = specimens_from_normal_skin_tree(syn) + + all_specimens = merge_specimen_dicts(drug_d, prot_d, rna_d, skin_d) + logging.info( + "Union: %d total specimens — %s", + len(all_specimens), _count_by_type(all_specimens), + ) + + # Per-source uniqueness diagnostics + drug_set, prot_set, rna_set, skin_set = ( + set(drug_d), set(prot_d), set(rna_d), set(skin_d), + ) + only_drug = drug_set - prot_set - rna_set - skin_set + only_prot = prot_set - drug_set - rna_set - skin_set + only_rna = rna_set - drug_set - prot_set - skin_set + only_skin = skin_set - drug_set - prot_set - rna_set + if only_drug: + logging.info(" Only in drug screen: %s", sorted(only_drug)) + if only_prot: + logging.info(" Only in proteomics: %s", sorted(only_prot)) + if only_rna: + logging.info(" Only in RNA: %s", sorted(only_rna)) + if only_skin: + logging.info(" Only in Normal Skin tree: %s", sorted(only_skin)) + + if not all_specimens: + logging.error("No specimens found from any source; aborting") + sys.exit(1) + + start_id = get_starting_id(args.prev_samples) + samples = build_samples_table(all_specimens, start_id) + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + samples.to_csv(args.output, index=False) + logging.info("Wrote %s (%d rows)", args.output, len(samples)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/02-omics-cnf.py b/coderbuild/cnf/02-omics-cnf.py new file mode 100644 index 00000000..bdbe0fac --- /dev/null +++ b/coderbuild/cnf/02-omics-cnf.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +""" +02-omics-cnf.py: generate cnf_transcriptomics.csv, cnf_proteomics.csv, + and cnf_phosphoproteomics.csv. + +For the cNF dataset: + +* Transcriptomics: raw gene-level TPM from per-cohort salmon.merged.gene_tpm.tsv + files (wide matrix: rows = genes, columns = specimens). Default cohorts: + syn66352931 — cNF_Cohort_1 (NF0017–NF0023) + syn70765053 — cNF_Cohort_2 (NF0025, NF0027, NF0035) + Override via env var CNF_RNA_COHORT_SYN_IDS (comma-separated Synapse IDs). + +* Proteomics: batch-corrected global proteomics from syn74815895 + (correctedAbundance column). Override via env var CNF_GLOBAL_PROT_SYN_ID. + +* Phosphoproteomics: batch-corrected phospho-proteomics from syn70078415 + (correctedAbundance column). Mapped to phosphosite_ids via a phosphosites.csv + reference file. Override via env var CNF_PHOSPHO_SYN_ID. + +* Protocol optimization samples are excluded from RNA via DROP_SAMPLE_SUBSTRINGS. + +Specimens are canonicalized via cnf_utils so that values like +"NF0018.T1.organoid" resolve to the canonical IDs in cnf_samples.csv. +Source rows whose specimen doesn't match any entry in cnf_samples.csv are +dropped silently via the inner join to sample_map. + +Usage: + python 02-omics-cnf.py + [--out_rna /tmp/cnf_transcriptomics.csv] + [--out_prot /tmp/cnf_proteomics.csv] + [--out_phospho /tmp/cnf_phosphoproteomics.csv] +""" + +import argparse +import logging +import os +import re +import sys + +import pandas as pd +import synapseclient + +from cnf_utils import canonicalize_specimen_column + + +# ---------------------------------------------------------------------------- +# Defaults / Synapse IDs +# ---------------------------------------------------------------------------- +_rna_env = os.environ.get("CNF_RNA_COHORT_SYN_IDS", "") +RNA_TPM_COHORT_SYN_IDS: list[str] = ( + [s.strip() for s in _rna_env.split(",") if s.strip()] + if _rna_env + else ["syn66352931", "syn70765053"] +) + +GLOBAL_PROT_SYN_ID = os.environ.get("CNF_GLOBAL_PROT_SYN_ID", "syn74815895") +PHOSPHO_SYN_ID = os.environ.get("CNF_PHOSPHO_SYN_ID", "syn70078415") + +# Protocol-optimization samples to drop from RNA. +DROP_SAMPLE_SUBSTRINGS: list[str] = [ + "cNF_organoid_DIA_G_02_11Feb25", + "cNF_organoid_DIA_G_05_11Feb25", + "cNF_organoid_DIA_G_06_11Feb25", + "cNF_organoid_DIA_P_02_29Jan25", + "cNF_organoid_DIA_P_05_11Feb25", + "cNF_organoid_DIA_P_06_11Feb25", +] + +STUDY = "cnf" +DEFAULT_OUT_RNA = "/tmp/cnf_transcriptomics.csv" +DEFAULT_OUT_PROT = "/tmp/cnf_proteomics.csv" +DEFAULT_OUT_PHOSPHO = "/tmp/cnf_phosphoproteomics.csv" + + +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Reference data loaders +# ---------------------------------------------------------------------------- +def load_gene_map(genes_path: str) -> pd.DataFrame: + """Load genes.csv. Required columns: gene_symbol, entrez_id.""" + df = pd.read_csv(genes_path) + needed = {"gene_symbol", "entrez_id"} + missing = needed - set(df.columns) + if missing: + raise KeyError(f"genes file missing columns: {missing}") + df = df.dropna(subset=["entrez_id", "gene_symbol"]).copy() + df["entrez_id"] = df["entrez_id"].astype(int) + df = df.drop_duplicates(subset="gene_symbol") + logging.info("Loaded %d gene_symbol → entrez_id mappings", len(df)) + return df[["gene_symbol", "entrez_id"]] + + +def load_sample_map(samples_path: str) -> pd.DataFrame: + """Load cnf_samples.csv; return (other_id, improve_sample_id) frame.""" + df = pd.read_csv(samples_path) + df = df[["other_id", "improve_sample_id"]].drop_duplicates() + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + logging.info("Loaded %d sample mappings", len(df)) + return df + + +def load_phosphosite_map(phosphosites_path: str) -> pd.DataFrame: + """Load phosphosites.csv; return (other_id, phosphosite_id) frame.""" + df = pd.read_csv(phosphosites_path) + needed = {"other_id", "phosphosite_id"} + missing = needed - set(df.columns) + if missing: + raise KeyError(f"phosphosites file missing columns: {missing}") + df = df[["other_id", "phosphosite_id"]].drop_duplicates() + df["phosphosite_id"] = df["phosphosite_id"].astype(int) + logging.info("Loaded %d phosphosite mappings", len(df)) + return df + + +def fetch_synapse_table(syn, syn_id: str) -> pd.DataFrame: + """Download a Synapse file and read it as CSV/TSV.""" + if not syn_id: + raise ValueError("Synapse ID is empty") + logging.info("Fetching %s from Synapse", syn_id) + entity = syn.get(syn_id) + path = entity.path + if path.endswith(".tsv") or path.endswith(".tsv.gz"): + return pd.read_csv(path, sep="\t") + return pd.read_csv(path) + + +# ---------------------------------------------------------------------------- +# Shared helpers +# ---------------------------------------------------------------------------- +def _col_stem(col: str) -> str: + """Return the basename without extension for a path-like column name.""" + base = re.sub(r"^.*[/\\]", "", col) + return re.sub(r"\.[^.]+$", "", base) + + +def _drop_protocol_cols(cols: list[str]) -> list[str]: + """Remove columns whose stem matches any DROP_SAMPLE_SUBSTRINGS entry.""" + kept = [] + for c in cols: + stem = _col_stem(c) + if any(sub in stem for sub in DROP_SAMPLE_SUBSTRINGS): + logging.info(" Dropping protocol optimization sample: %s", stem) + else: + kept.append(c) + return kept + + +def _normalize_col(df: pd.DataFrame, candidates: list[str], target: str) -> pd.DataFrame: + """Rename the first matching candidate column to target.""" + for c in candidates: + if c in df.columns: + return df.rename(columns={c: target}) + raise KeyError( + f"None of {candidates} found in columns {list(df.columns)[:20]}" + ) + + +# ---------------------------------------------------------------------------- +# Transcriptomics +# ---------------------------------------------------------------------------- +def fetch_wide_tpm_cohort(syn, syn_id: str) -> pd.DataFrame: + """Fetch a wide salmon.merged.gene_tpm.tsv and return long-format DataFrame. + + nf-core salmon output: gene_id (Ensembl), gene_name (symbol), then one + column per specimen. Returns (gene_symbol, specimen, transcriptomics, source). + """ + logging.info("Fetching cohort TPM matrix %s", syn_id) + entity = syn.get(syn_id) + path = entity.path + df = pd.read_csv(path, sep="\t") + + gene_col = None + for cname in ["gene_name", "gene_symbol", "Symbol", "GeneSymbol", "gene"]: + if cname in df.columns: + gene_col = cname + break + if gene_col is None: + gene_col = df.columns[0] + logging.warning( + "%s: no gene_name column found; using %s — most rows may fail the gene_map join", + syn_id, gene_col, + ) + + meta_cols = {"gene_id", "gene_name", "gene_symbol", "Symbol", + "GeneSymbol", "gene", "transcript_id"} + sample_cols = [c for c in df.columns if c not in meta_cols] + sample_cols = _drop_protocol_cols(sample_cols) + + if not sample_cols: + raise ValueError( + f"{syn_id}: no specimen columns found after excluding metadata " + f"and protocol optimization samples" + ) + + long = ( + df[[gene_col] + sample_cols] + .rename(columns={gene_col: "gene_symbol"}) + .melt(id_vars="gene_symbol", var_name="specimen", value_name="transcriptomics") + ) + long["source"] = "Synapse" + logging.info( + " %s: %d genes × %d specimens → %d long rows", + syn_id, df[gene_col].nunique(), len(sample_cols), len(long), + ) + return long + + +def build_transcriptomics( + syn, gene_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format transcriptomics table from per-cohort TPM matrices.""" + if not RNA_TPM_COHORT_SYN_IDS: + logging.error("RNA_TPM_COHORT_SYN_IDS is empty; skipping transcriptomics.") + return pd.DataFrame( + columns=["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"] + ) + + cohort_frames = [] + for syn_id in RNA_TPM_COHORT_SYN_IDS: + try: + cohort_frames.append(fetch_wide_tpm_cohort(syn, syn_id)) + except Exception as exc: + logging.warning("Skipping RNA cohort %s: %s", syn_id, exc) + + if not cohort_frames: + logging.error("All RNA cohort fetches failed; returning empty transcriptomics.") + return pd.DataFrame( + columns=["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"] + ) + + raw = pd.concat(cohort_frames, ignore_index=True) + logging.info("Combined %d RNA cohorts → %d total rows", len(cohort_frames), len(raw)) + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Transcriptomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + df = ( + raw.merge(gene_map, on="gene_symbol", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["study"] = STUDY + df["transcriptomics"] = pd.to_numeric(df["transcriptomics"], errors="coerce") + df = df.dropna(subset=["transcriptomics"]) + df = df[["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"]] + + logging.info( + "Built transcriptomics table: %d rows (%d samples × %d genes)", + len(df), df["improve_sample_id"].nunique(), df["entrez_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Proteomics +# ---------------------------------------------------------------------------- +def build_proteomics( + syn, gene_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format proteomics table from the batch-corrected Synapse file.""" + raw = fetch_synapse_table(syn, GLOBAL_PROT_SYN_ID) + raw = _normalize_col(raw, + ["Specimen", "specimenID", "specimen_id", "specimen", "specimen_norm", "sample_id"], + "specimen") + raw = _normalize_col(raw, + ["gene_symbol", "Gene", "GeneSymbol", "Symbol", "gene", "feature_id"], + "gene_symbol") + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Proteomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + raw = _normalize_col(raw, + ["correctedAbundance", "log_ratio", "logRatio", "proteomics", "value"], + "proteomics") + + df = ( + raw.merge(gene_map, on="gene_symbol", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["source"] = "Synapse" + df["study"] = STUDY + df["proteomics"] = pd.to_numeric(df["proteomics"], errors="coerce") + df = df.dropna(subset=["proteomics"]) + df = df[["entrez_id", "improve_sample_id", "source", "study", "proteomics"]] + + logging.info( + "Built proteomics table: %d rows (%d samples × %d genes)", + len(df), df["improve_sample_id"].nunique(), df["entrez_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Phosphoproteomics +# ---------------------------------------------------------------------------- +def build_phosphoproteomics( + syn, phosphosite_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format phosphoproteomics table from the batch-corrected Synapse file.""" + raw = fetch_synapse_table(syn, PHOSPHO_SYN_ID) + raw = _normalize_col(raw, + ["Specimen", "specimenID", "specimen_id", "specimen"], + "specimen") + raw = _normalize_col(raw, + ["site", "Site", "phosphosite", "feature_id"], + "site") + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Phosphoproteomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + raw = _normalize_col(raw, + ["correctedAbundance", "log_ratio", "logRatio", "phosphoproteomics", "value"], + "phosphoproteomics") + + df = ( + raw.merge(phosphosite_map, left_on="site", right_on="other_id", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["source"] = "Synapse" + df["study"] = STUDY + df["phosphoproteomics"] = pd.to_numeric(df["phosphoproteomics"], errors="coerce") + df = df.dropna(subset=["phosphoproteomics"]) + df = df[["phosphosite_id", "improve_sample_id", "source", "study", "phosphoproteomics"]] + + logging.info( + "Built phosphoproteomics table: %d rows (%d samples × %d sites)", + len(df), df["improve_sample_id"].nunique(), df["phosphosite_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("genes", help="Path to genes.csv (gene_symbol → entrez_id)") + parser.add_argument("samples", help="Path to cnf_samples.csv from build_samples") + parser.add_argument("phosphosites", help="Path to phosphosites.csv (other_id → phosphosite_id)") + parser.add_argument("--out_rna", default=DEFAULT_OUT_RNA) + parser.add_argument("--out_prot", default=DEFAULT_OUT_PROT) + parser.add_argument("--out_phospho", default=DEFAULT_OUT_PHOSPHO) + args = parser.parse_args() + + configure_logging() + + gene_map = load_gene_map(args.genes) + sample_map = load_sample_map(args.samples) + phosphosite_map = load_phosphosite_map(args.phosphosites) + + syn = synapseclient.Synapse() + syn.login() + + rna = build_transcriptomics(syn, gene_map, sample_map) + prot = build_proteomics(syn, gene_map, sample_map) + phospho = build_phosphoproteomics(syn, phosphosite_map, sample_map) + + for path in (args.out_rna, args.out_prot, args.out_phospho): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + + rna.to_csv(args.out_rna, index=False) + prot.to_csv(args.out_prot, index=False) + phospho.to_csv(args.out_phospho, index=False) + logging.info("Wrote %s (%d rows)", args.out_rna, len(rna)) + logging.info("Wrote %s (%d rows)", args.out_prot, len(prot)) + logging.info("Wrote %s (%d rows)", args.out_phospho, len(phospho)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/03-drugs-cnf.py b/coderbuild/cnf/03-drugs-cnf.py new file mode 100644 index 00000000..fe307ddb --- /dev/null +++ b/coderbuild/cnf/03-drugs-cnf.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 +""" +03-drugs-cnf.py: generate cnf_drugs.tsv and cnf_drug_descriptors.tsv. + +Pulls unique drug names from the cNF drug screen on Synapse, then calls the +standard coderdata utilities: + + coderbuild/utils/pubchem_retrieval.py + -> cnf_drugs.tsv + + coderbuild/utils/build_drug_descriptor_table.py + -> cnf_drug_descriptors.tsv + +Important implementation note: + pubchem_retrieval.py does not expose a command-line interface in the + provided version. It defines update_dataframe_and_write_tsv(), but running + `python pubchem_retrieval.py --input ... --output ...` exits 0 without + doing any work. Therefore this script imports pubchem_retrieval.py directly + and calls update_dataframe_and_write_tsv(). + +The descriptor builder is still run as a subprocess. It may call R/biomaRt +internally, which can fail when Ensembl is unavailable. To keep the drugs file +produced even when descriptors fail, this script: + + 1. Retries external/remote work with exponential backoff. + 2. Sets a generous timeout. + 3. Supports --skip-descriptors. + 4. Supports --only-descriptors for retrying descriptor generation later. + +Usage: + python 03-drugs-cnf.py --prev_drugs + [--out_drugs /tmp/cnf_drugs.tsv] + [--out_desc /tmp/cnf_drug_descriptors.tsv] + [--skip-descriptors] + [--only-descriptors] + [--retries 3] + [--timeout 1800] +""" + +import argparse +import importlib.util +import logging +import os +import signal +import subprocess +import sys +import tempfile +import time +import pandas as pd +import synapseclient + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- + +DRUG_SCREEN_TABLE = "syn51301431" + +PUBCHEM_SCRIPT = os.environ.get( + "CODERDATA_PUBCHEM_SCRIPT", + "/coderbuild/utils/pubchem_retrieval.py", +) + +DESCRIPTOR_SCRIPT = os.environ.get( + "CODERDATA_DESCRIPTOR_SCRIPT", + "/coderbuild/utils/build_drug_desc.py", +) + +DEFAULT_OUT_DRUGS = "/tmp/cnf_drugs.tsv" +DEFAULT_OUT_DESC = "/tmp/cnf_drug_descriptors.tsv" + +DEFAULT_RETRIES = 3 +DEFAULT_TIMEOUT = 1800 # seconds per attempt +RETRY_BACKOFF_S = 30 # 30s, 60s, 120s, ... + + +# ---------------------------------------------------------------------------- +# Logging +# ---------------------------------------------------------------------------- + +def configure_logging() -> None: + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Generic subprocess wrapper with retry +# ---------------------------------------------------------------------------- + +def run_with_retries( + cmd: list[str], + retries: int, + timeout: int, + label: str, +) -> bool: + """ + Run a subprocess, retrying on failure with exponential backoff. + + Returns True on success, False if all retries are exhausted. + """ + for attempt in range(1, retries + 1): + logging.info("[%s] attempt %d/%d", label, attempt, retries) + + try: + subprocess.run(cmd, check=True, timeout=timeout) + logging.info("[%s] succeeded on attempt %d", label, attempt) + return True + + except subprocess.TimeoutExpired: + logging.warning( + "[%s] timed out after %ds on attempt %d", + label, + timeout, + attempt, + ) + + except subprocess.CalledProcessError as exc: + logging.warning( + "[%s] failed with exit %d on attempt %d", + label, + exc.returncode, + attempt, + ) + + except Exception as exc: + logging.warning( + "[%s] unexpected error on attempt %d: %s", + label, + attempt, + exc, + ) + + if attempt < retries: + backoff = RETRY_BACKOFF_S * (2 ** (attempt - 1)) + logging.info("[%s] sleeping %ds before retry", label, backoff) + time.sleep(backoff) + + logging.error("[%s] exhausted %d retries", label, retries) + return False + + +# ---------------------------------------------------------------------------- +# PubChem utility loading/calling +# ---------------------------------------------------------------------------- + +def load_pubchem_module(path: str): + """ + Load pubchem_retrieval.py as a Python module. + + The provided pubchem_retrieval.py has no command-line entry point, so it + must be imported and called directly. + """ + if not os.path.exists(path): + raise FileNotFoundError(f"PubChem retrieval script not found: {path}") + + spec = importlib.util.spec_from_file_location( + "coderdata_pubchem_retrieval", + path, + ) + + if spec is None or spec.loader is None: + raise ImportError(f"Could not create import spec for {path}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + if not hasattr(module, "update_dataframe_and_write_tsv"): + raise AttributeError( + f"{path} does not define update_dataframe_and_write_tsv()" + ) + + return module + + +def call_pubchem_retrieval( + names: list[str], + prev_drug_files: list[str], + out_drugs: str, + retries: int, + timeout: int, +) -> bool: + """ + Generate the drugs TSV by calling pubchem_retrieval.update_dataframe_and_write_tsv(). + + This intentionally does not use subprocess because the supplied + pubchem_retrieval.py has no argparse/main block and exits 0 without writing + anything when run as a script. + """ + prev_arg = ",".join(prev_drug_files) if prev_drug_files else None + + for attempt in range(1, retries + 1): + logging.info("[pubchem_retrieval] attempt %d/%d", attempt, retries) + + ignore_chems = None + + try: + pubchem = load_pubchem_module(PUBCHEM_SCRIPT) + + with tempfile.NamedTemporaryFile( + "w", + suffix="_cnf_ignore_chems.txt", + delete=False, + ) as tmp_ignore: + ignore_chems = tmp_ignore.name + + df = pubchem.update_dataframe_and_write_tsv( + unique_names=names, + output_filename=out_drugs, + ignore_chems=ignore_chems, + batch_size=1, + isname=True, + time_limit=timeout, + prev_drug_filepaths=prev_arg, + restrict_to_raw_names=names, + ) + + # pubchem_retrieval.py sets signal.alarm(time_limit), but does not + # clear it. Clear it here so the descriptor step is not interrupted. + try: + signal.alarm(0) + except Exception: + pass + + if getattr(pubchem, "should_continue", True) is False: + raise TimeoutError( + f"pubchem_retrieval reached its time limit of {timeout}s" + ) + + if not os.path.exists(out_drugs): + raise FileNotFoundError( + f"pubchem_retrieval returned but did not create {out_drugs}" + ) + + if os.path.getsize(out_drugs) == 0: + raise RuntimeError( + f"pubchem_retrieval created {out_drugs}, but it is empty" + ) + + n_rows = len(df) if df is not None else "unknown" + + logging.info( + "[pubchem_retrieval] succeeded on attempt %d; wrote %s rows to %s", + attempt, + n_rows, + out_drugs, + ) + + return True + + except Exception as exc: + try: + signal.alarm(0) + except Exception: + pass + + logging.warning( + "[pubchem_retrieval] failed on attempt %d/%d: %s", + attempt, + retries, + exc, + ) + + if attempt < retries: + backoff = RETRY_BACKOFF_S * (2 ** (attempt - 1)) + logging.info( + "[pubchem_retrieval] sleeping %ds before retry", + backoff, + ) + time.sleep(backoff) + + finally: + if ignore_chems and os.path.exists(ignore_chems): + try: + os.unlink(ignore_chems) + except OSError: + pass + + logging.error("[pubchem_retrieval] exhausted %d retries", retries) + return False + + +# ---------------------------------------------------------------------------- +# Drug name collection +# ---------------------------------------------------------------------------- + +def collect_drug_names(syn) -> list[str]: + """ + Pull every cNF drug-screen file from Synapse and collect unique drug names. + """ + query = ( + "select id, individualID, specimenID " + f"from {DRUG_SCREEN_TABLE} " + "where dataType='drug screen'" + ) + + files = syn.tableQuery(query).asDataFrame() + + logging.info("Reading %d drug-screen files for drug names", len(files)) + + names: set[str] = set() + + for _, row in files.iterrows(): + try: + entity = syn.get(row["id"]) + df = pd.read_csv(entity.path) + + except Exception as exc: + logging.warning("Skipping file %s: %s", row["id"], exc) + continue + + if "Drug" not in df.columns: + logging.warning("File %s has no 'Drug' column; skipping", row["id"]) + continue + + names.update( + df["Drug"] + .dropna() + .astype(str) + .str.strip() + .tolist() + ) + + names = sorted(n for n in names if n and n.lower() != "nan") + + logging.info("Collected %d unique drug names", len(names)) + + return names + + +# ---------------------------------------------------------------------------- +# Descriptor utility +# ---------------------------------------------------------------------------- + +def call_descriptor_table( + drug_file: str, + out_desc: str, + retries: int, + timeout: int, +) -> bool: + """ + Run the descriptor table builder as a subprocess. + + This function assumes build_drug_descriptor_table.py has a CLI accepting + --input and --output. + """ + cmd = [ + "python", + DESCRIPTOR_SCRIPT, + "--drugtable", + drug_file, + "--desctable", + out_desc, + ] + + ok = run_with_retries( + cmd, + retries, + timeout, + "descriptor_table", + ) + + if ok and not os.path.exists(out_desc): + logging.error( + "descriptor_table exited successfully but did not create %s", + out_desc, + ) + return False + + return ok + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument( + "--prev_drugs", + default="", + help="Comma-delimited list of existing drug TSV files for ID reuse.", + ) + + parser.add_argument( + "--out_drugs", + default=DEFAULT_OUT_DRUGS, + help=f"Output drugs TSV. Default: {DEFAULT_OUT_DRUGS}", + ) + + parser.add_argument( + "--out_desc", + default=DEFAULT_OUT_DESC, + help=f"Output drug descriptors TSV. Default: {DEFAULT_OUT_DESC}", + ) + + parser.add_argument( + "--skip-descriptors", + action="store_true", + help=( + "Skip the descriptor step entirely. Use when the descriptor " + "utility or its biomaRt/Ensembl dependency is failing. The drugs " + "table is still produced." + ), + ) + + parser.add_argument( + "--only-descriptors", + action="store_true", + help=( + "Only run the descriptor step, expecting --out_drugs to already " + "exist. Useful for retrying after a previous descriptor failure." + ), + ) + + parser.add_argument( + "--retries", + type=int, + default=DEFAULT_RETRIES, + help=f"Number of retry attempts. Default: {DEFAULT_RETRIES}", + ) + + parser.add_argument( + "--timeout", + type=int, + default=DEFAULT_TIMEOUT, + help=f"Timeout in seconds per attempt. Default: {DEFAULT_TIMEOUT}", + ) + + args = parser.parse_args() + + configure_logging() + + if args.skip_descriptors and args.only_descriptors: + logging.error("Cannot combine --skip-descriptors with --only-descriptors") + return 2 + + for path in (args.out_drugs, args.out_desc): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + + # ------------------------------------------------------------------------ + # Drugs step + # ------------------------------------------------------------------------ + + if not args.only_descriptors: + syn = synapseclient.Synapse() + syn.login() + + names = collect_drug_names(syn) + + if not names: + logging.error("No drug names found; aborting") + return 1 + + prev = [ + p.strip() + for p in args.prev_drugs.split(",") + if p.strip() + ] + + ok = call_pubchem_retrieval( + names=names, + prev_drug_files=prev, + out_drugs=args.out_drugs, + retries=args.retries, + timeout=args.timeout, + ) + + if not ok: + logging.error("pubchem_retrieval failed; aborting") + return 1 + + if not os.path.exists(args.out_drugs): + logging.error( + "pubchem_retrieval did not produce %s", + args.out_drugs, + ) + return 1 + + logging.info("Wrote %s", args.out_drugs) + + # ------------------------------------------------------------------------ + # Descriptor step + # ------------------------------------------------------------------------ + + if args.skip_descriptors: + logging.warning( + "Skipping descriptor step. Re-run with --only-descriptors to " + "produce %s when descriptor dependencies are reachable.", + args.out_desc, + ) + return 0 + + if not os.path.exists(args.out_drugs): + logging.error( + "Drugs file %s missing; cannot run descriptors", + args.out_drugs, + ) + return 1 + + ok = call_descriptor_table( + drug_file=args.out_drugs, + out_desc=args.out_desc, + retries=args.retries, + timeout=args.timeout, + ) + + if not ok: + logging.error( + "Descriptor step failed after %d attempts. The drugs file %s is " + "complete; re-run with --only-descriptors when descriptor " + "dependencies are reachable to produce %s.", + args.retries, + args.out_drugs, + args.out_desc, + ) + + # Exit 0 because the primary drugs table was produced. Descriptors can + # be retried later. Use --skip-descriptors to suppress this explicitly. + return 0 + + logging.info("Wrote %s", args.out_desc) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/coderbuild/cnf/04-experiments-cnf.py b/coderbuild/cnf/04-experiments-cnf.py new file mode 100644 index 00000000..fe1e216b --- /dev/null +++ b/coderbuild/cnf/04-experiments-cnf.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +""" +04-experiments-cnf.py: generate cnf_experiments.tsv. + +Discovers every drug-screen viability file under three Synapse parent folders, +attaches a specimen ID by reading the file's `specimenID` annotation (with a +filename-pattern fallback), then per (specimen, drug): + + * Multi-concentration measurements are run through the standard + coderbuild/utils/fit_curve.py utility, producing fit_auc / fit_ic50 / + fit_einf / fit_hs / fit_r2. + * Single-concentration measurements at 1 μM are recorded with + metric = 'uM_viability' and value = Viability_percentage / 100. + Schema dependency: 'uM_viability' must be in the ResponseMetric enum. + See schema_patch.md. + +Specimen attribution policy: + 1. Trust the file's `specimenID` annotation if present. + 2. Fall back to parsing the filename (e.g. "NF0017_T2_Viabilities.csv" + → "NF0017_T2"). + 3. Run the result through cnf_utils.classify_specimen as a safety net + (handles dot-separators, suffixes, case variants). Files whose + specimen can't be classified are skipped with a warning. + +Dual-mapping: one viability file's results should be attributed to two +specimens. Handled via SPECIMEN_DUAL_MAPPINGS at the top of the script; +add new entries as more cases come up. + +Usage: + python 04-experiments-cnf.py + [--output /tmp/cnf_experiments.tsv] + [--parents syn51301414,syn51301420,syn51301426] +""" + +import argparse +import logging +import os +import re +import subprocess +import sys +import tempfile + +import pandas as pd +import synapseclient + +from cnf_utils import classify_specimen + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- +# Synapse folders to walk for drug-screen viability files +DEFAULT_DRUG_SCREEN_PARENTS = [ + "syn51301414", + "syn51301420", + "syn51301426", +] + +TIME = 120 +TIME_UNIT = "hours" +STUDY = "cnf" +SOURCE = "Synapse" +SINGLE_DOSE_UM = 1.0 +SINGLE_DOSE_METRIC = "uM_viability" + +FIT_CURVE_SCRIPT = os.environ.get( + "CODERDATA_FIT_CURVE_SCRIPT", "/coderbuild/utils/fit_curve.py") + +DEFAULT_OUTPUT = "/tmp/cnf_experiments.tsv" + +# Regex to recover specimen from filenames like "NF0017_T2_Viabilities.csv" +# or "NF0021_T1_Onalespid_1uM_Viabilities.csv" +FILENAME_SPECIMEN_RE = re.compile( + r"^(NF\d{4}(?:_T\d+)?(?:_[A-Za-z0-9._-]+?)?)" # specimen prefix + r"_(?:Viabilities|viabilities|Viability|drug_screen)" # known label + r"\.(?:csv|tsv|txt)$", + re.IGNORECASE, +) + +# ---------------------------------------------------------------------------- +# DUAL-MAPPING TABLE +# ---------------------------------------------------------------------------- +# Maps a canonical specimen to any additional specimens that should receive +# the same drug-response rows. Add new entries as more cases come up. +# +# Known case: NF0021_T1_Onalespid_1uM was screened against the same drug +# panel as NF0021_T1 and should be available under both labels. +SPECIMEN_DUAL_MAPPINGS: dict[str, list[str]] = { + "NF0021_T1_Onalespid_1uM": ["NF0021_T1"], +} + + +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Specimen attribution +# ---------------------------------------------------------------------------- +def specimen_from_annotation(syn, file_id: str) -> str | None: + """Return the file's specimenID annotation, or None if missing.""" + try: + anno = syn.getAnnotations(file_id) + except Exception as exc: + logging.warning("getAnnotations failed for %s: %s", file_id, exc) + return None + val = anno.get("specimenID") + if isinstance(val, list): + val = val[0] if val else None + if val is None: + return None + s = str(val).strip() + return s or None + + +def specimen_from_filename(name: str) -> str | None: + """Parse 'NF0017_T2_Viabilities.csv' → 'NF0017_T2'.""" + m = FILENAME_SPECIMEN_RE.match(name) + if m: + return m.group(1) + return None + + +def resolve_specimen(syn, file_id: str, file_name: str) -> str | None: + """Resolve a file to a canonical specimen ID, trying annotation first. + + Returns canonical_id (e.g. "NF0017_T2") or None if both annotation and + filename parsing fail or the result doesn't classify as a known sample + type. + """ + raw = specimen_from_annotation(syn, file_id) + if not raw: + raw = specimen_from_filename(file_name) + if not raw: + return None + + # Safety-net classification — handles dot-separators, suffixes, case + result = classify_specimen(raw) + if result is None: + # Annotation or filename produced something that doesn't fit any + # known sample pattern. + logging.warning( + "Could not classify specimen '%s' (file %s, %s); skipping", + raw, file_id, file_name, + ) + return None + return result[0] + + +def expand_dual_mappings(specimen: str) -> list[str]: + """Return [specimen] + any additional specimens it's dual-mapped to.""" + return [specimen] + SPECIMEN_DUAL_MAPPINGS.get(specimen, []) + + +# ---------------------------------------------------------------------------- +# Synapse walking +# ---------------------------------------------------------------------------- +def walk_for_viability_files(syn, parent_id: str) -> list[dict]: + """Recursively yield File entities under a Synapse folder. + + Returns a list of {id, name} dicts for files whose name suggests they + contain viability data (drug-screen CSV/TSV). + """ + try: + import synapseutils + except ImportError: + logging.error("synapseutils not available; cannot walk %s", parent_id) + return [] + + files = [] + for dirpath, _subfolders, file_list in synapseutils.walk(syn, parent_id): + for fname, fid in file_list: + # Filter to plausible viability files. Accept anything ending in + # csv/tsv that has 'viab' or 'drug' in the name; fall back to + # 'csv' if neither matches but the file is small enough. + lower = fname.lower() + if not lower.endswith((".csv", ".tsv", ".txt")): + continue + if any(tok in lower for tok in ("viab", "drug", "screen")): + files.append({"id": fid, "name": fname, + "parent_path": dirpath}) + # Don't auto-grab arbitrary CSVs — we only want drug screens. + return files + + +def discover_drug_files(syn, parents: list[str]) -> pd.DataFrame: + """Walk all parent folders and dedupe by Synapse ID.""" + all_files = [] + for parent in parents: + logging.info("Walking %s ...", parent) + files = walk_for_viability_files(syn, parent) + logging.info(" found %d candidate viability files", len(files)) + all_files.extend(files) + + if not all_files: + return pd.DataFrame(columns=["id", "name"]) + + df = pd.DataFrame(all_files).drop_duplicates(subset="id").reset_index(drop=True) + logging.info("Discovered %d unique viability files across %d parents", + len(df), len(parents)) + return df + + +# ---------------------------------------------------------------------------- +# Read and combine drug-screen files +# ---------------------------------------------------------------------------- +def pull_screen_files(syn, parents: list[str]) -> pd.DataFrame: + """Walk parents, read each viability CSV, attach canonical specimen. + + Files whose specimen can't be resolved are skipped with a warning. + Files attributed to a dual-mapped specimen produce duplicate rows for + every additional specimen in the mapping. + """ + files_df = discover_drug_files(syn, parents) + if files_df.empty: + raise RuntimeError("No drug-screen files found under any parent") + + frames = [] + for _, row in files_df.iterrows(): + fid, fname = row["id"], row["name"] + + canonical = resolve_specimen(syn, fid, fname) + if not canonical: + continue + + try: + df = pd.read_csv(syn.get(fid).path) + except Exception as exc: + logging.warning("Skipping file %s: %s", fid, exc) + continue + + # Verify required columns + required = {"Drug", "Concentration_uM", "Viability_percentage"} + missing = required - set(df.columns) + if missing: + logging.warning("File %s missing %s; skipping", + fname, sorted(missing)) + continue + + # Attribute to canonical + any dual-mapped specimens + for specimen in expand_dual_mappings(canonical): + attributed = df.copy() + attributed["specimen"] = specimen + attributed["source_file"] = fid + frames.append(attributed) + + if not frames: + raise RuntimeError("No drug-screen files could be read successfully") + + combined = pd.concat(frames, ignore_index=True) + n_specimens = combined["specimen"].nunique() + n_drugs = combined["Drug"].nunique() + logging.info( + "Combined %d drug-screen rows across %d specimens × %d drugs", + len(combined), n_specimens, n_drugs, + ) + return combined + + +# ---------------------------------------------------------------------------- +# I/O helpers +# ---------------------------------------------------------------------------- +def load_sample_map(samples_path: str) -> dict[str, int]: + df = pd.read_csv(samples_path) + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + return dict(zip(df["other_id"].astype(str), df["improve_sample_id"])) + + +def load_drug_map(drugs_path: str) -> dict[str, str]: + df = pd.read_csv(drugs_path, sep="\t") + if "chem_name" not in df.columns or "improve_drug_id" not in df.columns: + raise KeyError( + "drugs file missing required columns " + f"(have: {list(df.columns)})" + ) + mapping = { + str(name).strip().lower(): str(did) + for name, did in zip(df["chem_name"], df["improve_drug_id"]) + } + logging.info("Loaded %d drug name → improve_drug_id mappings", len(mapping)) + return mapping + + +def lookup_drug_id(name, drug_map: dict[str, str]) -> str | None: + if pd.isna(name): + return None + return drug_map.get(str(name).strip().lower()) + + +# ---------------------------------------------------------------------------- +# Single vs multi dose split +# ---------------------------------------------------------------------------- +def split_single_vs_multi(combined: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + """Per (specimen, drug), separate single-dose vs multi-dose.""" + counts = ( + combined.groupby(["specimen", "Drug"])["Concentration_uM"] + .nunique() + .reset_index(name="n_conc") + ) + multi_keys = counts[counts["n_conc"] > 1] + single_keys = counts[counts["n_conc"] == 1] + + multi = combined.merge(multi_keys[["specimen", "Drug"]], + on=["specimen", "Drug"], how="inner") + single = combined.merge(single_keys[["specimen", "Drug"]], + on=["specimen", "Drug"], how="inner") + # Single-dose rows must be at the canonical 1 μM concentration. + single = single[single["Concentration_uM"] == SINGLE_DOSE_UM].copy() + + logging.info("Multi-dose rows: %d (across %d sample-drug pairs)", + len(multi), len(multi_keys)) + logging.info("Single-dose rows: %d (across %d sample-drug pairs at %g μM)", + len(single), len(single_keys), SINGLE_DOSE_UM) + return multi, single + + +# ---------------------------------------------------------------------------- +# Curve fitting (multi-dose) and single-dose formatting +# ---------------------------------------------------------------------------- +def run_curve_fit(multi: pd.DataFrame, sample_map, drug_map) -> pd.DataFrame: + if multi.empty: + return pd.DataFrame() + + multi = multi.copy() + multi["improve_sample_id"] = multi["specimen"].map(sample_map) + multi["improve_drug_id"] = multi["Drug"].apply( + lambda n: lookup_drug_id(n, drug_map)) + + n_drop = (multi["improve_sample_id"].isna() | + multi["improve_drug_id"].isna()).sum() + if n_drop: + logging.warning("Dropping %d multi-dose rows with unmapped sample/drug", + n_drop) + multi = multi.dropna(subset=["improve_sample_id", "improve_drug_id"]) + if multi.empty: + return pd.DataFrame() + + # fit_curve.py expects DOSE in μM and GROWTH as viability percentage + multi["DOSE"] = multi["Concentration_uM"].astype(float) + 1e-4 + multi["GROWTH"] = multi["Viability_percentage"].astype(float) + multi["time"] = TIME + multi["time_unit"] = TIME_UNIT + multi["study"] = STUDY + multi["source"] = SOURCE + + cols = ["DOSE", "GROWTH", "study", "source", "improve_sample_id", + "Drug", "time", "time_unit"] + multi_for_fit = multi[cols].copy() + multi_for_fit["improve_sample_id"] = ( + multi_for_fit["improve_sample_id"].astype(int)) + + with tempfile.TemporaryDirectory() as tmpdir: + in_path = os.path.join(tmpdir, "drug_response.tsv") + out_prefix = os.path.join(tmpdir, "cnf_curve") + multi_for_fit.to_csv(in_path, sep="\t", index=False) + + cmd = ["python", FIT_CURVE_SCRIPT, + "--input", in_path, "--output", out_prefix] + logging.info("Running fit_curve: %s", " ".join(cmd)) + subprocess.run(cmd, check=True) + + # fit_curve.py writes .0 in the reference snippet's behavior + out_path = out_prefix + ".0" + if not os.path.exists(out_path): + alt = out_prefix + if os.path.exists(alt): + out_path = alt + else: + raise FileNotFoundError( + f"fit_curve.py did not produce output at {out_path}" + ) + fitted = pd.read_csv(out_path, sep="\t") + + # fit_curve.py renames Drug → improve_drug_id, so the column is present + # but holds the raw drug name, not the SMI_ ID. Always remap through drug_map. + if "improve_drug_id" in fitted.columns: + name_col = "improve_drug_id" + elif "Drug" in fitted.columns: + name_col = "Drug" + else: + raise KeyError( + f"fit_curve output has neither improve_drug_id nor Drug " + f"column (have: {list(fitted.columns)})" + ) + fitted["improve_drug_id"] = fitted[name_col].apply( + lambda n: lookup_drug_id(n, drug_map)) + + fitted = fitted.dropna(subset=["improve_drug_id"]) + n_metrics = (fitted["dose_response_metric"].nunique() + if "dose_response_metric" in fitted.columns else 0) + logging.info("Curve-fit produced %d rows across %d metrics", + len(fitted), n_metrics) + return fitted + + +def format_single_dose(single: pd.DataFrame, sample_map, drug_map) -> pd.DataFrame: + if single.empty: + return pd.DataFrame() + + df = single.copy() + df["improve_sample_id"] = df["specimen"].map(sample_map) + df["improve_drug_id"] = df["Drug"].apply( + lambda n: lookup_drug_id(n, drug_map)) + + n_drop = (df["improve_sample_id"].isna() | + df["improve_drug_id"].isna()).sum() + if n_drop: + logging.warning("Dropping %d single-dose rows with unmapped sample/drug", + n_drop) + df = df.dropna(subset=["improve_sample_id", "improve_drug_id"]).copy() + + df["dose_response_metric"] = SINGLE_DOSE_METRIC + df["dose_response_value"] = df["Viability_percentage"].astype(float) / 100.0 + df["time"] = TIME + df["time_unit"] = TIME_UNIT + df["study"] = STUDY + df["source"] = SOURCE + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + + cols = ["source", "improve_sample_id", "improve_drug_id", "study", + "time", "time_unit", "dose_response_metric", "dose_response_value"] + df = df[cols] + logging.info("Formatted %d single-dose rows", len(df)) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("samples", help="cnf_samples.csv from build_samples") + parser.add_argument("drugs", help="cnf_drugs.tsv from build_drugs") + parser.add_argument("--output", default=DEFAULT_OUTPUT) + parser.add_argument( + "--parents", default=",".join(DEFAULT_DRUG_SCREEN_PARENTS), + help="Comma-delimited Synapse folder IDs to walk for viability files", + ) + args = parser.parse_args() + + configure_logging() + + parents = [p.strip() for p in args.parents.split(",") if p.strip()] + if not parents: + logging.error("No parent folders provided") + sys.exit(2) + + sample_map = load_sample_map(args.samples) + drug_map = load_drug_map(args.drugs) + + syn = synapseclient.Synapse() + syn.login() + + combined = pull_screen_files(syn, parents) + multi, single = split_single_vs_multi(combined) + + fitted = run_curve_fit(multi, sample_map, drug_map) + single_df = format_single_dose(single, sample_map, drug_map) + + final = pd.concat([fitted, single_df], ignore_index=True) + cols = ["source", "improve_sample_id", "improve_drug_id", "study", + "time", "time_unit", "dose_response_metric", "dose_response_value"] + for c in cols: + if c not in final.columns: + final[c] = pd.NA + final = final[cols] + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + final.to_csv(args.output, sep="\t", index=False) + logging.info("Wrote %s (%d rows)", args.output, len(final)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/README.md b/coderbuild/cnf/README.md new file mode 100644 index 00000000..0755a952 --- /dev/null +++ b/coderbuild/cnf/README.md @@ -0,0 +1,169 @@ +# Cutaneous Neurofibroma (cnf) Organoid Drug Screen + +A coderdata dataset of cutaneous neurofibroma (cNF) patient-derived organoids +with drug-response measurements and matched multi-omic profiling. + +## Overview + +Cutaneous neurofibromas are benign nerve-sheath tumors that develop in patients +with neurofibromatosis type 1 (NF1). This dataset contains drug-screen +viability measurements for 238 small-molecule compounds across 23 cNF tumor +specimens from 10 patients, paired with bulk RNA-seq, global LC-MS/MS +proteomics, and batch-corrected phosphoproteomics for many of the same +specimens. + +| Property | Value | +|---|---| +| Cancer type | Cutaneous Neurofibroma | +| Model type | patient derived organoid | +| Patients | 10 | +| Specimens | 23 (untreated organoids; additional tissue and treated organoid samples also present) | +| Drugs | 238 | +| Modalities included | Transcriptomics (TPM), Proteomics (log ratio), Phosphoproteomics (log ratio) | + +## Multi-tumor patient structure + +Each patient (`NF0017`, `NF0018`, ...) contributed up to three independent +tumor specimens, named `NFxxxx_T1`, `NFxxxx_T2`, `NFxxxx_T3`. This dataset +treats each specimen as an independent sample (its own `improve_sample_id`). +The patient ID is preserved in the `other_names` column so downstream models +can group specimens by patient when needed (e.g. patient-level holdout). + +``` +improve_sample_id other_id common_name other_names other_id_source +2401 NF0019_T1 NF0019_T1 NF0019 NF1_cNF_Project +2402 NF0019_T2 NF0019_T2 NF0019 NF1_cNF_Project +2403 NF0019_T3 NF0019_T3 NF0019 NF1_cNF_Project +``` + +The sample table also includes matched normal skin (`NFxxxx_skin`), primary +tumor tissue (`NFxxxx_Tx_tissue`), and drug-treated organoids +(`NFxxxx_Tx_`). Each gets its own `improve_sample_id` and +`model_type`. Organoids (untreated) receive the lowest IDs, sorted by +specimen ID, to make drug-modelable samples easy to filter. + + +## Drug-response metrics + +The cNF screen mixes two designs: + +- **Multi-dose drugs**: a subset of drugs were screened at multiple + concentrations. These are run through `coderbuild/utils/fit_curve.py` and + contribute the standard schema metrics (`fit_auc`, `fit_ic50`, `fit_einf`, + `fit_hs`, `fit_r2`). + +- **Single-dose drugs**: remaining drugs were screened at a single 1 μM + concentration. These are recorded with `dose_response_metric = uM_viability` + and `dose_response_value = Viability_percentage / 100` (range 0–1, where + 1.0 = full viability and 0.0 = full kill). + +`uM_viability` is a value in the `ResponseMetric` enum. See the Schema +dependency section below. + +## SPECIMEN_DUAL_MAPPINGS edge case + +One viability file covers the treated organoid `NF0021_T1_Onalespid_1uM` but +its drug-response measurements should also be attributed to the untreated +`NF0021_T1`. The `SPECIMEN_DUAL_MAPPINGS` dict in `04-experiments-cnf.py` +handles this by duplicating the rows for every additional specimen listed: + +```python +SPECIMEN_DUAL_MAPPINGS = { + "NF0021_T1_Onalespid_1uM": ["NF0021_T1"], +} +``` + +Add new entries here if additional treated organoids need the same dual +attribution. Both specimens must already exist in `cnf_samples.csv` or the +duplicate rows will be dropped during the sample-map join. + +## Protocol optimization samples + +Six RNA columns in the cohort TPM matrices correspond to protocol optimization +runs rather than real specimens. They are excluded in `02-omics-cnf.py` before +the wide-to-long melt. Column stems matching any of these substrings are +dropped: + +```python +DROP_SAMPLE_SUBSTRINGS = [ + "cNF_organoid_DIA_G_02_11Feb25", + "cNF_organoid_DIA_G_05_11Feb25", + "cNF_organoid_DIA_G_06_11Feb25", + "cNF_organoid_DIA_P_02_29Jan25", + "cNF_organoid_DIA_P_05_11Feb25", + "cNF_organoid_DIA_P_06_11Feb25", +] +``` + +These are matched on the column stem (basename without extension), so both +plain paths and filename columns are handled correctly. If new protocol +optimization runs appear in future cohort data, add their column stems to +this list. + +## Phosphosites dependency and known unmatched sites + +`02-omics-cnf.py` requires `phosphosites.csv` as a positional argument. +The phosphosites reference must be built before the cNF omics step runs. +`build_dataset.py` and `build_all.py` enforce this sequencing. + +The phosphoproteomics output file (`cnf_phosphoproteomics.csv`) is produced +by an inner join between the raw Synapse phospho data and `phosphosites.csv`. +Sites that don't appear in the reference are silently dropped. As of the +current release, **12 sites** in the raw cNF phospho data (syn70078415) are +permanently unmatched because their gene symbols (`BAP18`, `C11orf96`, +`C14orf93`, `C1orf21`, `C2orf49`, etc). The remaining 2,974 of 2,986 unique sites +(99.6%) are present in the output. + +## Data sources + +| Modality | Synapse ID | Notes | +|---|---|---| +| Drug-screen file index | `syn51301431` (table) | `dataType='drug screen'` | +| Drug-screen file parents | `syn51301414`, `syn51301420`, `syn51301426` | One folder per cohort | +| RNA-seq TPM (Cohort 1) | `syn66352931` | `salmon.merged.gene_tpm.tsv` | +| RNA-seq TPM (Cohort 2) | `syn70765053` | `salmon.merged.gene_tpm.tsv` | +| Global proteomics | `syn74815895` | Batch-corrected; `correctedAbundance` column | +| Phospho-proteomics | `syn70078415` | Batch-corrected; `correctedAbundance` column; mapped via `phosphosites.csv` | +| Sample discovery (RNA) | `syn71333780` | All-conditions RNA file for sample enumeration only | +| Normal Skin folder | `syn74284682` | One child per patient for skin sample discovery | + +## Build pipeline + +The build follows coderdata conventions: four shell scripts wrapping Python +implementations, plus shared utilities, packaged in a Docker image. + +``` +coderbuild/cnf/ +├── 01-samples-cnf.py # mint improve_sample_ids, write cnf_samples.csv +├── build_samples.sh +├── 02-omics-cnf.py # write cnf_transcriptomics.csv, cnf_proteomics.csv, +├── build_omics.sh # cnf_phosphoproteomics.csv +├── 03-drugs-cnf.py # call pubchem_retrieval + descriptor utility +├── build_drugs.sh +├── 04-experiments-cnf.py # split single/multi-dose, run fit_curve, write cnf_experiments.tsv +├── build_exp.sh +├── cnf_utils.py # shared specimen canonicalization utilities +├── requirements.txt +└── README.md (this file) + +coderbuild/docker/ +└── Dockerfile.cnf +``` + +### Local build + +```bash +# From repository root +python coderbuild/build_dataset.py --build --validate --dataset cnf +``` + +In the full coderdata build, this dataset is orchestrated by `build_dataset.py` +/ `build_all.py`, which ensures `genes.csv`, `samples.csv`, `drugs.tsv`, and +`phosphosites.csv` are all present before starting the cNF containers. + + +## What's intentionally not included + +- **Mutations** - not yet generated for this dataset. +- **Copy number** - not yet generated for this dataset. + diff --git a/coderbuild/cnf/build_drugs.sh b/coderbuild/cnf/build_drugs.sh new file mode 100644 index 00000000..9402f178 --- /dev/null +++ b/coderbuild/cnf/build_drugs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# build_drugs.sh - wraps 03-drugs-cnf.py per coderdata convention. +# +# Usage: +# build_drugs.sh +# +# is a comma-delimited list of drug TSV files from +# previous datasets in the build sequence. Their improve_drug_id values +# get reused when the canonical SMILES match. + +set -euo pipefail + +PREV_DRUGS="${1:-}" + +python 03-drugs-cnf.py \ + --prev_drugs "$PREV_DRUGS" \ + --out_drugs /tmp/cnf_drugs.tsv \ + --out_desc /tmp/cnf_drug_descriptors.tsv + +python build_drug_desc.py --drugtable /tmp/cnf_drugs.tsv --desctable /tmp/cnf_drug_descriptors.tsv.gz diff --git a/coderbuild/cnf/build_exp.sh b/coderbuild/cnf/build_exp.sh new file mode 100644 index 00000000..18c8b107 --- /dev/null +++ b/coderbuild/cnf/build_exp.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# build_exp.sh - wraps 04-experiments-cnf.py per coderdata convention. +# +# Usage: +# build_exp.sh + +set -euo pipefail + +SAMPLES="${1:?Usage: build_exp.sh }" +DRUGS="${2:?Usage: build_exp.sh }" + +python 04-experiments-cnf.py "$SAMPLES" "$DRUGS" --output /tmp/cnf_experiments.tsv \ No newline at end of file diff --git a/coderbuild/cnf/build_omics.sh b/coderbuild/cnf/build_omics.sh new file mode 100644 index 00000000..7db0d062 --- /dev/null +++ b/coderbuild/cnf/build_omics.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# build_omics.sh - wraps 02-omics-cnf.py per coderdata convention. +# +# Usage: +# build_omics.sh +# +# - : genes.csv with gene_symbol → entrez_id mappings +# - : cnf_samples.csv produced by build_samples.sh +# - : phosphosites.csv (other_id → phosphosite_id) + +set -euo pipefail + +GENES="${1:?Usage: build_omics.sh }" +SAMPLES="${2:?Usage: build_omics.sh }" +PHOSPHOSITES="${3:?Usage: build_omics.sh }" + +python 02-omics-cnf.py "$GENES" "$SAMPLES" "$PHOSPHOSITES" \ + --out_rna /tmp/cnf_transcriptomics.csv \ + --out_prot /tmp/cnf_proteomics.csv \ + --out_phospho /tmp/cnf_phosphoproteomics.csv \ No newline at end of file diff --git a/coderbuild/cnf/build_samples.sh b/coderbuild/cnf/build_samples.sh new file mode 100644 index 00000000..37af2162 --- /dev/null +++ b/coderbuild/cnf/build_samples.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# build_samples.sh - wraps 01-samples-cnf.py per coderdata convention. +# +# Usage: +# build_samples.sh +# +# The previous samples file is the latest samples CSV from the coderdata +# build (with the highest improve_sample_id used so far). 01-samples-cnf.py +# reads it, finds the max ID, and increments from there. + +set -euo pipefail + +PREV_SAMPLES="${1:?Usage: build_samples.sh }" + +python 01-samples-cnf.py "$PREV_SAMPLES" --output /tmp/cnf_samples.csv \ No newline at end of file diff --git a/coderbuild/cnf/cnf_utils.py b/coderbuild/cnf/cnf_utils.py new file mode 100644 index 00000000..10ec38d2 --- /dev/null +++ b/coderbuild/cnf/cnf_utils.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +cnf_utils.py: shared specimen classification utilities for the cNF build scripts. + +Used by 01-samples-cnf.py, 02-omics-cnf.py, and 04-experiments-cnf.py to +canonicalize specimen strings so omics data, drug measurements, and the sample +table all use the same improve_sample_id keys. +""" + +import re + + +# ---------------------------------------------------------------------------- +# Sample type → coderdata model_type enum +# ---------------------------------------------------------------------------- +MODEL_TYPE_MAP = { + "organoid": "patient derived organoid", + "treated_organoid": "patient derived organoid", + "tumor_tissue": "tumor", + "normal_skin": "normal_tissue", +} + + + + +# ---------------------------------------------------------------------------- +# Regex constants +# ---------------------------------------------------------------------------- +# Bare tumor organoid: NFxxxx_Tx (4 patient digits, T + tumor digits) +TUMOR_RE = re.compile(r"^NF\d{4}_T\d+$") +# Patient prefix at the start of any specimen string +PATIENT_PREFIX_RE = re.compile(r"^(NF\d{4})") +# Normal skin marker: NFxxxx[_-]skin..., allowing trailing date/replicate junk +SKIN_RE = re.compile(r"^(NF\d{4})[_-]skin", re.IGNORECASE) +# Primary tumor tissue (not organoid culture): NFxxxx_Tx_tissue +TISSUE_RE = re.compile(r"^(NF\d{4}_T\d+)_tissue$", re.IGNORECASE) +# Treated organoid: NFxxxx_Tx_, e.g. NF0021_T1_Onalespid_1uM +TREATED_RE = re.compile(r"^NF\d{4}_T\d+_.+$") + + +# ---------------------------------------------------------------------------- +# Classification +# ---------------------------------------------------------------------------- +def classify_specimen(spec) -> tuple[str, str] | None: + """Classify a raw specimen string into (canonical_id, sample_type) or None. + + sample_type ∈ {"organoid", "treated_organoid", "tumor_tissue", "normal_skin"}. + + Canonical IDs: + organoid → NFxxxx_Tx (suffix stripped) + treated_organoid → NFxxxx_Tx_ (label preserved so + treated samples don't collide + with the untreated organoid) + tumor_tissue → NFxxxx_Tx_tissue (suffix preserved) + normal_skin → NFxxxx_skin (replicates collapsed per patient) + + Unrecognized prefixes and malformed inputs return None. + """ + if spec is None: + return None + s = str(spec).strip() + if not s or s.lower() == "nan": + return None + + # Convert dot-separated form (e.g. RNA file's "NF0017.T1.organoid") to + # underscore-separated. + s_norm = s.replace(".", "_") + + # Skin: any string starting with NFxxxx that contains a "skin" marker + skin_m = SKIN_RE.match(s_norm) + if skin_m: + return (f"{skin_m.group(1)}_skin", "normal_skin") + + # Tumor tissue: NFxxxx_Tx_tissue + tissue_m = TISSUE_RE.match(s_norm) + if tissue_m: + return (f"{tissue_m.group(1)}_tissue", "tumor_tissue") + + # Strip organoid suffix (handles "_organoid" and "_organoids") + s_stripped = re.sub(r"_organoids?$", "", s_norm, flags=re.IGNORECASE) + + # Bare organoid (untreated): NFxxxx_Tx exactly + if TUMOR_RE.match(s_stripped): + return (s_stripped, "organoid") + + # Treated organoid: NFxxxx_Tx_ + if TREATED_RE.match(s_stripped): + return (s_stripped, "treated_organoid") + + return None + + +def patient_from_specimen(specimen: str) -> str: + """Extract NFxxxx patient ID from a canonical specimen ID.""" + m = PATIENT_PREFIX_RE.match(specimen) + return m.group(1) if m else "" + + +def canonicalize_specimen_column(df, specimen_col: str): + """Apply classify_specimen to a dataframe's specimen column in place. + + Returns the same df with: + - rows whose specimen classifies dropped if None + - specimen_col values replaced with their canonical IDs + - a new column 'sample_type' added + + Used by build_omics.py and build_exp.py to map raw Synapse specimen + strings (e.g. "NF0018.T1.organoid", "NF0021.T1.Onalespid.1uM") to + canonical IDs that match the cnf_samples.csv `other_id` column. + """ + classifications = df[specimen_col].apply(classify_specimen) + keep = classifications.notna() + df = df[keep].copy() + classifications = classifications[keep] + df[specimen_col] = [c[0] for c in classifications] + df["sample_type"] = [c[1] for c in classifications] + return df diff --git a/coderbuild/cnf/requirements.txt b/coderbuild/cnf/requirements.txt new file mode 100644 index 00000000..a5677837 --- /dev/null +++ b/coderbuild/cnf/requirements.txt @@ -0,0 +1,9 @@ +pandas>=2.0 +numpy>=1.24,<2.0 +synapseclient>=4.0 +requests>=2.28 +scipy>=1.10 +matplotlib +scikit-learn +scipy +mordred \ No newline at end of file diff --git a/coderbuild/docker/Dockerfile.beataml b/coderbuild/docker/Dockerfile.beataml index d5b16b9f..63c3b9c6 100644 --- a/coderbuild/docker/Dockerfile.beataml +++ b/coderbuild/docker/Dockerfile.beataml @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.bladder b/coderbuild/docker/Dockerfile.bladder index 2cebce01..4312be7d 100644 --- a/coderbuild/docker/Dockerfile.bladder +++ b/coderbuild/docker/Dockerfile.bladder @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.broad_sanger_exp b/coderbuild/docker/Dockerfile.broad_sanger_exp index d25a90e3..21aed7e5 100755 --- a/coderbuild/docker/Dockerfile.broad_sanger_exp +++ b/coderbuild/docker/Dockerfile.broad_sanger_exp @@ -2,9 +2,57 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive -# Update and upgrade packages, then install required packages. + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + +# Install required packages. +# +# Do NOT add `apt-get upgrade -y` here. r-base:4.4.1 pins R at 4.4.1, but its +# apt sources point at Debian sid, where the r-cran-* packages have since been +# rebuilt against R 4.5.x. Upgrading therefore replaces the base image's +# consistent R library set with binaries built for a different R, and loading +# any of them fails with: +# +# /usr/lib/R/library/Matrix/libs/Matrix.so: undefined symbol: ANY_ATTRIB +# +# which cascades: Matrix breaks survival, which breaks reticulate, tidyverse +# and PharmacoGx, ~50 minutes into the image build. Verified: on a clean +# r-base:4.4.1 Matrix loads; after `apt-get upgrade -y` it does not. +# +# The upgrade is also unnecessary -- this apt list installs cleanly without it, +# and 4.4.1 already ships libssl3t64 3.6.3-1 so there is no openssl conflict. +# +# The lib*-dev entries below are build dependencies for CRAN packages that +# compile from source here. They are not optional: +# libuv1-dev fs. Without it fs fails to configure, taking down +# sass -> bslib -> rmarkdown/shiny -> DT/htmlwidgets -> +# piano -> CoreGx -> PharmacoGx. +# libfontconfig1-dev systemfonts (needs fontconfig + freetype2) +# libharfbuzz-dev textshaping +# libfribidi-dev textshaping +# libfreetype6-dev ragg -> tidyverse +# libpng/tiff/jpeg ragg image backends +# Each failure surfaces only as "dependency X is not available", an hour in, +# so the root cause is easy to misread as a CRAN outage. RUN apt-get update --fix-missing && \ - apt-get upgrade -y && \ apt-get install -y --fix-missing --allow-unauthenticated \ build-essential \ python3-pip \ @@ -15,7 +63,15 @@ RUN apt-get update --fix-missing && \ libglpk-dev \ libxml2-dev \ libpq-dev \ - ca-certificates && \ + ca-certificates \ + libuv1-dev \ + libfontconfig1-dev \ + libharfbuzz-dev \ + libfribidi-dev \ + libfreetype6-dev \ + libpng-dev \ + libtiff5-dev \ + libjpeg-dev && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Create and upgrade the Python virtual environment. diff --git a/coderbuild/docker/Dockerfile.broad_sanger_omics b/coderbuild/docker/Dockerfile.broad_sanger_omics index 6b6cfc2e..23f73ca8 100755 --- a/coderbuild/docker/Dockerfile.broad_sanger_omics +++ b/coderbuild/docker/Dockerfile.broad_sanger_omics @@ -2,9 +2,43 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive -# Update and upgrade packages, then install required packages. + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + +# Install required packages. +# +# Do NOT add `apt-get upgrade -y` here. r-base:4.4.1 pins R at 4.4.1, but its +# apt sources point at Debian sid, where the r-cran-* packages have since been +# rebuilt against R 4.5.x. Upgrading therefore replaces the base image's +# consistent R library set with binaries built for a different R, and loading +# any of them fails with: +# +# /usr/lib/R/library/Matrix/libs/Matrix.so: undefined symbol: ANY_ATTRIB +# +# which cascades through every package that depends on Matrix. Verified: on a +# clean r-base:4.4.1 Matrix loads; after `apt-get upgrade -y` it does not. +# +# The upgrade is also unnecessary -- this apt list installs cleanly without it, +# and 4.4.1 already ships libssl3t64 3.6.3-1 so there is no openssl conflict. RUN apt-get update --fix-missing && \ - apt-get upgrade -y && \ apt-get install -y --fix-missing --allow-unauthenticated \ build-essential \ python3-pip \ @@ -15,7 +49,15 @@ RUN apt-get update --fix-missing && \ libglpk-dev \ libxml2-dev \ libpq-dev \ - ca-certificates && \ + ca-certificates \ + libuv1-dev \ + libfontconfig1-dev \ + libharfbuzz-dev \ + libfribidi-dev \ + libfreetype6-dev \ + libpng-dev \ + libtiff5-dev \ + libjpeg-dev && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Create and upgrade the Python virtual environment. diff --git a/coderbuild/docker/Dockerfile.cnf b/coderbuild/docker/Dockerfile.cnf new file mode 100644 index 00000000..f4a99784 --- /dev/null +++ b/coderbuild/docker/Dockerfile.cnf @@ -0,0 +1,74 @@ +# Dockerfile.cnf - container for building the cNF (cutaneous neurofibroma) +# organoid drug-response dataset for inclusion in coderdata. +# +# Conventions (matching coderbuild/docker/Dockerfile.): +# * Working directory is /coderbuild/cnf +# * Standard utilities are mounted/copied to /coderbuild/utils (fit_curve.py, +# pubchem_retrieval.py, build_drug_desc.py) +# * Synapse auth via SYNAPSE_AUTH_TOKEN env var (passed at `docker run`) +# * Build scripts are invoked from build_dataset.py / build_all.py via the +# standard build_*.sh entry points +# +# Build: +# docker build -f coderbuild/docker/Dockerfile.cnf -t coderdata-cnf . +# +# Run (orchestrated by build_dataset.py / build_all.py): +# docker run --rm \ +# -e SYNAPSE_AUTH_TOKEN=$SYNAPSE_AUTH_TOKEN \ +# -e CNF_RNA_TPM_SYN_ID=$CNF_RNA_TPM_SYN_ID \ +# -v $(pwd)/local_data:/coderbuild/local_data \ +# coderdata-cnf bash build_samples.sh /coderbuild/local_data/prev_samples.csv + +FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + +# System deps (rdkit-pypi pulls in some C libs; basic build tools for scipy) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential wget ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Set up coderbuild layout +WORKDIR /coderbuild/cnf + +# Install Python deps. rdkit is brought in via the descriptor utility but +# pinned here so the descriptor step is reproducible. +COPY coderbuild/cnf/requirements.txt /coderbuild/cnf/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt rdkit-pypi + +# Copy the cNF build scripts +COPY coderbuild/cnf/01-samples-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/02-omics-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/03-drugs-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/04-experiments-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/build_samples.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_omics.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_drugs.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_exp.sh /coderbuild/cnf/ +COPY coderbuild/cnf/cnf_utils.py /coderbuild/cnf/ +COPY coderbuild/utils/build_drug_desc.py /coderbuild/cnf/ + +RUN chmod +x /coderbuild/cnf/*.sh + +# Standard coderdata utilities (fit_curve.py, pubchem_retrieval.py, +COPY coderbuild/utils /coderbuild/utils + diff --git a/coderbuild/docker/Dockerfile.colorectal b/coderbuild/docker/Dockerfile.colorectal index c4b7b872..506d1f35 100644 --- a/coderbuild/docker/Dockerfile.colorectal +++ b/coderbuild/docker/Dockerfile.colorectal @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.cptac b/coderbuild/docker/Dockerfile.cptac index 95d11391..652b7e56 100755 --- a/coderbuild/docker/Dockerfile.cptac +++ b/coderbuild/docker/Dockerfile.cptac @@ -1,4 +1,25 @@ FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + #ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --allow-insecure-repositories RUN apt-get install -y --allow-unauthenticated --fix-missing gcc-12 build-essential libpq-dev python3-venv diff --git a/coderbuild/docker/Dockerfile.genes b/coderbuild/docker/Dockerfile.genes index aa6c63af..ed87e121 100755 --- a/coderbuild/docker/Dockerfile.genes +++ b/coderbuild/docker/Dockerfile.genes @@ -1,5 +1,26 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + RUN apt-get update --fix-missing RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libglpk-dev libxml2-dev libpq-dev diff --git a/coderbuild/docker/Dockerfile.hcmi b/coderbuild/docker/Dockerfile.hcmi index 176f3c86..c465aeb7 100644 --- a/coderbuild/docker/Dockerfile.hcmi +++ b/coderbuild/docker/Dockerfile.hcmi @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.liver b/coderbuild/docker/Dockerfile.liver index ce0133b6..1b2e864a 100644 --- a/coderbuild/docker/Dockerfile.liver +++ b/coderbuild/docker/Dockerfile.liver @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.mpnst b/coderbuild/docker/Dockerfile.mpnst index 37c75f59..2f3eff70 100755 --- a/coderbuild/docker/Dockerfile.mpnst +++ b/coderbuild/docker/Dockerfile.mpnst @@ -1,8 +1,38 @@ -FROM r-base:4.3.3 +# r-base:4.4.1 to match every other R image in this build (bladder, colorectal, +# genes, liver, novartis, broad_sanger_exp, broad_sanger_omics). +# +# 4.3.3 could no longer build: its resident libssl3t64 (3.2.1-3) owns +# /usr/lib/x86_64-linux-gnu/ossl-modules/legacy.so, while Debian sid has moved +# that file into openssl-provider-legacy (3.6.3-1). Installing libssl-dev then +# failed the layer with "trying to overwrite ... which is also in package +# libssl3t64". 4.4.1 already ships libssl3t64 3.6.3-1, so libssl-dev installs +# cleanly with no dpkg workaround. +FROM r-base:4.4.1 # Set environment to noninteractive ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.mpnstpdx b/coderbuild/docker/Dockerfile.mpnstpdx index ddc6b1e3..08354c56 100755 --- a/coderbuild/docker/Dockerfile.mpnstpdx +++ b/coderbuild/docker/Dockerfile.mpnstpdx @@ -3,6 +3,27 @@ FROM r-base:4.3.2 # Set environment to noninteractive ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.novartis b/coderbuild/docker/Dockerfile.novartis index 9687b410..bf9edf79 100644 --- a/coderbuild/docker/Dockerfile.novartis +++ b/coderbuild/docker/Dockerfile.novartis @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.pancreatic b/coderbuild/docker/Dockerfile.pancreatic index bf797c9c..95f088e8 100644 --- a/coderbuild/docker/Dockerfile.pancreatic +++ b/coderbuild/docker/Dockerfile.pancreatic @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app diff --git a/coderbuild/docker/Dockerfile.phosphosites b/coderbuild/docker/Dockerfile.phosphosites new file mode 100644 index 00000000..c854644f --- /dev/null +++ b/coderbuild/docker/Dockerfile.phosphosites @@ -0,0 +1,35 @@ +FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /coderbuild/phosphosites + +RUN pip install --no-cache-dir pandas>=2.0 synapseclient>=4.0 openpyxl>=3.1 + +COPY coderbuild/phosphosites/00-buildPhosphositeFile.py /coderbuild/phosphosites/ +COPY coderbuild/phosphosites/build_phosphosites.sh /coderbuild/phosphosites/ + +RUN chmod +x /coderbuild/phosphosites/build_phosphosites.sh diff --git a/coderbuild/docker/Dockerfile.sarcoma b/coderbuild/docker/Dockerfile.sarcoma index 3b741c39..ac25f26a 100644 --- a/coderbuild/docker/Dockerfile.sarcoma +++ b/coderbuild/docker/Dockerfile.sarcoma @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.upload b/coderbuild/docker/Dockerfile.upload index e8cf61eb..2bc465f4 100644 --- a/coderbuild/docker/Dockerfile.upload +++ b/coderbuild/docker/Dockerfile.upload @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app RUN python -m pip install --upgrade pip pyyaml requests linkml diff --git a/coderbuild/docker/docker-compose.yml b/coderbuild/docker/docker-compose.yml index 51eb56ee..3e91615a 100644 --- a/coderbuild/docker/docker-compose.yml +++ b/coderbuild/docker/docker-compose.yml @@ -130,4 +130,4 @@ services: args: HTTPS_PROXY: ${HTTPS_PROXY} platform: linux/amd64 - image: novartis:latest \ No newline at end of file + image: novartis:latest diff --git a/coderbuild/genes/00-buildGeneFile.R b/coderbuild/genes/00-buildGeneFile.R index 5c113a94..f545ca4c 100755 --- a/coderbuild/genes/00-buildGeneFile.R +++ b/coderbuild/genes/00-buildGeneFile.R @@ -1,57 +1,306 @@ -##this uses the Hg38 database to seed the gene name mappings that we might want to use +## Build /tmp/genes.csv — entrez_id, gene_symbol, other_id, other_id_source. +## +## Uses org.Hs.eg.db for the local mappings (entrez ↔ symbol, ensembl gene, +## ensembl transcript) and biomaRt to filter to protein-coding genes. +## +## biomaRt's call to Ensembl frequently fails with mirror unavailability. +## To handle this, the Ensembl call is wrapped in a retry loop: +## - cycles through the three valid mirror choices ('www', 'useast', 'asia') +## - each cycle waits longer than the last (exponential backoff) +## - up to MAX_CYCLES total cycles before giving up +## +## If every mirror remains unreachable, the script falls back to a local +## protein-coding biotype list derived from org.Hs.eg.db's GO mapping +## (filtering out pseudogenes/RNAs that lack ENSG IDs in `ens`). The +## fallback is documented in the code so the lower coverage is visible. -#if(!require('org.Hs.eg.db')){ -# BiocManager::install('org.Hs.eg.db') -library(org.Hs.eg.db) -library(biomaRt) -#} +suppressPackageStartupMessages({ + library(org.Hs.eg.db) + library(biomaRt) + library(dplyr) +}) -library(dplyr) -##get entrez ids to symbol -entrez<-as.data.frame(org.Hs.egALIAS2EG) +# ---- Ensembl "new site" compatibility (REQUIRED) ---------------------------- +# +# In Aug 2026 Ensembl migrated all sites to the new ensembl.org. The new site +# performs Accept-header content negotiation on the legacy BioMart path: +# +# Accept: */* -> 308 redirect to the +# Accept: text/xml BioMart archive host +# Accept: application/json -> 404 {"details": +# Accept: application/json, text/xml, ... (httr) "No supported new +# Ensembl equivalent +# for this URL"} +# +# httr -- which biomaRt uses -- sends `application/json` FIRST by default, so +# every biomaRt call gets a hard 404 and biomaRt reports it as the misleading +# "Ensembl site unresponsive". This is deterministic: without this override the +# retry loop below can never succeed, no matter how many cycles it runs. +# +# Forcing Accept: text/xml makes Ensembl issue the 308 to the BioMart archive, +# which is what biomaRt actually wants. Verified 2026-08-28: with this line the +# protein-coding query succeeds; without it, it fails 100% of the time. +httr::set_config(httr::add_headers(Accept = "text/xml")) -sym <- as.data.frame(org.Hs.egSYMBOL) +# ---- Local mappings (no network) ------------------------------------------- +entrez <- as.data.frame(org.Hs.egALIAS2EG) +sym <- as.data.frame(org.Hs.egSYMBOL) +ens <- as.data.frame(org.Hs.egENSEMBL2EG) +enst <- as.data.frame(org.Hs.egENSEMBLTRANS) -##get entriz ids to ensembl -ens<-as.data.frame(org.Hs.egENSEMBL2EG) +# ---- biomaRt with retry / mirror cycling ----------------------------------- +# +# Strategy: +# For up to MAX_CYCLES, try each mirror in MIRRORS in order. +# On a successful useEnsembl() + getBM() call, return the result. +# On any failure, sleep for the current backoff and try the next mirror. +# Backoff doubles each cycle (30s → 60s → 120s → ...). +# +# Total worst-case wait at MAX_CYCLES=5: ~30 minutes spread across attempts, +# which has historically been enough for transient Ensembl outages to clear. +# +# Retries still matter even with the Accept fix above: during the Aug 2026 +# migration the BioMart backends are load-balanced and only some of them +# answer -- roughly 1 request in 5 returns real MartRegistry XML, the rest +# return a "Service unavailable" page (served, unhelpfully, with HTTP 200). +# Across MAX_CYCLES x MIRRORS attempts that is very likely to land. +# +# NOTE: 'uswest' is NOT a valid biomaRt mirror -- biomaRt rejects it with +# "Invalid mirror. Select a mirror from [www, useast, asia]" and silently falls +# back to www, wasting a quarter of every cycle. Only the three real mirrors +# are listed here. +MIRRORS <- c("www", "useast", "asia") +MAX_CYCLES <- 5 +BASE_SLEEP <- 30 # seconds; doubles each cycle -##get transcript ids as well -enst<-as.data.frame(org.Hs.egENSEMBLTRANS) +fetch_protein_coding <- function() { + for (cycle in seq_len(MAX_CYCLES)) { + sleep_s <- BASE_SLEEP * (2 ^ (cycle - 1)) + for (mirror in MIRRORS) { + message(sprintf("[%s] Cycle %d/%d, trying mirror '%s'...", + Sys.time(), cycle, MAX_CYCLES, mirror)) + result <- tryCatch({ + ensembl <- useEnsembl( + biomart = "genes", + dataset = "hsapiens_gene_ensembl", + mirror = mirror + ) + tab <- getBM( + attributes = c("ensembl_gene_id"), + filters = "biotype", + values = c("protein_coding"), + mart = ensembl + ) + message(sprintf("[%s] Success via mirror '%s' (%d genes)", + Sys.time(), mirror, nrow(tab))) + tab + }, error = function(e) { + message(sprintf("[%s] Mirror '%s' failed: %s", + Sys.time(), mirror, conditionMessage(e))) + NULL + }) + if (!is.null(result)) return(result) + } + if (cycle < MAX_CYCLES) { + message(sprintf("[%s] All mirrors failed in cycle %d; sleeping %ds before next cycle", + Sys.time(), cycle, sleep_s)) + Sys.sleep(sleep_s) + } + } + return(NULL) +} - #now we can filter by protein coding using biomart -ensembl <- useEnsembl(biomart = "genes", dataset = "hsapiens_gene_ensembl") +tab <- fetch_protein_coding() -tab <- getBM(attributes=c('ensembl_gene_id'),filters='biotype', values=c('protein_coding'),mart=ensembl) +# ---- Fallback 1: query BioMart directly over HTTP --------------------------- +# +# This is a FALLBACK ONLY. biomaRt above stays the primary path so that when +# Ensembl and biomaRt are both healthy we use the supported, maintained API. +# This runs only when every biomaRt mirror attempt has already failed. +# +# WHY IT IS NEEDED +# ---------------- +# As of 2026-09-01 biomaRt (2.62.1) cannot reach Ensembl for two independent +# reasons, neither of which is fixable from our side: +# +# 1. Ensembl's martservice no longer accepts POST, which is what biomaRt +# sends: +# POST https://www.ensembl.org/biomart/martservice -> 405 Method Not Allowed +# GET https://www.ensembl.org/biomart/martservice?query=... -> 200 +# +# 2. biomaRt's own health check, .test_ensembl(), performs a request against +# BOTH www.ensembl.org and useast.ensembl.org and raises if EITHER fails. +# useast currently returns 502 and asia returns 403, so a perfectly +# healthy www is still reported as "Ensembl site unresponsive". +# +# Both are being reported upstream (Huber-group-EMBL/biomaRt). When they are +# fixed, biomaRt will simply succeed above and this code will never execute -- +# so it is safe to leave in place, and it should NOT be promoted to the +# primary path. +# +# The query below is the exact equivalent of the getBM() call above: the +# hsapiens_gene_ensembl dataset, filtered to biotype = protein_coding, +# returning ensembl_gene_id. Verified 2026-09-01 to return 23,272 genes, +# matching what biomaRt returned when it last worked (2026-08-29). +# +# GET is safe here despite BioMart queries normally using POST: this query +# encodes to ~500 characters, far below the ~8 KB practical URL limit. A query +# with 30 attributes still only reaches ~1,700. +fetch_protein_coding_direct <- function() { + if (!requireNamespace("httr2", quietly = TRUE)) return(NULL) + query_xml <- paste0( + '', + '', + '', + '', + '', + '') -joined.df<-entrez|> - left_join(sym)|> - dplyr::rename(entrez_id='gene_id',gene_symbol='symbol',other_id='alias_symbol',gene_symbol='symbol')%>% - mutate(other_id_source='entrez_alias') + for (attempt in seq_len(3)) { + message(sprintf("[%s] Direct BioMart query, attempt %d/3...", Sys.time(), attempt)) + res <- tryCatch({ + req <- httr2::req_timeout( + httr2::req_url_query( + httr2::request("https://www.ensembl.org/biomart/martservice"), + query = query_xml), + 300) + body <- httr2::resp_body_string(httr2::req_perform(req)) -##now get aliases from ensembl -edf <- sym|> - inner_join(ens)|> - dplyr::rename(entrez_id='gene_id',gene_symbol='symbol',other_id='ensembl_id')%>% - mutate(other_id_source='ensembl_gene') + # BioMart signals failure in-band with a 200, so check the payload + # rather than trusting the status code. + if (grepl("Query ERROR|status.ensembl.org| - inner_join(enst)|> - dplyr::rename(entrez_id='gene_id',gene_symbol='symbol',other_id='trans_id')|> - subset(entrez_id%in%edf$entrez_id)|> -# subset(gene_symbol%in%ed.df$gene_symbol)|> - dplyr::mutate(other_id_source='ensembl_transcript') +# ---- Fallback 2: protein-coding inference from org.Hs.eg.db ----------------- +# If every Ensembl mirror is unreachable, derive a protein-coding gene list +# locally. org.Hs.eg.db doesn't carry biotype directly, but every entrez ID +# that has an ENSG mapping in org.Hs.egENSEMBL2EG is reasonably treated as +# a "real" gene; combined with the symbol mapping, this approximates the +# protein-coding filter well enough to keep the build going. +# +# The fallback genes.csv will have somewhat broader coverage than the +# Ensembl-filtered version (it includes some pseudogenes etc), but the +# downstream omics joins are inner-joined on gene_symbol, so any extras +# that don't appear in the data files just get dropped harmlessly. -prots<-subset(edf,other_id%in%tab$ensembl_gene_id) +# The fallback is OPT-IN. It used to run automatically, which meant a transient +# Ensembl problem would silently produce a genes.csv with a broader, +# non-canonical gene list -- and because the build then SUCCEEDED, that +# degraded file could flow all the way through validation to a published +# Figshare release with nobody noticing. Every downstream omics file joins +# against genes.csv, so this is not a local defect: it changes the whole +# release. +# +# Default is now to fail loudly. Set ALLOW_GENE_FALLBACK=1 to deliberately +# accept the degraded list (e.g. for a local debug build that is not going to +# be published). +if (is.null(tab)) { + allow_fallback <- nzchar(Sys.getenv("ALLOW_GENE_FALLBACK")) + if (!allow_fallback) { + stop(paste0( + "\n", + "=====================================================================\n", + " Ensembl unreachable -- REFUSING to build a degraded genes.csv\n", + "=====================================================================\n", + " All Ensembl mirrors failed after ", MAX_CYCLES, " cycles, AND the\n", + " direct BioMart HTTP query fallback also failed.\n", + "\n", + " A local org.Hs.eg.db fallback exists, but it produces a BROADER,\n", + " non-canonical gene list (it includes pseudogenes and other biotypes\n", + " that the Ensembl protein-coding filter excludes). Every omics file\n", + " joins against genes.csv, so publishing a release built on the\n", + " fallback would silently change every dataset.\n", + "\n", + " This build has therefore been stopped rather than continuing quietly.\n", + "\n", + " Options:\n", + " - Wait for Ensembl and re-run (check https://status.ensembl.org)\n", + " - For a local, NON-published debug build only, re-run with:\n", + " ALLOW_GENE_FALLBACK=1\n", + "\n", + " NOTE: if every attempt failed with a 404 mentioning \"No supported new\n", + " Ensembl equivalent\", the Accept-header override at the top of this\n", + " script is missing or was undone -- see the comment there.\n", + "=====================================================================\n" + )) + } + message("\n=== WARNING ===") + message("All Ensembl mirrors unreachable after ", MAX_CYCLES, + " cycles. ALLOW_GENE_FALLBACK is set, so using the local ", + "org.Hs.eg.db fallback.") + message("This produces a BROADER gene list than the canonical ", + "Ensembl-filtered version. DO NOT PUBLISH a release built this way; ", + "re-run when Ensembl is reachable.") + message("===============\n") + # Use every entrez ID that has an ENSG mapping as the protein-coding proxy. + tab <- data.frame(ensembl_gene_id = unique(ens$ensembl_id), + stringsAsFactors = FALSE) +} -full.df<-rbind(joined.df,edf,tdf)|> - subset(entrez_id%in%prots$entrez_id)|> - distinct() -#save to file and version -write.table(full.df,'/tmp/genes.csv',sep=',',row.names=F,quote=T) +# ---- Assemble joined alias / ensembl gene / ensembl transcript table ------- +joined.df <- entrez |> + left_join(sym, by = "gene_id") |> + dplyr::rename(entrez_id = "gene_id", + gene_symbol = "symbol", + other_id = "alias_symbol") |> + mutate(other_id_source = "entrez_alias") -##store this file somewhere! +edf <- sym |> + inner_join(ens, by = "gene_id") |> + dplyr::rename(entrez_id = "gene_id", + gene_symbol = "symbol", + other_id = "ensembl_id") |> + mutate(other_id_source = "ensembl_gene") +tdf <- sym |> + inner_join(enst, by = "gene_id") |> + dplyr::rename(entrez_id = "gene_id", + gene_symbol = "symbol", + other_id = "trans_id") |> + subset(entrez_id %in% edf$entrez_id) |> + dplyr::mutate(other_id_source = "ensembl_transcript") + +prots <- subset(edf, other_id %in% tab$ensembl_gene_id) + +full.df <- rbind(joined.df, edf, tdf) |> + subset(entrez_id %in% prots$entrez_id) |> + distinct() + + +# ---- Write ------------------------------------------------------------------ +out_path <- "/tmp/genes.csv" +write.table(full.df, out_path, sep = ",", row.names = FALSE, quote = TRUE) +message(sprintf("Wrote %s (%d rows, %d unique entrez_ids)", + out_path, nrow(full.df), length(unique(full.df$entrez_id)))) \ No newline at end of file diff --git a/coderbuild/genes/requirements.r b/coderbuild/genes/requirements.r index 0ccb7610..5196b7b6 100755 --- a/coderbuild/genes/requirements.r +++ b/coderbuild/genes/requirements.r @@ -1,4 +1,9 @@ install.packages("BiocManager") -BiocManager::install("org.Hs.eg.db",update=TRUE,ask=FALSE) -BiocManager::install("biomaRt",update=TRUE,ask=FALSE) +BiocManager::install(c("AnnotationDbi", "org.Hs.eg.db", "biomaRt"), update=FALSE, ask=FALSE) install.packages('https://cran.r-project.org/src/contrib/Archive/dbplyr/dbplyr_2.3.4.tar.gz', repos = NULL) +install.packages('tidyr') + +# Fail the Docker build layer if any required package didn't install +required_pkgs <- c("org.Hs.eg.db", "biomaRt", "tidyr") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/mpnst/00_sample_gen.R b/coderbuild/mpnst/00_sample_gen.R index db1f238e..73f59c30 100644 --- a/coderbuild/mpnst/00_sample_gen.R +++ b/coderbuild/mpnst/00_sample_gen.R @@ -1,3 +1,4 @@ +# 00_sample_gen.R # This script generate a new sample table based on previous dataset's sample file (taking the max improve_sample_id) # Load required libraries library(data.table) @@ -26,9 +27,14 @@ synapser::synLogin(authToken=synapse_token) manifest<-synapser::synTableQuery("select * from syn53503360")$asDataFrame()|> as.data.frame() -#Drop contaminated sample JH-2-009 +#Drop contaminated sample JH-2-009 and others with issues manifest <- manifest %>% - filter(Sample != "JH-2-009") + filter(Sample != "JH-2-009") %>% + filter(Sample != "WU-545") %>% + filter(Sample != "WU-536") %>% + filter(Sample != "WU-505") %>% + filter(Sample != "MN-1") %>% + filter(Sample != "MN-3") ###sample file has a strict schema @@ -56,7 +62,7 @@ sampTable<-manifest|> ##third, generate a sample for the MTs if they were generated pdxmt<-subset(sampTable,!is.na(MicroTissueDrugFolder)) -pdxmt$model_type=rep('xenograft derived organoid',nrow(pdxmt)) +pdxmt$model_type=rep('3D-MEDS',nrow(pdxmt)) print(pdxmt) main<-rbind(sampTable,pdxmt)|> diff --git a/coderbuild/mpnst/00b_sample_gen_treated.R b/coderbuild/mpnst/00b_sample_gen_treated.R new file mode 100644 index 00000000..63db50b1 --- /dev/null +++ b/coderbuild/mpnst/00b_sample_gen_treated.R @@ -0,0 +1,594 @@ +# 00_sample_gen_treated.R +# This script generates treated microtissue samples and appends them to an existing samples file. + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(readxl) +}) + +# ----------------------- +# logging helpers +# ----------------------- +.ts <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +.p <- function(...) cat(.ts(), paste0(...), "\n") + +print_df_head <- function(df, n = 8) { + if (is.null(df) || nrow(df) == 0) { + .p("[PRINT] <0 rows>") + return(invisible(NULL)) + } + n <- min(n, nrow(df)) + print(utils::head(df, n)) + invisible(NULL) +} + +# ----------------------- +# constants +# ----------------------- +CONST_CANCER <- "Malignant peripheral nerve sheath tumor" +CONST_SPECIES <- "Homo sapiens (Human)" +CONST_MODEL <- "3D-MEDS" +CONST_SOURCE <- "NF Data Portal" + +# NOTE: updated to correctly preserve hyphens in IDs (MN-2, JH-2-002, WU-225) +safe_token <- function(x) { + x <- as.character(x) + x <- trimws(x) + x <- gsub("\\s+", "_", x) + x <- gsub("[^[:alnum:]_.-]+", "_", x) + x +} + +# ----------------------- +# Canonicalize individual IDs to hyphen style (MN-2, JH-2-002, WU-225) +# ----------------------- +canonicalize_individual_id <- function(x) { + x <- trimws(as.character(x)) + x <- gsub("[_\\s]+", "-", x) # underscores/spaces -> hyphen + x <- gsub("-+", "-", x) # collapse repeated hyphens + + # Normalize common patterns (only changes if it matches) + x <- sub("^MN-?([0-9A-Za-z]+)$", "MN-\\1", x) + x <- sub("^WU-?([0-9A-Za-z]+)$", "WU-\\1", x) + x <- sub("^JH-?([0-9]+)-?([0-9A-Za-z]+)$", "JH-\\1-\\2", x) + + x +} + +# ----------------------- +# Migrate existing treated other_id prefixes from underscore->hyphen +# This is because some samples were mis-labeled. +# ----------------------- +migrate_treated_other_id_prefix <- function(other_id) { + x <- as.character(other_id) + is_treated <- grepl("_treated_microtissue$", x) + + y <- x + y[is_treated] <- sub("^MN_([0-9A-Za-z]+)_", "MN-\\1_", y[is_treated]) + y[is_treated] <- sub("^WU_([0-9A-Za-z]+)_", "WU-\\1_", y[is_treated]) + y[is_treated] <- sub("^JH_([0-9]+)_([0-9A-Za-z]+)_", "JH-\\1-\\2_", y[is_treated]) + + y +} + +# ----------------------- +# Timepoint normalizer (handles "24 HR", "8HR ", etc) +# ----------------------- +normalize_timepoint <- function(tp) { + tp <- trimws(as.character(tp)) + tpu <- toupper(tp) + tpu <- gsub("\\s+", "", tpu) + + out <- tp + out[tpu %in% c("8HR", "8H")] <- "8h" + out[tpu %in% c("24HR", "24H")] <- "24h" + out[tpu %in% c("0HR", "0H")] <- "0h" + + is_h <- grepl("^[0-9]+h$", tolower(out)) + out[is_h] <- tolower(out[is_h]) + out +} + +hours_from_timepoint <- function(tp_norm) { + tp_norm <- tolower(trimws(as.character(tp_norm))) + hrs <- gsub("h$", "", tp_norm) + suppressWarnings(as.integer(hrs)) +} + +# ----------------------- +# Drop controls: DMSO + untreated (and close variants) +# ----------------------- +is_dmso <- function(drug) { + d <- tolower(trimws(as.character(drug))) + grepl("\\bdmso\\b", d) || grepl("dimethyl", d) || grepl("sulfoxide", d) +} + +is_untreated <- function(drug) { + d <- tolower(trimws(as.character(drug))) + grepl("\\buntreated\\b", d) || grepl("no[_\\s-]*treat", d) || grepl("\\bvehicle\\b", d) +} + +is_control <- function(drug) { + isTRUE(is_dmso(drug)) || isTRUE(is_untreated(drug)) +} + +# ============================================================ +# Builder function: +# - Drops DMSO + untreated rows +# - Aggregates replicates: ONE row per treated sample +# - other_id = {individual_id}_{drug}_{hours}hr_treated_microtissue +# - other_names = comma-separated replicate specimen IDs +# ============================================================ +build_from_standard_map <- function(std_map, orig_samples, batch_key, next_id, require_base = TRUE) { + std_map <- as.data.table(std_map) + + # Canonicalize IDs so other_id uses MN-2 / JH-2-002 / WU-225 formatting + std_map[, individual_id := canonicalize_individual_id(individual_id)] + + # base_exists check (optional) + std_map[, base_exists := individual_id %in% orig_samples$common_name] + + # drop controls early (DMSO + untreated) + std_map[, drop_control := vapply(drug, is_control, logical(1))] + n_ctrl <- sum(std_map$drop_control %in% TRUE) + if (n_ctrl > 0) .p("[", batch_key, "] dropping control rows (DMSO/untreated): ", n_ctrl) + std_map <- std_map[drop_control == FALSE | is.na(drop_control)] + + debug_tbl <- std_map[, .( + batch = batch_key, + specimen_id, + individual_id, + drug, + timepoint, + base_exists + )] + + keep_tbl <- if (require_base) std_map[base_exists == TRUE] else copy(std_map) + + .p("[", batch_key, "] rows after drop controls: ", nrow(std_map)) + .p("[", batch_key, "] require_base=", require_base) + .p("[", batch_key, "] rows kept pre-aggregate: ", nrow(keep_tbl)) + .p("[", batch_key, "] preview:") + print_df_head(keep_tbl, 8) + + if (nrow(keep_tbl) == 0) { + return(list( + new_samples_tbl = data.table(), + map_tbl = data.table(), + debug_tbl = debug_tbl, + next_id = next_id + )) + } + + # ---- First aggregation: collapse exact (individual_id, drug, timepoint) rows + agg1 <- keep_tbl[ + , + .(specimen_ids = sort(unique(trimws(as.character(specimen_id))))), + by = .(individual_id, drug, timepoint) + ] + + # Remove NA/blank specimen IDs inside groups + agg1[, specimen_ids := lapply(specimen_ids, function(v) v[!is.na(v) & nzchar(v)])] + agg1 <- agg1[lengths(specimen_ids) > 0] + + agg1[, other_names := vapply(specimen_ids, function(v) paste(v, collapse = ", "), character(1))] + agg1[, hours := hours_from_timepoint(timepoint)] + agg1[is.na(hours), hours := -1L] + + # ---- Compute other_id_val per group (this is what ends up in samples.csv) + agg1[, other_id_val := paste0( + safe_token(individual_id), "_", + safe_token(drug), "_", + as.integer(hours), "hr_treated_microtissue" + )] + + # ---- SECOND aggregation: merge replicate IDs when other_id_val collides + agg <- agg1[ + , + .( + individual_id = individual_id[1], + drug = drug[1], + timepoint = timepoint[1], + hours = hours[1], + other_names = paste( + sort(unique(trimws(unlist(strsplit(paste(other_names, collapse = ", "), ",\\s*"))))), + collapse = ", " + ) + ), + by = .(other_id_val) + ] + + .p("[", batch_key, "] aggregate groups after merge-by-other_id: ", nrow(agg)) + .p("[", batch_key, "] aggregate head:") + print_df_head(agg[, .(other_id_val, individual_id, drug, timepoint, other_names, hours)], 8) + + # fast lookup of existing treated keys in orig_samples + orig_treated_keys <- paste0(as.character(orig_samples$other_id), "||", as.character(orig_samples$model_type)) + + map_tbl <- data.table( + batch = character(), + individual_id = character(), + drug = character(), + timepoint = character(), + hours = integer(), + other_id = character(), + other_names = character(), + treated_improve_sample_id = integer() + ) + + new_samples_tbl <- data.table( + other_id = character(), + common_name = character(), + other_id_source = character(), + other_names = character(), + cancer_type = character(), + species = character(), + model_type = character(), + improve_sample_id = integer() + ) + + for (i in seq_len(nrow(agg))) { + indiv <- as.character(agg$individual_id[i]) + drug <- as.character(agg$drug[i]) + tp <- as.character(agg$timepoint[i]) + hrs <- as.integer(agg$hours[i]) + repnames <- as.character(agg$other_names[i]) + + other_id_val <- as.character(agg$other_id_val[i]) + key <- paste0(other_id_val, "||", CONST_MODEL) + + # Skip if already exists in samples.csv + if (key %in% orig_treated_keys) { + .p("[", batch_key, "][SKIP] already exists other_id=", other_id_val) + next + } + + treated_id <- next_id + next_id <- next_id + 1L + + new_samples_tbl <- rbind( + new_samples_tbl, + data.table( + other_id = other_id_val, + common_name = indiv, + other_id_source = CONST_SOURCE, + other_names = repnames, + cancer_type = CONST_CANCER, + species = CONST_SPECIES, + model_type = CONST_MODEL, + improve_sample_id = treated_id + ), + fill = TRUE + ) + + map_tbl <- rbind( + map_tbl, + data.table( + batch = batch_key, + individual_id = indiv, + drug = drug, + timepoint = tp, + hours = hrs, + other_id = other_id_val, + other_names = repnames, + treated_improve_sample_id = treated_id + ), + fill = TRUE + ) + + .p("[", batch_key, "][ADD] ", other_id_val, " | reps={", repnames, "} -> ", treated_id) + } + + list( + new_samples_tbl = new_samples_tbl, + map_tbl = map_tbl, + debug_tbl = debug_tbl, + next_id = next_id + ) +} + +write_batch_outputs <- function(batch_key, debug_tbl, map_tbl) { + out_debug_path <- file.path("/tmp", sprintf("mpnst_samples_treated_%s_debug.csv", batch_key)) + out_map_path <- file.path("/tmp", sprintf("mpnst_samples_treated_%s_map.csv", batch_key)) + + .p("[WRITE] ", batch_key, " debug -> ", out_debug_path) + fwrite(debug_tbl, out_debug_path) + .p("[WRITE] OK") + + .p("[WRITE] ", batch_key, " map -> ", out_map_path) + fwrite(map_tbl, out_map_path) + .p("[WRITE] OK") +} + +# ============================================================ +# Batch 1 +# ============================================================ +BATCH1_KEY <- "batch1" +BATCH1_SAMPLEMAP_SYNID <- "syn64608368" + +fetch_batch1_map <- function() { + .p("[BATCH1] synGet(", BATCH1_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH1_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH1] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch1_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch1 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch1_map(fetch_batch1_map()), orig_samples, BATCH1_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 2 +# ============================================================ +BATCH2_KEY <- "batch2" +BATCH2_SAMPLEMAP_SYNID <- "syn66302373" # Update to syn64608372 once the corrected version is uploaded. + + +fetch_batch2_map <- function() { + .p("[BATCH2] synGet(", BATCH2_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH2_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH2] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch2_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + #fixed in metadata, so this is not needed anymore. + # MN2 -> MN-2 + # dt[individualID == "MN2", individualID := "MN-2"] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch2 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch2_map(fetch_batch2_map()), orig_samples, BATCH2_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 3 +# ============================================================ +BATCH3_KEY <- "batch3" +BATCH3_SAMPLEMAP_SYNID <- "syn66050299" + +fetch_batch3_map <- function() { + .p("[BATCH3] synGet(", BATCH3_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH3_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH3] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch3_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch3 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch3_map(fetch_batch3_map()), orig_samples, BATCH3_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 4 (CSV) +# ============================================================ +BATCH4_KEY <- "batch4" +BATCH4_SAMPLEMAP_SYNID <- "syn72518652" + +fetch_batch4_map <- function() { + .p("[BATCH4] synGet(", BATCH4_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH4_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH4] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch4_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + + # Expected columns in 01252026_Batch4_samplemap.csv: + # specimenID, individualID, Drug, Timepoint + if (!all(c("specimenID", "individualID", "Drug", "Timepoint") %in% names(dt))) { + .p("[BATCH4][WARN] Missing required columns. Found: ", paste(names(dt), collapse = " | ")) + return(data.table( + specimen_id = character(), + individual_id = character(), + drug = character(), + timepoint = character() + )) + } + + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + out <- dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] + + .p("[BATCH4] normalized dim: ", nrow(out), " x ", ncol(out)) + .p("[BATCH4] normalized head:") + print_df_head(out, 8) + + out +} + +run_batch4 <- function(orig_samples, next_id) { + build_from_standard_map( + normalize_batch4_map(fetch_batch4_map()), + orig_samples, + BATCH4_KEY, + next_id, + require_base = FALSE + ) +} + + +# ============================================================ +# Register all batches (order) +# ============================================================ +ALL_BATCH_RUNNERS <- list( + list(key=BATCH1_KEY, fn=run_batch1), + list(key=BATCH2_KEY, fn=run_batch2), + list(key=BATCH3_KEY, fn=run_batch3), + list(key=BATCH4_KEY, fn=run_batch4) +) + +# ============================================================ +# MAIN +# ============================================================ +args <- commandArgs(trailingOnly = TRUE) + +.p("============================================================") +.p("=== 00b_sample_gen_treated.R (RUN ALL BATCHES) ===") +.p("Timestamp: ", .ts()) +.p("Args: ", paste(args, collapse = " | ")) +.p("============================================================") + +if (length(args) < 1) { + stop( + "Usage: Rscript 00b_sample_gen_treated.R \n", + "Example: Rscript 00b_sample_gen_treated.R /tmp/mpnst_samples.csv", + call. = FALSE + ) +} + +prev_samples_path <- args[1] +.p("[ARGS] prev_samples_path=", prev_samples_path) +if (!file.exists(prev_samples_path)) stop("File not found: ", prev_samples_path, call. = FALSE) + +.p("[READ] fread(samples) ...") +orig_samples <- fread(prev_samples_path) +.p("[READ] orig_samples dim: ", nrow(orig_samples), " x ", ncol(orig_samples)) +print_df_head(orig_samples, 6) + +# ------------------------------------------------------------ +# Migrate already-existing treated other_ids to hyphen style +# so future runs match your desired format and don't skip forever +# ------------------------------------------------------------ +orig_samples[, other_id_old := other_id] +orig_samples[, other_id := migrate_treated_other_id_prefix(other_id)] + +n_changed <- sum(orig_samples$other_id != orig_samples$other_id_old, na.rm = TRUE) +if (n_changed > 0) .p("[MIGRATE] updated treated other_id prefix for ", n_changed, " row(s)") +orig_samples[, other_id_old := NULL] + +# If migration created duplicates (same other_id + model_type), merge other_names and drop dup rows +orig_samples[, dedup_key := paste(other_id, model_type, sep = "||")] +dup_keys <- orig_samples[duplicated(dedup_key), unique(dedup_key)] + +if (length(dup_keys) > 0) { + .p("[MIGRATE] duplicates after migration: ", length(dup_keys), " key(s). Merging other_names + dropping dup rows.") + + merged <- orig_samples[dedup_key %in% dup_keys, .( + merged_other_names = paste( + sort(unique(trimws(unlist(strsplit(paste(na.omit(other_names), collapse = ", "), ",\\s*"))))), + collapse = ", " + ) + ), by = dedup_key] + + orig_samples[merged, on = "dedup_key", other_names := i.merged_other_names] + orig_samples <- orig_samples[!duplicated(dedup_key)] +} + +orig_samples[, dedup_key := NULL] + +token <- Sys.getenv("SYNAPSE_AUTH_TOKEN") +.p("[ENV] SYNAPSE_AUTH_TOKEN present? ", nzchar(token)) +if (!nzchar(token)) stop("Please set SYNAPSE_AUTH_TOKEN in your environment.", call. = FALSE) + +.p("[SYN] synLogin() ...") +synLogin(authToken = token) +.p("[SYN] synLogin() OK") + +max_id <- suppressWarnings(max(as.integer(orig_samples$improve_sample_id), na.rm = TRUE)) +if (!is.finite(max_id)) max_id <- 0L +next_id <- max_id + 1L +.p("[IDS] max improve_sample_id=", max_id, " next_id=", next_id) + +.p("============================================================") +.p("[RUN] Running ", length(ALL_BATCH_RUNNERS), " batch runner(s) in order...") +.p("============================================================") + +for (item in ALL_BATCH_RUNNERS) { + batch_key <- item$key + runner <- item$fn + + res <- runner(orig_samples = orig_samples, next_id = next_id) + + if (!is.null(res$new_samples_tbl) && nrow(res$new_samples_tbl) > 0) { + orig_samples <- rbindlist(list(orig_samples, res$new_samples_tbl), use.names = TRUE, fill = TRUE) + + orig_samples[, dedup_key := paste(other_id, model_type, sep = "||")] + before <- nrow(orig_samples) + orig_samples <- orig_samples[!duplicated(dedup_key)] + after <- nrow(orig_samples) + if (after < before) .p("[", batch_key, "][WARN] post-append dedup removed ", before - after, " row(s)") + orig_samples[, dedup_key := NULL] + } + + next_id <- res$next_id + + .p("[RUN] Finished ", batch_key, " | added=", ifelse(is.null(res$new_samples_tbl), 0, nrow(res$new_samples_tbl))) + write_batch_outputs(batch_key, res$debug_tbl, res$map_tbl) +} + +.p("[WRITE] Writing updated samples -> ", prev_samples_path) +fwrite(orig_samples, prev_samples_path) +.p("[WRITE] OK") + +.p("============================================================") +.p("[DONE] Completed all batches. Updated samples at: ", prev_samples_path) +.p("============================================================") diff --git a/coderbuild/mpnst/01_combined_omics.R b/coderbuild/mpnst/01_combined_omics.R index 4706d0b8..498da51f 100644 --- a/coderbuild/mpnst/01_combined_omics.R +++ b/coderbuild/mpnst/01_combined_omics.R @@ -1,3 +1,4 @@ +# 01_combined_omics.R #!/usr/bin/env Rscript # Combined MPNST & MPNST-PDX Data Extraction Script @@ -31,7 +32,7 @@ genes_df <- fread(genes) # Subset by model type pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") tumor_samps<- filter(samples_df, model_type == "tumor") -mt_samps <- filter(samples_df, model_type == "xenograft derived organoid") # These end up being the same as pdx_samps in the manifest. +mt_samps <- filter(samples_df, model_type == "3D-MEDS") # These end up being the same as pdx_samps in the manifest. # Retrieve manifest table from Synapse manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% @@ -58,7 +59,7 @@ tumor_data <- manifest %>% mutate(Proteomics = "") %>% filter(!is.na(improve_sample_id)) -mt_data <- manifest %>% #Note, this is the same as pdx_data but I think we default to "xenograft derived organoid" if present (based on original files) +mt_data <- manifest %>% #Note, this is the same as pdx_data but I think we default to "3D-MEDS" if present (based on original files) select(common_name, starts_with("PDX")) %>% left_join(mt_samps, by = "common_name") %>% select(improve_sample_id, common_name, model_type, @@ -78,7 +79,7 @@ study_label <- function(type) { case_when( type == "patient derived xenograft" ~ "MPNST PDX", type == "tumor" ~ "MPNST Tumor", - type == "xenograft derived organoid" ~ "MPNST PDX MT", + type == "3D-MEDS" ~ "MPNST PDX MT", TRUE ~ "MPNST" ) } @@ -147,13 +148,28 @@ transcriptomics_list <- lapply( if (is.null(meta)) return(NULL) df <- tryCatch({ - fread(synGet(id)$path) %>% + raw <- fread(synGet(id)$path) %>% separate(Name, into = c("other_id","vers"), sep = "\\.") %>% - select(-vers) %>% + select(-vers) + + is_enst_input <- all(grepl("^ENST", raw$other_id)) + + mapped <- raw %>% left_join(genes_df) %>% - select(entrez_id, transcriptomics = TPM) %>% - filter(!is.na(entrez_id), transcriptomics != 0) %>% - distinct() + select(entrez_id, other_id, transcriptomics = TPM) %>% + filter(!is.na(entrez_id), transcriptomics != 0) + + if (is_enst_input) { + mapped <- mapped %>% + group_by(entrez_id) %>% + summarise(transcriptomics = sum(transcriptomics), .groups = "drop") + } else { + mapped <- mapped %>% + select(entrez_id, transcriptomics) %>% + distinct() + } + + mapped }, error = function(e) NULL) i_safe_extract( diff --git a/coderbuild/mpnst/01b_combined_omics_treated.R b/coderbuild/mpnst/01b_combined_omics_treated.R new file mode 100644 index 00000000..8125fc61 --- /dev/null +++ b/coderbuild/mpnst/01b_combined_omics_treated.R @@ -0,0 +1,219 @@ +# 01b_combined_omics_treated.R +#!/usr/bin/env Rscript + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(dplyr) + library(tidyr) +}) + +# ----------------------- +# logging helpers +# ----------------------- +.ts <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +.p <- function(...) cat(.ts(), paste0(...), "\n") + +# ----------------------- +# batch inputs (Synapse) +# ----------------------- +B1_SYN <- "syn64608366" # salmon.merged.gene_tpm_corrected_id.csv +B2_SYN <- "syn64608370" # salmon.merged.gene_tpm.tsv_corrected_ID.csv +B3_SYN <- "syn65887795" # salmon.merged.gene_tpm.tsv +B4_SYN <- "syn68898091" # salmon.merged.gene_tpm.tsv + +# ----------------------- +# helpers +# ----------------------- +study_label <- function(type) { + dplyr::case_when( + type == "patient derived xenograft" ~ "MPNST PDX", + type == "tumor" ~ "MPNST Tumor", + type == "3D-MEDS" ~ "MPNST PDX MT", + TRUE ~ "MPNST" + ) +} + +is_treated_microtissue <- function(other_id, other_names) { + oid <- as.character(other_id) + onm <- as.character(other_names) + grepl("treated_microtissue$", oid) && !is.na(onm) && nzchar(trimws(onm)) +} + +split_rep_ids <- function(x) { + if (is.na(x) || !nzchar(trimws(x))) return(character(0)) + v <- unlist(strsplit(x, ",")) + v <- trimws(v) + v <- v[nzchar(v)] + unique(v) +} + +guess_batch_from_rep <- function(rep_id) { + r <- as.character(rep_id) + if (grepl("^b1_", r)) return("b1") + if (grepl("^b2_", r)) return("b2") + if (grepl("^B3-", r) || grepl("^B3_", r)) return("b3") + if (grepl("^B4-", r) || grepl("^B4\\.", r)) return("b4") + return(NA_character_) +} + +rep_to_colname <- function(rep_id, batch) { + r <- as.character(rep_id) + if (batch %in% c("b1", "b2")) return(r) + if (batch == "b3") return(gsub("-", "_", r, fixed = FALSE)) # B3-10 -> B3_10 + if (batch == "b4") return(gsub("B4-", "B4.", r, fixed = TRUE)) # B4-10 -> B4.10 + return(r) +} + +read_tpm_matrix <- function(syn_id) { + .p("[SYN] synGet(", syn_id, ") ...") + pth <- synGet(syn_id)$path + .p("[READ] ", syn_id, " -> ", pth) + dt <- fread(pth) + stopifnot(all(c("gene_id", "gene_name") %in% names(dt))) + dt +} + +# returns a data.table with columns: +# gene_id, transcriptomics, improve_sample_id, source, study +extract_treated_from_batch <- function(mat, treated_tbl, batch_key) { + if (nrow(treated_tbl) == 0) return(data.table()) + + tmp <- copy(treated_tbl) + tmp[, rep_ids := lapply(other_names, split_rep_ids)] + tmp[, rep_batch := vapply(rep_ids, function(v) { + if (length(v) == 0) return(NA_character_) + guess_batch_from_rep(v[1]) + }, character(1))] + + tmp <- tmp[rep_batch == batch_key] + if (nrow(tmp) == 0) return(data.table()) + + tmp[, rep_cols := lapply(rep_ids, function(v) rep_to_colname(v, batch_key))] + + .p("[", batch_key, "] treated samples in this batch: ", nrow(tmp)) + + out_list <- vector("list", nrow(tmp)) + + for (i in seq_len(nrow(tmp))) { + samp_id <- tmp$improve_sample_id[i] + samp_model <- tmp$model_type[i] + cols <- tmp$rep_cols[[i]] + cols <- cols[cols %in% names(mat)] + + if (length(cols) == 0) { + .p("[", batch_key, "][SKIP] improve_sample_id=", samp_id, + " -> no replicate columns found in matrix") + next + } + + expr <- rowMeans(as.matrix(mat[, ..cols]), na.rm = TRUE) + + out_list[[i]] <- data.table( + gene_id = mat$gene_id, + transcriptomics = expr, + improve_sample_id = samp_id, + source = "NF Data Portal", + study = study_label(samp_model) + ) + } + + rbindlist(out_list, use.names = TRUE, fill = TRUE) +} + +# ----------------------- +# MAIN +# ----------------------- +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 3) { + stop("Usage: Rscript 01b_combined_omics_treated.R ", call. = FALSE) +} +PAT <- args[1] +samples <- args[2] +genes <- args[3] + +.p("============================================================") +.p("=== 01b_combined_omics_treated.R ===") +.p("Timestamp: ", .ts()) +.p("Args: ", paste(args, collapse = " | ")) +.p("============================================================") + +.p("[SYN] synLogin() ...") +synLogin(authToken = PAT) +.p("[SYN] synLogin() OK") + +.p("[READ] samples: ", samples) +samples_df <- fread(samples) + +treated <- samples_df[ + model_type == "3D-MEDS" & + vapply(seq_len(nrow(samples_df)), function(i) { + is_treated_microtissue(samples_df$other_id[i], samples_df$other_names[i]) + }, logical(1)), + .(other_names, improve_sample_id, model_type) +] + +.p("[SAMPLES] treated_microtissue rows: ", nrow(treated)) +if (nrow(treated) == 0) stop("No treated microtissue samples found in samples.csv", call. = FALSE) + +.p("[READ] genes: ", genes) +genes_df <- fread(genes) +stopifnot(all(c("other_id", "entrez_id") %in% names(genes_df))) + +# matrices +m1 <- read_tpm_matrix(B1_SYN) +m2 <- read_tpm_matrix(B2_SYN) +m3 <- read_tpm_matrix(B3_SYN) +m4 <- read_tpm_matrix(B4_SYN) + +# extract +t1 <- extract_treated_from_batch(m1, treated, "b1") +t2 <- extract_treated_from_batch(m2, treated, "b2") +t3 <- extract_treated_from_batch(m3, treated, "b3") +t4 <- extract_treated_from_batch(m4, treated, "b4") + +treated_long <- rbindlist(list(t1, t2, t3, t4), use.names = TRUE, fill = TRUE) +.p("[OUT] treated_long rows: ", nrow(treated_long)) +if (nrow(treated_long) == 0) stop("No treated data extracted (check replicate IDs vs column names).", call. = FALSE) + +# ENSEMBL gene_id has version -> strip to match genes_df$other_id +treated_long[, other_id := tstrsplit(gsub("\\s+", "", gene_id), "\\.", keep = 1)] + +treated_final <- treated_long %>% + left_join(genes_df, by = "other_id") %>% + dplyr::select(entrez_id, transcriptomics, improve_sample_id, source, study) %>% + filter(!is.na(entrez_id), transcriptomics != 0) %>% + distinct() + +# enforce strict schema & order +treated_final <- as.data.table(treated_final)[, .(entrez_id, transcriptomics, improve_sample_id, source, study)] + +treated_out_path <- file.path("/tmp", "mpnst_transcriptomics_treated_microtissue.csv") +.p("[WRITE] treated-only transcriptomics -> ", treated_out_path) +fwrite(treated_final, treated_out_path) +.p("[WRITE] OK") + +# optional append into main transcriptomics (same strict schema) +main_path <- file.path("/tmp", "mpnst_transcriptomics.csv") +if (file.exists(main_path)) { + .p("[APPEND] Found existing main transcriptomics: ", main_path) + main_df <- fread(main_path) + main_df <- main_df[, .(entrez_id, transcriptomics, improve_sample_id, source, study)] # enforce schema + + combined <- rbindlist(list(main_df, treated_final), use.names = TRUE, fill = TRUE) + + # de-dup conservatively on (entrez_id, improve_sample_id, source, study, transcriptomics) + combined[, dedup_key := paste(entrez_id, improve_sample_id, source, study, transcriptomics, sep = "||")] + combined <- combined[!duplicated(dedup_key)] + combined[, dedup_key := NULL] + + .p("[WRITE] overwriting main transcriptomics with appended treated rows -> ", main_path) + fwrite(combined, main_path) + .p("[WRITE] OK") +} else { + .p("[APPEND] No existing /tmp/mpnst_transcriptomics.csv found. Not appending.") +} + +.p("============================================================") +.p("[DONE] Wrote treated transcriptomics with strict schema.") +.p("============================================================") diff --git a/coderbuild/mpnst/02_get_drug_data.R b/coderbuild/mpnst/02_get_drug_data.R index deb0daa2..b0727542 100644 --- a/coderbuild/mpnst/02_get_drug_data.R +++ b/coderbuild/mpnst/02_get_drug_data.R @@ -1,6 +1,7 @@ +# 02_get_drug_data.R #!/usr/bin/env Rscript -# Combined Drug List Extraction for MPNST & MPNST‑PDX +# Combined Drug List Extraction for MPNST & MPNST-PDX library(data.table) library(dplyr) @@ -8,6 +9,40 @@ library(stringr) library(synapser) library(reticulate) +# ---------------------------- +# DEBUG helper +# ---------------------------- +debug_file_summary <- function(path, label) { + message("\n========== DEBUG: ", label, " ==========") + message("Path: ", path) + if (is.null(path) || is.na(path) || !file.exists(path)) { + message("File missing.") + return(invisible(NULL)) + } + + df <- tryCatch(fread(path, sep="\t", header=TRUE), error=function(e) { + message("fread failed: ", conditionMessage(e)) + return(NULL) + }) + if (is.null(df)) return(invisible(NULL)) + + message("Rows: ", nrow(df), " Cols: ", ncol(df)) + message("Columns: ", paste(names(df), collapse=", ")) + + if ("chem_name" %in% names(df)) { + cn <- unique(tolower(trimws(as.character(df$chem_name)))) + message("Unique chem_name (first 25): ", paste(head(cn, 25), collapse=", ")) + message("Has chem_name == 'verteporfin'? ", any(cn == "verteporfin", na.rm = TRUE)) + message("Has chem_name == 'wu713d62n9'? ", any(cn == "wu713d62n9", na.rm = TRUE)) + } + if ("pubchem_id" %in% names(df)) { + pc <- unique(trimws(as.character(df$pubchem_id))) + message("Unique pubchem_id (first 10): ", paste(head(pc, 10), collapse=", ")) + } + + invisible(df) +} + # 0) Args & login args <- commandArgs(trailingOnly = TRUE) if (length(args) < 1) { @@ -25,38 +60,54 @@ synLogin(authToken = token) manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% rename(common_name = Sample) -# 2) PDX‑sourced drugs via annotations -pdx_df <- manifest %>% - select(common_name, PDX_Drug_Data) %>% - distinct() %>% - filter(!is.na(PDX_Drug_Data)) +# 2) PDX-sourced drugs via Synapse IDs stored in the manifest drug column. +# Column was renamed PDX_Drug_Data → PDXDrugData in 2025; values may now be +# file names rather than Synapse IDs, so filter to syn* entries only. +pdx_col <- if ("PDXDrugData" %in% names(manifest)) "PDXDrugData" else + if ("PDX_Drug_Data" %in% names(manifest)) "PDX_Drug_Data" else NULL +pdx_drugs <- character(0) +if (!is.null(pdx_col)) { + pdx_df <- manifest %>% + select(common_name, all_of(pdx_col)) %>% + rename(pdx_data = all_of(pdx_col)) %>% + distinct() %>% + filter(!is.na(pdx_data)) -pdx_ids <- unique(unlist(strsplit(pdx_df$PDX_Drug_Data, ","))) -pdx_ids <- pdx_ids[ pdx_ids != "" & !is.na(pdx_ids) & pdx_ids != "NA" ] - -get_pdx_drugs <- function(synid) { - # Query the metadata table for this file's experimentalCondition - q <- sprintf( - "select experimentalCondition from syn21993642 where id='%s'", - synid - ) - df <- synTableQuery(q)$asDataFrame() - if (nrow(df)==0) return(character(0)) - # Split on semicolon, lowercase and drop empties - conds <- unlist(strsplit(df$experimentalCondition, ";")) - tolower(conds[conds!=""]) -} + pdx_ids <- unique(unlist(lapply(pdx_df$pdx_data, function(x) { + strsplit(paste(trimws(unlist(x)), collapse=","), ",")[[1]] + }))) + pdx_ids <- pdx_ids[ pdx_ids != "" & !is.na(pdx_ids) & pdx_ids != "NA" ] + pdx_ids <- pdx_ids[ grepl("^syn[0-9]+$", trimws(pdx_ids), ignore.case = TRUE) ] -pdx_drugs <- unique(unlist(lapply(pdx_ids, get_pdx_drugs))) -pdx_drugs <- setdiff(pdx_drugs, "control") + get_pdx_drugs <- function(synid) { + q <- sprintf( + "select experimentalCondition from syn21993642 where id='%s'", + synid + ) + df <- synTableQuery(q)$asDataFrame() + if (nrow(df)==0) return(character(0)) + conds <- unlist(strsplit(as.character(df$experimentalCondition), ";")) + tolower(conds[conds!=""]) + } + if (length(pdx_ids) > 0) { + pdx_drugs <- unique(unlist(lapply(pdx_ids, get_pdx_drugs))) + pdx_drugs <- setdiff(pdx_drugs, "control") + } else { + message("No Synapse IDs found in ", pdx_col, "; skipping PDX Synapse drug extraction.") + } +} else { + message("No PDX drug column found in manifest; skipping PDX drug extraction.") +} -# 3) MicroTissue‑sourced drugs via table "children" +# 3) MicroTissue-sourced drugs via table "children" mts_df <- manifest %>% select(common_name, MicroTissueDrugFolder) %>% filter(!is.na(MicroTissueDrugFolder)) -mts_ids <- unique(unlist(strsplit(mts_df$MicroTissueDrugFolder, ","))) +mts_ids <- unique(unlist(lapply(mts_df$MicroTissueDrugFolder, function(x) { + strsplit(paste(trimws(unlist(x)), collapse=","), ",")[[1]] +}))) mts_ids <- mts_ids[mts_ids != "" & !is.na(mts_ids) & mts_ids != "NA"] get_mts_drugs <- function(parentId) { @@ -69,17 +120,93 @@ get_mts_drugs <- function(parentId) { mts_drugs <- unique(unlist(lapply(mts_ids, get_mts_drugs))) +# 3.5) IMPROVE-sourced drugs via improve_sample_id (syn69801348) +improve_sample_drugs <- character(0) +improve_path <- tryCatch(synGet("syn69801348")$path, error=function(e) NA) +if (!is.na(improve_path) && file.exists(improve_path)) { + improve_tab <- fread(improve_path, sep="\t", header=TRUE, fill=TRUE) + if ("improve_sample_id" %in% names(improve_tab)) { + ids <- improve_tab$improve_sample_id %>% as.character() %>% unique() + ids <- ids[!is.na(ids) & ids != ""] + drug_part <- sub("^.*_", "", ids) + drug_tokens <- unlist(strsplit(drug_part, "\\+")) + improve_sample_drugs <- unique(tolower(trimws(drug_tokens))) + improve_sample_drugs <- improve_sample_drugs[improve_sample_drugs != ""] + } +} + +# 3.6) Add drugs from treatment lookup list (syn64608501) Full_name column +lookup_drugs <- character(0) +lookup_path <- tryCatch(synGet("syn64608501")$path, error=function(e) NA) +if (!is.na(lookup_path) && file.exists(lookup_path)) { + lookup_dt <- tryCatch( + fread(lookup_path, sep = "\t", header = TRUE, fill = TRUE), + error = function(e) NULL + ) + + if (is.null(lookup_dt)) { + lookup_dt <- tryCatch( + as.data.table(readxl::read_excel(lookup_path)), + error = function(e) NULL + ) + } + + if (!is.null(lookup_dt)) { + full_col <- NULL + if ("Full_name" %in% names(lookup_dt)) full_col <- "Full_name" + if (is.null(full_col) && "Full Name" %in% names(lookup_dt)) full_col <- "Full Name" + + if (!is.null(full_col)) { + lookup_drugs <- unique(tolower(trimws(as.character(lookup_dt[[full_col]])))) + lookup_drugs <- lookup_drugs[!is.na(lookup_drugs) & nzchar(lookup_drugs)] + lookup_drugs <- setdiff(lookup_drugs, c("control", "dimethyl_sulfoxide", "dimethyl sulfoxide", "dmso")) + } else { + message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") + message("WARNING: Could not find Full_name column in lookup file. Columns: ", + paste(names(lookup_dt), collapse = ", ")) + } + } else { + message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") + message("WARNING: Could not read lookup file at: ", lookup_path) + } +} else { + message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") + message("WARNING: lookup file missing/unavailable for syn64608501") +} + +message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") +message("lookup_path: ", ifelse(is.na(lookup_path), "", lookup_path)) +message("lookup_drugs count: ", length(lookup_drugs)) +if (length(lookup_drugs) > 0) { + message("First 40 lookup_drugs: ", paste(head(lookup_drugs, 40), collapse = ", ")) +} + # 4) Combine and fix bad names -all_drugs <- unique(c(pdx_drugs, mts_drugs)) +all_drugs <- unique(c(pdx_drugs, mts_drugs, improve_sample_drugs, lookup_drugs)) all_drugs[all_drugs == "pd901"] <- "pd-0325901" -# message("Combined drug list: ", paste(all_drugs, collapse=", ")) +all_drugs[all_drugs == "rmc4630"] <- "rmc-4630" + +# IMPORTANT: drop NA/empty +all_drugs <- all_drugs[!is.na(all_drugs) & nzchar(all_drugs)] + +# ---- NEW: replace verteporfin with WU713D62N9 so we can do ONE name-based call +# (keep exact casing you provided) +if (any(all_drugs == "verteporfin", na.rm = TRUE)) { + all_drugs[all_drugs == "verteporfin"] <- "WU713D62N9" +} -# 5) Read old‑drug files or initialize empty +message("\n========== DEBUG: DRUG LISTS ==========") +message("all_drugs count: ", length(all_drugs)) +message("Contains 'verteporfin' in all_drugs? ", any(all_drugs == "verteporfin", na.rm=TRUE)) +message("Contains 'WU713D62N9' in all_drugs? ", any(all_drugs == "WU713D62N9", na.rm=TRUE)) +message("First 40 all_drugs: ", paste(head(all_drugs, 40), collapse=", ")) + +# 5) Read old-drug files or initialize empty if (!is.na(olddrugfiles)) { paths <- strsplit(olddrugfiles, ",")[[1]] %>% trimws() old_list <- lapply(paths, function(f) { if (file.exists(f)) fread(f, sep="\t", header=TRUE) else { - warning("Missing old‑drug file: ", f) + warning("Missing old-drug file: ", f) NULL } }) @@ -101,36 +228,95 @@ if (!is.na(olddrugfiles)) { pubchem_id=character(), canSMILES=character(), InChIKey=character(), formula=character(), weight=numeric() ) - message("No old‑drug files provided; starting fresh") + message("No old-drug files provided; starting fresh") } # 6) Write placeholder fwrite(olddrugs, newdrugfile, sep="\t", quote=FALSE) message("Wrote placeholder to ", newdrugfile) -# 7) Augment via Python +# 7) Augment via Python (single call) ignore_file <- "/tmp/combined_drugs_ignore_chems.txt" -use_python("/opt/venv/bin/python3", required=TRUE) -# use_python("/Users/jaco059/miniconda3/bin/python3", required=TRUE) -# source_python("build/utils/pubchem_retrieval.py") +message("\n========== DEBUG: IGNORE FILE ==========") +message("ignore_file: ", ignore_file) +message("ignore_file exists? ", file.exists(ignore_file)) +if (file.exists(ignore_file)) { + ign <- tolower(trimws(readLines(ignore_file, warn=FALSE))) + message("ignore_file lines (first 50): ", paste(head(ign, 50), collapse=" | ")) + message("ignore contains 'verteporfin'? ", any(ign == "verteporfin")) + message("ignore contains 'wu713d62n9'? ", any(ign == "wu713d62n9")) +} + +use_python("/opt/venv/bin/python3", required=TRUE) source_python("pubchem_retrieval.py") -# prepare prev_drug_filepaths argument for python helper (NULL if none) prev_paths <- if (!is.na(olddrugfiles)) olddrugfiles else NULL -update_dataframe_and_write_tsv( - unique_names = all_drugs, - output_filename = newdrugfile, - ignore_chems = ignore_file, - batch_size = as.integer(50), - isname = TRUE, - prev_drug_filepaths = prev_paths, - restrict_to_raw_names = all_drugs -) - - -# 8) Final filter & save -tab <- fread(newdrugfile, sep="\t", header=TRUE) + +# reticulate-safe list conversion +name_drugs_py <- as.list(as.character(all_drugs)) + +if (length(all_drugs) > 0) { + message("\n========== DEBUG: RUN SINGLE NAME-BASED PYTHON CALL ==========") + message("Calling update_dataframe_and_write_tsv with isname=TRUE for ", length(all_drugs), " names.") + message("First 30 all_drugs: ", paste(head(all_drugs, 30), collapse=", ")) + + update_dataframe_and_write_tsv( + unique_names = name_drugs_py, + output_filename = newdrugfile, + ignore_chems = ignore_file, + batch_size = as.integer(50), + isname = TRUE, + prev_drug_filepaths = prev_paths, + restrict_to_raw_names = name_drugs_py + ) + + debug_file_summary(newdrugfile, "AFTER SINGLE NAME-BASED PYTHON CALL (newdrugfile)") +} else { + message("\n========== DEBUG: SKIP PYTHON CALL (all_drugs empty) ==========") +} + +# 8) Final unique & save (no CID merge) +message("\n========== DEBUG: MAIN OUTPUT BEFORE FINAL UNIQUE ==========") +debug_file_summary(newdrugfile, "MAIN OUTPUT BEFORE FINAL UNIQUE (newdrugfile)") + +tab <- fread(newdrugfile, sep="\t", header=TRUE) final_tab <- unique(tab) + +message("\n========== DEBUG: FINAL CHECK ==========") +message("Final rows after unique(): ", nrow(final_tab)) +if ("chem_name" %in% names(final_tab)) { + cnf <- unique(tolower(trimws(as.character(final_tab$chem_name)))) + message("FINAL has verteporfin? ", any(cnf == "verteporfin", na.rm=TRUE)) + message("FINAL has wu713d62n9? ", any(cnf == "wu713d62n9", na.rm=TRUE)) +} + +# 9) Ensure bare "verteporfin" exists as a chem_name row. +# PubChem synonyms often return only qualified variants like +# "verteporfin [usan:usp:inn:ban]" but not the bare name, which +# causes exact-join failures downstream in 03_get_experiments. +if ("chem_name" %in% names(final_tab)) { + cn_lower <- tolower(trimws(as.character(final_tab$chem_name))) + has_bare <- any(cn_lower == "verteporfin", na.rm = TRUE) + + if (!has_bare) { + # Look for any row whose chem_name contains "verteporfin" or "verteporphin" + match_idx <- which(grepl("verteporfin|verteporphin", cn_lower)) + if (length(match_idx) > 0) { + donor_row <- final_tab[match_idx[1], ] + donor_row[["chem_name"]] <- "verteporfin" + final_tab <- rbind(donor_row, final_tab) + message("Inserted bare 'verteporfin' alias row (from row with chem_name='", + as.character(final_tab$chem_name[match_idx[1] + 1]), # +1 because we prepended + "', improve_drug_id=", donor_row$improve_drug_id, ")") + } else { + message("WARNING: no verteporfin/verteporphin variant found in drug file; cannot insert bare alias.") + } + } else { + message("Bare 'verteporfin' already present in drug file; no insertion needed.") + } +} + + fwrite(final_tab, newdrugfile, sep="\t", quote=FALSE) -message("Wrote full synonyms list to ", newdrugfile) \ No newline at end of file +message("Wrote full synonyms list to ", newdrugfile) diff --git a/coderbuild/mpnst/03_get_experiments.R b/coderbuild/mpnst/03_get_experiments.R index a430cae8..c2921970 100644 --- a/coderbuild/mpnst/03_get_experiments.R +++ b/coderbuild/mpnst/03_get_experiments.R @@ -1,282 +1,473 @@ -library(data.table) -library(synapser) -library(dplyr) -library(stringr) -library(readr) -library(readxl) -library(tidyr) - -# Check that correct number of arguments are present -args <- commandArgs(trailingOnly = TRUE) -if (length(args) != 4) { - stop("Usage: Rscript 03_get_experiments.R ", call. = FALSE) -} -PAT <- args[1] -samples <- args[2] -drugfile <- args[3] -out_prefix <- args[4] - -synLogin(authToken = PAT) - -# Read in sampes file -samples_df <- fread(samples) %>% - select(improve_sample_id, common_name, model_type) %>% - distinct() - -pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") -mt_samps <- filter(samples_df, model_type == "xenograft derived organoid") - -# Get manifest table from Synapse -manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% - rename(common_name = Sample) %>% - as.data.table() - -# Helper Function to extract date and hour from experiment ID -extract_date_hour <- function(experiment_id) { - pattern <- "(\\d{6})_?(\\d{2,3})?" - m <- str_match(experiment_id, pattern) - date <- m[,2]; hour <- m[,3] - date[is.na(date)] <- NA - hour[is.na(hour)] <- 48 - list(date = date, hour = hour) -} - -# ──────────────────────────────────────────────── -# MicroTissue Experiments -# ──────────────────────────────────────────────── - -getDrugDataByParent <- function(parid, sampleId) { - q <- sprintf( - "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", - parid - ) - qtab <- synTableQuery(q)$asDataFrame() %>% - filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% - select(id, name, experimentalCondition) - do.call(rbind, lapply(qtab$id, function(x) { - info <- filter(qtab, id == x) - d <- extract_date_hour(info$name) - fread(synGet(x)$path) %>% - filter(response_type == "percent viability") %>% - transmute( - improve_sample_id = sampleId, - DOSE = (10^dosage) * 1e6, - GROWTH = response, - source = "NF Data Portal", - chem_name = compound_name, - study = paste0("MT ", d$date, " exp"), - time = d$hour - ) - })) -} - -# Create map of MicroTissue Drug Folders -mts_map <- manifest %>% - select(common_name, MicroTissueDrugFolder) %>% - inner_join(mt_samps, by = "common_name") %>% - separate_rows(MicroTissueDrugFolder, sep = ",") %>% - # keep exactly what old script did: drop only "NA" and actual NA - filter( - !is.na(MicroTissueDrugFolder), - MicroTissueDrugFolder != "NA" - ) %>% - select( - improve_sample_id, - folder = MicroTissueDrugFolder - ) - -# Fetch all MicroTissue drug response data -mt_data <- do.call(rbind, lapply(seq_len(nrow(mts_map)), function(i) { - sample_id <- mts_map$improve_sample_id[i] - folder <- mts_map$folder[i] - getDrugDataByParent(folder, sample_id) -})) - -drug_map <- fread(drugfile) %>% - select(improve_drug_id, chem_name) %>% - distinct() - -# Clean up drug names and join with drug_map -mt_curve <- mt_data %>% - mutate( - chem_name = tolower(chem_name), - chem_name = ifelse(chem_name == "pd901", "pd-0325901", chem_name) - ) %>% - left_join(drug_map, by = "chem_name") %>% - filter(!is.na(improve_drug_id)) %>% - transmute( - source = source, - improve_sample_id = improve_sample_id, - Drug = improve_drug_id, - study = study, - time = time, - time_unit = "hours", - DOSE = DOSE, - GROWTH = GROWTH - ) - -# Run curve fitting, Write MicroTissue curve data -fwrite(mt_curve, file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")), sep = "\t") - -message("Wrote MT curve data") - -# Write MT experiments file -system(sprintf( - "/opt/venv/bin/python fit_curve.py --input %s --output %s", - paste0("/tmp/", out_prefix, "_mt_curve_data.tsv"), - paste0("/tmp/", out_prefix, "_mt_experiments") -)) -file.rename( - paste0("/tmp/", out_prefix, "_mt_experiments.0"), - paste0("/tmp/", out_prefix, "_mt_experiments.tsv") -) -message("Wrote MT experiments") - -# ──────────────────────────────────────────────── -# PDX Experiments -# ──────────────────────────────────────────────── - -# Create a map of PDX Drug Data -# This will be used to fetch the drug data for each PDX sample -pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { - row <- manifest[i, ] - samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] - if (nrow(samp)==0 || is.na(row$PDX_Drug_Data) || row$PDX_Drug_Data %in% c("", "NA")) - return(NULL) - ids <- strsplit(row$PDX_Drug_Data, ",")[[1]] - ids <- trimws(ids[ids!=""]) - data.frame( - improve_sample_id = samp$improve_sample_id, - child_id = ids, - stringsAsFactors = FALSE - ) -})) - -# Create a dataframe of PDX metadata -pdx_meta <- do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { - sid <- pdx_map$improve_sample_id[i] - cid <- pdx_map$child_id[i] - pid <- synGet(cid)$parentId - if (is.null(pid) || pid=="") stop("no parentId for ", cid) - data.frame( - improve_sample_id = sid, - child_id = cid, - parentId = pid, - stringsAsFactors = FALSE - ) -})) - -all_pdx <- do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { - m <- pdx_meta[i, ] - pth <- synGet(m$child_id)$path - raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) - - # detect second‐drug column - sec_opts <- c("compound 2_name", "compound_2_name") - drug2_col <- intersect(sec_opts, names(raw))[1] - compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ - - df <- data.frame( - child_id = m$child_id, - specimen_id = raw$specimen_id, - compound_name = raw$compound_name, - compound_2_name = compound2, - experimental_time_point = raw$experimental_time_point, - experimental_time_point_unit = raw$experimental_time_point_unit, - assay_value = raw$assay_value, - stringsAsFactors = FALSE - ) - - df <- within(df, { - drug1 <- tolower(trimws(compound_name)) - drug2 <- tolower(trimws(compound_2_name)) - treatment <- ifelse( - is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), - "control", - ifelse(!is.na(drug2) & drug2 != "", - paste(drug1, drug2, sep = "+"), - drug1 - ) - ) - time <- experimental_time_point - time_unit <- experimental_time_point_unit - volume <- assay_value - }) - - df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] -})) - -# join on parentId and sample -pdx_data <- merge(all_pdx, pdx_meta, by="child_id") - -pdx_data <- subset(pdx_data, duplicated(child_id) | TRUE) -pdx_data <- within(pdx_data, { - experiment <- parentId - model_id <- improve_sample_id -}) - -# Filter out experiments missing a control -has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) -no_ctl_exps <- names(has_ctl)[!has_ctl] -pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] - -# Reorder final columns -pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", - "time","time_unit","volume","model_id")] - -# Correct doxorubinsin typo across all data -pdx_data$treatment <- gsub("doxorubinsin", - "doxorubicin", - pdx_data$treatment, - ignore.case = TRUE) - -# Drop any remaining NA rows -pdx_data <- na.omit(pdx_data) - -# write & fit -fwrite(pdx_data, file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")), sep = "\t") - - -message("Wrote PDX curve data") - -system(sprintf( - "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", - paste0("/tmp/", out_prefix, "_pdx_curve_data.tsv"), - drugfile, - paste0("/tmp/", out_prefix, "_pdx") -)) - - - -message("Wrote PDX experiments to ", "/tmp/", out_prefix, "_pdx_experiments.tsv and combinations") - - -# ──────────────────────────────────────────────── -# Combine all Experiments -# ──────────────────────────────────────────────── - -# Read MicroTissue experiments -mt_exp <- fread(paste0("/tmp/", out_prefix, "_mt_experiments.tsv")) %>% - mutate( - dose_response_value = as.character(dose_response_value) - ) - -# Read PDX experiments -pdx_exp <- fread(paste0("/tmp/", out_prefix, "_pdx_experiments.tsv")) %>% - mutate( - dose_response_value = as.character(dose_response_value) - ) - -# Join experiments into one. -all_exp <- bind_rows(mt_exp, pdx_exp) - -# Write out Experiments -fwrite(all_exp, paste0("/tmp/", out_prefix, "_experiments.tsv"), sep = "\t") -message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") - - -# Rename the Drug Combination data file to fit schema naming -file.rename( - paste0("/tmp/", out_prefix, "_pdx_combinations.tsv"), - paste0("/tmp/", out_prefix, "_combinations.tsv") -) \ No newline at end of file +# This is the original version of the script before adding treated-combo support. +# Not actually called in this branch, but kept for reference. + +# # 03_get_experiments.R +# #!/usr/bin/env Rscript + +# library(data.table) +# library(synapser) +# library(dplyr) +# library(stringr) +# library(readr) +# library(readxl) +# library(tidyr) + +# # ============================================================ +# # 03_get_experiments.R +# # - MT (original): unchanged logic, but forces time -> integer +# # - MT (treated): robust parsing + sample duplication mapping +# # - PDX: unchanged +# # ============================================================ + +# # ----------------------- +# # args +# # ----------------------- +# args <- commandArgs(trailingOnly = TRUE) +# if (length(args) != 4) { +# stop("Usage: Rscript 03_get_experiments.R ", call. = FALSE) +# } +# PAT <- args[1] +# samples <- args[2] +# drugfile <- args[3] +# out_prefix <- args[4] + +# synLogin(authToken = PAT) + +# # ----------------------- +# # helpers +# # ----------------------- +# norm_str <- function(x) tolower(trimws(as.character(x))) + +# # normalize drug strings for joining +# norm_drug <- function(x) { +# x <- norm_str(x) +# x <- ifelse(x == "pd901", "pd-0325901", x) +# x +# } + +# # make column names consistent across files +# std_names <- function(nms) { +# nms <- tolower(nms) +# nms <- gsub("[^a-z0-9]+", "_", nms) +# nms <- gsub("_+", "_", nms) +# nms <- gsub("^_|_$", "", nms) +# nms +# } + +# # choose first existing column from candidates +# pick_col <- function(df, candidates) { +# candidates <- std_names(candidates) +# nm <- names(df) +# hit <- intersect(candidates, nm) +# if (length(hit) == 0) return(NA_character_) +# hit[1] +# } + +# # safe annotation get +# get_ann <- function(synid) { +# a <- tryCatch(synGetAnnotations(synid)$annotations, error = function(e) NULL) +# if (is.null(a)) return(list()) +# a +# } + +# # ----------------------- +# # load inputs +# # ----------------------- +# samples_all <- fread(samples) %>% as.data.frame() + +# samples_df <- samples_all %>% +# dplyr::select(improve_sample_id, common_name, model_type) %>% +# distinct() + +# pdx_samps <- dplyr::filter(samples_df, model_type == "patient derived xenograft") +# mt_samps <- dplyr::filter(samples_df, model_type == "xenograft derived organoid") + +# drug_map <- fread(drugfile) %>% +# dplyr::select(improve_drug_id, chem_name) %>% +# distinct() %>% +# mutate(chem_name = norm_drug(chem_name)) + +# # ----------------------- +# # manifest (unchanged) +# # ----------------------- +# manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% +# rename(common_name = Sample) %>% +# as.data.table() + +# extract_date_hour <- function(experiment_id) { +# pattern <- "(\\d{6})_?(\\d{2,3})?" +# m <- str_match(experiment_id, pattern) +# date <- m[,2]; hour <- m[,3] +# date[is.na(date)] <- NA +# hour[is.na(hour)] <- 48 +# # force integer hour +# hour <- suppressWarnings(as.integer(hour)) +# hour[is.na(hour)] <- 48L +# list(date = date, hour = hour) +# } + +# # ============================================================ +# # 1) MT ORIGINAL (same logic; ensure time integer) +# # ============================================================ + +# getDrugDataByParent <- function(parid, sampleId) { +# q <- sprintf( +# "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", +# parid +# ) +# qtab <- synTableQuery(q)$asDataFrame() %>% +# filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% +# select(id, name, experimentalCondition) + +# do.call(rbind, lapply(qtab$id, function(x) { +# info <- filter(qtab, id == x) +# d <- extract_date_hour(info$name) + +# fread(synGet(x)$path) %>% +# filter(response_type == "percent viability") %>% +# transmute( +# improve_sample_id = sampleId, +# DOSE = (10^dosage) * 1e6, +# GROWTH = response, +# source = "NF Data Portal", +# chem_name = compound_name, +# study = paste0("MT ", d$date, " exp"), +# time = as.integer(d$hour) +# ) +# })) +# } + +# mts_map <- manifest %>% +# select(common_name, MicroTissueDrugFolder) %>% +# inner_join(mt_samps, by = "common_name") %>% +# separate_rows(MicroTissueDrugFolder, sep = ",") %>% +# filter(!is.na(MicroTissueDrugFolder), MicroTissueDrugFolder != "NA") %>% +# select(improve_sample_id, folder = MicroTissueDrugFolder) + +# mt_data <- do.call(rbind, lapply(seq_len(nrow(mts_map)), function(i) { +# sample_id <- mts_map$improve_sample_id[i] +# folder <- mts_map$folder[i] +# getDrugDataByParent(folder, sample_id) +# })) + +# mt_curve_orig <- mt_data %>% +# mutate( +# chem_name = norm_drug(chem_name), +# time = suppressWarnings(as.integer(time)) +# ) %>% +# left_join(drug_map, by = "chem_name") %>% +# filter(!is.na(improve_drug_id)) %>% +# transmute( +# source = source, +# improve_sample_id = improve_sample_id, +# Drug = improve_drug_id, +# study = study, +# time = as.integer(time), +# time_unit = "hours", +# DOSE = DOSE, +# GROWTH = GROWTH +# ) + +# message("Built MT original curve rows: ", nrow(mt_curve_orig)) + +# # ============================================================ +# # 2) MT TREATED (robust parsing + duplication mapping) +# # ============================================================ + +# # treated mapping from samples.csv: +# # extract drug_key from other_id using regex (no vectorized grepl patterns) +# treated_lookup <- samples_all %>% +# filter(!is.na(other_id), grepl("_treated_microtissue$", other_id)) %>% +# mutate( +# base_common_name = common_name, +# other_id_clean = as.character(other_id), +# # drug_key = part between first "_" and "_hr_treated_microtissue" +# drug_key = sub("^.*?_", "", other_id_clean), +# drug_key = sub("_\\d+hr_treated_microtissue$", "", drug_key), +# drug_key = norm_drug(drug_key) +# ) %>% +# select(base_common_name, drug_key, improve_sample_id) %>% +# distinct() + +# # generic fallback: xenograft derived organoid row for base_common_name (exclude treated rows) +# generic_xdo_lookup <- samples_all %>% +# filter( +# model_type == "xenograft derived organoid", +# !is.na(common_name), +# (is.na(other_id) | !grepl("_treated_microtissue$", other_id)) +# ) %>% +# select(base_common_name = common_name, generic_improve_sample_id = improve_sample_id) %>% +# distinct() + +# # enumerate candidate treated MT files from the same index table +# mt_index <- synTableQuery("select id,name,experimentalCondition,parentId from syn21993642")$asDataFrame() %>% +# filter(name != "synapse_storage_manifest.csv") %>% +# select(id, name, experimentalCondition) + +# # build treated curve rows for one file +# get_treated_curve_rows_one <- function(file_id, fallback_condition = NA_character_) { + +# ann <- get_ann(file_id) + +# # always map sample base name from individualID +# base_common_name <- NA_character_ +# if (!is.null(ann$individualID) && length(ann$individualID) > 0) { +# base_common_name <- as.character(ann$individualID[[1]]) +# } +# if (is.na(base_common_name) || base_common_name == "") return(NULL) + +# # drug for treated sample selection: experimentalCondition (annotation preferred) +# drug_raw <- NA_character_ +# if (!is.null(ann$experimentalCondition) && length(ann$experimentalCondition) > 0) { +# drug_raw <- as.character(ann$experimentalCondition[[1]]) +# } else if (!is.na(fallback_condition)) { +# drug_raw <- as.character(fallback_condition) +# } +# if (is.na(drug_raw) || drug_raw == "") return(NULL) + +# drug_key <- norm_drug(drug_raw) + +# # load file + standardize columns +# pth <- synGet(file_id)$path +# dt <- suppressWarnings(fread(pth)) +# if (nrow(dt) == 0) return(NULL) + +# setnames(dt, names(dt), std_names(names(dt))) + +# # pick key columns robustly +# col_resp_type <- pick_col(dt, c("response_type", "response type")) +# col_resp <- pick_col(dt, c("response", "value")) +# col_dose <- pick_col(dt, c("dosage", "dose", "log_m", "logm")) +# col_cmpd <- pick_col(dt, c("compound_name", "compound", "drug", "compoundname")) + +# if (is.na(col_resp_type) || is.na(col_resp) || is.na(col_dose)) return(NULL) + +# # percent viability only +# dt <- dt[get(col_resp_type) == "percent viability", ] +# if (nrow(dt) == 0) return(NULL) + +# # map improve_drug_id: +# # 1) annotation experimentalCondition +# # 2) fallback to file compound_name if present +# improve_drug_id <- NA_character_ + +# dm1 <- drug_map %>% filter(chem_name == drug_key) +# if (nrow(dm1) > 0) improve_drug_id <- dm1$improve_drug_id[1] + +# if (is.na(improve_drug_id) && !is.na(col_cmpd)) { +# chem2 <- norm_drug(dt[[col_cmpd]][1]) +# dm2 <- drug_map %>% filter(chem_name == chem2) +# if (nrow(dm2) > 0) improve_drug_id <- dm2$improve_drug_id[1] +# } + +# # if still unmapped drug, skip (can be relaxed later when you provide more mapping rules) +# if (is.na(improve_drug_id) || improve_drug_id == "") return(NULL) + +# # sample mapping: +# # - if treated match exists: DUPLICATE across all improve_sample_id matches +# # - else fallback to generic XDO sample for that individualID +# treated_matches <- treated_lookup %>% +# filter(base_common_name == !!base_common_name, drug_key == !!drug_key) + +# if (nrow(treated_matches) == 0) { +# gen <- generic_xdo_lookup %>% filter(base_common_name == !!base_common_name) +# if (nrow(gen) == 0) return(NULL) +# target_ids <- gen$generic_improve_sample_id +# } else { +# target_ids <- treated_matches$improve_sample_id +# } + +# # build output duplicated per target id +# out <- do.call(rbind, lapply(target_ids, function(sid) { +# data.frame( +# source = "NF Data Portal", +# improve_sample_id = sid, +# Drug = improve_drug_id, +# study = "MT treated", +# time = 48L, +# time_unit = "hours", +# DOSE = (10^(as.numeric(dt[[col_dose]]))) * 1e6, +# GROWTH = as.numeric(dt[[col_resp]]), +# stringsAsFactors = FALSE +# ) +# })) + +# out +# } + +# mt_curve_treated_list <- lapply(seq_len(nrow(mt_index)), function(i) { +# get_treated_curve_rows_one( +# file_id = mt_index$id[i], +# fallback_condition = mt_index$experimentalCondition[i] +# ) +# }) + +# mt_curve_treated <- do.call(rbind, mt_curve_treated_list) + +# if (is.null(mt_curve_treated) || nrow(mt_curve_treated) == 0) { +# mt_curve_treated <- data.frame( +# source = character(), +# improve_sample_id = integer(), +# Drug = character(), +# study = character(), +# time = integer(), +# time_unit = character(), +# DOSE = numeric(), +# GROWTH = numeric(), +# stringsAsFactors = FALSE +# ) +# } + +# mt_curve_treated <- mt_curve_treated %>% +# mutate(time = as.integer(time)) + +# message("Built MT treated curve rows: ", nrow(mt_curve_treated)) + +# # ============================================================ +# # 3) Combine MT and fit once +# # ============================================================ + +# # ensure time is integer on both sides (prevents your bind_rows crash) +# mt_curve_orig <- mt_curve_orig %>% mutate(time = as.integer(time)) +# mt_curve_treated <- mt_curve_treated %>% mutate(time = as.integer(time)) + +# mt_curve_all <- bind_rows(mt_curve_orig, mt_curve_treated) + +# fwrite(mt_curve_all, file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")), sep = "\t") +# message("Wrote MT curve data: /tmp/", out_prefix, "_mt_curve_data.tsv") + +# system(sprintf( +# "/opt/venv/bin/python fit_curve.py --input %s --output %s", +# paste0("/tmp/", out_prefix, "_mt_curve_data.tsv"), +# paste0("/tmp/", out_prefix, "_mt_experiments") +# )) + +# file.rename( +# paste0("/tmp/", out_prefix, "_mt_experiments.0"), +# paste0("/tmp/", out_prefix, "_mt_experiments.tsv") +# ) +# message("Wrote MT experiments") + +# # ============================================================ +# # 4) PDX (UNCHANGED) +# # ============================================================ + +# pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { +# row <- manifest[i, ] +# samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] +# if (nrow(samp)==0 || is.na(row$PDX_Drug_Data) || row$PDX_Drug_Data %in% c("", "NA")) +# return(NULL) +# ids <- strsplit(row$PDX_Drug_Data, ",")[[1]] +# ids <- trimws(ids[ids!=""]) +# data.frame( +# improve_sample_id = samp$improve_sample_id, +# child_id = ids, +# stringsAsFactors = FALSE +# ) +# })) + +# pdx_meta <- do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { +# sid <- pdx_map$improve_sample_id[i] +# cid <- pdx_map$child_id[i] +# pid <- synGet(cid)$parentId +# if (is.null(pid) || pid=="") stop("no parentId for ", cid) +# data.frame( +# improve_sample_id = sid, +# child_id = cid, +# parentId = pid, +# stringsAsFactors = FALSE +# ) +# })) + +# all_pdx <- do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { +# m <- pdx_meta[i, ] +# pth <- synGet(m$child_id)$path +# raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) + +# sec_opts <- c("compound 2_name", "compound_2_name") +# drug2_col <- intersect(sec_opts, names(raw))[1] +# compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ + +# df <- data.frame( +# child_id = m$child_id, +# specimen_id = raw$specimen_id, +# compound_name = raw$compound_name, +# compound_2_name = compound2, +# experimental_time_point = raw$experimental_time_point, +# experimental_time_point_unit = raw$experimental_time_point_unit, +# assay_value = raw$assay_value, +# stringsAsFactors = FALSE +# ) + +# df <- within(df, { +# drug1 <- tolower(trimws(compound_name)) +# drug2 <- tolower(trimws(compound_2_name)) +# treatment <- ifelse( +# is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), +# "control", +# ifelse(!is.na(drug2) & drug2 != "", +# paste(drug1, drug2, sep = "+"), +# drug1 +# ) +# ) +# time <- experimental_time_point +# time_unit <- experimental_time_point_unit +# volume <- assay_value +# }) + +# df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] +# })) + +# pdx_data <- merge(all_pdx, pdx_meta, by="child_id") + +# pdx_data <- subset(pdx_data, duplicated(child_id) | TRUE) +# pdx_data <- within(pdx_data, { +# experiment <- parentId +# model_id <- improve_sample_id +# }) + +# has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) +# no_ctl_exps <- names(has_ctl)[!has_ctl] +# pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] + +# pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", +# "time","time_unit","volume","model_id")] + +# pdx_data$treatment <- gsub("doxorubinsin", +# "doxorubicin", +# pdx_data$treatment, +# ignore.case = TRUE) + +# pdx_data <- na.omit(pdx_data) + +# fwrite(pdx_data, file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")), sep = "\t") +# message("Wrote PDX curve data") + +# system(sprintf( +# "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", +# paste0("/tmp/", out_prefix, "_pdx_curve_data.tsv"), +# drugfile, +# paste0("/tmp/", out_prefix, "_pdx") +# )) + +# message("Wrote PDX experiments to /tmp/", out_prefix, "_pdx_experiments.tsv and combinations") + +# # ============================================================ +# # 5) Combine all Experiments (unchanged) +# # ============================================================ + +# mt_exp <- fread(paste0("/tmp/", out_prefix, "_mt_experiments.tsv")) %>% +# mutate(dose_response_value = as.character(dose_response_value)) + +# pdx_exp <- fread(paste0("/tmp/", out_prefix, "_pdx_experiments.tsv")) %>% +# mutate(dose_response_value = as.character(dose_response_value)) + +# all_exp <- bind_rows(mt_exp, pdx_exp) + +# fwrite(all_exp, paste0("/tmp/", out_prefix, "_experiments.tsv"), sep = "\t") +# message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") + +# file.rename( +# paste0("/tmp/", out_prefix, "_pdx_combinations.tsv"), +# paste0("/tmp/", out_prefix, "_combinations.tsv") +# ) diff --git a/coderbuild/mpnst/03_get_experiments_orig.R b/coderbuild/mpnst/03_get_experiments_orig.R new file mode 100644 index 00000000..5aa42151 --- /dev/null +++ b/coderbuild/mpnst/03_get_experiments_orig.R @@ -0,0 +1,286 @@ +# This is an older version of the script. +# Not actually called in this branch, but kept for reference. + +# # 03_get_experiments_orig.R +# library(data.table) +# library(synapser) +# library(dplyr) +# library(stringr) +# library(readr) +# library(readxl) +# library(tidyr) + +# # Check that correct number of arguments are present +# args <- commandArgs(trailingOnly = TRUE) +# if (length(args) != 4) { +# stop("Usage: Rscript 03_get_experiments.R ", call. = FALSE) +# } +# PAT <- args[1] +# samples <- args[2] +# drugfile <- args[3] +# out_prefix <- args[4] + +# synLogin(authToken = PAT) + +# # Read in sampes file +# samples_df <- fread(samples) %>% +# select(improve_sample_id, common_name, model_type) %>% +# distinct() + +# pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") +# mt_samps <- filter(samples_df, model_type == "3D-MEDS") + +# # Get manifest table from Synapse +# manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% +# rename(common_name = Sample) %>% +# as.data.table() + +# # Helper Function to extract date and hour from experiment ID +# extract_date_hour <- function(experiment_id) { +# pattern <- "(\\d{6})_?(\\d{2,3})?" +# m <- str_match(experiment_id, pattern) +# date <- m[,2]; hour <- m[,3] +# date[is.na(date)] <- NA +# hour[is.na(hour)] <- 48 +# list(date = date, hour = hour) +# } + +# # ──────────────────────────────────────────────── +# # MicroTissue Experiments +# # ──────────────────────────────────────────────── + +# getDrugDataByParent <- function(parid, sampleId) { +# q <- sprintf( +# "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", +# parid +# ) +# qtab <- synTableQuery(q)$asDataFrame() %>% +# filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% +# select(id, name, experimentalCondition) +# do.call(rbind, lapply(qtab$id, function(x) { +# info <- filter(qtab, id == x) +# d <- extract_date_hour(info$name) +# fread(synGet(x)$path) %>% +# filter(response_type == "percent viability") %>% +# transmute( +# improve_sample_id = sampleId, +# DOSE = (10^dosage) * 1e6, +# GROWTH = response, +# source = "NF Data Portal", +# chem_name = compound_name, +# study = paste0("MT ", d$date, " exp"), +# time = d$hour +# ) +# })) +# } + +# # Create map of MicroTissue Drug Folders +# mts_map <- manifest %>% +# select(common_name, MicroTissueDrugFolder) %>% +# inner_join(mt_samps, by = "common_name") %>% +# separate_rows(MicroTissueDrugFolder, sep = ",") %>% +# # keep exactly what old script did: drop only "NA" and actual NA +# filter( +# !is.na(MicroTissueDrugFolder), +# MicroTissueDrugFolder != "NA" +# ) %>% +# select( +# improve_sample_id, +# folder = MicroTissueDrugFolder +# ) + +# # Fetch all MicroTissue drug response data +# mt_data <- do.call(rbind, lapply(seq_len(nrow(mts_map)), function(i) { +# sample_id <- mts_map$improve_sample_id[i] +# folder <- mts_map$folder[i] +# getDrugDataByParent(folder, sample_id) +# })) + +# drug_map <- fread(drugfile) %>% +# select(improve_drug_id, chem_name) %>% +# distinct() + +# # Clean up drug names and join with drug_map +# mt_curve <- mt_data %>% +# mutate( +# chem_name = tolower(chem_name), +# chem_name = ifelse(chem_name == "pd901", "pd-0325901", chem_name) +# ) %>% +# left_join(drug_map, by = "chem_name") %>% +# filter(!is.na(improve_drug_id)) %>% +# transmute( +# source = source, +# improve_sample_id = improve_sample_id, +# Drug = improve_drug_id, +# study = study, +# time = time, +# time_unit = "hours", +# DOSE = DOSE, +# GROWTH = GROWTH +# ) + +# # Run curve fitting, Write MicroTissue curve data +# fwrite(mt_curve, file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")), sep = "\t") + +# message("Wrote MT curve data") + +# # Write MT experiments file +# system(sprintf( +# "/opt/venv/bin/python fit_curve.py --input %s --output %s", +# paste0("/tmp/", out_prefix, "_mt_curve_data.tsv"), +# paste0("/tmp/", out_prefix, "_mt_experiments") +# )) +# file.rename( +# paste0("/tmp/", out_prefix, "_mt_experiments.0"), +# paste0("/tmp/", out_prefix, "_mt_experiments.tsv") +# ) +# message("Wrote MT experiments") + +# # ──────────────────────────────────────────────── +# # PDX Experiments +# # ──────────────────────────────────────────────── + +# # Create a map of PDX Drug Data +# # This will be used to fetch the drug data for each PDX sample +# pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { +# row <- manifest[i, ] +# samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] +# if (nrow(samp)==0 || is.na(row$PDX_Drug_Data) || row$PDX_Drug_Data %in% c("", "NA")) +# return(NULL) +# ids <- strsplit(row$PDX_Drug_Data, ",")[[1]] +# ids <- trimws(ids[ids!=""]) +# data.frame( +# improve_sample_id = samp$improve_sample_id, +# child_id = ids, +# stringsAsFactors = FALSE +# ) +# })) + +# # Create a dataframe of PDX metadata +# pdx_meta <- do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { +# sid <- pdx_map$improve_sample_id[i] +# cid <- pdx_map$child_id[i] +# pid <- synGet(cid)$parentId +# if (is.null(pid) || pid=="") stop("no parentId for ", cid) +# data.frame( +# improve_sample_id = sid, +# child_id = cid, +# parentId = pid, +# stringsAsFactors = FALSE +# ) +# })) + +# all_pdx <- do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { +# m <- pdx_meta[i, ] +# pth <- synGet(m$child_id)$path +# raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) + +# # detect second‐drug column +# sec_opts <- c("compound 2_name", "compound_2_name") +# drug2_col <- intersect(sec_opts, names(raw))[1] +# compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ + +# df <- data.frame( +# child_id = m$child_id, +# specimen_id = raw$specimen_id, +# compound_name = raw$compound_name, +# compound_2_name = compound2, +# experimental_time_point = raw$experimental_time_point, +# experimental_time_point_unit = raw$experimental_time_point_unit, +# assay_value = raw$assay_value, +# stringsAsFactors = FALSE +# ) + +# df <- within(df, { +# drug1 <- tolower(trimws(compound_name)) +# drug2 <- tolower(trimws(compound_2_name)) +# treatment <- ifelse( +# is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), +# "control", +# ifelse(!is.na(drug2) & drug2 != "", +# paste(drug1, drug2, sep = "+"), +# drug1 +# ) +# ) +# time <- experimental_time_point +# time_unit <- experimental_time_point_unit +# volume <- assay_value +# }) + +# df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] +# })) + +# # join on parentId and sample +# pdx_data <- merge(all_pdx, pdx_meta, by="child_id") + +# pdx_data <- subset(pdx_data, duplicated(child_id) | TRUE) +# pdx_data <- within(pdx_data, { +# experiment <- parentId +# model_id <- improve_sample_id +# }) + +# # Filter out experiments missing a control +# has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) +# no_ctl_exps <- names(has_ctl)[!has_ctl] +# pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] + +# # Reorder final columns +# pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", +# "time","time_unit","volume","model_id")] + +# # Correct doxorubinsin typo across all data +# pdx_data$treatment <- gsub("doxorubinsin", +# "doxorubicin", +# pdx_data$treatment, +# ignore.case = TRUE) + +# # Drop any remaining NA rows +# pdx_data <- na.omit(pdx_data) + +# # write & fit +# fwrite(pdx_data, file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")), sep = "\t") + + +# message("Wrote PDX curve data") + +# system(sprintf( +# "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", +# paste0("/tmp/", out_prefix, "_pdx_curve_data.tsv"), +# drugfile, +# paste0("/tmp/", out_prefix, "_pdx") +# )) + + + +# message("Wrote PDX experiments to ", "/tmp/", out_prefix, "_pdx_experiments.tsv and combinations") + + +# # ──────────────────────────────────────────────── +# # Combine all Experiments +# # ──────────────────────────────────────────────── + +# # Read MicroTissue experiments +# mt_exp <- fread(paste0("/tmp/", out_prefix, "_mt_experiments.tsv")) %>% +# mutate( +# dose_response_value = as.character(dose_response_value) +# ) + +# # Read PDX experiments +# pdx_exp <- fread(paste0("/tmp/", out_prefix, "_pdx_experiments.tsv")) %>% +# mutate( +# dose_response_value = as.character(dose_response_value) +# ) + +# # Join experiments into one. +# all_exp <- bind_rows(mt_exp, pdx_exp) + +# # Write out Experiments +# fwrite(all_exp, paste0("/tmp/", out_prefix, "_experiments.tsv"), sep = "\t") +# message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") + + +# # Rename the Drug Combination data file to fit schema naming +# file.rename( +# paste0("/tmp/", out_prefix, "_pdx_combinations.tsv"), +# paste0("/tmp/", out_prefix, "_combinations.tsv") +# ) \ No newline at end of file diff --git a/coderbuild/mpnst/03_get_experiments_with_treated.R b/coderbuild/mpnst/03_get_experiments_with_treated.R new file mode 100644 index 00000000..8af7fa3b --- /dev/null +++ b/coderbuild/mpnst/03_get_experiments_with_treated.R @@ -0,0 +1,1640 @@ +#!/usr/bin/env Rscript +# 03_get_experiments_with_treated_combos.R +# +# DEBUG VERSION +# - Adds verbose print statements throughout MT + treated-microtissue code paths + helpers +# - Adds safe readers/guards so empty MT outputs don't crash downstream (instead they print diagnostics) +# + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(dplyr) + library(stringr) + library(readr) + library(readxl) + library(tidyr) +}) + +# ============================================================ +# FAST DEBUG SWITCH - skip ahead to later treated section (save time debugging) +# ============================================================ +SKIP_FAST <- FALSE # set to TRUE to skip MT/PDX/combine sections + +# ============================================================ +# PER-SECTION SKIP SWITCHES +# ============================================================ +SKIP_MT <- FALSE +SKIP_PDX <- FALSE +SKIP_COMBINE <- FALSE +SKIP_EXACT <- FALSE + +# Backward-compat: SKIP_FAST skips all heavy sections +if (isTRUE(SKIP_FAST)) { + SKIP_MT <- TRUE + SKIP_PDX <- TRUE + SKIP_COMBINE <- TRUE + SKIP_EXACT <- TRUE +} + +# ============================================================ +# DEBUG CONTROLS +# ============================================================ +DEBUG <- TRUE +DEBUG_MAX_ITEMS <- 10 # limit loop prints +DEBUG_HEAD_N <- 6 + +# ----------------------- +# args +# ----------------------- +args <- commandArgs(trailingOnly = TRUE) +if (length(args) != 4) { + stop("Usage: Rscript 03_get_experiments_with_treated_combos.R ", call. = FALSE) +} +PAT <- args[1] +samples <- args[2] +drugfile <- args[3] +out_prefix <- args[4] + +# ----------------------- +# helpers +# ----------------------- +cat0 <- function(...) cat(..., "\n", sep = "") +hr <- function() cat0(strrep("─", 80)) + +dbg_time <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +dbg <- function(...) { if (isTRUE(DEBUG)) cat0(dbg_time(), " | ", paste0(...)) } +dbg_hr <- function(title = NULL) { + if (!isTRUE(DEBUG)) return(invisible(NULL)) + hr() + if (!is.null(title)) cat0(dbg_time(), " | ", title) + hr() +} + +dbg_df <- function(df, label = "df") { + if (!isTRUE(DEBUG)) return(invisible(NULL)) + if (is.null(df)) { + dbg("[", label, "] is NULL") + return(invisible(NULL)) + } + dbg("[", label, "] rows=", nrow(df), " cols=", ncol(df)) + dbg("[", label, "] cols: ", paste(names(df), collapse = ", ")) + if (nrow(df) > 0) print(utils::head(df, DEBUG_HEAD_N)) + invisible(NULL) +} + +dbg_file <- function(path, label = "file") { + if (!isTRUE(DEBUG)) return(invisible(NULL)) + dbg("[", label, "] path=", path) + if (is.null(path) || is.na(path) || path == "" || !file.exists(path)) { + dbg("[", label, "] MISSING") + return(invisible(NULL)) + } + sz <- file.info(path)$size + dbg("[", label, "] exists size=", sz) + if (!is.na(sz) && sz > 0) { + dt <- tryCatch(data.table::fread(path, nrows = DEBUG_HEAD_N), error = function(e) NULL) + if (!is.null(dt)) { + dbg("[", label, "] peek rows=", nrow(dt), " cols=", ncol(dt)) + dbg("[", label, "] cols: ", paste(names(dt), collapse = ", ")) + print(dt) + } else { + dbg("[", label, "] peek fread failed") + } + } + invisible(NULL) +} + +dbg_stop <- function(...) { + dbg("FATAL: ", paste0(...)) + stop(paste0(...), call. = FALSE) +} + +# safer fread: returns data.frame() if missing/empty/unreadable +safe_fread_df <- function(path, label = "safe_fread") { + if (is.null(path) || is.na(path) || path == "" || !file.exists(path)) { + dbg("[", label, "] missing: ", path) + return(data.frame()) + } + sz <- file.info(path)$size + if (is.na(sz) || sz == 0) { + dbg("[", label, "] EMPTY (0 bytes): ", path) + return(data.frame()) + } + dt <- tryCatch(data.table::fread(path), error = function(e) { + dbg("[", label, "] fread ERROR: ", conditionMessage(e)) + return(NULL) + }) + if (is.null(dt)) return(data.frame()) + as.data.frame(dt) +} + +norm_str <- function(x) tolower(trimws(as.character(x))) + +norm_drug <- function(x) { + x <- norm_str(x) + x <- ifelse(x == "pd901", "pd-0325901", x) + x +} + +safe_token <- function(x) { + x <- as.character(x) + x <- trimws(x) + x <- gsub("\\s+", "_", x) + x <- gsub("[^A-Za-z0-9_\\-\\.]+", "_", x) + x +} + +warn_first_by <- function(df, group_cols, label) { + if (is.null(df) || nrow(df) == 0) { + dbg("[", label, "] warn_first_by: input empty") + return(df) + } + + counts <- df %>% + group_by(across(all_of(group_cols))) %>% + summarise(n = n(), .groups = "drop") + + dup_n <- sum(counts$n > 1) + if (dup_n > 0) { + warning(sprintf("[%s] Found %d duplicated key group(s); using first() for each group.", label, dup_n)) + if (isTRUE(DEBUG)) { + dbg("[", label, "] duplicated groups (top):") + print(utils::head(counts[counts$n > 1, ], DEBUG_HEAD_N)) + } + } + + df %>% + group_by(across(all_of(group_cols))) %>% + slice(1) %>% + ungroup() +} + +extract_date_hour <- function(experiment_id) { + pattern <- "(\\d{6})_?(\\d{2,3})?" + m <- str_match(experiment_id, pattern) + date <- m[,2]; hour <- m[,3] + date[is.na(date)] <- NA + hour[is.na(hour)] <- 48 + list(date = date, hour = hour) +} + +canon_name <- function(x) { + x <- norm_drug(x) + x <- gsub("\\s+", "", x) + x <- gsub("_", "", x) + x <- gsub("-", "", x) + x +} + +canon_id <- function(x) { + x <- toupper(trimws(as.character(x))) + x <- gsub("\\s+", "", x) + x <- gsub("_", "", x) + x <- gsub("-", "", x) + x +} + +split_plus_once <- function(x) { + x <- as.character(x) + if (!grepl("\\+", x)) return(c(NA_character_, NA_character_)) + parts <- strsplit(x, "\\+", fixed = FALSE)[[1]] + if (length(parts) < 2) return(c(NA_character_, NA_character_)) + if (length(parts) > 2) { + a <- parts[1] + b <- paste(parts[-1], collapse = "+") + return(c(a, b)) + } + c(parts[1], parts[2]) +} + +parse_combo_key <- function(combo_key) { + x <- as.character(combo_key) + if (!grepl("_", x) || !grepl("\\+", x)) { + return(list(base = NA_character_, drug1 = NA_character_, drug2 = NA_character_)) + } + base <- sub("^([^_]+)_.*$", "\\1", x) + combo_str <- sub("^[^_]+_", "", x) + dp <- split_plus_once(combo_str) + list(base = base, drug1 = dp[1], drug2 = dp[2]) +} + +escape_regex <- function(x) { + x <- as.character(x) + x <- gsub("([][{}()+*^$|\\\\?.])", "\\\\\\1", x) + x +} + +# ============================================================ +# fallback mapping (regex search in chem_name) +# ============================================================ +fallback_id_from_chem_name <- function(pattern_raw, chem_tbl) { + if (is.null(pattern_raw) || is.na(pattern_raw)) return(NA_character_) + p <- trimws(as.character(pattern_raw)) + if (p == "" || tolower(p) %in% c("na","n/a","nan")) return(NA_character_) + + patt <- escape_regex(p) + hits <- chem_tbl %>% + filter(grepl(patt, chem_name, ignore.case = TRUE)) %>% + pull(improve_drug_id) + + if (length(hits) > 0) return(as.character(hits[1])) + NA_character_ +} + +fill_ids_with_fallback <- function(df, raw_col, id_col, chem_tbl, label) { + stopifnot(raw_col %in% names(df), id_col %in% names(df)) + + missing_idx <- which(is.na(df[[id_col]]) | df[[id_col]] == "") + if (length(missing_idx) == 0) { + dbg(label, ": no missing ", id_col, " to fill.") + return(df) + } + + dbg(label, ": attempting fallback fill for ", length(missing_idx), + " missing ", id_col, " values using regex search in chem_name.") + + filled <- vapply(df[[raw_col]][missing_idx], fallback_id_from_chem_name, character(1), chem_tbl = chem_tbl) + + replace_idx <- missing_idx[!is.na(filled)] + if (length(replace_idx) > 0) { + df[[id_col]][replace_idx] <- filled[!is.na(filled)] + } + + dbg(label, ": fallback filled ", length(replace_idx), + " of ", length(missing_idx), " missing ", id_col, ".") + df +} + +# ============================================================ +# treated token extraction (supports underscores) +# ============================================================ +base_to_otherid_prefix <- function(common_name) { + gsub("-", "_", trimws(as.character(common_name))) +} + +extract_treated_token_from_other_id <- function(other_id, common_name) { + oid <- trimws(as.character(other_id)) + cn <- trimws(as.character(common_name)) + + if (is.na(oid) || oid == "" || is.na(cn) || cn == "") { + if (isTRUE(DEBUG)) dbg("[token_extract] invalid inputs oid/cn: oid=", oid, " cn=", cn) + return(NA_character_) + } + + treated_suffix_re <- "_(8hr|24hr)_treated_microtissue$" + no_suffix <- sub(treated_suffix_re, "", oid) + if (identical(no_suffix, oid)) { + if (isTRUE(DEBUG)) dbg("[token_extract] suffix not found: ", oid) + return(NA_character_) + } + + base_prefixes <- unique(c(cn, gsub("-", "_", cn))) + drug <- no_suffix + stripped <- FALSE + for (bp in base_prefixes) { + drug2 <- sub(paste0("^", escape_regex(bp), "_"), "", drug) + if (!identical(drug2, drug)) { + drug <- drug2 + stripped <- TRUE + break + } + } + if (!stripped && isTRUE(DEBUG)) { + dbg("[token_extract] base prefix NOT stripped (unexpected?) oid=", oid, " cn=", cn, " no_suffix=", no_suffix) + } + + drug <- trimws(drug) + if (drug == "" || tolower(drug) %in% c("na","n/a","nan")) { + if (isTRUE(DEBUG)) dbg("[token_extract] empty/NA drug after parse: oid=", oid, " cn=", cn, " parsed=", drug) + return(NA_character_) + } + drug +} + +# ============================================================ +# START +# ============================================================ +dbg_hr("SCRIPT START") +dbg("SKIP_FAST=", SKIP_FAST) +dbg("SKIP_MT=", SKIP_MT) +dbg("SKIP_PDX=", SKIP_PDX) +dbg("SKIP_COMBINE=", SKIP_COMBINE) +dbg("SKIP_EXACT=", SKIP_EXACT) +dbg("samples=", samples) +dbg("drugfile=", drugfile) +dbg("out_prefix=", out_prefix) + +synLogin(authToken = PAT) +dbg("Synapse login OK") + +# ----------------------- +# load inputs +# ----------------------- +dbg_hr("LOADING INPUTS") +samples_all <- fread(samples) %>% as.data.frame() +dbg_df(samples_all, "samples_all") +if (!all(c("improve_sample_id","common_name","model_type") %in% names(samples_all))) { + dbg_stop("samples.csv missing required columns: improve_sample_id, common_name, model_type") +} + +samples_df <- samples_all %>% + select(improve_sample_id, common_name, model_type) %>% + distinct() +dbg_df(samples_df, "samples_df") + +pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") +mt_samps <- filter(samples_df, model_type == "3D-MEDS") +dbg_df(pdx_samps, "pdx_samps") +dbg_df(mt_samps, "mt_samps") + +drug_map <- fread(drugfile) %>% + select(improve_drug_id, chem_name) %>% + distinct() %>% + mutate( + improve_drug_id = as.character(improve_drug_id), + chem_name = norm_drug(chem_name) + ) +dbg_df(drug_map, "drug_map") +dbg("drug_map unique chem_name=", length(unique(drug_map$chem_name))) + +manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% + rename(common_name = Sample) %>% + as.data.table() +if ("PDXDrugData" %in% names(manifest) && !"PDX_Drug_Data" %in% names(manifest)) + setnames(manifest, "PDXDrugData", "PDX_Drug_Data") +dbg_df(as.data.frame(manifest), "manifest") +if (!all(c("common_name","MicroTissueDrugFolder") %in% names(manifest))) { + dbg("manifest columns: ", paste(names(manifest), collapse=", ")) + dbg_stop("manifest missing MicroTissueDrugFolder and/or common_name") +} + +# ============================================================ +# OPTIONAL: MT / PDX / Combine +# ============================================================ +if (!(isTRUE(SKIP_MT) && isTRUE(SKIP_PDX) && isTRUE(SKIP_COMBINE) && isTRUE(SKIP_EXACT))) { + + # ──────────────────────────────────────────────── + # MicroTissue Experiments + # ──────────────────────────────────────────────── + dbg_hr("MT SECTION") + + getDrugDataByParent <- function(parid, sampleId) { + dbg_hr(paste0("getDrugDataByParent(parid=", parid, ", sampleId=", sampleId, ")")) + q <- sprintf( + "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", + parid + ) + dbg("MT query: ", q) + + qtab <- tryCatch({ + synTableQuery(q)$asDataFrame() + }, error = function(e) { + dbg("[MT] synTableQuery ERROR for parentId=", parid, " : ", conditionMessage(e)) + return(NULL) + }) + + if (is.null(qtab) || nrow(qtab) == 0) { + dbg("[MT] qtab empty for parentId=", parid) + return(NULL) + } + + dbg_df(qtab, "MT qtab raw") + qtab <- qtab %>% + filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% + select(id, name, experimentalCondition) + + dbg_df(qtab, "MT qtab filtered") + if (nrow(qtab) == 0) { + dbg("[MT] qtab filtered to 0 rows for parentId=", parid, " (all NA experimentalCondition?)") + return(NULL) + } + + # Loop over child files + out_list <- vector("list", length = nrow(qtab)) + for (ii in seq_len(nrow(qtab))) { + x <- qtab$id[ii] + info <- qtab[ii, , drop = FALSE] + d <- extract_date_hour(info$name) + + dbg_hr(paste0("[MT child ", ii, "/", nrow(qtab), "] synID=", x, " name=", info$name)) + sget <- tryCatch(synGet(x), error = function(e) { + dbg("[MT] synGet ERROR for ", x, ": ", conditionMessage(e)) + return(NULL) + }) + if (is.null(sget)) { + out_list[[ii]] <- NULL + next + } + + pth <- sget$path + dbg_file(pth, label = paste0("MT child file ", x)) + + raw <- tryCatch({ + data.table::fread(pth) + }, error = function(e) { + dbg("[MT] fread ERROR for ", x, " path=", pth, " : ", conditionMessage(e)) + return(NULL) + }) + if (is.null(raw) || nrow(raw) == 0) { + dbg("[MT] raw empty for ", x) + out_list[[ii]] <- NULL + next + } + + dbg("[MT] raw rows=", nrow(raw), " cols=", ncol(raw)) + dbg("[MT] raw cols: ", paste(names(raw), collapse=", ")) + if ("response_type" %in% names(raw)) { + dbg("[MT] response_type top:") + print(utils::head(sort(table(raw$response_type), decreasing = TRUE), DEBUG_HEAD_N)) + } else { + dbg("[MT] WARNING: raw missing response_type column") + } + + # Must have these columns for downstream + needed <- c("response_type","dosage","response","compound_name") + miss <- setdiff(needed, names(raw)) + if (length(miss) > 0) { + dbg("[MT] MISSING required cols in ", x, ": ", paste(miss, collapse=", ")) + out_list[[ii]] <- NULL + next + } + + raw2 <- raw %>% + filter(response_type == "percent viability") + + dbg("[MT] rows after filter(response_type=='percent viability') = ", nrow(raw2)) + if (nrow(raw2) == 0) { + dbg("[MT] NOTE: no 'percent viability' rows in ", x, " (check response_type values)") + out_list[[ii]] <- NULL + next + } + + # Transmute final + tmp <- raw2 %>% + transmute( + improve_sample_id = sampleId, + DOSE = (10^dosage) * 1e6, + GROWTH = response, + source = "NF Data Portal", + chem_name = compound_name, + study = paste0("MT ", d$date, " exp"), + time = d$hour + ) + + dbg_df(tmp, label = paste0("MT tmp ", x)) + out_list[[ii]] <- tmp + } + + out <- dplyr::bind_rows(out_list) + dbg_df(out, label = paste0("MT OUT parentId=", parid)) + if (nrow(out) == 0) { + dbg("[MT] parentId=", parid, " produced 0 rows total") + return(NULL) + } + out + } + + # Build MT folder map + mts_map <- manifest %>% + select(common_name, MicroTissueDrugFolder) %>% + mutate(MicroTissueDrugFolder = sapply(MicroTissueDrugFolder, function(x) { + if (is.null(x) || (length(x)==1 && is.na(x))) return(NA_character_) + paste(trimws(unlist(x)), collapse=",") + })) %>% + inner_join(mt_samps, by = "common_name") %>% + separate_rows(MicroTissueDrugFolder, sep = ",") %>% + filter(!is.na(MicroTissueDrugFolder), MicroTissueDrugFolder != "NA") %>% + select(improve_sample_id, folder = MicroTissueDrugFolder) + + dbg_df(as.data.frame(mts_map), "mts_map") + dbg("mts_map unique folders=", length(unique(mts_map$folder))) + + if (nrow(mts_map) == 0) { + if (!isTRUE(SKIP_MT)) { + dbg_stop("MT: mts_map is empty. Likely manifest join failed or MicroTissueDrugFolder missing/NA.") + } else { + dbg("MT: mts_map is empty (SKIP_MT=TRUE); continuing.") + } + } + + if (!isTRUE(SKIP_MT)) { + + # Pull MT data for each folder + mt_list <- vector("list", length = nrow(mts_map)) + for (i in seq_len(nrow(mts_map))) { + sample_id <- mts_map$improve_sample_id[i] + folder <- mts_map$folder[i] + + if (isTRUE(DEBUG) && i <= DEBUG_MAX_ITEMS) { + dbg_hr(paste0("MT MAP ROW ", i, "/", nrow(mts_map), " sample_id=", sample_id, " folder=", folder)) + } + + mt_list[[i]] <- getDrugDataByParent(folder, sample_id) + } + + mt_data <- dplyr::bind_rows(mt_list) + dbg_df(mt_data, "mt_data") + + # If mt_data empty, dump context + stop (this is the earliest real failure) + if (nrow(mt_data) == 0) { + dbg("MT FAILURE: mt_data is empty. Writing debug dumps.") + dbg_out1 <- file.path("/tmp", paste0(out_prefix, "_DEBUG_mts_map.tsv")) + dbg_out2 <- file.path("/tmp", paste0(out_prefix, "_DEBUG_manifest_mt_subset.tsv")) + fwrite(as.data.table(mts_map), dbg_out1, sep = "\t") + fwrite(as.data.table(manifest %>% as.data.frame() %>% filter(common_name %in% mt_samps$common_name)), + dbg_out2, sep = "\t") + dbg("Wrote: ", dbg_out1) + dbg("Wrote: ", dbg_out2) + dbg_stop("MT produced 0 rows. Investigate printed logs above + debug TSVs in /tmp.") + } + + # Join to drug map + mt_curve <- mt_data %>% + mutate(chem_name = norm_drug(chem_name)) + + # show join coverage BEFORE filtering NA improve_drug_id + mt_joined <- mt_curve %>% + left_join(drug_map, by = "chem_name") + + dbg_df(mt_joined, "mt_joined") + n_unmapped <- sum(is.na(mt_joined$improve_drug_id)) + dbg("MT join unmapped chem_name rows=", n_unmapped, " / ", nrow(mt_joined)) + if (n_unmapped > 0) { + top_unmapped <- mt_joined %>% + filter(is.na(improve_drug_id)) %>% + count(chem_name, sort = TRUE) %>% + head(20) + dbg("MT top unmapped chem_name (up to 20):") + print(top_unmapped) + dbg_unmapped_path <- file.path("/tmp", paste0(out_prefix, "_DEBUG_mt_unmapped_chem_name.tsv")) + fwrite(as.data.table(mt_joined %>% filter(is.na(improve_drug_id))), dbg_unmapped_path, sep = "\t") + dbg("Wrote: ", dbg_unmapped_path) + } + + # Fallback: regex match for chem_names that didn't exact-join + # (e.g. raw "verteporfin" vs drug_map "verteporfin [usan:usp:inn:ban]") + mt_joined <- fill_ids_with_fallback( + as.data.frame(mt_joined), + raw_col = "chem_name", + id_col = "improve_drug_id", + chem_tbl = drug_map, + label = "MT" + ) + + mt_curve <- mt_joined %>% + filter(!is.na(improve_drug_id)) %>% + transmute( + source = source, + improve_sample_id = improve_sample_id, + Drug = improve_drug_id, + study = study, + time = time, + time_unit = "hours", + DOSE = DOSE, + GROWTH = GROWTH + ) + + dbg_df(mt_curve, "mt_curve (final)") + + out_mt_curve_path <- file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")) + fwrite(mt_curve, out_mt_curve_path, sep = "\t") + dbg_file(out_mt_curve_path, "MT curve output") + message("Wrote MT curve data") + + if (nrow(mt_curve) == 0) { + dbg_stop("MT curve is EMPTY after join/filter. Root cause is likely chem_name mismatch vs drugfile.tsv. See DEBUG_mt_unmapped_chem_name.tsv.") + } + + # Fit curves + out_mt_prefix <- paste0("/tmp/", out_prefix, "_mt_experiments") + cmd <- sprintf("/opt/venv/bin/python fit_curve.py --input %s --output %s", + out_mt_curve_path, out_mt_prefix) + dbg_hr("RUN fit_curve.py (MT)") + dbg("CMD: ", cmd) + rc <- system(cmd) + dbg("fit_curve.py return code: ", rc) + + # fit_curve.py writes ".0" + mt_exp0 <- paste0(out_mt_prefix, ".0") + dbg_file(mt_exp0, "MT experiments .0 (pre-rename)") + if (!file.exists(mt_exp0) || file.info(mt_exp0)$size == 0) { + dbg_stop("fit_curve.py did not produce a non-empty MT experiments file: ", mt_exp0, + " (check fit_curve.py logs + MT curve input)") + } + + mt_exp_path <- paste0("/tmp/", out_prefix, "_mt_experiments.tsv") + ok_rename <- file.rename(mt_exp0, mt_exp_path) + dbg("rename ", mt_exp0, " -> ", mt_exp_path, " ok=", ok_rename) + dbg_file(mt_exp_path, "MT experiments TSV") + + # Average MT experiments across dates for each unique sample/drug/time/metric + mt_exp_dt <- tryCatch(data.table::fread(mt_exp_path), error = function(e) data.table::data.table()) + if (nrow(mt_exp_dt) > 0) { + if ("Drug" %in% names(mt_exp_dt) && !"improve_drug_id" %in% names(mt_exp_dt)) { + data.table::setnames(mt_exp_dt, "Drug", "improve_drug_id") + } + + req_cols <- c("source","improve_sample_id","improve_drug_id","study","time","time_unit","dose_response_metric","dose_response_value") + if (all(req_cols %in% names(mt_exp_dt))) { + + # optional sanity check (no filtering) + if (any(!grepl("^MT\\s*\\d{6}\\b", as.character(mt_exp_dt$study), ignore.case = TRUE))) { + warning("[MT collapse] Found unexpected study values in mt_exp_path; still averaging all rows in this MT file.") + } + + mt_exp_dt[, dose_response_value := suppressWarnings(as.numeric(as.character(dose_response_value)))] + mt_exp_dt <- mt_exp_dt[is.finite(dose_response_value)] + + key_cols <- c("source","improve_sample_id","improve_drug_id","time","time_unit","dose_response_metric") + + mt_agg <- mt_exp_dt[, .( + study = "MPNST Microtissue", + dose_response_value = mean(dose_response_value, na.rm = TRUE) + ), by = key_cols] + + data.table::fwrite(mt_agg, mt_exp_path, sep = "\t") + } + } + + + message("Wrote MT experiments") + + } else { + dbg_hr("SKIP_MT=TRUE") + cat0("SKIP_MT=TRUE: skipping MT section.") + } + + # ──────────────────────────────────────────────── + # PDX Experiments + # ──────────────────────────────────────────────── + if (!isTRUE(SKIP_PDX)) { + + dbg_hr("PDX SECTION") + + pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { + row <- manifest[i, ] + samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] + if (nrow(samp) == 0) return(NULL) + raw <- tryCatch(paste(trimws(unlist(row$PDX_Drug_Data)), collapse=","), error=function(e) "") + ids <- regmatches(raw, gregexpr("syn[0-9]+", raw, ignore.case=TRUE))[[1]] + if (length(ids) == 0) return(NULL) + data.frame( + improve_sample_id = samp$improve_sample_id, + child_id = ids, + stringsAsFactors = FALSE + ) + })) + if (is.null(pdx_map) || nrow(pdx_map) == 0) { + dbg("PDX: pdx_map empty (no PDX drug data)") + } else { + dbg_df(pdx_map, "pdx_map") + } + + pdx_meta <- if (!is.null(pdx_map) && nrow(pdx_map) > 0) { + do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { + sid <- pdx_map$improve_sample_id[i] + cid <- pdx_map$child_id[i] + pid <- synGet(cid)$parentId + if (is.null(pid) || pid=="") stop("no parentId for ", cid) + data.frame( + improve_sample_id = sid, + child_id = cid, + parentId = pid, + stringsAsFactors = FALSE + ) + })) + } else { + data.frame() + } + + all_pdx <- if (nrow(pdx_meta) > 0) { + do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { + m <- pdx_meta[i, ] + pth <- synGet(m$child_id)$path + raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) + + sec_opts <- c("compound 2_name", "compound_2_name") + drug2_col <- intersect(sec_opts, names(raw))[1] + compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ + + df <- data.frame( + child_id = m$child_id, + specimen_id = raw$specimen_id, + compound_name = raw$compound_name, + compound_2_name = compound2, + experimental_time_point = raw$experimental_time_point, + experimental_time_point_unit = raw$experimental_time_point_unit, + assay_value = raw$assay_value, + stringsAsFactors = FALSE + ) + + df <- within(df, { + drug1 <- tolower(trimws(compound_name)) + drug2 <- tolower(trimws(compound_2_name)) + treatment <- ifelse( + is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), + "control", + ifelse(!is.na(drug2) & drug2 != "", + paste(drug1, drug2, sep = "+"), + drug1 + ) + ) + time <- experimental_time_point + time_unit <- experimental_time_point_unit + volume <- assay_value + }) + + df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] + })) + } else { + data.frame() + } + + pdx_data <- if (nrow(all_pdx) > 0) merge(all_pdx, pdx_meta, by="child_id") else data.frame() + if (nrow(pdx_data) > 0) { + pdx_data <- within(pdx_data, { + experiment <- parentId + model_id <- improve_sample_id + }) + + has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) + pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] + + pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", + "time","time_unit","volume","model_id")] + + pdx_data$treatment <- gsub("doxorubinsin", "doxorubicin", pdx_data$treatment, ignore.case = TRUE) + pdx_data <- na.omit(pdx_data) + + out_pdx_curve <- file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")) + fwrite(pdx_data, out_pdx_curve, sep = "\t") + dbg_file(out_pdx_curve, "PDX curve output") + message("Wrote PDX curve data") + + cmd2 <- sprintf( + "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", + out_pdx_curve, + drugfile, + paste0("/tmp/", out_prefix, "_pdx") + ) + dbg("CMD: ", cmd2) + rc2 <- system(cmd2) + dbg("calc_pdx_metrics.py return code: ", rc2) + message("Wrote PDX experiments to /tmp/", out_prefix, "_pdx_experiments.tsv and combinations") + } else { + dbg("PDX: pdx_data empty; skipping PDX metrics") + } + + } else { + dbg_hr("SKIP_PDX=TRUE") + cat0("SKIP_PDX=TRUE: skipping PDX section.") + } + + # ──────────────────────────────────────────────── + # Combine all Experiments + # ──────────────────────────────────────────────── + if (!isTRUE(SKIP_COMBINE)) { + + dbg_hr("COMBINE MT + PDX EXPERIMENTS") + + mt_exp_path2 <- paste0("/tmp/", out_prefix, "_mt_experiments.tsv") + pdx_exp_path2 <- paste0("/tmp/", out_prefix, "_pdx_experiments.tsv") + + mt_exp <- safe_fread_df(mt_exp_path2, label = "read MT experiments") + pdx_exp <- safe_fread_df(pdx_exp_path2, label = "read PDX experiments") + + dbg_df(mt_exp, "mt_exp (read)") + dbg_df(pdx_exp, "pdx_exp (read)") + + # Make sure dose_response_value exists so mutate won't crash + if (!("dose_response_value" %in% names(mt_exp))) mt_exp$dose_response_value <- NA + if (!("dose_response_value" %in% names(pdx_exp))) pdx_exp$dose_response_value <- NA + + mt_exp <- mt_exp %>% mutate(dose_response_value = as.character(dose_response_value)) + pdx_exp <- pdx_exp %>% mutate(dose_response_value = as.character(dose_response_value)) + + all_exp <- bind_rows(mt_exp, pdx_exp) + dbg_df(all_exp, "all_exp (combined)") + + out_all <- paste0("/tmp/", out_prefix, "_experiments.tsv") + fwrite(all_exp, out_all, sep = "\t") + dbg_file(out_all, "combined experiments TSV") + message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") + + # rename combinations + pdx_combo_src <- paste0("/tmp/", out_prefix, "_pdx_combinations.tsv") + combo_dst <- paste0("/tmp/", out_prefix, "_combinations.tsv") + if (file.exists(pdx_combo_src)) { + okc <- file.rename(pdx_combo_src, combo_dst) + dbg("rename combos ", pdx_combo_src, " -> ", combo_dst, " ok=", okc) + } else { + dbg("PDX combinations file missing (ok if no PDX): ", pdx_combo_src) + } + + } else { + dbg_hr("SKIP_COMBINE=TRUE") + cat0("SKIP_COMBINE=TRUE: skipping combine section.") + } + + # ──────────────────────────────────────────────── + # EXACT Experiments + # - Average replicates within each file + # - Run curve fitting per date via study = "EXACT {DATE}" + # - Average fitted metrics across dates back to study = "EXACT" + # ──────────────────────────────────────────────── + if (!isTRUE(SKIP_EXACT)) { + + dbg_hr("EXACT SECTION") + + EXACT_PARENT <- "syn65473019" # Folder (Processed Data) + EXACT_FILEVIEW <- "syn52369043" # FileView table used by your python + + get_exact_fileview <- function(parent_id) { + q <- sprintf("SELECT * FROM %s WHERE parentId = '%s'", EXACT_FILEVIEW, parent_id) + dbg("EXACT FileView query: ", q) + + tab <- tryCatch({ + synTableQuery(q)$asDataFrame() + }, error = function(e) { + dbg("[EXACT] synTableQuery(fileview) ERROR: ", conditionMessage(e)) + return(NULL) + }) + + if (is.null(tab) || nrow(tab) == 0) { + dbg("[EXACT] FileView returned 0 rows for parentId=", parent_id) + return(data.frame()) + } + + dbg_df(tab, "EXACT fileview raw") + + tab <- tab %>% + filter(!is.na(name)) %>% + filter(name != "synapse_storage_manifest.csv") %>% + filter(grepl("\\.csv$", name, ignore.case = TRUE)) %>% + filter(!grepl("250122\\.csv$", name, ignore.case = TRUE)) + + dbg_df(tab, "EXACT fileview filtered") + tab + } + + pick_specimen_col <- function(df) { + opts <- c("specimenID","specimenId","specimen_id","improve_sample_id") + hit <- intersect(opts, names(df)) + if (length(hit) == 0) return(NA_character_) + hit[1] + } + + exact_files <- get_exact_fileview(EXACT_PARENT) + + exact_samples_map <- samples_all %>% + transmute( + common_name = trimws(as.character(common_name)), + model_type = as.character(model_type), + improve_sample_id = suppressWarnings(as.integer(improve_sample_id)) + ) %>% + distinct() + + map_exact_sampleName_to_improve_id <- function(sampleName) { + sn <- trimws(as.character(sampleName)) + if (is.na(sn) || sn == "" || tolower(sn) %in% c("na","n/a","nan")) return(NA_integer_) + + sn_canon <- canon_id(sn) + + hit1 <- exact_samples_map$improve_sample_id[ + exact_samples_map$model_type == "3D-MEDS" & + canon_id(exact_samples_map$common_name) == sn_canon + ] + if (length(hit1) > 0 && !is.na(hit1[1])) return(as.integer(hit1[1])) + + hit2 <- exact_samples_map$improve_sample_id[ + exact_samples_map$model_type == "patient derived xenograft" & + canon_id(exact_samples_map$common_name) == sn_canon + ] + if (length(hit2) > 0 && !is.na(hit2[1])) return(as.integer(hit2[1])) + + NA_integer_ + } + + exact_data <- data.frame() + if (nrow(exact_files) == 0) { + dbg("EXACT: no files found under ", EXACT_PARENT, "; skipping EXACT.") + } else { + + spec_col <- pick_specimen_col(exact_files) + if (is.na(spec_col)) { + dbg("[EXACT] WARNING: FileView has no specimenID column; will fall back to sampleName inside CSVs.") + exact_files$.__specimen <- NA_character_ + spec_col <- ".__specimen" + } + + exact_list <- vector("list", length = nrow(exact_files)) + + for (i in seq_len(nrow(exact_files))) { + file_id <- as.character(exact_files$id[i]) + file_name <- as.character(exact_files$name[i]) + specimen <- exact_files[[spec_col]][i] + + if (isTRUE(DEBUG) && i <= DEBUG_MAX_ITEMS) { + dbg_hr(paste0("EXACT FILE ", i, "/", nrow(exact_files), + " id=", file_id, " name=", file_name, + " specimen=", specimen)) + } + + sget <- tryCatch(synGet(file_id), error = function(e) { + dbg("[EXACT] synGet ERROR for ", file_id, ": ", conditionMessage(e)) + return(NULL) + }) + if (is.null(sget)) { + exact_list[[i]] <- NULL + next + } + + pth <- sget$path + dbg_file(pth, label = paste0("EXACT file ", file_id)) + + raw <- tryCatch({ + data.table::fread(pth) + }, error = function(e) { + dbg("[EXACT] fread ERROR for ", file_id, " path=", pth, " : ", conditionMessage(e)) + return(NULL) + }) + if (is.null(raw) || nrow(raw) == 0) { + dbg("[EXACT] raw empty for ", file_id) + exact_list[[i]] <- NULL + next + } + + raw_df <- as.data.frame(raw) + + # Optional: only keep processed rows if column exists + if ("dataSubtype" %in% names(raw_df)) { + raw_df <- raw_df[raw_df$dataSubtype == "processed", , drop = FALSE] + dbg("[EXACT] rows after dataSubtype=='processed': ", nrow(raw_df)) + if (nrow(raw_df) == 0) { + exact_list[[i]] <- NULL + next + } + } + + needed <- c("drugName","concentration","percentViability","timePoint","timePointUnit") + miss <- setdiff(needed, names(raw_df)) + if (length(miss) > 0) { + dbg("[EXACT] MISSING required cols in ", file_id, ": ", paste(miss, collapse=", ")) + exact_list[[i]] <- NULL + next + } + + # Determine improve_sample_id for this file (map sampleName -> mpnst_samples.csv other_id) + raw_df$improve_sample_id <- NA_integer_ + if ("sampleName" %in% names(raw_df)) { + raw_df$improve_sample_id <- vapply(raw_df$sampleName, map_exact_sampleName_to_improve_id, integer(1)) + } + + if (all(is.na(raw_df$improve_sample_id))) { + dbg("[EXACT] WARNING: sampleName->improve_sample_id mapping failed for ", file_id, " name=", file_name) + if ("sampleName" %in% names(raw_df)) { + dbg("[EXACT] sampleName values (unique, up to 20):") + print(head(unique(raw_df$sampleName), 20)) + } + exact_list[[i]] <- NULL + next + } + + # Extract 6-digit date token from filename + file_date <- str_match(file_name, "(\\d{6})\\.csv$")[,2] + if (is.na(file_date)) file_date <- str_match(file_name, "(\\d{6})")[,2] + + # Average replicates WITHIN FILE + raw_avg <- raw_df %>% + mutate( + improve_sample_id = suppressWarnings(as.integer(improve_sample_id)), + drugName = as.character(drugName), + concentration = suppressWarnings(as.numeric(concentration)), + percentViability = suppressWarnings(as.numeric(percentViability)), + timePoint = suppressWarnings(as.integer(timePoint)), + timePointUnit = as.character(timePointUnit) + ) %>% + filter(!is.na(improve_sample_id), !is.na(drugName), !is.na(concentration), !is.na(percentViability), !is.na(timePoint)) %>% + group_by(improve_sample_id, drugName, concentration, timePoint, timePointUnit) %>% + summarise(percentViability = mean(percentViability, na.rm = TRUE), .groups = "drop") + + tmp <- raw_avg %>% + transmute( + improve_sample_id = improve_sample_id, + DOSE = concentration, # µM + GROWTH = percentViability, # percent viability (0-100) + source = "NF Data Portal", + chem_name = norm_drug(drugName), + study = paste0("EXACT ", file_date), + time = timePoint, + time_unit = timePointUnit + ) + + dbg_df(tmp, label = paste0("EXACT tmp ", file_id)) + exact_list[[i]] <- tmp + } + + exact_data <- dplyr::bind_rows(exact_list) + } + + dbg_df(exact_data, "exact_data (raw)") + + if (!is.null(exact_data) && nrow(exact_data) > 0) { + + # map to improve_drug_id using drug_map (join on normalized chem_name) + exact_joined <- exact_data %>% + left_join(drug_map, by = "chem_name") + + dbg_df(exact_joined, "exact_joined") + n_unmapped_ex <- sum(is.na(exact_joined$improve_drug_id)) + dbg("EXACT join unmapped chem_name rows=", n_unmapped_ex, " / ", nrow(exact_joined)) + if (n_unmapped_ex > 0) { + top_unmapped_ex <- exact_joined %>% + filter(is.na(improve_drug_id)) %>% + count(chem_name, sort = TRUE) %>% + head(20) + dbg("EXACT top unmapped chem_name (up to 20):") + print(top_unmapped_ex) + dbg_unmapped_ex_path <- file.path("/tmp", paste0(out_prefix, "_DEBUG_exact_unmapped_chem_name.tsv")) + fwrite(as.data.table(exact_joined %>% filter(is.na(improve_drug_id))), dbg_unmapped_ex_path, sep = "\t") + dbg("Wrote: ", dbg_unmapped_ex_path) + } + + # Fallback: regex match for chem_names that didn't exact-join + # (e.g. raw "verteporfin" vs drug_map "verteporfin [usan:usp:inn:ban]") + exact_joined <- fill_ids_with_fallback( + as.data.frame(exact_joined), + raw_col = "chem_name", + id_col = "improve_drug_id", + chem_tbl = drug_map, + label = "EXACT" + ) + + exact_curve <- exact_joined %>% + filter(!is.na(improve_drug_id)) %>% + transmute( + source = source, + improve_sample_id = improve_sample_id, + Drug = improve_drug_id, + study = study, # "EXACT {DATE}" for per-date fitting + time = time, + time_unit = time_unit, + DOSE = DOSE, + GROWTH = GROWTH + ) + + dbg_df(exact_curve, "exact_curve (final)") + + out_exact_curve_path <- file.path("/tmp", paste0(out_prefix, "_exact_curve_data.tsv")) + fwrite(exact_curve, out_exact_curve_path, sep = "\t") + dbg_file(out_exact_curve_path, "EXACT curve output") + message("Wrote EXACT curve data") + + if (nrow(exact_curve) > 0) { + # Fit curves (per-date via study) + out_exact_prefix <- paste0("/tmp/", out_prefix, "_exact_experiments") + cmd_ex <- sprintf("/opt/venv/bin/python fit_curve.py --input %s --output %s", + out_exact_curve_path, out_exact_prefix) + dbg_hr("RUN fit_curve.py (EXACT)") + dbg("CMD: ", cmd_ex) + rc_ex <- system(cmd_ex) + dbg("fit_curve.py return code: ", rc_ex) + + exact_exp0 <- paste0(out_exact_prefix, ".0") + dbg_file(exact_exp0, "EXACT experiments .0 (pre-rename)") + if (!file.exists(exact_exp0) || file.info(exact_exp0)$size == 0) { + dbg_stop("fit_curve.py did not produce a non-empty EXACT experiments file: ", exact_exp0, + " (check fit_curve.py logs + EXACT curve input)") + } + + + exact_exp_path <- paste0("/tmp/", out_prefix, "_exact_experiments.tsv") + + ok_rename_ex <- file.rename(exact_exp0, exact_exp_path) + if (!isTRUE(ok_rename_ex)) { + ok_copy <- file.copy(exact_exp0, exact_exp_path, overwrite = TRUE) + if (isTRUE(ok_copy)) unlink(exact_exp0) + ok_rename_ex <- ok_copy + } + dbg("rename ", exact_exp0, " -> ", exact_exp_path, " ok=", ok_rename_ex) + dbg_file(exact_exp_path, "EXACT experiments TSV") + + # Average across dates -> study="EXACT" + exact_exp_dt <- tryCatch(data.table::fread(exact_exp_path), error = function(e) data.table::data.table()) + if (nrow(exact_exp_dt) > 0) { + if ("Drug" %in% names(exact_exp_dt) && !"improve_drug_id" %in% names(exact_exp_dt)) { + data.table::setnames(exact_exp_dt, "Drug", "improve_drug_id") + } + + req_cols_ex <- c("source","improve_sample_id","improve_drug_id","study","time","time_unit","dose_response_metric","dose_response_value") + if (all(req_cols_ex %in% names(exact_exp_dt))) { + exact_exp_dt[, dose_response_value := suppressWarnings(as.numeric(as.character(dose_response_value)))] + exact_exp_dt <- exact_exp_dt[is.finite(dose_response_value)] + + key_cols_ex <- c("source","improve_sample_id","improve_drug_id","time","time_unit","dose_response_metric") + + exact_agg <- exact_exp_dt[, .( + study = "EXACT", + dose_response_value = mean(dose_response_value, na.rm = TRUE) + ), by = key_cols_ex] + + data.table::fwrite(exact_agg, exact_exp_path, sep = "\t") + } + } + + message("Wrote EXACT experiments") + + # Append EXACT into combined experiments + exp_path2 <- file.path("/tmp", paste0(out_prefix, "_experiments.tsv")) + if (file.exists(exp_path2) && file.exists(exact_exp_path)) { + exp_dt2 <- data.table::fread(exp_path2) + ex_dt2 <- data.table::fread(exact_exp_path) + + missing_cols2 <- setdiff(names(exp_dt2), names(ex_dt2)) + for (cc in missing_cols2) ex_dt2[[cc]] <- NA + + extra_cols2 <- setdiff(names(ex_dt2), names(exp_dt2)) + if (length(extra_cols2) > 0) ex_dt2 <- ex_dt2[, setdiff(names(ex_dt2), extra_cols2), with = FALSE] + + data.table::setcolorder(ex_dt2, names(exp_dt2)) + + for (cc in intersect(c("improve_sample_id", "time"), names(exp_dt2))) { + exp_dt2[[cc]] <- suppressWarnings(as.integer(exp_dt2[[cc]])) + ex_dt2[[cc]] <- suppressWarnings(as.integer(ex_dt2[[cc]])) + } + for (cc in intersect(c("dose_response_value"), names(exp_dt2))) { + exp_dt2[[cc]] <- suppressWarnings(as.numeric(exp_dt2[[cc]])) + ex_dt2[[cc]] <- suppressWarnings(as.numeric(ex_dt2[[cc]])) + } + + before_n2 <- nrow(exp_dt2) + add_n2 <- nrow(ex_dt2) + + final_dt2 <- data.table::rbindlist(list(exp_dt2, ex_dt2), use.names = TRUE, fill = TRUE) + final_dt2 <- unique(final_dt2) + + bak_path2 <- paste0(exp_path2, ".bak_exact") + file.copy(exp_path2, bak_path2, overwrite = TRUE) + data.table::fwrite(final_dt2, exp_path2, sep = "\t") + + message(sprintf( + "Appended EXACT into %s: +%d rows (was %d, now %d). Backup: %s", + exp_path2, add_n2, before_n2, nrow(final_dt2), bak_path2 + )) + } + } + + } else { + dbg("EXACT: no rows found; skipping curve fit + append.") + } + + } else { + dbg_hr("SKIP_EXACT=TRUE") + cat0("SKIP_EXACT=TRUE: skipping EXACT section.") + } + + +} else { + dbg_hr("ALL SKIP SWITCHES=TRUE") + cat0("All skip switches are TRUE: skipping MT + PDX + combine + EXACT sections (debugging treated combos only).") + cat0("Will still read: samples.csv, drugfile.tsv, and syn69801348.") +} + +# ============================================================ +# Treated-combination experiments from syn69801348 (auc ONLY - note we can change to fit_auc if needed) +# ============================================================ +TREATED_COMBO_SYNID <- "syn69801348" +dbg_hr(paste0("TREATED COMBO SECTION from ", TREATED_COMBO_SYNID)) + +combo_path <- synGet(TREATED_COMBO_SYNID)$path +dbg_file(combo_path, "syn69801348 downloaded file") +combo_raw <- fread(combo_path) + +needed_cols <- c("source","improve_sample_id","improve_drug_id","study","time","time_unit","dose_response_metric","dose_response_value") +missing_cols <- setdiff(needed_cols, names(combo_raw)) +if (length(missing_cols) > 0) { + stop("syn69801348 is missing required columns: ", paste(missing_cols, collapse=", "), call. = FALSE) +} + +dbg_df(as.data.frame(combo_raw), "combo_raw (pre-filter)") + +combo_raw <- combo_raw %>% + mutate( + combo_key = as.character(improve_sample_id), + improve_sample_id = as.character(improve_sample_id), + improve_drug_id = as.character(improve_drug_id), + study = as.character(study), + time = suppressWarnings(as.integer(time)), + time_unit = as.character(time_unit), + dose_response_metric = as.character(dose_response_metric), + dose_response_value = suppressWarnings(as.numeric(dose_response_value)) + ) %>% + filter(tolower(dose_response_metric) == "auc") %>% # This is auc only - can change to fit_auc if needed. + filter(!is.na(time), !is.na(dose_response_value)) %>% + distinct() + +dbg("syn69801348 rows (auc): ", nrow(combo_raw)) +dbg("combo_raw cols: ", paste(names(combo_raw), collapse=", ")) +dbg("combo_raw head():") +print(head(combo_raw, 6)) + +# ---- parse combo_key into base_sample + drug1/drug2 from combo_key +if (nrow(combo_raw) > 0) { + parsed <- lapply(combo_raw$combo_key, parse_combo_key) + combo_raw$base_sample <- vapply(parsed, `[[`, character(1), "base") + combo_raw$drug1_raw <- vapply(parsed, `[[`, character(1), "drug1") + combo_raw$drug2_raw <- vapply(parsed, `[[`, character(1), "drug2") +} else { + combo_raw$base_sample <- character() + combo_raw$drug1_raw <- character() + combo_raw$drug2_raw <- character() +} + +parse_fail <- combo_raw %>% filter(is.na(base_sample) | is.na(drug1_raw) | is.na(drug2_raw)) +if (nrow(parse_fail) > 0) { + warning("Some combo_key values failed parsing (showing up to 10):") + print(head(parse_fail$combo_key, 10)) + dbg <- file.path("/tmp", paste0(out_prefix, "_treated_parse_fail.tsv")) + fwrite(as.data.table(parse_fail), dbg, sep = "\t") + cat0("Wrote: ", dbg) +} +combo_raw <- combo_raw %>% filter(!is.na(base_sample), !is.na(drug1_raw), !is.na(drug2_raw)) +dbg("rows after combo_key parse filter: ", nrow(combo_raw)) + +combo_raw <- combo_raw %>% + mutate( + drug1_canon = canon_name(drug1_raw), + drug2_canon = canon_name(drug2_raw), + drug_canon = canon_name(improve_drug_id), + is_combo_row = grepl("\\+", improve_drug_id) + ) + +# ============================================================ +# Treated samples mapping +# ============================================================ +dbg_hr("TREATED SAMPLES MAPPING") + +treated_samples_long <- samples_all %>% + mutate( + other_id = as.character(other_id), + model_type = as.character(model_type), + improve_sample_id = as.integer(improve_sample_id), + common_name = as.character(common_name) + ) %>% + filter( + model_type == "3D-MEDS", + !is.na(other_id), + grepl("hr_treated_microtissue$", other_id) + ) %>% + mutate( + base_common_name = common_name, + base_prefix = base_to_otherid_prefix(common_name), + treated_token = mapply(extract_treated_token_from_other_id, other_id, common_name), + treated_token_u = toupper(safe_token(treated_token)) + ) %>% + select( + base_common_name, + treated_token, + treated_token_u, + treated_other_id = other_id, + treated_improve_sample_id = improve_sample_id + ) %>% + distinct() + +dbg("treated_samples_long rows: ", nrow(treated_samples_long)) +dbg("treated_samples_long head():") +print(head(treated_samples_long, 10)) + +failed_token <- treated_samples_long %>% filter(is.na(treated_token) | treated_token == "") +if (nrow(failed_token) > 0) { + warning("Some treated other_id rows failed treated_token extraction (showing up to 20):") + print(head(failed_token, 20)) + dbg_fail_tok <- file.path("/tmp", paste0(out_prefix, "_treated_token_extract_fail.tsv")) + fwrite(as.data.table(failed_token), dbg_fail_tok, sep = "\t") + cat0("Wrote token extraction failures: ", dbg_fail_tok) +} + +# ============================================================ +# Build chem_tbl + mapping token -> improve_drug_id +# ============================================================ +dbg_hr("TREATED TOKEN -> DRUG ID MAPPING") + +chem_tbl <- drug_map %>% + transmute( + improve_drug_id = as.character(improve_drug_id), + chem_name = as.character(chem_name), + chem_canon = canon_name(chem_name), + id_canon = canon_id(improve_drug_id), + abbrev3 = substr(toupper(gsub("[^A-Za-z]+","",chem_name)), 1, 3), + chem_token_u = toupper(safe_token(chem_name)) + ) %>% + distinct() + +dbg_df(chem_tbl, "chem_tbl") + +treated_tokens <- treated_samples_long %>% + distinct(treated_token, treated_token_u) %>% + mutate( + treated_token_canon = canon_name(treated_token), + treated_token3 = substr(toupper(gsub("[^A-Za-z]+","",treated_token)), 1, 3) + ) + +dbg_df(treated_tokens, "treated_tokens") + +# These were updated on synapse so this isn't needed now. +manual_override <- c( + # "DOX" = "doxorubicin", + # "TNO" = "TNO155" +) + +override_tbl <- data.frame( + treated_token_u = character(0), + override_val = character(0), + override_id = character(0), + stringsAsFactors = FALSE +) + +if (length(manual_override) > 0) { + override_tbl <- data.frame( + treated_token_u = names(manual_override), + override_val = as.character(manual_override), + stringsAsFactors = FALSE + ) + resolve_override_to_id <- function(v) { + if (is.na(v) || trimws(v) == "") return(NA_character_) + v0 <- as.character(v) + hit_id <- chem_tbl$improve_drug_id[chem_tbl$id_canon == canon_id(v0)] + if (length(hit_id) > 0) return(hit_id[1]) + hit_nm <- chem_tbl$improve_drug_id[chem_tbl$chem_canon == canon_name(v0)] + if (length(hit_nm) > 0) return(hit_nm[1]) + NA_character_ + } + override_tbl$override_id <- vapply(override_tbl$override_val, resolve_override_to_id, character(1)) +} + +map_by_canon <- treated_tokens %>% + left_join(chem_tbl %>% select(improve_drug_id, chem_canon), + by = c("treated_token_canon" = "chem_canon")) %>% + transmute(treated_token_u, canon_id = improve_drug_id) + +map_by_token <- treated_tokens %>% + left_join(chem_tbl %>% select(improve_drug_id, chem_token_u), + by = c("treated_token_u" = "chem_token_u")) %>% + transmute(treated_token_u, token_id = improve_drug_id) + +map_by_abbrev3 <- treated_tokens %>% + left_join(chem_tbl %>% select(improve_drug_id, abbrev3), + by = c("treated_token3" = "abbrev3"), relationship = "many-to-many") %>% + transmute(treated_token_u, abbr_id = improve_drug_id) + +map_by_canon <- warn_first_by(map_by_canon, c("treated_token_u"), "treated_drug_map_canon") +map_by_token <- warn_first_by(map_by_token, c("treated_token_u"), "treated_drug_map_token") +map_by_abbrev3 <- warn_first_by(map_by_abbrev3, c("treated_token_u"), "treated_drug_map_abbrev3") + +drug_token_map <- treated_tokens %>% + left_join(override_tbl %>% select(treated_token_u, override_id), by = "treated_token_u") %>% + left_join(map_by_canon, by = "treated_token_u") %>% + left_join(map_by_token, by = "treated_token_u") %>% + left_join(map_by_abbrev3, by = "treated_token_u") %>% + mutate(mapped_improve_drug_id = dplyr::coalesce(override_id, canon_id, token_id, abbr_id)) %>% + select(treated_token, treated_token_u, mapped_improve_drug_id) + +drug_token_map <- warn_first_by(drug_token_map, c("treated_token_u"), "treated_drug_map_final") + +mapping_out <- file.path("/tmp", paste0(out_prefix, "_treated_drug_mapping.tsv")) +fwrite(as.data.table(drug_token_map), mapping_out, sep = "\t") +dbg("Wrote treated drug mapping debug: ", mapping_out) +dbg_df(drug_token_map, "drug_token_map") + +treated_samples_mapped <- treated_samples_long %>% + left_join(drug_token_map %>% select(treated_token_u, mapped_improve_drug_id), by = "treated_token_u") %>% + filter(!is.na(mapped_improve_drug_id)) %>% + mutate(mapped_improve_drug_id = as.character(mapped_improve_drug_id)) + +dbg("treated_samples_mapped rows: ", nrow(treated_samples_mapped)) +dbg_df(treated_samples_mapped, "treated_samples_mapped") + +# ============================================================ +# Build pairing tables +# ============================================================ +dbg_hr("PAIRING auc SINGLE vs COMBO") + +ctx_cols <- c("combo_key","base_sample","time","source","study","time_unit") + +combo_ctx <- combo_raw %>% + filter(is_combo_row) %>% + transmute( + combo_key, base_sample, + drug1_raw, drug2_raw, drug1_canon, drug2_canon, + time, source, study, time_unit, + auc_combo = dose_response_value + ) + +single_ctx <- combo_raw %>% + filter(!is_combo_row) %>% + transmute( + combo_key, base_sample, + time, source, study, time_unit, + drugA_raw = improve_drug_id, + drugA_canon = drug_canon, + auc_A = dose_response_value + ) + +dbg_df(combo_ctx, "combo_ctx") +dbg_df(single_ctx, "single_ctx") + +combo_ctx <- warn_first_by(combo_ctx, ctx_cols, "syn69801348_combo_auc") +single_ctx <- warn_first_by(single_ctx, c(ctx_cols,"drugA_canon"), "syn69801348_single_auc") + +paired <- single_ctx %>% + inner_join(combo_ctx, by = ctx_cols) %>% + mutate( + drugB_raw = dplyr::case_when( + drugA_canon == drug1_canon ~ drug2_raw, + drugA_canon == drug2_canon ~ drug1_raw, + TRUE ~ NA_character_ + ), + drugB_canon = canon_name(drugB_raw), + delta_auc = auc_A - auc_combo + ) + +dbg("paired rows: ", nrow(paired)) +dbg_df(paired, "paired") + +dbg_pairs <- file.path("/tmp", paste0(out_prefix, "_treated_pairs_debug.tsv")) +fwrite(as.data.table(paired), dbg_pairs, sep = "\t") +dbg("Wrote paired debug: ", dbg_pairs) + +if (nrow(paired) == 0) { + dbg_stop("paired is empty. Investigate /tmp/*_treated_pairs_debug.tsv and parse failures.") +} + +# ============================================================ +# Map canonical -> improve_drug_id (primary), then fallback regex +# ============================================================ +dbg_hr("MAP drugA/drugB to improve_drug_id") + +chem_to_id <- chem_tbl %>% distinct(chem_canon, improve_drug_id) +chem_to_id <- warn_first_by(chem_to_id, c("chem_canon"), "chem_to_id") + +paired <- paired %>% + left_join(chem_to_id %>% rename(drugA_id = improve_drug_id), by = c("drugA_canon" = "chem_canon")) %>% + left_join(chem_to_id %>% rename(drugB_id = improve_drug_id), by = c("drugB_canon" = "chem_canon")) + +missingA_before <- sum(is.na(paired$drugA_id)) +missingB_before <- sum(is.na(paired$drugB_id)) +dbg("Missing drugA_id before fallback: ", missingA_before, " / ", nrow(paired)) +dbg("Missing drugB_id before fallback: ", missingB_before, " / ", nrow(paired)) + +paired <- fill_ids_with_fallback(paired, raw_col = "drugA_raw", id_col = "drugA_id", chem_tbl = chem_tbl, label = "drugA_id") +paired <- fill_ids_with_fallback(paired, raw_col = "drugB_raw", id_col = "drugB_id", chem_tbl = chem_tbl, label = "drugB_id") + +missingA_after <- sum(is.na(paired$drugA_id)) +missingB_after <- sum(is.na(paired$drugB_id)) +dbg("Missing drugA_id after fallback: ", missingA_after, " / ", nrow(paired)) +dbg("Missing drugB_id after fallback: ", missingB_after, " / ", nrow(paired)) + +fallback_dbg <- paired %>% + transmute( + combo_key, base_sample, time, source, study, time_unit, + drugA_raw, drugA_canon, drugA_id, + drugB_raw, drugB_canon, drugB_id, + delta_auc, + drugA_id_missing = is.na(drugA_id), + drugB_id_missing = is.na(drugB_id) + ) + +fallback_dbg_path <- file.path("/tmp", paste0(out_prefix, "_treated_drug_id_fallback_debug.tsv")) +fwrite(as.data.table(fallback_dbg), fallback_dbg_path, sep = "\t") +dbg("Wrote drug-id fallback debug: ", fallback_dbg_path) + +dbg("paired with IDs mapped head():") +print(head(paired %>% select(combo_key, base_sample, time, drugA_raw, drugA_id, drugB_raw, drugB_id, delta_auc), 10)) + +# ============================================================ +# Join to treated samples +# ============================================================ +dbg_hr("JOIN paired -> treated samples") + +treated_joined <- paired %>% + filter(!is.na(drugA_id), !is.na(drugB_id)) %>% + inner_join( + treated_samples_mapped %>% + transmute( + base_sample = base_common_name, + drugA_id = mapped_improve_drug_id, + treated_improve_sample_id, + treated_other_id, + treated_token + ), + by = c("base_sample","drugA_id") + ) + +dbg("treated_joined rows: ", nrow(treated_joined)) +dbg_df(treated_joined, "treated_joined") + +dbg_joined <- file.path("/tmp", paste0(out_prefix, "_treated_joined_debug.tsv")) +fwrite(as.data.table(treated_joined), dbg_joined, sep = "\t") +dbg("Wrote treated_joined debug: ", dbg_joined) + +if (nrow(treated_joined) == 0) { + dbg("EARLY EXIT: treated_joined is empty. Likely token->drugA_id mapping mismatch.") + dbg("Example base_sample from paired: ", paste(head(unique(paired$base_sample), 10), collapse=", ")) + dbg("Example base_common_name from treated_samples_mapped: ", paste(head(unique(treated_samples_mapped$base_common_name), 10), collapse=", ")) + dbg("Example drugA_id from paired: ", paste(head(unique(paired$drugA_id), 10), collapse=", ")) + dbg("Example mapped_improve_drug_id: ", paste(head(unique(treated_samples_mapped$mapped_improve_drug_id), 10), collapse=", ")) + quit(save="no", status=0) +} + +# ============================================================ +# Outputs +# ============================================================ +dbg_hr("WRITE TREATED OUTPUTS") + +treated_out_ids <- treated_joined %>% + transmute( + source = source, + improve_sample_id = as.integer(treated_improve_sample_id), + improve_drug_id = as.character(drugB_id), + study = study, + time = as.integer(time), + time_unit = time_unit, + dose_response_metric = "delta_auc", + dose_response_value = as.numeric(delta_auc) + ) %>% + distinct() + +treated_ids_path <- file.path("/tmp", paste0(out_prefix, "_experiment_combinations_treated.tsv")) +fwrite(as.data.table(treated_out_ids), treated_ids_path, sep = "\t") +dbg("Wrote treated combination experiments (IDs): ", treated_ids_path, " [rows=", nrow(treated_out_ids), "]") +dbg_file(treated_ids_path, "treated IDs TSV") + +treated_out_names <- treated_joined %>% + transmute( + source = source, + improve_sample_id = as.character(combo_key), + improve_drug_id = as.character(drugB_raw), + study = study, + time = as.integer(time), + time_unit = time_unit, + dose_response_metric = "delta_auc", + dose_response_value = as.numeric(delta_auc) + ) %>% + distinct() + +treated_names_path <- file.path("/tmp", paste0(out_prefix, "_experiment_combinations_treated_names.tsv")) +fwrite(as.data.table(treated_out_names), treated_names_path, sep = "\t") +dbg("Wrote treated combination experiments (NAMES): ", treated_names_path, " [rows=", nrow(treated_out_names), "]") +dbg_file(treated_names_path, "treated NAMES TSV") + +# ============================================================ +# FINAL: append treated-combo experiments into main experiments +# ============================================================ +dbg_hr("FINAL APPEND treated -> experiments") + +exp_path <- file.path("/tmp", paste0(out_prefix, "_experiments.tsv")) +trt_path <- treated_ids_path + +if (!file.exists(exp_path)) stop("Missing experiments file: ", exp_path, call. = FALSE) +if (!file.exists(trt_path)) stop("Missing treated-combos file: ", trt_path, call. = FALSE) + +exp_dt <- data.table::fread(exp_path) +trt_dt <- data.table::fread(trt_path) + +dbg("exp_dt rows=", nrow(exp_dt), " cols=", ncol(exp_dt)) +dbg("trt_dt rows=", nrow(trt_dt), " cols=", ncol(trt_dt)) + +missing_cols <- setdiff(names(exp_dt), names(trt_dt)) +for (cc in missing_cols) trt_dt[[cc]] <- NA + +extra_cols <- setdiff(names(trt_dt), names(exp_dt)) +if (length(extra_cols) > 0) trt_dt <- trt_dt[, setdiff(names(trt_dt), extra_cols), with = FALSE] + +data.table::setcolorder(trt_dt, names(exp_dt)) + +for (cc in intersect(c("improve_sample_id", "time"), names(exp_dt))) { + exp_dt[[cc]] <- suppressWarnings(as.integer(exp_dt[[cc]])) + trt_dt[[cc]] <- suppressWarnings(as.integer(trt_dt[[cc]])) +} +for (cc in intersect(c("dose_response_value"), names(exp_dt))) { + exp_dt[[cc]] <- suppressWarnings(as.numeric(exp_dt[[cc]])) + trt_dt[[cc]] <- suppressWarnings(as.numeric(trt_dt[[cc]])) +} + +before_n <- nrow(exp_dt) +add_n <- nrow(trt_dt) + +final_dt <- data.table::rbindlist(list(exp_dt, trt_dt), use.names = TRUE, fill = TRUE) +final_dt <- unique(final_dt) + +bak_path <- paste0(exp_path, ".bak") +file.copy(exp_path, bak_path, overwrite = TRUE) +data.table::fwrite(final_dt, exp_path, sep = "\t") + +message(sprintf( + "Appended treated combos into %s: +%d rows (was %d, now %d). Backup: %s", + exp_path, add_n, before_n, nrow(final_dt), bak_path +)) + +dbg_hr("SCRIPT END") \ No newline at end of file diff --git a/coderbuild/mpnst/build_drugs.sh b/coderbuild/mpnst/build_drugs.sh index 821bc46c..d0932886 100644 --- a/coderbuild/mpnst/build_drugs.sh +++ b/coderbuild/mpnst/build_drugs.sh @@ -1,3 +1,4 @@ +# build_drugs.sh #!/bin/bash #set -euo pipefail diff --git a/coderbuild/mpnst/build_exp.sh b/coderbuild/mpnst/build_exp.sh index 14506cfe..2620c8eb 100644 --- a/coderbuild/mpnst/build_exp.sh +++ b/coderbuild/mpnst/build_exp.sh @@ -1,9 +1,10 @@ +# build_exp.sh #!/bin/bash set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR -echo "Running 03_get_experiments.R with $SYNAPSE_AUTH_TOKEN, $1, and $2." -Rscript 03_get_experiments.R $SYNAPSE_AUTH_TOKEN $1 $2 mpnst -rm /tmp/mpnst_pdx_experiments.tsv /tmp/mpnst_mt_experiments.tsv /tmp/mpnst_mt_curve_data.tsv /tmp/mpnst_pdx_curve_data.tsv +echo "Running 03_get_experiments_with_treated.R with $SYNAPSE_AUTH_TOKEN, $1, and $2." +Rscript 03_get_experiments_with_treated.R $SYNAPSE_AUTH_TOKEN $1 $2 mpnst +# rm /tmp/mpnst_pdx_experiments.tsv /tmp/mpnst_mt_experiments.tsv /tmp/mpnst_mt_curve_data.tsv /tmp/mpnst_pdx_curve_data.tsv diff --git a/coderbuild/mpnst/build_omics.sh b/coderbuild/mpnst/build_omics.sh index d6d2cec7..c6b0853c 100644 --- a/coderbuild/mpnst/build_omics.sh +++ b/coderbuild/mpnst/build_omics.sh @@ -1,3 +1,4 @@ +# build_omics.sh #!/bin/bash set -euo pipefail @@ -5,3 +6,5 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit echo "Running 01_combined_omics.R with $SYNAPSE_AUTH_TOKEN, $2, and $1." Rscript 01_combined_omics.R $SYNAPSE_AUTH_TOKEN $2 $1 +echo "Running 01b_combined_omics_treated.R with PAT, samples=$2, genes=$1" +Rscript 01b_combined_omics_treated.R "$SYNAPSE_AUTH_TOKEN" "$2" "$1" \ No newline at end of file diff --git a/coderbuild/mpnst/build_samples.sh b/coderbuild/mpnst/build_samples.sh index c9c079fa..832ed895 100644 --- a/coderbuild/mpnst/build_samples.sh +++ b/coderbuild/mpnst/build_samples.sh @@ -1,3 +1,4 @@ +# build_samples.sh #!/bin/bash set -euo pipefail @@ -5,3 +6,4 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit echo "Running 00_sample_gen.R with $1." Rscript 00_sample_gen.R $1 +Rscript 00b_sample_gen_treated.R /tmp/mpnst_samples.csv diff --git a/coderbuild/mpnst/requirements.r b/coderbuild/mpnst/requirements.r index e8bfac35..44e0ea0f 100755 --- a/coderbuild/mpnst/requirements.r +++ b/coderbuild/mpnst/requirements.r @@ -11,3 +11,12 @@ install.packages("stringr") install.packages("tidyr") install.packages("readr") install.packages("readxl") + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("reticulate", "remotes", "rjson", "synapser", "dplyr", "data.table", "R.utils", "stringr", "tidyr", "readr", "readxl") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/phosphosites/00-buildPhosphositeFile.py b/coderbuild/phosphosites/00-buildPhosphositeFile.py new file mode 100644 index 00000000..09292dec --- /dev/null +++ b/coderbuild/phosphosites/00-buildPhosphositeFile.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 +""" +00-buildPhosphositeFile.py: build the phosphosites reference CSV from three sources: + + 1. Ochoa et al. (2020) Nat Biotechnol 38, 365-373: ~116k curated human + phosphosites from Supplementary Table S3 (Springer static content). + https://static-content.springer.com/esm/art%3A10.1038%2Fs41587-019-0344-3/ + MediaObjects/41587_2019_344_MOESM4_ESM.xlsx + + 2. UniProt PTM annotations: all reviewed human proteins with phospho + modifications from the UniProt REST API (~41k sites, ~12k unique to UniProt). + + 3. Synapse supplement(s): raw phosphoproteomics CSV/TSV files on Synapse + whose site column is parsed to add experiment-specific sites not in the + databases above (--synapse_supplements syn70078415 ...). + +Sources 1 and 2 are always fetched. Source 3 is optional but recommended. + +UniProt accession -> gene_symbol: UniProt REST API (TSV, accession+gene_names). +gene_symbol -> entrez_id: genes.csv. + +Site notation: - + e.g. AAAS-S495s (s=phosphoserine, t=phosphothreonine, y=phosphotyrosine) + +Output columns: + phosphosite_id, entrez_id, gene_symbol, residue, position, modification, other_id + +Usage: + python 00-buildPhosphositeFile.py + [--out /tmp/phosphosites.csv] + [--prev /tmp/prev_phosphosites.csv] + [--supplement site_file.csv ...] + [--site_col site] + [--synapse_supplements syn70078415 ...] + [--synapse_site_col site] + [--ochoa_url ] + [--uniprot_gene_url ] + [--uniprot_ptm_url ] +""" + +import argparse +import csv +import io +import json +import logging +import re +import sys +import tempfile +import time +import urllib.request + +import openpyxl +import pandas as pd + + +OCHOA_URL = ( + "https://static-content.springer.com/esm/" + "art%3A10.1038%2Fs41587-019-0344-3/MediaObjects/" + "41587_2019_344_MOESM4_ESM.xlsx" +) + +UNIPROT_GENE_URL = ( + "https://rest.uniprot.org/uniprotkb/search" + "?query=reviewed:true+AND+organism_id:9606" + "&fields=accession,gene_names" + "&format=tsv" + "&size=500" +) + +UNIPROT_PTM_URL = ( + "https://rest.uniprot.org/uniprotkb/search" + "?query=reviewed:true+AND+organism_id:9606+AND+ft_mod_res:phospho*" + "&fields=accession,gene_names,ft_mod_res" + "&format=json" + "&size=200" +) + +_RESIDUE_MOD = {"S": "s", "T": "t", "Y": "y"} +_PHOSPHO_DESC_MAP = { + "phosphoserine": ("S", "s"), + "phosphothreonine": ("T", "t"), + "phosphotyrosine": ("Y", "y"), +} +_SITE_RE = re.compile(r'^(.+)-([STY])(\d+)([a-z]+)$', re.IGNORECASE) +_SUPP_MOD_CODES = {'s': 'phospho', 't': 'phospho', 'y': 'phospho'} + +_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0" + ), + "Accept": "*/*", +} + + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# --------------------------------------------------------------------------- +# Reference data +# --------------------------------------------------------------------------- +def load_gene_map(genes_path: str) -> pd.DataFrame: + df = pd.read_csv(genes_path) + needed = {'gene_symbol', 'entrez_id'} + if needed - set(df.columns): + raise KeyError(f"genes file missing columns: {needed - set(df.columns)}") + df = df.dropna(subset=['gene_symbol', 'entrez_id']).copy() + df['entrez_id'] = df['entrez_id'].astype(int) + df = df.drop_duplicates(subset='gene_symbol')[['gene_symbol', 'entrez_id']] + logging.info("Loaded %d gene_symbol → entrez_id mappings", len(df)) + return df + + +# --------------------------------------------------------------------------- +# Network helper +# --------------------------------------------------------------------------- +def _fetch_url(url: str, retries: int = 3, timeout: int = 120) -> bytes: + req = urllib.request.Request(url, headers=_HEADERS) + for attempt in range(1, retries + 1): + try: + logging.info(" GET %s (attempt %d/%d)", url, attempt, retries) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read() + except Exception as exc: + logging.warning(" Attempt %d failed: %s", attempt, exc) + if attempt < retries: + time.sleep(5 * attempt) + raise RuntimeError(f"Failed to fetch {url} after {retries} attempts") + + +def _fetch_json_page(url: str, timeout: int = 120) -> tuple[list[dict], str | None]: + req = urllib.request.Request(url, headers={**_HEADERS, "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode('utf-8')) + link_header = resp.headers.get('Link', '') + entries = data.get('results', []) + m = re.search(r'<([^>]+)>;\s*rel="next"', link_header) + return entries, m.group(1) if m else None + + +# --------------------------------------------------------------------------- +# Source 1: Ochoa et al. supplementary table +# --------------------------------------------------------------------------- +def fetch_ochoa_phosphosites(url: str) -> list[dict]: + """Download Ochoa MOESM4 xlsx and return raw phosphosite records.""" + logging.info("Downloading Ochoa et al. supplementary phosphoproteome") + data = _fetch_url(url) + + with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tmp: + tmp.write(data) + tmp_path = tmp.name + + logging.info("Parsing xlsx (%d MB) …", len(data) // 1_000_000) + wb = openpyxl.load_workbook(tmp_path, read_only=True, data_only=True) + + sheet_name = "annotated_phosphoproteome" + if sheet_name not in wb.sheetnames: + wb.close() + raise ValueError( + f"Sheet '{sheet_name}' not found. Available: {wb.sheetnames}" + ) + + ws = wb[sheet_name] + records, header, uni_i, pos_i, res_i, skipped = [], None, None, None, None, 0 + + for row in ws.iter_rows(values_only=True): + if header is None: + header = [str(c).strip() if c is not None else "" for c in row] + try: + uni_i = header.index("uniprot") + pos_i = header.index("position") + res_i = header.index("residue") + except ValueError as exc: + wb.close() + raise ValueError(f"Required column missing in Ochoa sheet: {exc}") + logging.info( + "Ochoa columns: uniprot=%d, position=%d, residue=%d", + uni_i, pos_i, res_i, + ) + continue + + acc = str(row[uni_i]).strip() if row[uni_i] is not None else "" + try: + pos = int(row[pos_i]) + except (TypeError, ValueError): + skipped += 1 + continue + + res_raw = str(row[res_i]).strip().upper() if row[res_i] is not None else "" + residue = res_raw[0] if res_raw and res_raw[0] in _RESIDUE_MOD else None + + if residue is None or not acc: + skipped += 1 + continue + + records.append({ + "accession": acc, + "residue": residue, + "position": pos, + "modification": "phospho", + "mod_code": _RESIDUE_MOD[residue], + }) + + wb.close() + logging.info( + "Ochoa: %d records parsed (%d skipped)", len(records), skipped + ) + return records + + +# --------------------------------------------------------------------------- +# Source 2: UniProt PTM annotations +# --------------------------------------------------------------------------- +def fetch_uniprot_ptm_sites(url: str) -> list[dict]: + """Fetch phosphorylation sites from UniProt Modified residue features. + + Returns records with gene_symbol (already resolved) instead of accession. + """ + logging.info("Fetching UniProt PTM phosphorylation sites") + all_records = [] + next_url: str | None = url + page = 0 + + while next_url: + page += 1 + if page % 10 == 1: + logging.info(" UniProt PTM page %d (%d records so far)", page, len(all_records)) + + for attempt in range(1, 4): + try: + entries, next_url = _fetch_json_page(next_url) + break + except Exception as exc: + logging.warning(" Page %d attempt %d failed: %s", page, attempt, exc) + if attempt == 3: + raise + time.sleep(5 * attempt) + + for entry in entries: + gene = "" + for g in entry.get("genes", []): + if "geneName" in g: + gene = g["geneName"].get("value", "").upper() + break + if not gene: + continue + for feat in entry.get("features", []): + if feat.get("type") != "Modified residue": + continue + desc = feat.get("description", "").lower() + residue, mod_code = None, None + for key, (aa, mc) in _PHOSPHO_DESC_MAP.items(): + if desc.startswith(key): + residue, mod_code = aa, mc + break + if residue is None: + continue + pos = feat.get("location", {}).get("start", {}).get("value") + if pos is None: + continue + all_records.append({ + "gene_symbol": gene, + "residue": residue, + "position": int(pos), + "modification": "phospho", + "mod_code": mod_code, + "other_id": f"{gene}-{residue}{pos}{mod_code}", + }) + + logging.info( + "UniProt PTM: %d records from %d pages", len(all_records), page + ) + return all_records + + +# --------------------------------------------------------------------------- +# UniProt accession → gene_symbol +# --------------------------------------------------------------------------- +def fetch_uniprot_gene_map(url: str) -> dict[str, str]: + """Return accession → primary gene_symbol for all reviewed human proteins.""" + logging.info("Fetching UniProt accession → gene_symbol map") + acc_to_gene: dict[str, str] = {} + next_url: str | None = url + page = 0 + + while next_url: + page += 1 + if page % 10 == 1: + logging.info( + " UniProt gene map page %d (%d entries so far)", page, len(acc_to_gene) + ) + for attempt in range(1, 4): + try: + req = urllib.request.Request(next_url, headers=_HEADERS) + with urllib.request.urlopen(req, timeout=120) as resp: + raw = resp.read() + link_header = resp.headers.get("Link", "") + break + except Exception as exc: + logging.warning(" Page %d attempt %d failed: %s", page, attempt, exc) + if attempt == 3: + raise + time.sleep(5 * attempt) + + reader = csv.DictReader( + io.StringIO(raw.decode("utf-8", errors="replace")), delimiter="\t" + ) + for row in reader: + acc = row.get("Entry", row.get("accession", "")).strip() + gene_names = row.get("Gene Names", row.get("gene_names", "")).strip() + if acc and gene_names: + acc_to_gene[acc] = gene_names.split()[0].upper() + + m = re.search(r'<([^>]+)>;\s*rel="next"', link_header) + next_url = m.group(1) if m else None + + logging.info( + "UniProt gene map: %d entries (%d pages)", len(acc_to_gene), page + ) + return acc_to_gene + + +# --------------------------------------------------------------------------- +# Source 3: Synapse supplement(s) +# --------------------------------------------------------------------------- +def fetch_synapse_supplement(syn_id: str, site_col: str) -> list[dict]: + """Download a Synapse file and parse its site strings as supplement records.""" + import synapseclient + logging.info("Fetching Synapse supplement %s", syn_id) + syn = synapseclient.Synapse() + syn.login(silent=True) + entity = syn.get(syn_id) + path = entity.path + sep = "\t" if path.endswith((".tsv", ".tsv.gz")) else "," + + import gzip as _gzip + open_fn = _gzip.open if path.endswith(".gz") else open + + records = [] + skipped = 0 + seen: set[str] = set() + resolved_col: str | None = None + + with open_fn(path, "rt", encoding="utf-8", errors="replace") as f: + reader = csv.DictReader(f, delimiter=sep) + for row in reader: + # Auto-detect site column on first row if requested column missing + if resolved_col is None: + if site_col in row: + resolved_col = site_col + else: + for candidate in ["site", "Site", "phosphosite", "feature_id"]: + if candidate in row: + resolved_col = candidate + logging.info( + " Synapse %s: using column '%s' for sites", + syn_id, resolved_col, + ) + break + if resolved_col is None: + logging.warning( + " Synapse %s: could not find site column (tried %s). " + "Available: %s", + syn_id, site_col, list(row.keys())[:10], + ) + return [] + + site = row.get(resolved_col, "").strip() + if not site or site in seen: + continue + seen.add(site) + m = _SITE_RE.match(site) + if not m: + skipped += 1 + continue + mod_code = m.group(4).lower() + records.append({ + "gene_symbol": m.group(1).upper(), + "residue": m.group(2).upper(), + "position": int(m.group(3)), + "modification": _SUPP_MOD_CODES.get(mod_code, f"phospho_{mod_code}"), + "mod_code": mod_code, + "other_id": site, + }) + + logging.info( + "Synapse %s: %d unique sites parsed (%d skipped)", syn_id, len(records), skipped + ) + return records + + +# --------------------------------------------------------------------------- +# Local file supplement +# --------------------------------------------------------------------------- +def parse_supplement_sites(path: str, site_col: str) -> list[dict]: + records = [] + skipped = 0 + sep = "\t" if path.endswith(".tsv") else "," + with open(path, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f, delimiter=sep) + seen: set[str] = set() + for row in reader: + site = row.get(site_col, "").strip() + if not site or site in seen: + continue + seen.add(site) + m = _SITE_RE.match(site) + if not m: + skipped += 1 + continue + mod_code = m.group(4).lower() + records.append({ + "gene_symbol": m.group(1).upper(), + "residue": m.group(2).upper(), + "position": int(m.group(3)), + "modification": _SUPP_MOD_CODES.get(mod_code, f"phospho_{mod_code}"), + "mod_code": mod_code, + "other_id": site, + }) + logging.info( + "Supplement %s: %d unique sites (%d skipped)", path, len(records), skipped + ) + return records + + +# --------------------------------------------------------------------------- +# Assembly +# --------------------------------------------------------------------------- +def build_phosphosite_table( + ochoa_records: list[dict], + acc_to_gene: dict[str, str], + uniprot_records: list[dict], + gene_map: pd.DataFrame, + supplement_records: list[dict], + prev: pd.DataFrame | None, +) -> pd.DataFrame: + gene_to_entrez = gene_map.set_index("gene_symbol")["entrez_id"].to_dict() + rows: list[dict] = [] + existing_ids: set[str] = set() + + def _add(gene: str, residue: str, position: int, modification: str, mod_code: str) -> None: + other_id = f"{gene}-{residue}{position}{mod_code}" + if other_id in existing_ids: + return + entrez = gene_to_entrez.get(gene) + if entrez is None: + return + rows.append({ + "gene_symbol": gene, + "entrez_id": entrez, + "residue": residue, + "position": position, + "modification": modification, + "other_id": other_id, + }) + existing_ids.add(other_id) + + # --- Ochoa (primary) --- + no_gene_ochoa = 0 + for rec in ochoa_records: + gene = acc_to_gene.get(rec["accession"], "") + if not gene: + no_gene_ochoa += 1 + continue + _add(gene, rec["residue"], rec["position"], rec["modification"], rec["mod_code"]) + logging.info( + "Ochoa: %d sites added (%d with no gene symbol)", len(rows), no_gene_ochoa + ) + + before_uniprot = len(rows) + # --- UniProt PTM (secondary) --- + for rec in uniprot_records: + _add( + rec["gene_symbol"], rec["residue"], rec["position"], + rec["modification"], rec["mod_code"], + ) + logging.info( + "UniProt PTM: +%d new sites (total %d)", len(rows) - before_uniprot, len(rows) + ) + + before_supp = len(rows) + # --- Supplements (local files + Synapse) --- + for rec in supplement_records: + _add( + rec["gene_symbol"], rec["residue"], rec["position"], + rec["modification"], rec["mod_code"], + ) + if len(rows) > before_supp: + logging.info( + "Supplements: +%d new sites (total %d)", len(rows) - before_supp, len(rows) + ) + + df = pd.DataFrame(rows) + + # --- Assign / preserve phosphosite_ids --- + if prev is not None and not prev.empty: + prev_map = prev.set_index("other_id")["phosphosite_id"].to_dict() + next_id = int(prev["phosphosite_id"].max()) + 1 + else: + prev_map = {} + next_id = 1 + + ids = [] + for site in df["other_id"]: + if site in prev_map: + ids.append(prev_map[site]) + else: + ids.append(next_id) + prev_map[site] = next_id + next_id += 1 + df["phosphosite_id"] = ids + + cols = [ + "phosphosite_id", "entrez_id", "gene_symbol", + "residue", "position", "modification", "other_id", + ] + df = df[cols].sort_values("phosphosite_id").reset_index(drop=True) + logging.info( + "Final phosphosites table: %d rows, %d unique genes", + len(df), df["gene_symbol"].nunique(), + ) + return df + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("genes", help="genes.csv (gene_symbol → entrez_id)") + parser.add_argument("--out", default="/tmp/phosphosites.csv") + parser.add_argument("--prev", default=None, + help="Previous phosphosites.csv to preserve IDs") + parser.add_argument("--supplement", nargs="*", default=[], + help="Local CSV/TSV file(s) with site strings") + parser.add_argument("--site_col", default="site", + help="Column name for site strings in local supplement files") + parser.add_argument("--synapse_supplements", nargs="*", default=[], + help="Synapse IDs of raw phospho files to add as supplements " + "(e.g. syn70078415)") + parser.add_argument("--synapse_site_col", default="site", + help="Site column name in Synapse supplement files") + parser.add_argument("--ochoa_url", default=OCHOA_URL) + parser.add_argument("--uniprot_gene_url", default=UNIPROT_GENE_URL) + parser.add_argument("--uniprot_ptm_url", default=UNIPROT_PTM_URL) + args = parser.parse_args() + + configure_logging() + + gene_map = load_gene_map(args.genes) + + prev = None + if args.prev: + prev = pd.read_csv(args.prev) + logging.info("Loaded %d previous phosphosites from %s", len(prev), args.prev) + + ochoa_records = fetch_ochoa_phosphosites(args.ochoa_url) + acc_to_gene = fetch_uniprot_gene_map(args.uniprot_gene_url) + uniprot_records = fetch_uniprot_ptm_sites(args.uniprot_ptm_url) + + supp_records: list[dict] = [] + for path in (args.supplement or []): + supp_records.extend(parse_supplement_sites(path, args.site_col)) + for syn_id in (args.synapse_supplements or []): + try: + supp_records.extend(fetch_synapse_supplement(syn_id, args.synapse_site_col)) + except Exception as exc: + logging.warning("Synapse supplement %s failed: %s — skipping", syn_id, exc) + + df = build_phosphosite_table( + ochoa_records, acc_to_gene, uniprot_records, gene_map, supp_records, prev + ) + + df.to_csv(args.out, index=False) + logging.info( + "Wrote %s (%d rows, %d unique genes)", + args.out, len(df), df["gene_symbol"].nunique(), + ) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/phosphosites/README.md b/coderbuild/phosphosites/README.md new file mode 100644 index 00000000..e2cf788f --- /dev/null +++ b/coderbuild/phosphosites/README.md @@ -0,0 +1,97 @@ +# Phosphosites Reference Builder + +Produces `phosphosites.csv`, the global human phosphosite reference used by +dataset omics pipelines (currently: cNF) to map raw phospho measurements to +stable integer identifiers. + +## Sources + +Three sources are merged in priority order. A site from an earlier source is +never overwritten by a later one (deduplication on `other_id`). + +| Priority | Source | Sites (approx.) | +|---|---|---| +| 1 | Ochoa et al. (2020) Nat Biotechnol 38:365-373, Supp. Table S3 | ~116k | +| 2 | UniProt PTM REST API | ~12k additional | +| 3 | Synapse supplement `syn70078415` (cNF raw phospho data) | ~188 additional | + +## Site notation + +``` +- +e.g. AAAS-S495s (s=phosphoserine, t=phosphothreonine, y=phosphotyrosine) +``` + +`other_id` in `phosphosites.csv` stores these strings and is the join key for +raw phospho data. + +## Output columns + +`phosphosite_id`, `entrez_id`, `gene_symbol`, `residue`, `position`, +`modification`, `other_id` + +## Files + +``` +coderbuild/phosphosites/ +├── 00-buildPhosphositeFile.py # Builder script +└── build_phosphosites.sh # Docker entry point +coderbuild/docker/ +└── Dockerfile.phosphosites +``` + +## Builder — `00-buildPhosphositeFile.py` + +**Ochoa (primary):** Downloads `41587_2019_344_MOESM4_ESM.xlsx` from the +Springer static CDN (no login required; requires `openpyxl`). Reads the +`annotated_phosphoproteome` sheet (`uniprot`, `position`, `residue` columns). +UniProt accessions are resolved to gene symbols via a separate UniProt REST +call (TSV, all reviewed human proteins, paginated via `Link: rel="next"` +header). + +**UniProt PTM (secondary):** Downloads `Modified residue` features with +phosphoserine/phosphothreonine/phosphotyrosine descriptions from the UniProt +REST API (JSON, same Link-header pagination). Returns gene symbols directly. + +**Synapse supplement (`--synapse_supplements`):** Downloads a raw phospho +file from Synapse, auto-detects the site column (`site`, `Site`, +`phosphosite`, or `feature_id`), and appends any `GENE-ResPositionmod` sites +not already in the reference. Requires `SYNAPSE_AUTH_TOKEN`. Failures are +logged as warnings and do not abort the build. + +### Key arguments + +| Argument | Default | Description | +|---|---|---| +| `genes` (positional) | — | Path to `genes.csv` | +| `--out` | `/tmp/phosphosites.csv` | Output path | +| `--prev` | None | Previous CSV to preserve `phosphosite_id` values across builds | +| `--synapse_supplements` | None | Synapse IDs of raw phospho files (space-separated) | +| `--synapse_site_col` | `site` | Site column name in Synapse supplement files | +| `--supplement` | None | Local CSV/TSV file(s) with site strings | +| `--site_col` | `site` | Site column name in local supplement files | + +## Build sequencing + +Phosphosites must run **after** `genes.csv` is ready. `build_dataset.py` and +`build_all.py` wait on the genes future before submitting the phosphosites +container. + +## ID stability + +Pass `--prev ` to preserve IDs across rebuilds. New +sites receive IDs starting from `max(prev.phosphosite_id) + 1` — same pattern +as `improve_sample_id` and `improve_drug_id`. + +## Local run (Only for debugging purposes, this is already build into build_all.py script) + +```bash +pip install pandas synapseclient openpyxl + +export SYNAPSE_AUTH_TOKEN= +python coderbuild/phosphosites/00-buildPhosphositeFile.py \ + local/genes.csv \ + --out local/phosphosites.csv \ + --synapse_supplements syn70078415 \ + --synapse_site_col site +``` diff --git a/coderbuild/phosphosites/build_phosphosites.sh b/coderbuild/phosphosites/build_phosphosites.sh new file mode 100644 index 00000000..4662546d --- /dev/null +++ b/coderbuild/phosphosites/build_phosphosites.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# build_phosphosites.sh: build the global phosphosites reference CSV. +# +# Usage: +# build_phosphosites.sh [] +# +# Sources (in priority order): +# 1. Ochoa et al. (2020) Nat Biotechnol Supplementary Table S3 (~116k sites) +# 2. UniProt PTM annotations (~12k additional sites unique to UniProt) +# 3. Synapse supplement: syn70078415 (cnf raw phospho — captures experiment- +# specific sites not yet in either database) + +set -euo pipefail + +GENES="${1:?Usage: build_phosphosites.sh []}" +PREV="${2:-}" + +PREV_ARG="" +if [ -n "$PREV" ] && [ -f "$PREV" ]; then + PREV_ARG="--prev $PREV" +fi + +python /coderbuild/phosphosites/00-buildPhosphositeFile.py \ + "$GENES" \ + --out /tmp/phosphosites.csv \ + --synapse_supplements syn70078415 \ + --synapse_site_col site \ + $PREV_ARG + +echo "Phosphosites built from Ochoa + UniProt PTM + Synapse supplement." diff --git a/coderbuild/utils/align_drug_descriptors.py b/coderbuild/utils/align_drug_descriptors.py index 0e190af9..6dac8265 100644 --- a/coderbuild/utils/align_drug_descriptors.py +++ b/coderbuild/utils/align_drug_descriptors.py @@ -1,9 +1,45 @@ #!/usr/bin/env python3 +"""Harmonize drug_descriptor values across datasets. + +For each (improve_drug_id, structural_descriptor) seen across the per-dataset +`*_drug_descriptors.tsv[.gz]` files, the FIRST value encountered (files processed +in sorted order) is treated as canonical, and any later file whose value differs +is rewritten to match. + +Memory-bounded implementation +----------------------------- +The reference map is kept in an on-disk SQLite database, not a Python dict, so the +step scales to very large descriptor files (e.g. nci60 ~89M rows / 2.7 GB). The +previous in-memory dict held every (drug_id, descriptor) -> value pair at once and +was OOM-killed by the container on large builds. + +Fingerprint descriptors are skipped from harmonization: a fingerprint is a +deterministic function of chemical structure, so the same drug always yields the +same value -- there is nothing to harmonize, and these rows dominate both memory +and file size. They are passed through unchanged. +""" import os +import sys import gzip import shutil import csv import argparse +import sqlite3 +import tempfile + +# Descriptor values (e.g. Morgan fingerprints) can be long strings. +csv.field_size_limit(min(sys.maxsize, 2**31 - 1)) + +# Descriptor names (case-insensitive substring match) excluded from harmonization. +SKIP_DESCRIPTOR_SUBSTRINGS = ("fingerprint",) + +_INSERT_BATCH = 100_000 + + +def _skip_descriptor(name): + n = name.lower() + return any(s in n for s in SKIP_DESCRIPTOR_SUBSTRINGS) + # Helper scripts def decompress_gz_if_needed(path): @@ -15,6 +51,7 @@ def decompress_gz_if_needed(path): return out, True return path, False + def recompress_if_needed(decompressed, was_gz, original): """If was_gz, recompress decompressed back to original and remove decompressed.""" if was_gz: @@ -22,6 +59,7 @@ def recompress_if_needed(decompressed, was_gz, original): shutil.copyfileobj(f_in, f_out) os.remove(decompressed) + def find_descriptor_files(directory): """Find all *_drug_descriptors.tsv and .tsv.gz files in directory.""" files = [] @@ -32,23 +70,39 @@ def find_descriptor_files(directory): # Actual work -def build_reference_map(files): - """Build map from (drug_id, descriptor) with first found value.""" - ref = {} +def build_reference_db(files, conn): + """Populate SQLite `ref` with the first-seen value per (drug_id, descriptor). + + INSERT OR IGNORE keeps the first value encountered, matching the original + 'first found wins' semantics. Fingerprint descriptors are not stored. + """ + cur = conn.cursor() + cur.execute( + "CREATE TABLE ref (drug_id TEXT, descriptor TEXT, value TEXT, " + "PRIMARY KEY (drug_id, descriptor)) WITHOUT ROWID" + ) for fp in files: path, gz = decompress_gz_if_needed(fp) with open(path, newline='', encoding='utf-8') as f: reader = csv.DictReader(f, delimiter='\t') + batch = [] for row in reader: - key = (row['improve_drug_id'], row['structural_descriptor']) - val = row['descriptor_value'] - if key not in ref: - ref[key] = val + desc = row['structural_descriptor'] + if _skip_descriptor(desc): + continue + batch.append((row['improve_drug_id'], desc, row['descriptor_value'])) + if len(batch) >= _INSERT_BATCH: + cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch) + batch.clear() + if batch: + cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch) + conn.commit() recompress_if_needed(path, gz, fp) - return ref -def rewrite_files(files, ref): - """Go back and rewrite any mismatches in-place.""" + +def rewrite_files(files, conn): + """Rewrite any mismatched descriptor_value in-place to the canonical value.""" + cur = conn.cursor() for fp in files: path, gz = decompress_gz_if_needed(fp) tmp = path + '.tmp' @@ -62,13 +116,16 @@ def rewrite_files(files, ref): writer.writeheader() for row in reader: - key = (row['improve_drug_id'], row['structural_descriptor']) - correct = ref.get(key) - if correct is not None and row['descriptor_value'] != correct: - # print(f"Fixing {key} in {os.path.basename(fp)}: " - # f"{row['descriptor_value']} to {correct}") - row['descriptor_value'] = correct - changed = True + desc = row['structural_descriptor'] + if not _skip_descriptor(desc): + cur.execute( + "SELECT value FROM ref WHERE drug_id=? AND descriptor=?", + (row['improve_drug_id'], desc), + ) + res = cur.fetchone() + if res is not None and row['descriptor_value'] != res[0]: + row['descriptor_value'] = res[0] + changed = True writer.writerow(row) if changed: @@ -78,21 +135,31 @@ def rewrite_files(files, ref): recompress_if_needed(path, gz, fp) + def main(): parser = argparse.ArgumentParser( description="Harmonize drug_descriptor values across multiple files." ) parser.add_argument('--local_dir', default='.', help='Folder containing *_drug_descriptors.tsv[.gz]') - parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input + parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input args = parser.parse_args() files = find_descriptor_files(args.local_dir) if not files: print("No drug_descriptor files found in", args.local_dir) return - ref = build_reference_map(files) - rewrite_files(files, ref) + with tempfile.TemporaryDirectory() as tmpdir: + conn = sqlite3.connect(os.path.join(tmpdir, "descriptors.sqlite")) + # Temp DB: durability is irrelevant, so favour speed. + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + try: + build_reference_db(files, conn) + rewrite_files(files, conn) + finally: + conn.close() print("Done.") + if __name__ == '__main__': main() diff --git a/coderbuild/utils/build_drug_desc.py b/coderbuild/utils/build_drug_desc.py index bbeeb81e..3a2c63ba 100644 --- a/coderbuild/utils/build_drug_desc.py +++ b/coderbuild/utils/build_drug_desc.py @@ -6,6 +6,39 @@ import argparse +import os +import tempfile + +# --------------------------------------------------------------------------- +# Keep multiprocessing's scratch files OFF the bind mount. +# +# This MUST run before multiprocessing creates any worker, so it sits above the +# imports that pull it in. +# +# build_all.py bind-mounts the host's local/ directory at /tmp inside every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", and forkserver opens a Unix domain socket +# under tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS +# file sharing does not support chmod() on a socket there, so worker startup +# died with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-8ai8jp5x/sock-2c162e8922a7' +# +# Mordred computes descriptors in parallel, so this killed the drugs step -- +# and it did so at the very END of build_drugs.sh, after nci60, the PSets and +# the join had all completed, wasting the whole multi-hour drug build on every +# attempt. It is deterministic, so retries do not help. +# +# Pointing TMPDIR at container-local storage fixes it at the source. Verified: +# with /tmp as a bind mount a Pool raises EINVAL; with this set it succeeds. +# Moving the socket is preferred over forcing start_method="fork", which would +# reintroduce the fork-with-threads hazard that motivated the upstream change. +_MP_TMPDIR = os.environ.get("CODERDATA_MP_TMPDIR", "/opt/mp_tmp") +os.makedirs(_MP_TMPDIR, exist_ok=True) +os.environ["TMPDIR"] = _MP_TMPDIR +tempfile.tempdir = _MP_TMPDIR +# --------------------------------------------------------------------------- + from rdkit import Chem from rdkit.Chem import AllChem from rdkit.Chem import rdFingerprintGenerator @@ -14,6 +47,7 @@ import numpy as np from mordred import Calculator, descriptors import multiprocessing +import gzip # Remove all of the Deprecation warnings. There is a github issue to update the code and 50k lines of warnings in the build log so I'm hiding them for now. from rdkit import RDLogger @@ -73,50 +107,97 @@ def smiles_to_mordred(smiles,nproc=2): return longtab +## Number of SMILES whose descriptors are held in memory at once. +## +## Mordred returns a WIDE frame of ~1,600 object-dtype descriptors per molecule, +## which pd.melt() then explodes into rows: at 56,890 SMILES that is roughly 92 +## MILLION rows, before the merge, the concat and two more full-frame copies in +## the cleaning step. Build v23 was OOM-killed doing exactly that +## ("build_drugs.sh: line 21: Killed") on a 23.4GiB Docker VM. +## +## v19 completed the same step at 46,319 SMILES, so this was never comfortable +## -- a 23% growth in the drug table was enough to cross the limit. Chunking +## makes peak memory independent of how many drugs the release contains, so it +## does not silently creep back up to the ceiling next time. +SMILES_CHUNK = int(os.environ.get("DRUG_DESC_SMILES_CHUNK", "2000")) + + +def _chunks(seq, n): + for i in range(0, len(seq), n): + yield seq[i:i + n] + + +def _clean_block(block): + """Coerce bad values and drop malformed ids for one block of rows.""" + # Mordred puts the exception object into a cell when a descriptor cannot be + # computed for a molecule, so str() of the error lands in the table (e.g. the + # "module 'numpy' has no attribute 'float'" AttributeError from np.float being + # removed in NumPy >= 1.24, or messages containing "invalid"/"missing"/"min..."). + # Every Mordred descriptor is numeric, so coerce any non-numeric value to + # "NaN" -- this robustly catches ANY such error string, unlike the previous + # fixed substring blocklist which missed the numpy AttributeError. Morgan + # fingerprints are legitimate bit strings and are left untouched. + block['descriptor_value'] = block['descriptor_value'].astype(str) + is_fingerprint = block['structural_descriptor'] == 'morgan fingerprint' + numeric = pd.to_numeric(block['descriptor_value'], errors='coerce') + + # Render every Mordred value through float, so the text written does not + # depend on where the chunk boundaries fall. Mordred returns ints for some + # descriptors; whether a given column also picks up a NaN decides if pandas + # holds it as int64 ("0") or float64 ("0.0"). That makes the SAME descriptor + # serialise differently depending on how many molecules share a chunk, which + # would make the output non-deterministic. Fingerprints are bit strings and + # are left untouched. + block.loc[~is_fingerprint, 'descriptor_value'] = ( + numeric[~is_fingerprint] + .map(lambda v: "NaN" if pd.isna(v) else repr(float(v))) + ) + + # Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens. + block['improve_drug_id'] = block['improve_drug_id'].astype(str).str.strip() + return block[block['improve_drug_id'].str.match(r'^SMI_\d+$')] + + def main(): parser = argparse.ArgumentParser('Build drug descriptor table') parser.add_argument('--drugtable',dest='drugtable') parser.add_argument('--desctable',dest='outtable') - args = parser.parse_args() + args = parser.parse_args() cores = multiprocessing.cpu_count() ncors = cores-1 - # print("Running with "+str(ncors)+' out of '+str(cores)+' processors') - # print('Adding drug table for '+args.drugtable) tab = pd.read_csv(args.drugtable,sep='\t') cansmiles = [a for a in set(tab.canSMILES) if str(a)!='nan'] - # isosmiles = list(set(tab.isoSMILES)) - morgs = smiles_to_fingerprint(cansmiles) ids = pd.DataFrame(tab[['improve_drug_id','canSMILES']]).drop_duplicates() - # print("IDS columns:", ids.columns.tolist()) - # print("MORGS columns:", morgs.columns.tolist()) - id_morg = ids.rename({"canSMILES":'smile'},axis=1).merge(morgs)[['improve_drug_id','structural_descriptor','descriptor_value']] - - mords = smiles_to_mordred(cansmiles,nproc=ncors) - - id_mord = ids.rename({'canSMILES':'smile'},axis=1).merge(mords)[['improve_drug_id','structural_descriptor','descriptor_value']] - - full = pd.concat([id_morg,id_mord],axis=0) - - # Convert any values that contain the following strings to NA. I think this covers all of the cases, but add here if more are found. - strings_to_replace = ["min", "max", "invalid", "multiple", "missing"] - pattern = '|'.join(strings_to_replace) - full['descriptor_value'] = full['descriptor_value'].astype(str) - full.loc[full['descriptor_value'].str.contains(pattern, case=False, na=False), 'descriptor_value'] = "NaN" - - # Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens. - full['improve_drug_id'] = full['improve_drug_id'].astype(str).str.strip() - mask = full['improve_drug_id'].str.match(r'^SMI_\d+$') - n_dropped = (~mask).sum() - # if n_dropped: - # print(f"Dropping {n_dropped} malformed improve_drug_id rows.") - full = full[mask].copy() - - - full.to_csv(args.outtable,sep='\t',index=False,compression='gzip') + ids = ids.rename({"canSMILES":'smile'},axis=1) + del tab + + cols = ['improve_drug_id','structural_descriptor','descriptor_value'] + total = len(cansmiles) + print(f'Building descriptors for {total} SMILES in chunks of {SMILES_CHUNK}', + flush=True) + + # Stream each block straight to the gzip output. Morgan fingerprints are + # emitted first, then Mordred descriptors, preserving the original row order. + with gzip.open(args.outtable, 'wt', newline='') as out: + header = True + for label, fn in (('morgan', smiles_to_fingerprint), + ('mordred', lambda c: smiles_to_mordred(c, nproc=ncors))): + for i, chunk in enumerate(_chunks(cansmiles, SMILES_CHUNK), start=1): + desc = fn(chunk) + if desc is None or desc.empty: + continue + block = ids.merge(desc)[cols] + block = _clean_block(block) + block.to_csv(out, sep='\t', index=False, header=header) + header = False + done = min(i * SMILES_CHUNK, total) + print(f' [{label}] {done}/{total} SMILES -> {len(block)} rows', + flush=True) + del desc, block if __name__=='__main__': main() diff --git a/coderbuild/utils/calc_pdx_metrics.py b/coderbuild/utils/calc_pdx_metrics.py index 347e1f78..ec7eb54d 100755 --- a/coderbuild/utils/calc_pdx_metrics.py +++ b/coderbuild/utils/calc_pdx_metrics.py @@ -111,7 +111,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): exdf["volume_change"] = tumor_volume_change(volume) exdf["average_response"] = avg_response(exdf["volume_change"]) - nxdf = {'metric':'mRESCIST','value':None} + nxdf = {'metric':'mrecist','value':None} df = pd.DataFrame({"time": time, "volume": volume}) df = df[df['time'] >= min_time] @@ -131,7 +131,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): best_average_response = exdf["best_average_response"][0] mrecist = None - nxdf = {'metric':'mRESCIST','value':mrecist} + nxdf = {'metric':'mrecist','value':mrecist} # exdf["mRECIST"] = mrecist if best_response is not None and best_average_response is not None: @@ -148,7 +148,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): mrecist = "CR" # exdf["mRECIST"] = mrecist - nxdf = {'metric':'mRESCIST','value':mrecist} + nxdf = {'metric':'mrecist','value':mrecist} if not return_detail: return nxdf @@ -211,7 +211,7 @@ def TGI(contr_volume, treat_volume,time): # Simulated batch response class object rtx = { - "metric": "TGI", + "metric": "tgi", "value": tgi, 'time': np.max(time) } diff --git a/coderbuild/utils/check_schema.py b/coderbuild/utils/check_schema.py index f547e54d..a0daba46 100644 --- a/coderbuild/utils/check_schema.py +++ b/coderbuild/utils/check_schema.py @@ -1,89 +1,128 @@ -import subprocess import argparse -import concurrent.futures +import logging +import os import sys +from concurrent.futures import ProcessPoolExecutor + import yaml -import os -def run_validations_for_dataset(dataset_name, validations): - """ - Runs validations for a dataset and returns whether all validations passed. +# Path to the LinkML schema, relative to the repo root (the working directory +# used both locally and inside the `upload` Docker image). +SCHEMA_PATH = os.path.join('schema', 'coderdata.yaml') + +# One Validator per worker process. Constructing a Validator compiles the +# schema (and, lazily, the per-target-class JSON Schema) exactly once and +# caches it, so reusing a single instance across many files avoids the +# per-file process startup + schema recompilation that dominated runtime when +# this script shelled out to `linkml-validate` once per file. +# +# The plugin configuration below reproduces the exact defaults of the +# `linkml-validate` CLI (JsonschemaValidationPlugin with closed=True, and +# strict=False), so validation is identical to the previous implementation -- +# same engine, same closed-world JSON Schema, same results. +_VALIDATOR = None + - Parameters: - dataset_name (str): The name of the dataset. - validations (list): List of validations, each a dict with 'target_class' and 'file'. +def _init_worker(): + """Initializer run once per worker process: build the shared Validator.""" + global _VALIDATOR + # LinkML emits chatty INFO logs while importing the schema; silence them so + # the validation output stays readable. + logging.disable(logging.INFO) + from linkml.validator import Validator + from linkml.validator.plugins import JsonschemaValidationPlugin - Returns: - tuple: (dataset_name, bool) where bool is True if all validations passed, False otherwise. + _VALIDATOR = Validator( + SCHEMA_PATH, + validation_plugins=[JsonschemaValidationPlugin(closed=True)], + ) + + +def _validate_one(task): + """Validate a single file against its target class using the shared Validator. + + Returns (dataset_name, target_class, file_path, [error_messages]). """ - validation_failed = False - for validation in validations: - target_class = validation['target_class'] - file_path = validation['file'] - print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...") - - # Run the validation command - try: - result = subprocess.run( - ['linkml-validate', '--schema', 'schema/coderdata.yaml', '--target-class', target_class, file_path], - stdout=sys.stdout, stderr=sys.stderr, text=True - ) - - # Print output and error from command - if result.stdout: - print(result.stdout) - if result.stderr: - print(f"Error in validating {target_class} in {file_path}:", result.stderr) - # Check the exit status - if result.returncode != 0: - print(f"Validation failed for {target_class} in file {file_path}.") - validation_failed = True - else: - print(f"Validation succeeded for {target_class} in file {file_path}.") - except Exception as e: - print(f"An error occurred while validating {target_class} in {file_path}: {e}") - validation_failed = True + dataset_name, target_class, file_path = task + from linkml.validator.loaders import default_loader_for_file + + # A declared file that was never produced is reported as a clean failure + # rather than crashing the whole validation run with a FileNotFoundError. + if not os.path.exists(file_path): + return ( + dataset_name, target_class, file_path, + [f"[ERROR] declared file not found (was it produced by the build?): {file_path}"], + ) + + loader = default_loader_for_file(file_path) + errors = [] + for result in _VALIDATOR.iter_results_from_source(loader, target_class): + severity = getattr(result.severity, 'value', str(result.severity)) + message = ( + f"[{severity}] [{loader.source}/{result.instance_index}] " + f"{result.message}" + ) + # linkml-validate exits non-zero only on ERROR (or worse); mirror that + # so a WARNING/INFO does not fail the build. + if str(severity).upper() in ('ERROR', 'FATAL', 'CRITICAL'): + errors.append(message) + return (dataset_name, target_class, file_path, errors) - if validation_failed: - print(f"One or more validations failed for dataset {dataset_name}.") - return (dataset_name, False) - else: - print(f"All validations succeeded for dataset {dataset_name}.") - return (dataset_name, True) def main(): - parser = argparse.ArgumentParser(description="Run schema validations for specified datasets.") - parser.add_argument('-d', '--datasets', nargs='*', help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")', default=None) + parser = argparse.ArgumentParser( + description="Run schema validations for specified datasets." + ) + parser.add_argument( + '-d', '--datasets', nargs='*', + help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")', + default=None, + ) args = parser.parse_args() - # Read the config file config_path = os.path.join('schema', 'expected_files.yaml') with open(config_path, 'r') as f: config = yaml.safe_load(f) - available_datasets = config['datasets'].keys() + available_datasets = list(config['datasets'].keys()) datasets_to_validate = args.datasets if args.datasets else available_datasets - datasets_to_validate = [dataset for dataset in datasets_to_validate if dataset in available_datasets] + datasets_to_validate = [d for d in datasets_to_validate if d in available_datasets] print(f"Datasets to validate: {datasets_to_validate}") - all_passed = True - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = { - executor.submit( - run_validations_for_dataset, - dataset, - config['datasets'][dataset] - ): dataset for dataset in datasets_to_validate - } - - for future in concurrent.futures.as_completed(futures): - dataset_name, result = future.result() - if not result: - all_passed = False - print(f"Validation failed for dataset {dataset_name}") + # Flatten to per-file tasks so all files validate in parallel across + # processes (the previous version only parallelized across datasets). + tasks = [] + for dataset in datasets_to_validate: + for validation in config['datasets'][dataset]: + tasks.append((dataset, validation['target_class'], validation['file'])) + + # Track pass/fail per dataset so the summary matches the old output. + dataset_failed = {d: False for d in datasets_to_validate} + + # Cap workers: each worker can load a multi-GB decompressed omics file into + # memory, so bound concurrency to avoid exhausting RAM on large builds. + max_workers = min(len(tasks), os.cpu_count() or 1, 8) or 1 + with ProcessPoolExecutor(max_workers=max_workers, initializer=_init_worker) as executor: + for dataset_name, target_class, file_path, errors in executor.map(_validate_one, tasks): + print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...") + if errors: + for message in errors: + print(message) + print(f"Validation failed for {target_class} in file {file_path}.") + dataset_failed[dataset_name] = True else: - print(f"Validation passed for dataset {dataset_name}") + print(f"Validation succeeded for {target_class} in file {file_path}.") + + all_passed = True + for dataset in datasets_to_validate: + if dataset_failed[dataset]: + all_passed = False + print(f"One or more validations failed for dataset {dataset}.") + print(f"Validation failed for dataset {dataset}") + else: + print(f"All validations succeeded for dataset {dataset}.") + print(f"Validation passed for dataset {dataset}") if all_passed: print("All schema validations passed successfully.") @@ -92,5 +131,6 @@ def main(): print("Some schema validations failed.") sys.exit(1) + if __name__ == '__main__': main() diff --git a/coderbuild/utils/download_utils.py b/coderbuild/utils/download_utils.py new file mode 100644 index 00000000..833c2060 --- /dev/null +++ b/coderbuild/utils/download_utils.py @@ -0,0 +1,95 @@ +"""Resumable, verified HTTP downloads for the build scripts. + +The NCI wiki serves DOSERESP.zip at ~346MB. A plain urlopen().read() of that +either succeeds or raises + + http.client.IncompleteRead: IncompleteRead(32601897 bytes read, + 313702904 more expected) + +which is exactly how build v32 lost the whole nci60 dataset: the download +truncated at 32MB, 04b-nci60-updated.py exited 1, the caller only warned, and +nci60DoseResponse was never written -- so curve fitting saw 7 studies instead +of 8 and nci60 shipped as eight empty files that still validated clean. + +Retrying from zero is not enough for a file this size on a link that drops +part-way; it tends to fail again at a similar point. Each attempt therefore +RESUMES with a Range request. Verified that wiki.nci.nih.gov sends +accept-ranges: bytes and answers a Range request with 206. +""" + +import os +import time +from urllib import request, error + +# 1, 3, 10 and 15 minutes, matching coderbuild/utils/retry_utils.R. +RETRY_SLEEPS = (60, 180, 600, 900) + +# The NCI wiki blocks Python's default User-Agent. +BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + + +def _remote_size(url, headers): + try: + req = request.Request(url, headers=headers, method="HEAD") + with request.urlopen(req, timeout=60) as resp: + n = resp.headers.get("Content-Length") + return int(n) if n else None + except Exception: + return None + + +def retrieve_url(url, dest, sleeps=RETRY_SLEEPS, headers=None): + """Download `url` to `dest`, resuming an interrupted transfer. + + Bytes accumulate in ".part", which is renamed onto `dest` only once + the finished size matches the server's Content-Length, so a truncated file + never appears at the destination. The ".part" is kept between attempts -- + deleting it would mean a flaky link could never accumulate a large file. + """ + headers = dict(headers or {"User-Agent": BROWSER_UA}) + part = dest + ".part" + expected = _remote_size(url, headers) + + last_error = None + for attempt in range(len(sleeps) + 1): + have = os.path.getsize(part) if os.path.exists(part) else 0 + try: + hdrs = dict(headers) + if have: + hdrs["Range"] = f"bytes={have}-" + req = request.Request(url, headers=hdrs) + with request.urlopen(req, timeout=120) as resp: + resumed = resp.status == 206 + if have and not resumed: + # Server ignored the Range; start over rather than append + # onto bytes it is about to send again. + have = 0 + mode = "ab" if (have and resumed) else "wb" + with open(part, mode) as f: + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + f.write(chunk) + + got = os.path.getsize(part) + if expected is not None and got != expected: + raise IOError(f"incomplete: {got} of {expected} bytes") + os.replace(part, dest) + return dest + + except (error.URLError, error.HTTPError, IOError, OSError) as e: + last_error = e + got = os.path.getsize(part) if os.path.exists(part) else 0 + if attempt < len(sleeps): + wait = sleeps[attempt] + print(f" download of {url} failed at {got:,} bytes ({e}); " + f"attempt {attempt + 1}/{len(sleeps) + 1}, resuming in " + f"{wait // 60} min", flush=True) + time.sleep(wait) + + raise RuntimeError( + f"Failed to download {url} after {len(sleeps) + 1} attempts: {last_error}") diff --git a/coderbuild/utils/fetch_depmap.py b/coderbuild/utils/fetch_depmap.py new file mode 100644 index 00000000..799e79a7 --- /dev/null +++ b/coderbuild/utils/fetch_depmap.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +Fetch the DepMap files required by the broad_sanger build from Synapse. + +WHY THIS EXISTS +--------------- +DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in front +of https://depmap.org/portal/api/download/files. That endpoint now returns an +HTML challenge page with HTTP 200, so the previous programmatic download silently +produced garbage: read_csv() parsed the HTML, and the build failed downstream with +a confusing "object 'filename' not found". DepMap also explicitly asks that the +portal not be scraped, so this is not something to work around. + +coderdata therefore keeps a STATIC COPY of the required DepMap files on Synapse. +This script downloads them into a container-local scratch dir (see +DEPMAP_WORKDIR) before the broad_sanger samples and omics R scripts run. Those +R scripts read them from there, delete each one once consumed, and never touch +the DepMap portal. + +UPDATING FOR A NEW DEPMAP RELEASE (must be done by hand) +-------------------------------------------------------- +1. Download the files from the DepMap download page: + https://depmap.org/portal/data_page/?tab=allData + Accept the terms, then use the download button on each file below. +2. Upload them into the "DepMap Raw" Synapse folder + https://www.synapse.org/Synapse:syn75028495 + as NEW VERSIONS of the existing entities listed in DEPMAP_SYN_IDS. + Uploading as a new version keeps the syn ID stable, so nothing in this file + needs to change. (If you create brand-new Synapse entities instead, update + the IDs in DEPMAP_SYN_IDS below.) +3. Bump DEPMAP_RELEASE below to the new release name. +4. Run the build with --depmap-ready to confirm the Synapse copy is current. + +Repository: https://github.com/PNNL-CompBio/coderdata + +STANDALONE USE +-------------- +Run this directly to verify Synapse access and the configured IDs before +launching a long build: + + SYNAPSE_AUTH_TOKEN=... python3 fetch_depmap.py --dest /tmp/depmap_check +""" + +import argparse +import os +import sys + +# --------------------------------------------------------------------------- +# Configuration -- edit DEPMAP_RELEASE and DEPMAP_SYN_IDS when DepMap ships a +# new release (see "UPDATING FOR A NEW DEPMAP RELEASE" above). +# +# Every value can also be overridden by environment variable, which is how +# other coderdata builders (cnf, liver, bladder, ...) parameterise Synapse IDs. +# --------------------------------------------------------------------------- +DEPMAP_RELEASE = os.environ.get('DEPMAP_RELEASE', '26Q1') + +# Synapse entity IDs for the four files the broad_sanger build consumes. +# These live in the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +# +# Upload each new DepMap release as a NEW VERSION of these same entities and +# the IDs below never need to change -- syn.get() always returns the latest +# version. +DEPMAP_SYNAPSE_FOLDER = 'syn75028495' # "DepMap Raw" -- for reference/uploads + +DEPMAP_SYN_IDS = { + 'Model.csv': + os.environ.get('DEPMAP_MODEL_SYN_ID', 'syn75028594'), + 'OmicsSomaticMutations.csv': + os.environ.get('DEPMAP_MUTATIONS_SYN_ID', 'syn77171195'), + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv': + os.environ.get('DEPMAP_TRANSCRIPTOMICS_SYN_ID', 'syn77171175'), + 'PortalOmicsCNGeneLog2.csv': + os.environ.get('DEPMAP_COPY_NUMBER_SYN_ID', 'syn77171184'), +} + +DOWNLOAD_PAGE = 'https://depmap.org/portal/data_page/?tab=allData' +REPO_URL = 'https://github.com/PNNL-CompBio/coderdata' + +# Container-local scratch directory for the downloaded DepMap files. +# +# Deliberately NOT under /tmp: build_all.py bind-mounts the host's local/ +# directory at /tmp inside every container, so anything written there lands on +# the host (and, here, inside a OneDrive-synced folder) and survives the run. +# These files are transient build inputs -- they are consumed by the R scripts +# and deleted -- so they belong in the container's own filesystem, which is +# discarded when the `docker run --rm` exits. +DEPMAP_WORKDIR = os.environ.get('DEPMAP_DIR', '/opt/depmap_data') + + +def default_dest(): + """Container-local directory the R scripts read the DepMap files from.""" + return DEPMAP_WORKDIR + + +def _unconfigured(): + """Names of files whose Synapse ID has not been set yet.""" + return [name for name, sid in DEPMAP_SYN_IDS.items() if not sid.strip()] + + +def main(): + parser = argparse.ArgumentParser( + description='Download the static DepMap files for broad_sanger from Synapse.' + ) + parser.add_argument('--dest', default=None, + help='Directory to download into ' + '(default: $DEPMAP_DIR or /opt/depmap_data, container-local)') + parser.add_argument('--print-dest', action='store_true', + help='Print the download directory and exit. Lets the ' + 'build shell scripts derive DEPMAP_DIR without ' + 'duplicating the release name.') + args = parser.parse_args() + + dest = args.dest or default_dest() + + if args.print_dest: + print(dest) + return + + missing_ids = _unconfigured() + if missing_ids: + sys.exit( + '\n' + '=====================================================================\n' + ' DepMap Synapse IDs are not configured\n' + '=====================================================================\n' + f' No Synapse entity ID is set for:\n' + + ''.join(f' - {n}\n' for n in missing_ids) + + '\n' + ' DepMap files can no longer be downloaded programmatically from the\n' + ' DepMap portal, so coderdata reads a static copy from Synapse.\n' + '\n' + ' To fix this:\n' + f' 1. Download the files from {DOWNLOAD_PAGE}\n' + ' 2. Upload them to Synapse\n' + ' 3. Put their syn IDs in DEPMAP_SYN_IDS in\n' + ' coderbuild/utils/fetch_depmap.py (or set the matching\n' + ' DEPMAP_*_SYN_ID environment variables)\n' + '\n' + f' Repository: {REPO_URL}\n' + '=====================================================================\n' + ) + + if not os.environ.get('SYNAPSE_AUTH_TOKEN'): + sys.exit('SYNAPSE_AUTH_TOKEN is not set; cannot download DepMap files ' + 'from Synapse.') + + try: + import synapseclient + except ImportError: + sys.exit("synapseclient is not installed in this image; it is required " + "to fetch the DepMap files from Synapse. Add 'synapseclient' " + "to coderbuild/broad_sanger/requirements.txt and rebuild.") + + os.makedirs(dest, exist_ok=True) + + syn = synapseclient.Synapse() + syn.login() + + print(f'Fetching DepMap {DEPMAP_RELEASE} files from Synapse into {dest}') + for fname, syn_id in DEPMAP_SYN_IDS.items(): + print(f' {fname} <- {syn_id}') + # Always re-download so the copy used by the build provably matches + # Synapse. `dest` is container-local, so this costs no host disk and is + # discarded when the container exits. + entity = syn.get( + syn_id, + downloadLocation=dest, + ifcollision='overwrite.local', + ) + + # Synapse names the downloaded file after the entity, which may differ + # from the name the R scripts expect. Normalise it. + got = getattr(entity, 'path', None) + if not got or not os.path.exists(got): + sys.exit(f'Synapse returned no downloaded file for {fname} ({syn_id}). ' + f'Check that the entity exists and that this account has ' + f'download permission on {DEPMAP_SYNAPSE_FOLDER}.') + want = os.path.join(dest, fname) + if os.path.abspath(got) != os.path.abspath(want): + os.replace(got, want) + + size = os.path.getsize(want) + if size == 0: + sys.exit(f'Downloaded {fname} from {syn_id} but it is empty.') + print(f' -> {want} ({size / 1e6:.1f} MB)') + + print(f'All {len(DEPMAP_SYN_IDS)} DepMap {DEPMAP_RELEASE} files ready in {dest}') + + +if __name__ == '__main__': + main() diff --git a/coderbuild/utils/fit_curve.py b/coderbuild/utils/fit_curve.py index c712cc3d..f050e268 100755 --- a/coderbuild/utils/fit_curve.py +++ b/coderbuild/utils/fit_curve.py @@ -17,6 +17,10 @@ from sklearn.metrics import r2_score from scipy.optimize import curve_fit import multiprocessing +from datetime import datetime + +def _log(msg): + print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] {msg}', flush=True) #import uno_data as ud @@ -225,26 +229,69 @@ def process_single_drug(name_group_tuple): metrics = compute_fit_metrics(xdata, ydata, popt, pcov) return name, metrics -def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None): - cols = ['source', 'improve_sample_id', 'Drug', 'study','time','time_unit'] - groups = df.groupby(cols) - count = count or (4484081 - start) - groups = islice(groups, start, start+count) +def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None, + workers=None, chunk_size=50000): + cols = ['source', 'improve_sample_id', 'Drug', 'study', 'time', 'time_unit'] cores = multiprocessing.cpu_count() - poolsize = round(cores-1) - print('we have '+str(cores)+' cores and '+str(poolsize)+' threads') - with multiprocessing.Pool(processes=poolsize) as pool: - results = pool.map(process_single_drug, groups) - - with open(f'{fname}.{start}', 'w') as f: - header = None - for result in results: - name, metrics = result - if header is None: - header = cols + metrics.index.tolist() - print(sep.join(header), file=f) - print(sep.join(str(n) for n in name), end=sep, file=f) - print(sep.join(f'{x:.4g}' for x in metrics), file=f) + poolsize = workers if workers is not None else max(1, round(cores * 3 / 4)) + + # Stream lazily — never materialise more than one chunk at a time so + # fork() workers only copy chunk-sized data, not the entire groupby result. + groups_iter = islice(df.groupby(cols), start, start + count if count else None) + + output_file = f'{fname}.{start}' + if os.path.exists(output_file): + os.remove(output_file) + header_written = False + groups_done = 0 + chunks_skipped = 0 + chunk_idx = 0 + + _log(f'fitting {fname}: {cores} cores, {poolsize} workers, chunk_size={chunk_size}') + + while True: + chunk = list(islice(groups_iter, chunk_size)) + if not chunk: + break + chunk_idx += 1 + chunk_workers = poolsize + + for attempt in range(1, 4): + try: + _log(f' chunk {chunk_idx} ' + f'(groups {groups_done + 1}-{groups_done + len(chunk)}, ' + f'attempt {attempt}/3, {chunk_workers} workers)') + with multiprocessing.Pool(processes=chunk_workers) as pool: + results = pool.map(process_single_drug, chunk) + + with open(output_file, 'a') as f: + for result in results: + name, metrics = result + if not header_written: + header = cols + metrics.index.tolist() + print(sep.join(header), file=f) + header_written = True + print(sep.join(str(n) for n in name), end=sep, file=f) + print(sep.join(f'{x:.4g}' for x in metrics), file=f) + + groups_done += len(chunk) + _log(f' chunk {chunk_idx} complete ({groups_done} groups so far)') + break + + except Exception as e: + _log(f' chunk {chunk_idx} failed ' + f'(attempt {attempt}/3): {type(e).__name__}: {e}') + if attempt < 3: + chunk_workers = max(1, chunk_workers // 2) + _log(f' retrying with {chunk_workers} workers') + else: + _log(f' WARNING: chunk {chunk_idx} skipped after 3 failed attempts') + groups_done += len(chunk) + chunks_skipped += 1 + + _log(f'fitting complete: {fname} — {groups_done} groups, ' + f'{chunks_skipped} chunk(s) skipped, output: {output_file}') + return chunks_skipped def main(): @@ -261,9 +308,11 @@ def main(): parser.add_argument('--output', help='prefix of output file') parser.add_argument('--beataml', action='store_true', help='Include this if for BeatAML') parser.add_argument('--debug',action='store_true',default=False) - + parser.add_argument('--workers', type=int, default=None, help='Number of parallel worker processes (default: 3/4 of available CPUs)') + parser.add_argument('--chunk_size', type=int, default=50000, help='Max drug-groups per pool.map call (default: 50000; reduce for very large datasets)') + args = parser.parse_args() - print(args.input) + _log(f'loading {args.input}') df_all = pd.read_table(args.input) if args.debug: df_all = df_all.iloc[0:1000000] @@ -274,12 +323,18 @@ def main(): df_all.DOSE = -1.0 * np.log10(df_all.DOSE/1000000.0) ##need data to be between 0 and 1, not 0 and 100 df_all.GROWTH=df_all.GROWTH/100.00 - print(df_all.head) + _log(f'loaded {len(df_all)} rows — starting curve fitting') fname = args.output or 'combined_single_response_agg' - process_df_part(df_all, fname, beataml=args.beataml)#, start=args.start, count=args.count) - -# if args.beataml == False: + skipped = process_df_part(df_all, fname, beataml=args.beataml, workers=args.workers, + chunk_size=args.chunk_size) + # Free the large input DataFrame before the schema-formatting step + del df_all + import gc; gc.collect() + _log(f'done: {fname}') format_coderd_schema(fname+'.0') + if skipped: + _log(f'ERROR: {skipped} chunk(s) permanently skipped — output is incomplete') + sys.exit(1) if __name__ == '__main__': main() diff --git a/coderbuild/utils/map_improve_drug_ids.py b/coderbuild/utils/map_improve_drug_ids.py index 691a26dc..13db3c8d 100644 --- a/coderbuild/utils/map_improve_drug_ids.py +++ b/coderbuild/utils/map_improve_drug_ids.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 import json import os +import sys import csv import argparse from datetime import datetime import gzip import shutil +# Drug synonym fields can legitimately be long, and a quoting bug in +# pubchem_retrieval.py (fixed 2026-08-29) historically inflated some chem_name +# values well past Python's default 131072-byte csv field limit. Matches the +# limit already set in align_drug_descriptors.py so the two stay consistent. +csv.field_size_limit(min(sys.maxsize, 2**31 - 1)) + ##### Load, make structure, save functions for improve_drug_mapping.json def load_mapping(mapping_file='improve_drug_mapping.json'): """ diff --git a/coderbuild/utils/pinned_psets.R b/coderbuild/utils/pinned_psets.R new file mode 100644 index 00000000..36003949 --- /dev/null +++ b/coderbuild/utils/pinned_psets.R @@ -0,0 +1,115 @@ +## --------------------------------------------------------------------------- +## Pinned PharmacoGx PSet list for the broad_sanger build. +## +## WHY THIS EXISTS +## --------------- +## PharmacoGx::availablePSets() is a LIVE list served by ORCESTRA. New PSets +## appear without notice, and the build previously consumed whatever that call +## returned. That is unsafe for a published dataset in two distinct ways: +## +## 1. A new PSet with an INCOMPATIBLE schema breaks the build. +## In Aug 2026 a second CTRPv2 PSet ("CTRPv2.1_2016") appeared. Its +## sensitivityInfo() has neither `treatmentid` nor `drugid` -- it uses +## `Compound.Name` / `Broad.Compound.ID` / `master_cpd_id` instead. The +## drugs step found 0 chemicals for it and then crashed in +## update_dataframe_and_write_tsv with +## "TypeError: 'NoneType' object is not iterable". +## +## 2. A new PSet with a COMPATIBLE schema silently changes the published +## data between releases, with no diff and no reviewer. +## +## Both the drugs step (03-createDrugFile.R) and the experiments step +## (04a-drugResponseData.R) resolve PSets through this file, so they can never +## disagree about which PSets are in a release. A mismatch there would produce +## experiments referencing drugs that do not exist in the drugs file, which +## fails schema validation late and confusingly. +## +## ADDING A NEW PSet +## ----------------- +## Adding a PSet changes the published dataset, so it is a deliberate, +## reviewed change -- not something that should happen because upstream +## published something new. To add one: +## 1. Confirm its sensitivityInfo() exposes `treatmentid` or `drugid` +## (or add an explicit column mapping in 03-createDrugFile.R). +## 2. Confirm the experiments step can extract dose-response from it. +## 3. Add the exact `PSet Name` below and note the release it entered in. +## +## CURRENT PINS +## ------------ +## These are exactly the PSets used to build coderdata v2.3 (verified against +## the v2.3 build log), so v2.4 keeps the same PharmacoGx provenance. +## +## KNOWN AVAILABLE BUT DELIBERATELY EXCLUDED: +## CTRPv2.1_2016 -- DO NOT ADD. It is not new data; it is the SAME CTRP v2 +## data as CTRPv2_2015 with different curation. Verified +## 2026-08-28 by loading both PSets: +## +## sensitivityInfo rows : 395263 vs 395263 (identical) +## unique samples : 887 vs 887 (identical) +## unique experiment_id : 907 vs 907 (identical +## SETS -- identical==TRUE) +## unique treatments : 544 vs 545 +## +## The only real difference is naming convention. +## CTRPv2_2015 is PharmacoGx-standardised (`treatmentid`, +## `sampleid`, INN names like "adavosertib", hyphenated +## codes like "azd-1480"). CTRPv2.1_2016 keeps raw CTRP +## columns (`Compound.Name`, `master_ccl_id`) and Broad +## code names ("abt-199", "azd1480"). Case-insensitively +## 481 of ~545 treatment names match outright; most of the +## remainder are the same compound spelled differently +## (azd-1480 / azd1480, azd-7545 / azd7545). +## +## Including it would therefore DUPLICATE all 395,263 +## measurements and mint a second improve_drug_id for +## compounds we already have under a different spelling -- +## a data-quality regression, not an enrichment. +## --------------------------------------------------------------------------- + +PINNED_PSETS <- c( + "CTRPv2_2015", + "CCLE_2015", + "CCLE_2019", + "FIMM_2016", + "gCSI_2019", + "GDSC_2020(v1-8.2)", + "GDSC_2020(v2-8.2)", + "PRISM_2020" +) + + +## Return the PSet names for one dataset, restricted to the pinned list. +## +## Loudly reports any PSet that upstream now offers but which is not pinned, +## so a new upstream release is visible in the build log rather than silently +## included or silently dropped. +psets_for_dataset <- function(all.dsets, cel) { + available <- all.dsets |> + subset(`Dataset Name` == cel) |> + (\(d) d[["PSet Name"]])() |> + unlist() |> + unique() + + keep <- available[available %in% PINNED_PSETS] + skipped <- setdiff(available, PINNED_PSETS) + + if (length(skipped) > 0) { + message(sprintf( + paste0("[pinned_psets] NOTE: dataset '%s' has %d PSet(s) available ", + "upstream that are NOT pinned and are being SKIPPED: %s\n", + " If this PSet should be part of the release, add it to ", + "PINNED_PSETS in coderbuild/utils/pinned_psets.R after verifying ", + "its schema in BOTH the drugs and experiments steps."), + cel, length(skipped), paste(skipped, collapse = ", "))) + } + + if (length(keep) == 0) { + stop(sprintf( + paste0("No pinned PSet found for dataset '%s'. Available upstream: %s. ", + "Either upstream renamed/removed a PSet, or PINNED_PSETS in ", + "coderbuild/utils/pinned_psets.R is out of date."), + cel, if (length(available)) paste(available, collapse = ", ") else "(none)")) + } + + keep +} diff --git a/coderbuild/utils/pubchem_retrieval.py b/coderbuild/utils/pubchem_retrieval.py index a72c5c68..a3c95873 100644 --- a/coderbuild/utils/pubchem_retrieval.py +++ b/coderbuild/utils/pubchem_retrieval.py @@ -1,11 +1,16 @@ import pandas as pd import requests +import csv from concurrent.futures import ThreadPoolExecutor, as_completed import os import threading import time import signal import sys +from datetime import datetime + +def _ts(): + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Global variables request_counter = 0 @@ -17,67 +22,131 @@ existing_structures = dict() existing_pubchemids = set() +# --------------------------------------------------------------------------- +# Local PubChem data cache — populated from previous-run drug files so that +# retrieve_drug_info can return cached properties+synonyms without HTTP requests. +# Keys: pubchem_id (str) AND chem_name (str.lower()). +# Values: {'SMILES': ..., 'InChIKey': ..., 'MolecularFormula': ..., +# 'MolecularWeight': ..., 'CID': ..., 'synonyms': [...]} +# improve_drug_id is NOT stored here; ID assignment is always fresh. +# --------------------------------------------------------------------------- +_pubchem_data_cache = {} + -def fetch_url(url, retries=3, backoff_factor=1): +def load_pubchem_cache(file_paths): + """Populate _pubchem_data_cache from previously built drug TSV files. + + Call this before the main drug-build loop to avoid re-querying PubChem for + drugs whose properties are already known. ID assignment is unaffected. + """ + global _pubchem_data_cache + if isinstance(file_paths, str): + file_paths = [p.strip() for p in file_paths.split(',') if p.strip()] + loaded = 0 + for path in file_paths: + if not os.path.exists(path): + print(f"[{_ts()}] Cache hint: file not found, skipping: {path}") + continue + try: + df = pd.read_csv(path, sep='\t', quoting=3) + except Exception: + try: + df = pd.read_csv(path, sep='\t') + except Exception as e: + print(f"[{_ts()}] Could not read cache file {path}: {e}") + continue + needed = {'pubchem_id', 'chem_name', 'canSMILES', 'InChIKey', 'formula', 'weight'} + if not needed.issubset(set(df.columns)): + print(f"[{_ts()}] Cache file {path} missing expected columns, skipping.") + continue + for cid, group in df.groupby('pubchem_id', dropna=True): + synonyms = [str(n) for n in group['chem_name'].dropna().tolist()] + row = group.iloc[0] + entry = { + 'SMILES': str(row['canSMILES']), + 'InChIKey': str(row['InChIKey']), + 'MolecularFormula': str(row['formula']), + 'MolecularWeight': str(row['weight']), + 'CID': str(cid), + 'synonyms': synonyms, + } + _pubchem_data_cache[str(cid)] = entry + for syn in synonyms: + _pubchem_data_cache[str(syn).lower()] = entry + loaded += 1 + print(f"[{_ts()}] PubChem cache: {loaded} drugs loaded from {len(file_paths)} file(s).") + + +# Auto-load from hint file written by build_all.py before the container starts. +_CACHE_HINT_FILE = "/tmp/prev_drug_files.txt" +if os.path.exists(_CACHE_HINT_FILE): + try: + _hint_paths = [l.strip() for l in open(_CACHE_HINT_FILE) if l.strip()] + if _hint_paths: + load_pubchem_cache(_hint_paths) + except Exception as _cache_err: + print(f"[{_ts()}] Warning: could not load PubChem cache from {_CACHE_HINT_FILE}: {_cache_err}") + + +def fetch_url(url, retries=4, backoff_factor=1): """ Fetches a URL with retry mechanism and backoff. - - Parameters: - - url (str): The URL to fetch. - - retries (int): Number of retry attempts. - - backoff_factor (float): Factor to calculate backoff time. - - Returns: - - dict: JSON response if successful. - - Raises: - - Exception: If all retry attempts fail. + + 503 responses use a 30s base backoff (honouring Retry-After if present) + because PubChem rate-throttles aggressively and a 1s retry just gets + another 503. """ global last_request_time, lock, request_counter with lock: current_time = time.time() - # Reset counter if more than 1 second has passed if current_time - last_request_time >= 1: request_counter = 0 last_request_time = current_time - - # Wait if the limit is reached - while request_counter >= 4: - time.sleep(0.2) # Sleep a bit to check again + while request_counter >= 3: # stay at ≤3 req/s; PubChem limit is 5 + time.sleep(0.1) current_time = time.time() if current_time - last_request_time >= 1: request_counter = 0 last_request_time = current_time - request_counter += 1 - - for attempt in range(retries + 1): # Total attempts = retries + 1 + + for attempt in range(retries + 1): + status_code = None + retry_after_header = None try: - response = requests.get(url, timeout=10) - if response.status_code == 200: - return response.json() - if response.status_code == 404: # permanent, no existing CID/name + response = requests.get(url, timeout=30) + status_code = response.status_code + if status_code == 200: + return response.json() + if status_code == 404: raise FileNotFoundError("404") - raise Exception(f"Failed to fetch {url}, Status Code: {response.status_code}") + retry_after_header = response.headers.get('Retry-After', '') + raise Exception(f"Failed to fetch {url}, Status Code: {status_code}") + except FileNotFoundError: + raise except Exception as exc: - if attempt < retries: - wait = backoff_factor * (2 ** attempt) - print(f"Attempt {attempt + 1} for URL {url} failed with error: {exc}. Retrying in {wait} seconds...") - time.sleep(wait) - else: - print(f"All {retries + 1} attempts failed for URL {url}.") + if attempt >= retries: + print(f"[{_ts()}] All {retries + 1} attempts failed for URL {url}.") raise + if status_code == 503: + wait = (int(retry_after_header) + if retry_after_header and retry_after_header.isdigit() + else min(15 * (attempt + 1), 60)) + else: + wait = min(15 * (attempt + 1), 60) + print(f"[{_ts()}] Attempt {attempt + 1} for URL {url} failed with error: {exc}. Retrying in {wait} seconds...") + time.sleep(wait) def retrieve_drug_info(compound, ignore_chems, isname=True): """ - Retrieves information for a given compound from PubChem. + Retrieves information for a given compound from PubChem (or local cache). Parameters: - compound (str or int): Name or CID of the compound. - ignore_chems (str): File path to log ignored compounds. - isname (bool): True if the compound is a name, False if it's a CID. - + Returns: - list: List of dictionaries containing drug information, or None if unsuccessful. """ @@ -85,77 +154,85 @@ def retrieve_drug_info(compound, ignore_chems, isname=True): if pd.isna(compound): return None - if isname: - urls = { - "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", - "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/synonyms/JSON" + # --- Check local cache before making any HTTP requests --- + cache_key = str(compound).lower() if isname else str(compound) + cached = _pubchem_data_cache.get(cache_key) + if cached is not None: + properties = { + 'SMILES': cached['SMILES'], + 'InChIKey': cached['InChIKey'], + 'MolecularFormula': cached['MolecularFormula'], + 'MolecularWeight': cached['MolecularWeight'], + 'CID': cached['CID'], } + synonyms_list = cached['synonyms'] else: - urls = { - "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", - "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/synonyms/JSON" - } - - with ThreadPoolExecutor(max_workers=4) as executor: - future_to_url = {executor.submit(fetch_url, url): key for key, url in urls.items()} - results = {} + # --- Normal PubChem fetch via HTTP --- + if isname: + urls = { + "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", + "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/synonyms/JSON" + } + else: + urls = { + "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", + "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/synonyms/JSON" + } + + with ThreadPoolExecutor(max_workers=4) as executor: + future_to_url = {executor.submit(fetch_url, url): key for key, url in urls.items()} + results = {} + for future in as_completed(future_to_url): + key = future_to_url[future] + try: + results[key] = future.result() + except FileNotFoundError: + print(f"[{_ts()}] {compound} not found in PubChem. Adding to ignore list.") + with open(ignore_chems, "a") as f: + f.write(f"{compound}\n") + return None + except Exception as exc: + print(f"[{_ts()}] {compound} generated a transient exception: {exc}") + return None + + if not all(key in results for key in ["properties", "synonyms"]): + return None - for future in as_completed(future_to_url): - key = future_to_url[future] - try: - data = future.result() - results[key] = data - - except FileNotFoundError: # only 404s are added - print(f"{compound} not found in PubChem. Adding to ignore list.") - with open(ignore_chems, "a") as f: - f.write(f"{compound}\n") - return None - except Exception as exc: # transient error, don't blacklist - print(f"{compound} generated a transient exception: {exc}") - return None - - if all(key in results for key in ["properties", "synonyms"]): properties = results["properties"]['PropertyTable']['Properties'][0] synonyms_list = results["synonyms"]['InformationList']['Information'][0]['Synonym'] - # Check if this compound or any of its synonyms already has an assigned improve_drug_id - new_syns = set() - if isname: - sl = synonyms_list + [compound] - else: - sl = synonyms_list - for synonym in sl: - synonym_lower = str(synonym).lower() - if synonym_lower not in existing_synonyms: - new_syns.add(synonym_lower) - if len(new_syns) == 0: # Ensure there are new synonyms before proceeding - return None - for synonym in new_syns: - synonym_lower = str(synonym).lower() - existing_synonyms.add(synonym_lower) - - # Check for structure - if properties['SMILES'] in existing_structures.keys(): - print(f'Found structure for {compound}') - SMI_assignment = existing_structures[properties['SMILES']] - else: - if improve_drug_id == 0: - improve_drug_id = 1 - SMI_assignment = f"SMI_{improve_drug_id}" - existing_structures[properties['SMILES']] = SMI_assignment - improve_drug_id += 1 - - #print(new_syns) - data_for_tsv = [{ - 'improve_drug_id': SMI_assignment, - 'name': str(synonym).lower(), - **properties - } for synonym in new_syns] - - return data_for_tsv - else: + # --- Shared: synonym dedup + SMI ID assignment (identical for both paths) --- + new_syns = set() + sl = synonyms_list + ([compound] if isname else []) + for synonym in sl: + synonym_lower = str(synonym).lower() + if synonym_lower not in existing_synonyms: + new_syns.add(synonym_lower) + if len(new_syns) == 0: return None + for synonym in new_syns: + existing_synonyms.add(str(synonym).lower()) + + if properties['SMILES'] in existing_structures: + print(f'[{_ts()}] Found structure for {compound}') + SMI_assignment = existing_structures[properties['SMILES']] + else: + if improve_drug_id == 0: + improve_drug_id = 1 + SMI_assignment = f"SMI_{improve_drug_id}" + existing_structures[properties['SMILES']] = SMI_assignment + improve_drug_id += 1 + + # Sanitise every value: these rows are later written to TSV with plain + # f.write() and no quoting, so an embedded tab/newline/quote would corrupt + # the record. See _sanitise_field(). + data_for_tsv = [{ + 'improve_drug_id': SMI_assignment, + 'name': _sanitise_field(str(synonym).lower()), + **{k: _sanitise_field(v) for k, v in properties.items()} + } for synonym in new_syns] + + return data_for_tsv def fetch_data_for_batch(batch, ignore_chems, isname): @@ -166,18 +243,91 @@ def fetch_data_for_batch(batch, ignore_chems, isname): - batch (list): List of compound names or CIDs. - ignore_chems (str): File path to log ignored compounds. - isname (bool): True if compounds are names, False if they're CIDs. - + Returns: - list: Combined list of drug information for the batch. """ all_data = [] - for compound_name in batch: - data = retrieve_drug_info(compound_name, ignore_chems, isname) + for compound in batch: + data = retrieve_drug_info(compound, ignore_chems, isname) if data: all_data.extend(data) return all_data +DRUG_TSV_COLUMNS = 7 # improve_drug_id, chem_name, pubchem_id, canSMILES, + # InChIKey, formula, weight + + +def _sanitise_field(value): + """Make a value safe to write into a raw (unescaped) TSV cell. + + Several writers in this pipeline emit TSV with plain f.write() and no + quoting, so any tab, carriage return, newline or double quote inside a + value silently corrupts the row: the record gains extra columns and every + later reader fails with + + pandas.errors.ParserError: Expected 7 fields in line N, saw 12 + + That is what broke the 2026-09-02 build after ~10 hours -- a single + malformed synonym row for SMI_20392 in nci60_drugs.tsv, whose weight cell + ended up as `367.4"` followed by five stray tabs. + + Drug synonyms carry no meaning in these characters, so replacing them is + lossless for our purposes and makes a structurally broken row impossible. + """ + if value is None: + return "" + s = str(value) + for ch in ("\t", "\r", "\n"): + s = s.replace(ch, " ") + s = s.replace('"', "") + return s.strip() + + +def _read_drug_tsv(path, **kwargs): + """Read a drug TSV, tolerating rows padded with trailing EMPTY columns. + + Historic files (written before _sanitise_field existed) can contain rows + with extra trailing tabs. Those extra cells hold no data, so trimming them + loses nothing -- unlike dropping the row, which would silently remove a + drug from the release. + + A row with extra NON-empty cells is a different matter: that is real data + in an unexpected shape, so this raises with the offending line number and + content rather than guessing. + """ + try: + return pd.read_csv(path, sep="\t", **kwargs) + except pd.errors.ParserError: + pass # fall through to the repairing path below + + print(f"[{_ts()}] {path}: malformed row(s) detected; inspecting.") + cleaned, repaired, bad = [], 0, [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for lineno, raw in enumerate(fh, start=1): + parts = raw.rstrip("\n").split("\t") + if len(parts) > DRUG_TSV_COLUMNS: + extra = parts[DRUG_TSV_COLUMNS:] + if any(x.strip() for x in extra): + bad.append((lineno, raw[:200])) + continue + parts = parts[:DRUG_TSV_COLUMNS] + repaired += 1 + cleaned.append("\t".join(parts)) + + if bad: + detail = "\n".join(f" line {n}: {t!r}" for n, t in bad[:5]) + raise ValueError( + f"{path} contains {len(bad)} row(s) with unexpected NON-EMPTY extra " + f"columns. Refusing to guess at their meaning, because silently " + f"dropping them would remove drugs from the release.\n{detail}") + + print(f"[{_ts()}] {path}: trimmed trailing empty columns from {repaired} row(s).") + from io import StringIO + return pd.read_csv(StringIO("\n".join(cleaned) + "\n"), sep="\t", **kwargs) + + def read_existing_data(output_filename): """ Reads existing data from the output file to prevent duplication. @@ -190,13 +340,13 @@ def read_existing_data(output_filename): """ global improve_drug_id, existing_synonyms, existing_structures, existing_pubchemids try: - df = pd.read_csv(output_filename, sep='\t', quoting=3) + df = _read_drug_tsv(output_filename, quoting=3) existing_synonyms = set([str(a).lower() for a in set(df.chem_name)]) existing_pubchemids = set([str(a) for a in df['pubchem_id']]) max_id = df['improve_drug_id'].str.extract(r'SMI_(\d+)').astype(float).max() improve_drug_id = int(max_id[0]) + 1 if pd.notna(max_id[0]) else 1 existing_structures = {row['canSMILES']: row['improve_drug_id'] for _, row in df.iterrows()} - print(f'Read in {len(existing_synonyms)} drug names and {len(existing_pubchemids)} pubchem IDs') + print(f'[{_ts()}] Read in {len(existing_synonyms)} drug names and {len(existing_pubchemids)} pubchem IDs') except FileNotFoundError: return {} @@ -206,7 +356,7 @@ def timeout_handler(signum, frame): Handles timeouts by setting the global `should_continue` flag to False. """ global should_continue - print("Time limit reached, exiting gracefully...") + print(f"[{_ts()}] Time limit reached, exiting gracefully...") should_continue = False @@ -223,7 +373,7 @@ def _load_prev_drugs_union(prevDrugFilepath: str) -> pd.DataFrame: dfs = [] for p in paths: if not os.path.exists(p): - print(f"Warning: previous drug file '{p}' not found; skipping.") + print(f"[{_ts()}] Warning: previous drug file '{p}' not found; skipping.") continue try: if p.lower().endswith(".tsv"): @@ -232,7 +382,7 @@ def _load_prev_drugs_union(prevDrugFilepath: str) -> pd.DataFrame: df = pd.read_csv(p) dfs.append(df) except Exception as e: - print(f"Warning: failed to read previous drug file '{p}': {e}; skipping.") + print(f"[{_ts()}] Warning: failed to read previous drug file '{p}': {e}; skipping.") if not dfs: return pd.DataFrame(columns=["improve_drug_id", "chem_name", "pubchem_id", "canSMILES", "InChIKey", "formula", "weight"]) @@ -304,15 +454,15 @@ def update_dataframe_and_write_tsv(unique_names, else: restrict_set = raw_names # default filtering - print(f"Starting with {len(raw_names)} provided {'names' if isname else 'IDs'}; restricting output to {len(restrict_set)} of them.") + print(f"[{_ts()}] Starting with {len(raw_names)} provided {'names' if isname else 'IDs'}; restricting output to {len(restrict_set)} of them.") # --- 1) read existing output to bootstrap state --- - print(f"Reading existing data from {output_filename}") + print(f"[{_ts()}] Reading existing data from {output_filename}") # capture existing output file (if any) to include in base existing_output_df = pd.DataFrame() if os.path.exists(output_filename): try: - existing_output_df = pd.read_csv(output_filename, sep="\t", quoting=3) + existing_output_df = _read_drug_tsv(output_filename, quoting=3) except Exception: existing_output_df = pd.read_csv(output_filename, sep="\t") # read_existing_data populates globals (synonyms, pubchemids, and sets improve_drug_id based on output) @@ -327,7 +477,7 @@ def update_dataframe_and_write_tsv(unique_names, desired_start = max(existing_output_max, prev_union_max) + 1 if improve_drug_id < desired_start: improve_drug_id = desired_start - print(f"SMI numbering will start from {improve_drug_id} (max prior was {desired_start - 1})") + print(f"[{_ts()}] SMI numbering will start from {improve_drug_id} (max prior was {desired_start - 1})") # build seen names/IDs (to avoid re-query) seen_names = set(existing_synonyms) @@ -341,10 +491,10 @@ def update_dataframe_and_write_tsv(unique_names, # --- 3) determine new candidates to query --- if isname: candidates = raw_names - seen_names - print(f"{len(raw_names)} raw names provided; {len(seen_names)} already seen; {len(candidates)} new to fetch.") + print(f"[{_ts()}] {len(raw_names)} raw names provided; {len(seen_names)} already seen; {len(candidates)} new to fetch.") else: candidates = raw_names - seen_pubchemids - print(f"{len(raw_names)} raw IDs provided; {len(seen_pubchemids)} already seen; {len(candidates)} new to fetch.") + print(f"[{_ts()}] {len(raw_names)} raw IDs provided; {len(seen_pubchemids)} already seen; {len(candidates)} new to fetch.") # apply ignore_chems filtering ignore_chem_set = set() @@ -353,10 +503,13 @@ def update_dataframe_and_write_tsv(unique_names, for line in file: ignore_chem_set.add(line.strip()) candidates = set(candidates) - ignore_chem_set - print(f"{len(candidates)} candidates remain after removing ignored.") + print(f"[{_ts()}] {len(candidates)} candidates remain after removing ignored.") # --- 4) make a temp union file with previous union + existing output --- - temp_file = output_filename.replace(".tsv", "_temp.tsv") + if output_filename.endswith(".tsv"): + temp_file = output_filename[:-4] + "_temp.tsv" + else: + temp_file = output_filename + "_temp" base_dfs = [] if not prev_union_df.empty: base_dfs.append(prev_union_df) @@ -380,34 +533,76 @@ def update_dataframe_and_write_tsv(unique_names, # create empty temp file so fetch logic can append headers open(temp_file, "a").close() - # --- 5) fetch new ones in batches and append to temp_file --- - candidates_list = list(candidates) - for i in range(0, len(candidates_list), batch_size): - if not should_continue: + # --- 5) fetch new ones in batches and append to temp_file (3 sweeps) --- + remaining_candidates = set(candidates) + + for sweep in range(3): + if not should_continue or not remaining_candidates: break - batch = candidates_list[i : i + batch_size] - data = fetch_data_for_batch(batch, ignore_chems, isname) - if data: - file_exists = os.path.isfile(temp_file) - mode = "a" if file_exists else "w" - with open(temp_file, mode) as f: - if os.path.getsize(temp_file) == 0: - f.write("improve_drug_id\tchem_name\tpubchem_id\tcanSMILES\tInChIKey\tformula\tweight\n") - for entry in data: - f.write( - f"{entry['improve_drug_id']}\t{entry['name']}\t{entry.get('CID', '')}\t" - f"{entry['SMILES']}\t{entry['InChIKey']}\t" - f"{entry['MolecularFormula']}\t{entry['MolecularWeight']}\n" - ) - with open(ignore_chems, "a") as ig_f: - for entry in data: - if isname: - ig_f.write(f"{entry['name']}\n") - else: - ig_f.write(f"{entry.get('CID', '')}\n") + + if sweep > 0: + print(f"[{_ts()}] Sweep {sweep + 1}/3: waiting 60 seconds before retry pass...") + time.sleep(60) + + # refresh ignore set + ignore_chem_set = set() + if os.path.exists(ignore_chems): + with open(ignore_chems, "r") as f: + for line in f: + ignore_chem_set.add(line.strip()) + + # remove already-fetched candidates by reading what landed in temp_file + if os.path.exists(temp_file) and os.path.getsize(temp_file) > 0: + try: + # QUOTE_NONE: temp_file is written with raw f.write() + tmp_df = pd.read_csv(temp_file, sep="\t", quoting=3) + if not isname and "pubchem_id" in tmp_df.columns: + fetched = {str(x) for x in tmp_df["pubchem_id"].dropna()} + remaining_candidates -= fetched + elif isname and "chem_name" in tmp_df.columns: + fetched = {str(x).lower() for x in tmp_df["chem_name"].dropna()} + remaining_candidates -= fetched + except Exception: + pass + + remaining_candidates -= ignore_chem_set + + print(f"[{_ts()}] Sweep {sweep + 1}/3: {len(remaining_candidates)} candidates to fetch.") + + candidates_list = list(remaining_candidates) + for i in range(0, len(candidates_list), batch_size): + if not should_continue: + break + batch = candidates_list[i : i + batch_size] + data = fetch_data_for_batch(batch, ignore_chems, isname) + if data: + file_exists = os.path.isfile(temp_file) + mode = "a" if file_exists else "w" + with open(temp_file, mode) as f: + if os.path.getsize(temp_file) == 0: + f.write("improve_drug_id\tchem_name\tpubchem_id\tcanSMILES\tInChIKey\tformula\tweight\n") + for entry in data: + # Every cell goes through _sanitise_field: this write + # is unescaped, so a stray tab/newline/quote here is + # what produced the malformed nci60_drugs.tsv row that + # failed the 2026-09-02 build. + f.write("\t".join(_sanitise_field(x) for x in ( + entry['improve_drug_id'], entry['name'], + entry.get('CID', ''), entry['SMILES'], + entry['InChIKey'], entry['MolecularFormula'], + entry['MolecularWeight'])) + "\n") + with open(ignore_chems, "a") as ig_f: + for entry in data: + if isname: + ig_f.write(f"{entry['name']}\n") + else: + ig_f.write(f"{entry.get('CID', '')}\n") # --- 6) load combined temp results --- - combined = pd.read_csv(temp_file, sep="\t") + # temp_file is written with raw f.write() (no quoting), so it must be read + # with QUOTE_NONE. Reading it with pandas' default QUOTE_MINIMAL would + # mis-parse any synonym containing a double quote. + combined = _read_drug_tsv(temp_file, quoting=3) # Determine previous max (before new fetches) to identify newly assigned SMI IDs previous_max = desired_start - 1 # desired_start was max(existing_output_max, prev_union_max) + 1 @@ -437,18 +632,40 @@ def update_dataframe_and_write_tsv(unique_names, if keep_ids: final_df = combined[combined["improve_drug_id"].isin(keep_ids)].copy() else: - print("Warning: no relevant drugs were retained/fetched for the restriction set.") + print(f"[{_ts()}] Warning: no relevant drugs were retained/fetched for the restriction set.") final_df = pd.DataFrame(columns=combined.columns) # --- 10) write final filtered output --- final_df.drop_duplicates(inplace=True) - final_df.to_csv(output_filename, sep="\t", index=False) + # QUOTE_NONE/quotechar=None is REQUIRED here, not cosmetic. + # + # These files are read back with quoting=3 (QUOTE_NONE) -- by + # read_existing_data(), by _load_prev_drugs_union(), and by the next build + # via --prev_drugs. Writing with pandas' default QUOTE_MINIMAL wraps any + # field containing a double quote and doubles its internal quotes, but the + # QUOTE_NONE reader does not strip the wrapper or collapse the doubling. + # So every read/write cycle DOUBLES the quote count in that field. + # + # Real drug synonyms contain quotes (e.g. cimetidine's + # n-cyano-n'-methyl-n''-[2-...]guanidine + # ), so this compounded silently across releases: v2.3 shipped a chem_name + # of ~2^16 characters, and by v2.4 the same field had reached ~2^24 (16 MB), + # inflating ctrpv2/nci60/prism_drugs.tsv to ~300 MB each and finally + # breaking map_improve_drug_ids.py with + # _csv.Error: field larger than field limit (131072) + # + # Writing QUOTE_NONE makes the writer symmetric with the readers, so a + # value survives any number of round trips unchanged. Verified stable over + # repeated cycles. Drug names never contain tabs, so QUOTE_NONE is safe for + # this TSV. + final_df.to_csv(output_filename, sep="\t", index=False, + quoting=csv.QUOTE_NONE, quotechar=None) if os.path.exists(temp_file): try: os.remove(temp_file) except OSError as e: - print(f"Warning: failed to delete temp file {temp_file}: {e}") + print(f"[{_ts()}] Warning: failed to delete temp file {temp_file}: {e}") return final_df diff --git a/coderbuild/utils/retry_utils.R b/coderbuild/utils/retry_utils.R new file mode 100644 index 00000000..780ac479 --- /dev/null +++ b/coderbuild/utils/retry_utils.R @@ -0,0 +1,445 @@ +## --------------------------------------------------------------------------- +## Retry helpers for flaky network fetches. +## +## WHY THIS EXISTS +## --------------- +## A full build makes hundreds of requests to a dozen third-party hosts over +## roughly a day. Any one of them being briefly unavailable used to fail the +## whole step, and because build_all.py restarts a failed step from the top, +## a single transient error could throw away many hours of work. +## +## Observed in real runs: +## 2026-09-02 Zenodo returned 504 Gateway Timeout on PSet_gCSI2019.rds, +## after CTRPv2 and GDSC had already downloaded (one PSet is +## 1.5 GB). The drugs step had run for ~17 hours across three +## attempts before giving up. The URL was fine minutes later. +## 2026-08-30 cog.sanger.ac.uk failed twice with an OpenSSL "unexpected +## eof" before succeeding on the third attempt. +## +## These are transient upstream problems, not build defects, so the build +## should ride them out rather than fail. +## +## NOTE: this only covers fetches that were previously unprotected. +## 02-broadSangerOmics.R already has robust_download_httr2() for its own +## downloads, and the Python builders have their own retry logic. +## --------------------------------------------------------------------------- + +## Waits between retries, in seconds: 1, 3, 10, then 15 minutes. +## Five attempts total (the initial one plus four retries), ~29 minutes of +## waiting if every attempt fails. Long enough to ride out a real upstream +## incident rather than only a momentary blip. +RETRY_SLEEPS <- c(60, 180, 600, 900) + +## Run `expr` (a zero-argument function), retrying on error. +## +## IMPORTANT: if every attempt fails this STOPS. It never returns a partial or +## empty result, and callers must not catch the error to "carry on without +## this dataset" -- a dataset must never be silently dropped from a release. +## Exhausting the retries is meant to fail the step, and therefore the build, +## so the problem is visible rather than shipped. +## +## what - human-readable label used in log lines +## sleeps - vector of waits between attempts; length(sleeps)+1 attempts +with_retries <- function(expr, what = "operation", sleeps = RETRY_SLEEPS) { + tries <- length(sleeps) + 1L + last_err <- NULL + for (attempt in seq_len(tries)) { + result <- tryCatch( + list(ok = TRUE, value = expr()), + error = function(e) list(ok = FALSE, msg = conditionMessage(e))) + + if (isTRUE(result$ok)) { + if (attempt > 1) { + message(sprintf("[%s] %s: succeeded on attempt %d/%d", + Sys.time(), what, attempt, tries)) + } + return(result$value) + } + + last_err <- result$msg + if (attempt < tries) { + sleep_s <- sleeps[attempt] + message(sprintf("[%s] %s: attempt %d/%d failed (%s); retrying in %d min", + Sys.time(), what, attempt, tries, + substr(gsub("[\r\n]+", " ", last_err), 1, 120), + round(sleep_s / 60))) + Sys.sleep(sleep_s) + } + } + stop(sprintf(paste0( + "%s failed after %d attempts over ~%d minutes. Last error: %s\n", + "Failing the build rather than continuing without this data."), + what, tries, round(sum(sleeps) / 60), last_err)) +} + + +## Download a PharmacoGx PSet with retries, caching it so that a later attempt +## -- including a completely fresh container started by build_all.py's own +## retry loop -- does not re-download it. +## +## PSETS are large (CTRPv2 is 228 MB, one of them is 1.5 GB). downloadPSet() +## skips a file that is already present in saveDir, so pointing saveDir at a +## directory under /tmp (which build_all.py bind-mounts to the host's local/) +## makes the download survive across container runs. A subdirectory is used +## deliberately: build_all.py's intermediate cleanup globs `local/*.*` and +## skips directories, so the cache is not swept away mid-build. +PSET_CACHE_DIR <- Sys.getenv("PSET_CACHE_DIR", unset = "/tmp/pset_cache") + +## Download a PSet, retrying, and never let a partial file poison the cache. +## +## PharmacoGx::downloadPSet only downloads when the file is ABSENT: +## +## if (!file.exists(file.path(saveDir, pSetFileName))) { download.file(...) } +## pSet <- readRDS(file.path(saveDir, pSetFileName)) +## +## So an interrupted download leaves a truncated .rds that downloadPSet then +## reuses forever: every later attempt skips the download and fails instantly +## inside readRDS() with "error reading from connection". That is not a network +## error, and no amount of retrying -- or waiting for the remote host to come +## back -- can clear it. The cache poisons itself permanently. +## +## This happened on 2026-09-09. A Zenodo outage truncated CCLE_2015.rds at +## 2.4GB, and the drugs step then failed 15 times (5 retries x 3 step attempts) +## with an identical error, while the other five PSets downloaded fine. +## +## The fix is to delete the cached file whenever an attempt fails, so the next +## attempt performs a real download. We deliberately do NOT try to validate the +## file instead: the truncated CCLE file is still a readable gzip stream that +## R's gzfile() walks to the end without complaint, so a cheap integrity check +## reports it healthy. Only readRDS() detects it, and that would mean +## deserialising several GB just to decide whether to keep a file we are about +## to re-download anyway. +## Download any URL to `dest`, resuming an interrupted transfer. +## +## Generic counterpart to the PSet fetcher below, for the plain file downloads +## in the omics steps. Those are the largest transfers in the whole pipeline -- +## Sanger ships rnaseq_all at ~897MB and WES_pureCN_CNV_genes at ~935MB -- and +## httr2's req_retry() restarts the whole request rather than resuming it. On a +## link that drops part-way, restarting a 935MB file simply fails again at a +## similar point: build v24 died exactly that way on a 112MB file, breaking +## 12MB in, three times. +## +## wget -c issues a Range request and continues from what is already on disk. +## Bytes accumulate in ".part", which is renamed onto `dest` only once the +## size matches the server's, so a partial file never appears at the +## destination. The ".part" is deliberately kept between attempts -- deleting it +## would mean a flaky link could never accumulate a large file. +download_file_resumable <- function(url, dest, sleeps = RETRY_SLEEPS, + what = NULL) { + what <- if (is.null(what)) basename(dest) else what + part <- paste0(dest, ".part") + + expected <- suppressWarnings(tryCatch({ + out <- system2("wget", c("--spider", "-S", "--timeout=30", "--tries=2", + shQuote(url)), stdout = TRUE, stderr = TRUE) + hits <- grep("Content-Length:", out, value = TRUE, ignore.case = TRUE) + if (length(hits)) { + as.numeric(sub(".*[Cc]ontent-[Ll]ength:[[:space:]]*([0-9]+).*", "\\1", + hits[length(hits)])) + } else NA_real_ + }, error = function(e) NA_real_)) + + with_retries( + function() { + status <- suppressWarnings(tryCatch( + system2("wget", c("-c", "--tries=10", "--waitretry=15", + "--read-timeout=60", "--timeout=30", + "--progress=dot:giga", + "-O", shQuote(part), shQuote(url))), + error = function(e) 1L)) + ok <- identical(as.integer(status), 0L) + + if (!file.exists(part)) { + stop(sprintf("download of %s produced no file", what)) + } + got <- file.info(part)$size + + if (!is.na(expected)) { + if (got < expected) { + stop(sprintf("%s incomplete (%s of %s bytes); next attempt resumes", + what, format(got, big.mark = ","), + format(expected, big.mark = ","))) + } + if (got > expected) { + unlink(part) + stop(sprintf("%s is oversized (%s vs %s bytes); discarded", what, + format(got, big.mark = ","), + format(expected, big.mark = ","))) + } + } else if (!ok) { + stop(sprintf("%s transfer did not complete (%s bytes so far, remote size unknown); next attempt resumes", + what, format(got, big.mark = ","))) + } + + if (!file.rename(part, dest)) { + stop(sprintf("could not move %s into place", what)) + } + message(sprintf("[%s] %s: downloaded %s bytes", Sys.time(), what, + format(got, big.mark = ","))) + invisible(dest) + }, + what = paste0("download('", what, "')"), + sleeps = sleeps) +} + + +## Look up a PSet's download URL from the PharmacoGx catalogue. +pset_download_url <- function(pset_name) { + tbl <- tryCatch(PharmacoGx::availablePSets(), error = function(e) NULL) + if (is.null(tbl)) return(NA_character_) + i <- match(pset_name, as.character(tbl[, "PSet Name"])) + if (is.na(i)) return(NA_character_) + as.character(tbl[i, "Download"]) +} + +## Fetch the publisher's size and MD5 for a PSet file. +## +## PSets are hosted on Zenodo, whose REST API publishes both. Returns a list +## with NA entries when the URL is not a Zenodo record or the API is +## unreachable, so callers degrade to size-only checking rather than failing. +pset_remote_meta <- function(url) { + none <- list(size = NA_real_, md5 = NA_character_) + if (is.na(url) || !nzchar(url)) return(none) + rec <- sub(".*/records?/([0-9]+)/files/.*", "\\1", url) + if (identical(rec, url) || !grepl("^[0-9]+$", rec)) return(none) + fname <- basename(sub("\\?.*$", "", url)) + api <- paste0("https://zenodo.org/api/records/", rec) + txt <- suppressWarnings(tryCatch( + system2("wget", c("-qO-", "--timeout=30", "--tries=3", shQuote(api)), stdout = TRUE), + error = function(e) character(0))) + if (!length(txt)) return(none) + j <- tryCatch(jsonlite::fromJSON(paste(txt, collapse = "")), error = function(e) NULL) + if (is.null(j) || is.null(j$files) || !nrow(j$files)) return(none) + i <- match(fname, j$files$key) + if (is.na(i)) return(none) + list(size = as.numeric(j$files$size[i]), + md5 = sub("^md5:", "", as.character(j$files$checksum[i]))) +} + +## Fetch with wget -c, which RESUMES an interrupted transfer via an HTTP range +## request instead of starting over. +## +## The CCLE PSet is 2.8GB and the connection can drop between 400MB and 1.1GB, +## so a non-resuming download never reaches the end: each retry throws away +## everything it had and dies at roughly the same place. Verified 2026-09-09 +## that Zenodo answers range requests with 206 PARTIAL_CONTENT. +fetch_pset_resumable <- function(url, dest) { + if (is.na(url) || !nzchar(url)) return(FALSE) + status <- suppressWarnings(tryCatch( + system2("wget", c("-c", "--tries=10", "--waitretry=15", "--read-timeout=60", + "--timeout=30", "--progress=dot:giga", + "-O", shQuote(dest), shQuote(url))), + error = function(e) 1L)) + identical(as.integer(status), 0L) +} + +## Load a cached PSet the way PharmacoGx::downloadPSet would, minus its final +## saveRDS() -- that re-save is only a caching optimisation, and it is what +## makes a verified file stop matching its published checksum. +## +## An unreadable file is discarded so the caller can fetch a clean copy. This +## is the last line of defence for files cached before verification existed. +pset_load_cached <- function(cached, pset_name) { + pSet <- tryCatch(readRDS(cached), error = function(e) { + message(sprintf("[%s] cached PSet '%s' is unreadable (%s); discarding it.", + Sys.time(), pset_name, conditionMessage(e))) + unlink(cached) + stop(e) + }) + updateObject(pSet) +} + +## Attempts with PharmacoGx's own downloader before the fallback kicks in. +## Short on purpose: long enough to ride out a blip, short enough that a real +## outage reaches the resumable path in minutes rather than half an hour. +PRIMARY_SLEEPS <- utils::head(RETRY_SLEEPS, 2) + +## Download a PSet. +## +## Primary path: PharmacoGx::downloadPSet, so normal builds behave exactly as +## upstream intends and we do not diverge from the library without cause. +## +## Fallback path (only after the primary has exhausted PRIMARY_SLEEPS): fetch +## with wget -c into ".rds.part" and rename to ".rds" only once +## size and MD5 match what Zenodo publishes. That separation is what makes +## resuming safe -- a ".part" file is by construction an interrupted download, +## so resuming it is always correct, while ".rds" is by construction +## complete, so it is never resumed onto. +## +## The fallback exists because downloadPSet cannot recover from two situations +## seen in real builds: +## +## - It only downloads when the file is ABSENT, so an interrupted transfer +## leaves a truncated .rds that it then reuses forever; every later attempt +## fails instantly in readRDS() with "error reading from connection". A +## Zenodo outage on 2026-09-09 truncated CCLE_2015.rds at 2.4GB of 2.8GB +## and the drugs step then failed 15 times identically. +## - It does not resume. CCLE is 2.8GB and the connection can drop between +## 400MB and 1.1GB, so every attempt discarded its progress and died in +## roughly the same place. +download_pset_retry <- function(pset_name, sleeps = RETRY_SLEEPS) { + if (!dir.exists(PSET_CACHE_DIR)) { + dir.create(PSET_CACHE_DIR, recursive = TRUE, showWarnings = FALSE) + } + cached <- file.path(PSET_CACHE_DIR, paste0(pset_name, ".rds")) + part <- paste0(cached, ".part") + + ## ---- Primary: PharmacoGx's own downloader ------------------------------- + primary <- tryCatch( + with_retries( + function() { + ## Deliberately does NOT delete the cached file on failure. If the + ## file is good and the error was transient, deleting it would throw + ## away a multi-GB download for nothing. If the file IS truncated, + ## downloadPSet reuses it blindly and the remaining primary attempts + ## fail instantly on the same bytes -- which is harmless and reaches + ## the fallback sooner. Either way the fallback sorts it out. + PharmacoGx::downloadPSet(pset_name, + saveDir = PSET_CACHE_DIR, + timeout = 10000) + }, + what = paste0("downloadPSet('", pset_name, "')"), + sleeps = PRIMARY_SLEEPS), + error = function(e) NULL) + + if (!is.null(primary)) return(primary) + + ## ---- Fallback: resumable, checksum-verified fetch ------------------------ + message(sprintf("[%s] PSet '%s': falling back to a resumable, checksum-verified download.", + Sys.time(), pset_name)) + ## Note we do not delete `cached` here. A file left by the primary may be + ## perfectly good -- the primary can fail for reasons unrelated to it -- and + ## discarding a multi-GB download on suspicion is expensive. pset_load_cached() + ## below is the gate: if the file is unreadable it is removed there, and the + ## next attempt downloads a clean, checksum-verified copy. Resuming onto it is + ## impossible either way, because the fallback only ever resumes ".part". + + with_retries( + function() { + if (!file.exists(cached)) { + url <- pset_download_url(pset_name) + if (is.na(url) || !nzchar(url)) { + stop(sprintf("no download URL for PSet '%s'", pset_name)) + } + meta <- pset_remote_meta(url) + complete <- fetch_pset_resumable(url, part) + + if (!file.exists(part)) { + stop(sprintf("download of PSet '%s' produced no file", pset_name)) + } + got <- file.info(part)$size + + ## A ".part" file is promoted to the final path ONLY on positive + ## evidence that it is complete. Without that evidence it stays a + ## ".part", so the next attempt RESUMES it. + ## + ## This is what failed on 2026-09-10 for CCLE_2019. Zenodo was timing + ## out, so the metadata lookup timed out too and both checks below were + ## skipped as NA. The partial file was then renamed into place, + ## readRDS rejected it, and the file was deleted -- destroying the + ## downloaded bytes on every cycle. Five attempts over 29 minutes each + ## restarted from zero and the step could never finish. + ## + ## wget's exit status is the evidence when the API is unreachable: it + ## is 0 only when the whole file was retrieved. + if (!is.na(meta$size)) { + if (got < meta$size) { + stop(sprintf("PSet '%s' incomplete (%s of %s bytes); next attempt resumes", + pset_name, format(got, big.mark = ","), + format(meta$size, big.mark = ","))) + } + if (got > meta$size) { + unlink(part) + stop(sprintf("PSet '%s' is oversized (%s vs %s bytes); discarded", + pset_name, format(got, big.mark = ","), + format(meta$size, big.mark = ","))) + } + } else if (!complete) { + ## No published size AND wget did not finish: assume partial. Keeping + ## it is safe -- the worst case is one wasted resume attempt, whereas + ## discarding it guarantees the download can never accumulate. + stop(sprintf("PSet '%s' transfer did not complete (%s bytes so far, remote size unknown); next attempt resumes", + pset_name, format(got, big.mark = ","))) + } + + if (!is.na(meta$md5)) { + local_md5 <- unname(tools::md5sum(part)) + if (!identical(local_md5, meta$md5)) { + unlink(part) + stop(sprintf("PSet '%s' failed checksum (got %s, published %s); discarded", + pset_name, local_md5, meta$md5)) + } + message(sprintf("[%s] PSet '%s' verified against published MD5 %s", + Sys.time(), pset_name, meta$md5)) + } else if (!is.na(meta$size)) { + message(sprintf("[%s] PSet '%s': no published checksum; verified against published size %s bytes.", + Sys.time(), pset_name, format(meta$size, big.mark = ","))) + } else { + message(sprintf("[%s] PSet '%s': no published size or checksum available; accepted on a complete transfer, readRDS is the remaining check.", + Sys.time(), pset_name)) + } + + if (!file.rename(part, cached)) { + stop(sprintf("could not move verified PSet '%s' into place", pset_name)) + } + } + pset_load_cached(cached, pset_name) + }, + what = paste0("downloadPSet('", pset_name, "') [resumable fallback]"), + sleeps = sleeps) +} + + +## --------------------------------------------------------------------------- +## Per-PSet completion markers. +## +## Retrying the download is not enough on its own. build_all.py restarts a +## failed step from the very beginning, so before these markers existed, a +## failure on the 5th of 6 PSets meant re-downloading and re-processing the +## first four -- which is how one Zenodo 504 turned into a 17-hour failure +## across three attempts. +## +## A marker is written only after a PSet's results have been merged into the +## cumulative output, so a half-finished PSet is never marked done. Markers +## live under /tmp (build_all.py's bind mount to the host's local/) alongside +## the cumulative output they describe, so the two are cleared together and +## cannot disagree: wiping local/ resets both, and keeping local/ keeps both. +## +## Net effect: a rerun re-does only the PSet that actually failed. +## --------------------------------------------------------------------------- +PSET_STATE_DIR <- Sys.getenv("PSET_STATE_DIR", + unset = file.path(PSET_CACHE_DIR, "completed")) + +pset_is_done <- function(tag) { + file.exists(file.path(PSET_STATE_DIR, paste0(make.names(tag), ".done"))) +} + +mark_pset_done <- function(tag) { + if (!dir.exists(PSET_STATE_DIR)) { + dir.create(PSET_STATE_DIR, recursive = TRUE, showWarnings = FALSE) + } + file.create(file.path(PSET_STATE_DIR, paste0(make.names(tag), ".done"))) + invisible(TRUE) +} + + +## Canonical path for a cached per-study dose-response table (04a). +## +## Both the writer and the reader must agree on this name. They previously did +## not -- the writer used paste0(tolower(studyName),'DoseResponse') while the +## reader looked for paste0(cel,'doseResponse'), so the resume check never +## matched and every attempt recomputed everything. Routing both through one +## function makes that class of mismatch impossible. +## +## Kept under /tmp so it survives between container attempts, same rationale +## as the PSet cache above. +DOSE_RESPONSE_CACHE_DIR <- Sys.getenv("DOSE_RESPONSE_CACHE_DIR", + unset = "/tmp/dose_response_cache") + +dose_response_cache_path <- function(study_name) { + if (!dir.exists(DOSE_RESPONSE_CACHE_DIR)) { + dir.create(DOSE_RESPONSE_CACHE_DIR, recursive = TRUE, showWarnings = FALSE) + } + file.path(DOSE_RESPONSE_CACHE_DIR, + paste0(tolower(study_name), "_doseResponse.tsv")) +} diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 8c3bb542..8f41b453 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -44,6 +44,8 @@ def __init__( name: Optional[str]=None, transcriptomics: Optional[pd.DataFrame]=None, proteomics: Optional[pd.DataFrame]=None, + phosphoproteomics: Optional[pd.DataFrame]=None, + phosphosites: Optional[pd.DataFrame]=None, mutations: Optional[pd.DataFrame]=None, copy_number: Optional[pd.DataFrame]=None, samples: Optional[pd.DataFrame]=None, @@ -95,6 +97,8 @@ def __init__( self.name = name self.transcriptomics = transcriptomics self.proteomics = proteomics + self.phosphoproteomics = phosphoproteomics + self.phosphosites = phosphosites self.mutations = mutations self.copy_number = copy_number self.samples = samples @@ -151,6 +155,32 @@ def proteomics(self): del self._proteomics + @property + def phosphoproteomics(self): + return self._phosphoproteomics + + @phosphoproteomics.setter + def phosphoproteomics(self, value): + self._phosphoproteomics = value + + @phosphoproteomics.deleter + def phosphoproteomics(self): + del self._phosphoproteomics + + + @property + def phosphosites(self): + return self._phosphosites + + @phosphosites.setter + def phosphosites(self, value): + self._phosphosites = value + + @phosphosites.deleter + def phosphosites(self): + del self._phosphosites + + @property def mutations(self): return self._mutations @@ -513,6 +543,8 @@ def types(self) -> list: data_types = [ 'transcriptomics', 'proteomics', + 'phosphoproteomics', + 'phosphosites', 'mutations', 'copy_number', 'samples', @@ -521,6 +553,7 @@ def types(self) -> list: 'experiments', 'methylation', 'metabolomics', + 'combinations', 'genes', ] data_types_present = [] @@ -609,6 +642,7 @@ def load( data_types_to_load = ( 'transcriptomics', 'proteomics', + 'phosphoproteomics', 'mutations', 'copy_number', 'samples', @@ -618,7 +652,11 @@ def load( 'experiments', 'methylation', 'metabolomics', + 'combinations', + # reference tables are loaded last so they can be subset to the + # identifiers present in the data types loaded above 'genes', + 'phosphosites', ) if type(local_path) is not Path: @@ -649,6 +687,9 @@ def load( for p in local_path.glob(f'genes*'): if p.name.endswith(accepted_file_endings) and p.is_file(): files['genes'] = p + for p in local_path.glob(f'phosphosites*'): + if p.name.endswith(accepted_file_endings) and p.is_file(): + files['phosphosites'] = p for dataset_type in data_types_to_load: if dataset_type not in files: @@ -659,7 +700,7 @@ def load( ) continue file = files[dataset_type] - if dataset_type != 'genes': + if dataset_type not in ('genes', 'phosphosites'): print( f"Importing '{dataset_type}' from {file} ...", end=' ', @@ -668,6 +709,34 @@ def load( if hasattr(dataset, dataset_type): setattr(dataset, dataset_type, _load_file(file)) print("DONE", file=sys.stderr) + elif dataset_type == 'phosphosites': + ''' + The phosphosites table is a universal reference (analogous + to 'genes') mapping phosphosite_id -> gene / residue info. + It is only relevant when the dataset has phosphoproteomics, + and is subset to the phosphosite_ids present in that data. + ''' + if dataset.phosphoproteomics is None: + print( + f"'phosphosites' skipped for {name} " + f"(no phosphoproteomics present)", + file=sys.stderr + ) + continue + print( + f"Importing 'phosphosites' from {file} ...", + end=' ', + file=sys.stderr + ) + dataset.phosphosites = _load_file(file) + if 'phosphosite_id' in dataset.phosphoproteomics.columns: + site_ids = set( + dataset.phosphoproteomics['phosphosite_id'].unique() + ) + dataset.phosphosites = dataset.phosphosites[ + dataset.phosphosites['phosphosite_id'].isin(site_ids) + ] + print("DONE", file=sys.stderr) else: ''' The genes dataset available in the online repository is diff --git a/coderdata/download/downloader.py b/coderdata/download/downloader.py index 552cd9ba..b6b367f9 100644 --- a/coderdata/download/downloader.py +++ b/coderdata/download/downloader.py @@ -110,7 +110,9 @@ def download( if name != "all": filtered_files = [ f for f in all_files - if (f.get('name', '').casefold().startswith(name)) or ('genes' in f.get('name', '').casefold()) + if (f.get('name', '').casefold().startswith(name)) + or ('genes' in f.get('name', '').casefold()) + or ('phosphosites' in f.get('name', '').casefold()) ] else: filtered_files = all_files diff --git a/schema/coderdata.yaml b/schema/coderdata.yaml index 819e9a8f..e34f7d35 100755 --- a/schema/coderdata.yaml +++ b/schema/coderdata.yaml @@ -13,6 +13,10 @@ slots: entrez_id: description: Gene id according to NCBI range: integer + phosphosite_id: + description: Unique identifier for a phosphorylation site (gene + residue + position) + identifier: true + range: integer improve_sample_id: description: Unique sample identifier for this dataset, generated by this package identifier: true @@ -49,6 +53,24 @@ classes: gene_symbol: required: true description: HGNC Gene symbol or synomym + Phosphosite: + description: Reference table of phosphorylation sites; each row is a unique gene + residue + position combination. + slots: + - phosphosite_id + - entrez_id + - other_id + - other_id_source + attributes: + gene_symbol: + required: true + description: HGNC gene symbol + residue: + description: Phosphorylated amino acid residue (S, T, or Y) + position: + range: integer + description: Position of the residue in the canonical protein sequence + modification: + description: Type of modification (e.g. phospho) Sample: description: Unique samples identified for each experiment, used to cross reference omics data to drug data. slots: @@ -122,6 +144,16 @@ classes: proteomics: range: float description: Log-normalized log ratio of proteomics measurements of sample at baseline. + Phosphoproteomics: + slots: + - phosphosite_id + - improve_sample_id + - source + - study + attributes: + phosphoproteomics: + range: float + description: Log-normalized, batch-corrected phosphoproteomics abundance value (correctedAbundance). Copy Number: slots: - entrez_id @@ -157,7 +189,7 @@ classes: - time_unit attributes: dose_response_metric: - range: CurveMetric + range: ResponseMetric description: Metric by which dose response value is measured dose_response_value: description: Value of metric @@ -187,6 +219,8 @@ classes: - improve_sample_id - source - study + - time + - time_unit attributes: improve_drug_1: description: improve_drug_id of first drug @@ -195,6 +229,7 @@ classes: drug_combination_metric: description: metric calculated for synergy, or other metric of two drugs drug_combination_value: + range: linkml:Any description: value of metric for synergy or combination enums: ResponseMetric: @@ -233,6 +268,19 @@ enums: description: The tumor growth inhibition (TGI) between the control and treatment time-volume curves. kulgap: description: kl divergence based metric + abc: + description: Area between curves for PDX in-vivo response + lmm: + description: Linear mixed model coefficient for PDX tumor growth response + uM_viability: + description: + Single-dose viability fraction at 1 μM. Values near 1.0 indicate no + effect (full viability); values near 0.0 indicate full kill. Values + above 1.0 are possible when treated cells proliferate beyond the + untreated control. Used for organoid drug screens where dose-response + curves were not collected for that drug. + delta_auc: + description: Difference in AUC between single-drug and combination treatment; used for MPNST treated-microtissue combination experiments. CellPerturbation: permissible_values: gene_ko: @@ -248,6 +296,8 @@ enums: xenograft derived organoid: patient derived organoid: ex vivo: + normal_tissue: + 3D-MEDS: Variant: permissible_values: 3'UTR: diff --git a/schema/expected_files.yaml b/schema/expected_files.yaml index 3de46533..2ca97376 100644 --- a/schema/expected_files.yaml +++ b/schema/expected_files.yaml @@ -42,6 +42,8 @@ datasets: file: /tmp/mpnst_drugs.tsv - target_class: Drug Descriptor file: /tmp/mpnst_drug_descriptors.tsv + - target_class: Combinations + file: /tmp/mpnst_combinations.tsv cptac: - target_class: Sample @@ -294,4 +296,33 @@ datasets: - target_class: Drug file: /tmp/novartis_drugs.tsv - target_class: Drug Descriptor - file: /tmp/novartis_drug_descriptors.tsv \ No newline at end of file + file: /tmp/novartis_drug_descriptors.tsv + + + cnf: + - target_class: Sample + file: /tmp/cnf_samples.csv + - target_class: Transcriptomics + file: /tmp/cnf_transcriptomics.csv + - target_class: Proteomics + file: /tmp/cnf_proteomics.csv + - target_class: Phosphoproteomics + file: /tmp/cnf_phosphoproteomics.csv + # - target_class: Mutations + # file: /tmp/cnf_mutations.csv + # - target_class: Copy Number + # file: /tmp/cnf_copy_number.csv + - target_class: Experiments + file: /tmp/cnf_experiments.tsv + - target_class: Drug + file: /tmp/cnf_drugs.tsv + - target_class: Drug Descriptor + file: /tmp/cnf_drug_descriptors.tsv + + phosphosites: + - target_class: Phosphosite + file: /tmp/phosphosites.csv + + genes: + - target_class: Gene + file: /tmp/genes.csv \ No newline at end of file diff --git a/scripts/prepare_data_for_improve.py b/scripts/prepare_data_for_improve.py index e567e331..b74d14cc 100644 --- a/scripts/prepare_data_for_improve.py +++ b/scripts/prepare_data_for_improve.py @@ -194,21 +194,21 @@ def process_datasets(args): # Logic for PDX datasets that don't have `auc` but mRECIST (note # the typo currently in the `drugresponse_metric` column). - elif experiments_raw['dose_response_metric'].isin(['mRESCIST']).any(): + elif experiments_raw['dose_response_metric'].isin(['mrecist']).any(): experiment = data_sets[data_set].format( data_type='experiments', shape='wide', metrics=[ - 'mRESCIST', + 'mrecist', ], ) # conversion logic from mRECIST -> auc - experiment.loc[experiment['mRESCIST'] == 'CR', 'mRESCIST'] = "0.1" - experiment.loc[experiment['mRESCIST'] == 'PR', 'mRESCIST'] = "0.2" - experiment.loc[experiment['mRESCIST'] == 'SD', 'mRESCIST'] = "0.5" - experiment.loc[experiment['mRESCIST'] == 'PD', 'mRESCIST'] = "1.0" + experiment.loc[experiment['mrecist'] == 'CR', 'mrecist'] = "0.1" + experiment.loc[experiment['mrecist'] == 'PR', 'mrecist'] = "0.2" + experiment.loc[experiment['mrecist'] == 'SD', 'mrecist'] = "0.5" + experiment.loc[experiment['mrecist'] == 'PD', 'mrecist'] = "1.0" - experiment.rename(columns={'mRESCIST': 'auc'}, inplace=True) + experiment.rename(columns={'mrecist': 'auc'}, inplace=True) experiments.append(experiment) # The remaining datasets should have `auc` as