Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
6f14e3a
Add resilient download, retry, and DepMap-fetch utilities
jjacobson95 Sep 21, 2026
40a31cf
Harden the drug-descriptor and curve-fitting pipeline
jjacobson95 Sep 21, 2026
7713186
Update broad_sanger sample, omics and drug processing
jjacobson95 Sep 21, 2026
24af077
Make the genes build resilient to Ensembl outages and API changes
jjacobson95 Sep 21, 2026
a09ee71
Add phosphosites reference builder
jjacobson95 Sep 21, 2026
89fe050
Add the cNF organoid dataset: samples and omics
jjacobson95 Sep 21, 2026
dfd74c8
Add MPNST treated drug-combination experiments
jjacobson95 Sep 21, 2026
1c4f112
Add MPNST treated-sample and treated-omics support
jjacobson95 Sep 21, 2026
55ec7e1
Add the cNF organoid dataset: drugs and experiments
jjacobson95 Sep 21, 2026
aa4dfb5
Fix Docker image builds and keep scratch off the bind mount
jjacobson95 Sep 21, 2026
5be9072
Add build resume support and DepMap readiness gate
jjacobson95 Sep 21, 2026
31d396c
Declare new dataset outputs and schema classes
jjacobson95 Sep 21, 2026
301e73f
Merge pull request #487 from PNNL-CompBio/mpnst-treated-samples-and-o…
sgosline Sep 23, 2026
989d46a
Merge pull request #490 from PNNL-CompBio/build-orchestration-and-safety
sgosline Sep 23, 2026
d550f70
Speed up and harden schema validation and ID mapping
jjacobson95 Sep 21, 2026
038dbd1
Update package dataset metadata and loader
jjacobson95 Sep 21, 2026
aeffc1d
Merge remote-tracking branch 'origin/cnf-dataset-drugs-and-experiment…
jjacobson95 Sep 30, 2026
ae075ca
Merge remote-tracking branch 'origin/docker-build-fixes' into build-2…
jjacobson95 Sep 30, 2026
f47eae8
Merge remote-tracking branch 'origin/scripts-migration-296' into buil…
jjacobson95 Sep 30, 2026
e15484b
Merge remote-tracking branch 'origin/package-updates-2.4' into build-…
jjacobson95 Sep 30, 2026
5d393e3
Merge remote-tracking branch 'origin/docs-api-reference-render' into …
jjacobson95 Sep 30, 2026
b244524
Merge remote-tracking branch 'origin/docs-deploy-pages-501' into buil…
jjacobson95 Sep 30, 2026
6bfb1f5
Merge remote-tracking branch 'origin/docs-curve-metrics-197' into bui…
jjacobson95 Sep 30, 2026
4667779
Merge remote-tracking branch 'origin/docs-dataset-stats-216' into bui…
jjacobson95 Sep 30, 2026
e0ca058
Merge branch 'copy-number-harmonization' into build-2.4-cn
jjacobson95 Sep 30, 2026
24ca8ac
Build only: keep DepMap CN as log2(relative CN + 1); open-ended copy_…
jjacobson95 Sep 30, 2026
f80b490
Merge branch 'copy-number-harmonization' into build-2.4-cn
jjacobson95 Sep 30, 2026
7d5db2b
Build only: use 0 as the lower copy_call bin edge in prepare_data_for…
jjacobson95 Sep 30, 2026
c3c9c43
Remove cnf and phosphosites from docker-compose.yml
jjacobson95 Oct 1, 2026
fc39a54
Merge branch 'copy-number-harmonization' into build-2.4-cn
jjacobson95 Oct 2, 2026
ef24b3c
Fix broad_sanger omics resume sentinel
jjacobson95 Oct 5, 2026
59dec34
Merge branch 'docker-build-fixes' into build-2.4-cn
jjacobson95 Oct 5, 2026
b527f6e
Harden resumed experiments and HCMI mutation parsing
jjacobson95 Oct 5, 2026
e596113
Merge branch 'docker-build-fixes' into build-2.4-cn
jjacobson95 Oct 5, 2026
7045f5e
Keep dataset-specific fixes out of build resume changes
jjacobson95 Oct 5, 2026
16cd85c
Merge branch 'docker-build-fixes' into build-2.4-cn
jjacobson95 Oct 5, 2026
a15a9d7
Merge branch 'copy-number-harmonization' into build-2.4-cn
jjacobson95 Oct 5, 2026
4e8cd7e
Merge main into build-2.4-cn
jjacobson95 Oct 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,11 @@ build/lib
build/local
coderdata/_version.py
local/

# Build logs and PID files.
# Build logs capture full `docker run` command lines. Those command lines pass
# credentials as `-e NAME=value`, so older logs contain a live
# SYNAPSE_AUTH_TOKEN in cleartext (build_all_v19.log had 47 copies).
# coderbuild/build_all.py now redacts them via _redact(), but never commit these.
*.log
build_v*.pid
88 changes: 80 additions & 8 deletions coderbuild/broad_sanger/01-broadSangerSamples.R
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ library(dplyr)
library(readr)
library(XML)

## Retry helpers for flaky third-party downloads
source("retry_utils.R")

##the only thing that Priyanka has here is TRP identifiers, so collecting those
# tab<-read.table('DepMap_Argonne_Mapping.csv',sep=',',header=T)%>%
# dplyr::select(Argonne_ID,DepMap_ID)%>%
Expand All @@ -22,18 +25,83 @@ library(XML)
###[1] "Panc-05-04" "L3-3" "Panc-02-03" "Panc-10-05" "G-292-clone-A141B1" "Hs-729" "Panc-03-27" "Panc-04-03" "Hs-688-A-T"
###[10] "Panc-08-13" "PE-CA-PJ34-clone-C12" "Hep-3B2-1-7" "PE-CA-PJ41-clone-D2" "PE-CA-PJ49" "Ishikawa-Heraklio-02-ER"

##here are all the models in depmap 23Q2, downloded on 10/11/2023
depmap_models<-readr::read_csv('https://figshare.com/ndownloader/files/40448834')#|>
## ---------------------------------------------------------------------------
## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE.
##
## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in
## front of https://depmap.org/portal/api/download/files. That endpoint now
## returns an HTML challenge page with HTTP 200, so read_csv() silently parses
## the HTML and every downstream column lookup fails. It cannot be scripted
## around, and DepMap explicitly asks that the portal not be scraped.
##
## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in
## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495
##
## build_samples.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script,
## which downloads them from Synapse into $DEPMAP_DIR. This script only reads
## from disk -- it never contacts the DepMap portal.
##
## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE:
## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData
## (accept the terms, then use the download button on each file)
## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities
## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py
## 4. Re-run the build with --depmap-ready
##
## Currently pinned to: DepMap Public 26Q1
## Files required by coderdata (4 total, ~1.29 GB):
## Model.csv (samples - this script)
## OmicsSomaticMutations.csv (omics)
## OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv (omics)
## PortalOmicsCNGeneLog2.csv (omics)
## ---------------------------------------------------------------------------
## DEPMAP_DIR is exported by build_samples.sh from fetch_depmap.py, so the
## release name is defined in exactly one place. The fallback keeps this script
## runnable standalone for debugging.
DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data")
depmap_model_file <- file.path(DEPMAP_DIR, "Model.csv")

if (!file.exists(depmap_model_file)) {
stop(sprintf(paste0(
"Required DepMap file not found: %s\n",
"It should have been downloaded from Synapse by fetch_depmap.py before\n",
"this script ran. DepMap files can no longer be downloaded from the DepMap\n",
"portal (Cloudflare challenge), so they are read from the Synapse folder\n",
"syn75028495. To refresh them for a new release, download from\n",
"https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."),
depmap_model_file))
}

message("Using DepMap file from Synapse copy: ", depmap_model_file)
depmap_models<-readr::read_csv(depmap_model_file)#|>
# dplyr::rename(DepMap_ID='ModelID')
## Transient container-local input -- drop it once it is in memory.
unlink(depmap_model_file)

sanger_models<-readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv")
## cog.sanger.ac.uk intermittently drops the connection mid-transfer
## (OpenSSL "unexpected eof"); this failed twice before succeeding on the
## third attempt on 2026-08-30.
sanger_models<-with_retries(
function() readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv",
show_col_types = FALSE),
what = "download Sanger model_list")

print(paste("Downloaded",nrow(depmap_models),'dep map identifiers and',nrow(sanger_models),'sanger models'))

##query for cellosaurus automagically to get loadest version
url='https://ftp.expasy.org/databases/cellosaurus/cellosaurus.xml'
if(!file.exists('/tmp/cell.xml'))
curl_download(url,'/tmp/cell.xml',quiet=TRUE)#curl(url, "r", h)
## 672 MB download; a partial file would poison every later run because the
## existence check below would treat it as complete. Download to a temp path
## and only move it into place once it has fully arrived.
if(!file.exists('/tmp/cell.xml')) {
with_retries(function() {
tmp_cello <- '/tmp/cell.xml.partial'
curl_download(url, tmp_cello, quiet=TRUE)
if (file.info(tmp_cello)$size < 1e8)
stop(sprintf('cellosaurus download truncated (%s bytes)', file.info(tmp_cello)$size))
file.rename(tmp_cello, '/tmp/cell.xml')
}, what = "download cellosaurus.xml")
}
cello<-XML::xmlParse('/tmp/cell.xml')
cdf<-XML::xmlToList(cello)

Expand Down Expand Up @@ -72,14 +140,14 @@ print(paste('Got',nrow(full.res),'human cellosaurus samples'))
allmod<-sanger_models|>
dplyr::rename(ModelID='BROAD_ID')|>
left_join(depmap_models)|>
dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type)|>
dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type,ModelType)|>
subset(!is.na(RRID))|> ##only take those we can map to cellosaurus
distinct()

##we missed a handful here, so let's add back
missedmod<-subset(depmap_models,ModelID%in%setdiff(depmap_models$ModelID,allmod$Depmap_ID))|>
subset(!is.na(RRID))|>
dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype')|>
dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype',ModelType)|>
mutate(PatientID=NA)

allmod<-rbind(allmod,missedmod)
Expand Down Expand Up @@ -122,7 +190,11 @@ long.df<-full.df%>%
dplyr::select(-c(StrippedCellLineName))|>
tidyr::pivot_longer(cols=c(PatientID,DepMap,Sanger,CCLE,COSMIC,Cellosaurus),names_to='other_id_source',
values_to='other_id')%>%
mutate(model_type='cell line')%>%
mutate(model_type=dplyr::case_when(
ModelType=='Organoid' ~ 'patient derived organoid',
TRUE ~ 'cell line'
))%>%
dplyr::select(-ModelType)%>%
subset(!is.na(other_id))%>%
subset(other_id!="")

Expand Down
Loading