Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 80 additions & 8 deletions coderbuild/broad_sanger/01-broadSangerSamples.R
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ library(dplyr)
library(readr)
library(XML)

## Retry helpers for flaky third-party downloads
source("retry_utils.R")

##the only thing that Priyanka has here is TRP identifiers, so collecting those
# tab<-read.table('DepMap_Argonne_Mapping.csv',sep=',',header=T)%>%
# dplyr::select(Argonne_ID,DepMap_ID)%>%
Expand All @@ -22,18 +25,83 @@ library(XML)
###[1] "Panc-05-04" "L3-3" "Panc-02-03" "Panc-10-05" "G-292-clone-A141B1" "Hs-729" "Panc-03-27" "Panc-04-03" "Hs-688-A-T"
###[10] "Panc-08-13" "PE-CA-PJ34-clone-C12" "Hep-3B2-1-7" "PE-CA-PJ41-clone-D2" "PE-CA-PJ49" "Ishikawa-Heraklio-02-ER"

##here are all the models in depmap 23Q2, downloded on 10/11/2023
depmap_models<-readr::read_csv('https://figshare.com/ndownloader/files/40448834')#|>
## ---------------------------------------------------------------------------
## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE.
##
## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in
## front of https://depmap.org/portal/api/download/files. That endpoint now
## returns an HTML challenge page with HTTP 200, so read_csv() silently parses
## the HTML and every downstream column lookup fails. It cannot be scripted
## around, and DepMap explicitly asks that the portal not be scraped.
##
## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in
## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495
##
## build_samples.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script,
## which downloads them from Synapse into $DEPMAP_DIR. This script only reads
## from disk -- it never contacts the DepMap portal.
##
## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE:
## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData
## (accept the terms, then use the download button on each file)
## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities
## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py
## 4. Re-run the build with --depmap-ready
##
## Currently pinned to: DepMap Public 26Q1
## Files required by coderdata (4 total, ~1.29 GB):
## Model.csv (samples - this script)
## OmicsSomaticMutations.csv (omics)
## OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv (omics)
## PortalOmicsCNGeneLog2.csv (omics)
## ---------------------------------------------------------------------------
## DEPMAP_DIR is exported by build_samples.sh from fetch_depmap.py, so the
## release name is defined in exactly one place. The fallback keeps this script
## runnable standalone for debugging.
DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data")
depmap_model_file <- file.path(DEPMAP_DIR, "Model.csv")

if (!file.exists(depmap_model_file)) {
stop(sprintf(paste0(
"Required DepMap file not found: %s\n",
"It should have been downloaded from Synapse by fetch_depmap.py before\n",
"this script ran. DepMap files can no longer be downloaded from the DepMap\n",
"portal (Cloudflare challenge), so they are read from the Synapse folder\n",
"syn75028495. To refresh them for a new release, download from\n",
"https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."),
depmap_model_file))
}

message("Using DepMap file from Synapse copy: ", depmap_model_file)
depmap_models<-readr::read_csv(depmap_model_file)#|>
# dplyr::rename(DepMap_ID='ModelID')
## Transient container-local input -- drop it once it is in memory.
unlink(depmap_model_file)

sanger_models<-readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv")
## cog.sanger.ac.uk intermittently drops the connection mid-transfer
## (OpenSSL "unexpected eof"); this failed twice before succeeding on the
## third attempt on 2026-08-30.
sanger_models<-with_retries(
function() readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv",
show_col_types = FALSE),
what = "download Sanger model_list")

print(paste("Downloaded",nrow(depmap_models),'dep map identifiers and',nrow(sanger_models),'sanger models'))

##query for cellosaurus automagically to get loadest version
url='https://ftp.expasy.org/databases/cellosaurus/cellosaurus.xml'
if(!file.exists('/tmp/cell.xml'))
curl_download(url,'/tmp/cell.xml',quiet=TRUE)#curl(url, "r", h)
## 672 MB download; a partial file would poison every later run because the
## existence check below would treat it as complete. Download to a temp path
## and only move it into place once it has fully arrived.
if(!file.exists('/tmp/cell.xml')) {
with_retries(function() {
tmp_cello <- '/tmp/cell.xml.partial'
curl_download(url, tmp_cello, quiet=TRUE)
if (file.info(tmp_cello)$size < 1e8)
stop(sprintf('cellosaurus download truncated (%s bytes)', file.info(tmp_cello)$size))
file.rename(tmp_cello, '/tmp/cell.xml')
}, what = "download cellosaurus.xml")
}
cello<-XML::xmlParse('/tmp/cell.xml')
cdf<-XML::xmlToList(cello)

Expand Down Expand Up @@ -72,14 +140,14 @@ print(paste('Got',nrow(full.res),'human cellosaurus samples'))
allmod<-sanger_models|>
dplyr::rename(ModelID='BROAD_ID')|>
left_join(depmap_models)|>
dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type)|>
dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type,ModelType)|>
subset(!is.na(RRID))|> ##only take those we can map to cellosaurus
distinct()

##we missed a handful here, so let's add back
missedmod<-subset(depmap_models,ModelID%in%setdiff(depmap_models$ModelID,allmod$Depmap_ID))|>
subset(!is.na(RRID))|>
dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype')|>
dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype',ModelType)|>
mutate(PatientID=NA)

allmod<-rbind(allmod,missedmod)
Expand Down Expand Up @@ -122,7 +190,11 @@ long.df<-full.df%>%
dplyr::select(-c(StrippedCellLineName))|>
tidyr::pivot_longer(cols=c(PatientID,DepMap,Sanger,CCLE,COSMIC,Cellosaurus),names_to='other_id_source',
values_to='other_id')%>%
mutate(model_type='cell line')%>%
mutate(model_type=dplyr::case_when(
ModelType=='Organoid' ~ 'patient derived organoid',
TRUE ~ 'cell line'
))%>%
dplyr::select(-ModelType)%>%
subset(!is.na(other_id))%>%
subset(other_id!="")

Expand Down
Loading