From 6f14e3ac6e1084c035b9198327e522be6c5c0905 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:49:59 -0700 Subject: [PATCH 01/20] Add resilient download, retry, and DepMap-fetch utilities --- coderbuild/utils/download_utils.py | 95 ++++++ coderbuild/utils/fetch_depmap.py | 191 +++++++++++++ coderbuild/utils/pinned_psets.R | 115 ++++++++ coderbuild/utils/retry_utils.R | 445 +++++++++++++++++++++++++++++ 4 files changed, 846 insertions(+) create mode 100644 coderbuild/utils/download_utils.py create mode 100644 coderbuild/utils/fetch_depmap.py create mode 100644 coderbuild/utils/pinned_psets.R create mode 100644 coderbuild/utils/retry_utils.R diff --git a/coderbuild/utils/download_utils.py b/coderbuild/utils/download_utils.py new file mode 100644 index 00000000..833c2060 --- /dev/null +++ b/coderbuild/utils/download_utils.py @@ -0,0 +1,95 @@ +"""Resumable, verified HTTP downloads for the build scripts. + +The NCI wiki serves DOSERESP.zip at ~346MB. A plain urlopen().read() of that +either succeeds or raises + + http.client.IncompleteRead: IncompleteRead(32601897 bytes read, + 313702904 more expected) + +which is exactly how build v32 lost the whole nci60 dataset: the download +truncated at 32MB, 04b-nci60-updated.py exited 1, the caller only warned, and +nci60DoseResponse was never written -- so curve fitting saw 7 studies instead +of 8 and nci60 shipped as eight empty files that still validated clean. + +Retrying from zero is not enough for a file this size on a link that drops +part-way; it tends to fail again at a similar point. Each attempt therefore +RESUMES with a Range request. Verified that wiki.nci.nih.gov sends +accept-ranges: bytes and answers a Range request with 206. +""" + +import os +import time +from urllib import request, error + +# 1, 3, 10 and 15 minutes, matching coderbuild/utils/retry_utils.R. +RETRY_SLEEPS = (60, 180, 600, 900) + +# The NCI wiki blocks Python's default User-Agent. +BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + + +def _remote_size(url, headers): + try: + req = request.Request(url, headers=headers, method="HEAD") + with request.urlopen(req, timeout=60) as resp: + n = resp.headers.get("Content-Length") + return int(n) if n else None + except Exception: + return None + + +def retrieve_url(url, dest, sleeps=RETRY_SLEEPS, headers=None): + """Download `url` to `dest`, resuming an interrupted transfer. + + Bytes accumulate in ".part", which is renamed onto `dest` only once + the finished size matches the server's Content-Length, so a truncated file + never appears at the destination. The ".part" is kept between attempts -- + deleting it would mean a flaky link could never accumulate a large file. + """ + headers = dict(headers or {"User-Agent": BROWSER_UA}) + part = dest + ".part" + expected = _remote_size(url, headers) + + last_error = None + for attempt in range(len(sleeps) + 1): + have = os.path.getsize(part) if os.path.exists(part) else 0 + try: + hdrs = dict(headers) + if have: + hdrs["Range"] = f"bytes={have}-" + req = request.Request(url, headers=hdrs) + with request.urlopen(req, timeout=120) as resp: + resumed = resp.status == 206 + if have and not resumed: + # Server ignored the Range; start over rather than append + # onto bytes it is about to send again. + have = 0 + mode = "ab" if (have and resumed) else "wb" + with open(part, mode) as f: + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + f.write(chunk) + + got = os.path.getsize(part) + if expected is not None and got != expected: + raise IOError(f"incomplete: {got} of {expected} bytes") + os.replace(part, dest) + return dest + + except (error.URLError, error.HTTPError, IOError, OSError) as e: + last_error = e + got = os.path.getsize(part) if os.path.exists(part) else 0 + if attempt < len(sleeps): + wait = sleeps[attempt] + print(f" download of {url} failed at {got:,} bytes ({e}); " + f"attempt {attempt + 1}/{len(sleeps) + 1}, resuming in " + f"{wait // 60} min", flush=True) + time.sleep(wait) + + raise RuntimeError( + f"Failed to download {url} after {len(sleeps) + 1} attempts: {last_error}") diff --git a/coderbuild/utils/fetch_depmap.py b/coderbuild/utils/fetch_depmap.py new file mode 100644 index 00000000..799e79a7 --- /dev/null +++ b/coderbuild/utils/fetch_depmap.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +Fetch the DepMap files required by the broad_sanger build from Synapse. + +WHY THIS EXISTS +--------------- +DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in front +of https://depmap.org/portal/api/download/files. That endpoint now returns an +HTML challenge page with HTTP 200, so the previous programmatic download silently +produced garbage: read_csv() parsed the HTML, and the build failed downstream with +a confusing "object 'filename' not found". DepMap also explicitly asks that the +portal not be scraped, so this is not something to work around. + +coderdata therefore keeps a STATIC COPY of the required DepMap files on Synapse. +This script downloads them into a container-local scratch dir (see +DEPMAP_WORKDIR) before the broad_sanger samples and omics R scripts run. Those +R scripts read them from there, delete each one once consumed, and never touch +the DepMap portal. + +UPDATING FOR A NEW DEPMAP RELEASE (must be done by hand) +-------------------------------------------------------- +1. Download the files from the DepMap download page: + https://depmap.org/portal/data_page/?tab=allData + Accept the terms, then use the download button on each file below. +2. Upload them into the "DepMap Raw" Synapse folder + https://www.synapse.org/Synapse:syn75028495 + as NEW VERSIONS of the existing entities listed in DEPMAP_SYN_IDS. + Uploading as a new version keeps the syn ID stable, so nothing in this file + needs to change. (If you create brand-new Synapse entities instead, update + the IDs in DEPMAP_SYN_IDS below.) +3. Bump DEPMAP_RELEASE below to the new release name. +4. Run the build with --depmap-ready to confirm the Synapse copy is current. + +Repository: https://github.com/PNNL-CompBio/coderdata + +STANDALONE USE +-------------- +Run this directly to verify Synapse access and the configured IDs before +launching a long build: + + SYNAPSE_AUTH_TOKEN=... python3 fetch_depmap.py --dest /tmp/depmap_check +""" + +import argparse +import os +import sys + +# --------------------------------------------------------------------------- +# Configuration -- edit DEPMAP_RELEASE and DEPMAP_SYN_IDS when DepMap ships a +# new release (see "UPDATING FOR A NEW DEPMAP RELEASE" above). +# +# Every value can also be overridden by environment variable, which is how +# other coderdata builders (cnf, liver, bladder, ...) parameterise Synapse IDs. +# --------------------------------------------------------------------------- +DEPMAP_RELEASE = os.environ.get('DEPMAP_RELEASE', '26Q1') + +# Synapse entity IDs for the four files the broad_sanger build consumes. +# These live in the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +# +# Upload each new DepMap release as a NEW VERSION of these same entities and +# the IDs below never need to change -- syn.get() always returns the latest +# version. +DEPMAP_SYNAPSE_FOLDER = 'syn75028495' # "DepMap Raw" -- for reference/uploads + +DEPMAP_SYN_IDS = { + 'Model.csv': + os.environ.get('DEPMAP_MODEL_SYN_ID', 'syn75028594'), + 'OmicsSomaticMutations.csv': + os.environ.get('DEPMAP_MUTATIONS_SYN_ID', 'syn77171195'), + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv': + os.environ.get('DEPMAP_TRANSCRIPTOMICS_SYN_ID', 'syn77171175'), + 'PortalOmicsCNGeneLog2.csv': + os.environ.get('DEPMAP_COPY_NUMBER_SYN_ID', 'syn77171184'), +} + +DOWNLOAD_PAGE = 'https://depmap.org/portal/data_page/?tab=allData' +REPO_URL = 'https://github.com/PNNL-CompBio/coderdata' + +# Container-local scratch directory for the downloaded DepMap files. +# +# Deliberately NOT under /tmp: build_all.py bind-mounts the host's local/ +# directory at /tmp inside every container, so anything written there lands on +# the host (and, here, inside a OneDrive-synced folder) and survives the run. +# These files are transient build inputs -- they are consumed by the R scripts +# and deleted -- so they belong in the container's own filesystem, which is +# discarded when the `docker run --rm` exits. +DEPMAP_WORKDIR = os.environ.get('DEPMAP_DIR', '/opt/depmap_data') + + +def default_dest(): + """Container-local directory the R scripts read the DepMap files from.""" + return DEPMAP_WORKDIR + + +def _unconfigured(): + """Names of files whose Synapse ID has not been set yet.""" + return [name for name, sid in DEPMAP_SYN_IDS.items() if not sid.strip()] + + +def main(): + parser = argparse.ArgumentParser( + description='Download the static DepMap files for broad_sanger from Synapse.' + ) + parser.add_argument('--dest', default=None, + help='Directory to download into ' + '(default: $DEPMAP_DIR or /opt/depmap_data, container-local)') + parser.add_argument('--print-dest', action='store_true', + help='Print the download directory and exit. Lets the ' + 'build shell scripts derive DEPMAP_DIR without ' + 'duplicating the release name.') + args = parser.parse_args() + + dest = args.dest or default_dest() + + if args.print_dest: + print(dest) + return + + missing_ids = _unconfigured() + if missing_ids: + sys.exit( + '\n' + '=====================================================================\n' + ' DepMap Synapse IDs are not configured\n' + '=====================================================================\n' + f' No Synapse entity ID is set for:\n' + + ''.join(f' - {n}\n' for n in missing_ids) + + '\n' + ' DepMap files can no longer be downloaded programmatically from the\n' + ' DepMap portal, so coderdata reads a static copy from Synapse.\n' + '\n' + ' To fix this:\n' + f' 1. Download the files from {DOWNLOAD_PAGE}\n' + ' 2. Upload them to Synapse\n' + ' 3. Put their syn IDs in DEPMAP_SYN_IDS in\n' + ' coderbuild/utils/fetch_depmap.py (or set the matching\n' + ' DEPMAP_*_SYN_ID environment variables)\n' + '\n' + f' Repository: {REPO_URL}\n' + '=====================================================================\n' + ) + + if not os.environ.get('SYNAPSE_AUTH_TOKEN'): + sys.exit('SYNAPSE_AUTH_TOKEN is not set; cannot download DepMap files ' + 'from Synapse.') + + try: + import synapseclient + except ImportError: + sys.exit("synapseclient is not installed in this image; it is required " + "to fetch the DepMap files from Synapse. Add 'synapseclient' " + "to coderbuild/broad_sanger/requirements.txt and rebuild.") + + os.makedirs(dest, exist_ok=True) + + syn = synapseclient.Synapse() + syn.login() + + print(f'Fetching DepMap {DEPMAP_RELEASE} files from Synapse into {dest}') + for fname, syn_id in DEPMAP_SYN_IDS.items(): + print(f' {fname} <- {syn_id}') + # Always re-download so the copy used by the build provably matches + # Synapse. `dest` is container-local, so this costs no host disk and is + # discarded when the container exits. + entity = syn.get( + syn_id, + downloadLocation=dest, + ifcollision='overwrite.local', + ) + + # Synapse names the downloaded file after the entity, which may differ + # from the name the R scripts expect. Normalise it. + got = getattr(entity, 'path', None) + if not got or not os.path.exists(got): + sys.exit(f'Synapse returned no downloaded file for {fname} ({syn_id}). ' + f'Check that the entity exists and that this account has ' + f'download permission on {DEPMAP_SYNAPSE_FOLDER}.') + want = os.path.join(dest, fname) + if os.path.abspath(got) != os.path.abspath(want): + os.replace(got, want) + + size = os.path.getsize(want) + if size == 0: + sys.exit(f'Downloaded {fname} from {syn_id} but it is empty.') + print(f' -> {want} ({size / 1e6:.1f} MB)') + + print(f'All {len(DEPMAP_SYN_IDS)} DepMap {DEPMAP_RELEASE} files ready in {dest}') + + +if __name__ == '__main__': + main() diff --git a/coderbuild/utils/pinned_psets.R b/coderbuild/utils/pinned_psets.R new file mode 100644 index 00000000..36003949 --- /dev/null +++ b/coderbuild/utils/pinned_psets.R @@ -0,0 +1,115 @@ +## --------------------------------------------------------------------------- +## Pinned PharmacoGx PSet list for the broad_sanger build. +## +## WHY THIS EXISTS +## --------------- +## PharmacoGx::availablePSets() is a LIVE list served by ORCESTRA. New PSets +## appear without notice, and the build previously consumed whatever that call +## returned. That is unsafe for a published dataset in two distinct ways: +## +## 1. A new PSet with an INCOMPATIBLE schema breaks the build. +## In Aug 2026 a second CTRPv2 PSet ("CTRPv2.1_2016") appeared. Its +## sensitivityInfo() has neither `treatmentid` nor `drugid` -- it uses +## `Compound.Name` / `Broad.Compound.ID` / `master_cpd_id` instead. The +## drugs step found 0 chemicals for it and then crashed in +## update_dataframe_and_write_tsv with +## "TypeError: 'NoneType' object is not iterable". +## +## 2. A new PSet with a COMPATIBLE schema silently changes the published +## data between releases, with no diff and no reviewer. +## +## Both the drugs step (03-createDrugFile.R) and the experiments step +## (04a-drugResponseData.R) resolve PSets through this file, so they can never +## disagree about which PSets are in a release. A mismatch there would produce +## experiments referencing drugs that do not exist in the drugs file, which +## fails schema validation late and confusingly. +## +## ADDING A NEW PSet +## ----------------- +## Adding a PSet changes the published dataset, so it is a deliberate, +## reviewed change -- not something that should happen because upstream +## published something new. To add one: +## 1. Confirm its sensitivityInfo() exposes `treatmentid` or `drugid` +## (or add an explicit column mapping in 03-createDrugFile.R). +## 2. Confirm the experiments step can extract dose-response from it. +## 3. Add the exact `PSet Name` below and note the release it entered in. +## +## CURRENT PINS +## ------------ +## These are exactly the PSets used to build coderdata v2.3 (verified against +## the v2.3 build log), so v2.4 keeps the same PharmacoGx provenance. +## +## KNOWN AVAILABLE BUT DELIBERATELY EXCLUDED: +## CTRPv2.1_2016 -- DO NOT ADD. It is not new data; it is the SAME CTRP v2 +## data as CTRPv2_2015 with different curation. Verified +## 2026-08-28 by loading both PSets: +## +## sensitivityInfo rows : 395263 vs 395263 (identical) +## unique samples : 887 vs 887 (identical) +## unique experiment_id : 907 vs 907 (identical +## SETS -- identical==TRUE) +## unique treatments : 544 vs 545 +## +## The only real difference is naming convention. +## CTRPv2_2015 is PharmacoGx-standardised (`treatmentid`, +## `sampleid`, INN names like "adavosertib", hyphenated +## codes like "azd-1480"). CTRPv2.1_2016 keeps raw CTRP +## columns (`Compound.Name`, `master_ccl_id`) and Broad +## code names ("abt-199", "azd1480"). Case-insensitively +## 481 of ~545 treatment names match outright; most of the +## remainder are the same compound spelled differently +## (azd-1480 / azd1480, azd-7545 / azd7545). +## +## Including it would therefore DUPLICATE all 395,263 +## measurements and mint a second improve_drug_id for +## compounds we already have under a different spelling -- +## a data-quality regression, not an enrichment. +## --------------------------------------------------------------------------- + +PINNED_PSETS <- c( + "CTRPv2_2015", + "CCLE_2015", + "CCLE_2019", + "FIMM_2016", + "gCSI_2019", + "GDSC_2020(v1-8.2)", + "GDSC_2020(v2-8.2)", + "PRISM_2020" +) + + +## Return the PSet names for one dataset, restricted to the pinned list. +## +## Loudly reports any PSet that upstream now offers but which is not pinned, +## so a new upstream release is visible in the build log rather than silently +## included or silently dropped. +psets_for_dataset <- function(all.dsets, cel) { + available <- all.dsets |> + subset(`Dataset Name` == cel) |> + (\(d) d[["PSet Name"]])() |> + unlist() |> + unique() + + keep <- available[available %in% PINNED_PSETS] + skipped <- setdiff(available, PINNED_PSETS) + + if (length(skipped) > 0) { + message(sprintf( + paste0("[pinned_psets] NOTE: dataset '%s' has %d PSet(s) available ", + "upstream that are NOT pinned and are being SKIPPED: %s\n", + " If this PSet should be part of the release, add it to ", + "PINNED_PSETS in coderbuild/utils/pinned_psets.R after verifying ", + "its schema in BOTH the drugs and experiments steps."), + cel, length(skipped), paste(skipped, collapse = ", "))) + } + + if (length(keep) == 0) { + stop(sprintf( + paste0("No pinned PSet found for dataset '%s'. Available upstream: %s. ", + "Either upstream renamed/removed a PSet, or PINNED_PSETS in ", + "coderbuild/utils/pinned_psets.R is out of date."), + cel, if (length(available)) paste(available, collapse = ", ") else "(none)")) + } + + keep +} diff --git a/coderbuild/utils/retry_utils.R b/coderbuild/utils/retry_utils.R new file mode 100644 index 00000000..780ac479 --- /dev/null +++ b/coderbuild/utils/retry_utils.R @@ -0,0 +1,445 @@ +## --------------------------------------------------------------------------- +## Retry helpers for flaky network fetches. +## +## WHY THIS EXISTS +## --------------- +## A full build makes hundreds of requests to a dozen third-party hosts over +## roughly a day. Any one of them being briefly unavailable used to fail the +## whole step, and because build_all.py restarts a failed step from the top, +## a single transient error could throw away many hours of work. +## +## Observed in real runs: +## 2026-09-02 Zenodo returned 504 Gateway Timeout on PSet_gCSI2019.rds, +## after CTRPv2 and GDSC had already downloaded (one PSet is +## 1.5 GB). The drugs step had run for ~17 hours across three +## attempts before giving up. The URL was fine minutes later. +## 2026-08-30 cog.sanger.ac.uk failed twice with an OpenSSL "unexpected +## eof" before succeeding on the third attempt. +## +## These are transient upstream problems, not build defects, so the build +## should ride them out rather than fail. +## +## NOTE: this only covers fetches that were previously unprotected. +## 02-broadSangerOmics.R already has robust_download_httr2() for its own +## downloads, and the Python builders have their own retry logic. +## --------------------------------------------------------------------------- + +## Waits between retries, in seconds: 1, 3, 10, then 15 minutes. +## Five attempts total (the initial one plus four retries), ~29 minutes of +## waiting if every attempt fails. Long enough to ride out a real upstream +## incident rather than only a momentary blip. +RETRY_SLEEPS <- c(60, 180, 600, 900) + +## Run `expr` (a zero-argument function), retrying on error. +## +## IMPORTANT: if every attempt fails this STOPS. It never returns a partial or +## empty result, and callers must not catch the error to "carry on without +## this dataset" -- a dataset must never be silently dropped from a release. +## Exhausting the retries is meant to fail the step, and therefore the build, +## so the problem is visible rather than shipped. +## +## what - human-readable label used in log lines +## sleeps - vector of waits between attempts; length(sleeps)+1 attempts +with_retries <- function(expr, what = "operation", sleeps = RETRY_SLEEPS) { + tries <- length(sleeps) + 1L + last_err <- NULL + for (attempt in seq_len(tries)) { + result <- tryCatch( + list(ok = TRUE, value = expr()), + error = function(e) list(ok = FALSE, msg = conditionMessage(e))) + + if (isTRUE(result$ok)) { + if (attempt > 1) { + message(sprintf("[%s] %s: succeeded on attempt %d/%d", + Sys.time(), what, attempt, tries)) + } + return(result$value) + } + + last_err <- result$msg + if (attempt < tries) { + sleep_s <- sleeps[attempt] + message(sprintf("[%s] %s: attempt %d/%d failed (%s); retrying in %d min", + Sys.time(), what, attempt, tries, + substr(gsub("[\r\n]+", " ", last_err), 1, 120), + round(sleep_s / 60))) + Sys.sleep(sleep_s) + } + } + stop(sprintf(paste0( + "%s failed after %d attempts over ~%d minutes. Last error: %s\n", + "Failing the build rather than continuing without this data."), + what, tries, round(sum(sleeps) / 60), last_err)) +} + + +## Download a PharmacoGx PSet with retries, caching it so that a later attempt +## -- including a completely fresh container started by build_all.py's own +## retry loop -- does not re-download it. +## +## PSETS are large (CTRPv2 is 228 MB, one of them is 1.5 GB). downloadPSet() +## skips a file that is already present in saveDir, so pointing saveDir at a +## directory under /tmp (which build_all.py bind-mounts to the host's local/) +## makes the download survive across container runs. A subdirectory is used +## deliberately: build_all.py's intermediate cleanup globs `local/*.*` and +## skips directories, so the cache is not swept away mid-build. +PSET_CACHE_DIR <- Sys.getenv("PSET_CACHE_DIR", unset = "/tmp/pset_cache") + +## Download a PSet, retrying, and never let a partial file poison the cache. +## +## PharmacoGx::downloadPSet only downloads when the file is ABSENT: +## +## if (!file.exists(file.path(saveDir, pSetFileName))) { download.file(...) } +## pSet <- readRDS(file.path(saveDir, pSetFileName)) +## +## So an interrupted download leaves a truncated .rds that downloadPSet then +## reuses forever: every later attempt skips the download and fails instantly +## inside readRDS() with "error reading from connection". That is not a network +## error, and no amount of retrying -- or waiting for the remote host to come +## back -- can clear it. The cache poisons itself permanently. +## +## This happened on 2026-09-09. A Zenodo outage truncated CCLE_2015.rds at +## 2.4GB, and the drugs step then failed 15 times (5 retries x 3 step attempts) +## with an identical error, while the other five PSets downloaded fine. +## +## The fix is to delete the cached file whenever an attempt fails, so the next +## attempt performs a real download. We deliberately do NOT try to validate the +## file instead: the truncated CCLE file is still a readable gzip stream that +## R's gzfile() walks to the end without complaint, so a cheap integrity check +## reports it healthy. Only readRDS() detects it, and that would mean +## deserialising several GB just to decide whether to keep a file we are about +## to re-download anyway. +## Download any URL to `dest`, resuming an interrupted transfer. +## +## Generic counterpart to the PSet fetcher below, for the plain file downloads +## in the omics steps. Those are the largest transfers in the whole pipeline -- +## Sanger ships rnaseq_all at ~897MB and WES_pureCN_CNV_genes at ~935MB -- and +## httr2's req_retry() restarts the whole request rather than resuming it. On a +## link that drops part-way, restarting a 935MB file simply fails again at a +## similar point: build v24 died exactly that way on a 112MB file, breaking +## 12MB in, three times. +## +## wget -c issues a Range request and continues from what is already on disk. +## Bytes accumulate in ".part", which is renamed onto `dest` only once the +## size matches the server's, so a partial file never appears at the +## destination. The ".part" is deliberately kept between attempts -- deleting it +## would mean a flaky link could never accumulate a large file. +download_file_resumable <- function(url, dest, sleeps = RETRY_SLEEPS, + what = NULL) { + what <- if (is.null(what)) basename(dest) else what + part <- paste0(dest, ".part") + + expected <- suppressWarnings(tryCatch({ + out <- system2("wget", c("--spider", "-S", "--timeout=30", "--tries=2", + shQuote(url)), stdout = TRUE, stderr = TRUE) + hits <- grep("Content-Length:", out, value = TRUE, ignore.case = TRUE) + if (length(hits)) { + as.numeric(sub(".*[Cc]ontent-[Ll]ength:[[:space:]]*([0-9]+).*", "\\1", + hits[length(hits)])) + } else NA_real_ + }, error = function(e) NA_real_)) + + with_retries( + function() { + status <- suppressWarnings(tryCatch( + system2("wget", c("-c", "--tries=10", "--waitretry=15", + "--read-timeout=60", "--timeout=30", + "--progress=dot:giga", + "-O", shQuote(part), shQuote(url))), + error = function(e) 1L)) + ok <- identical(as.integer(status), 0L) + + if (!file.exists(part)) { + stop(sprintf("download of %s produced no file", what)) + } + got <- file.info(part)$size + + if (!is.na(expected)) { + if (got < expected) { + stop(sprintf("%s incomplete (%s of %s bytes); next attempt resumes", + what, format(got, big.mark = ","), + format(expected, big.mark = ","))) + } + if (got > expected) { + unlink(part) + stop(sprintf("%s is oversized (%s vs %s bytes); discarded", what, + format(got, big.mark = ","), + format(expected, big.mark = ","))) + } + } else if (!ok) { + stop(sprintf("%s transfer did not complete (%s bytes so far, remote size unknown); next attempt resumes", + what, format(got, big.mark = ","))) + } + + if (!file.rename(part, dest)) { + stop(sprintf("could not move %s into place", what)) + } + message(sprintf("[%s] %s: downloaded %s bytes", Sys.time(), what, + format(got, big.mark = ","))) + invisible(dest) + }, + what = paste0("download('", what, "')"), + sleeps = sleeps) +} + + +## Look up a PSet's download URL from the PharmacoGx catalogue. +pset_download_url <- function(pset_name) { + tbl <- tryCatch(PharmacoGx::availablePSets(), error = function(e) NULL) + if (is.null(tbl)) return(NA_character_) + i <- match(pset_name, as.character(tbl[, "PSet Name"])) + if (is.na(i)) return(NA_character_) + as.character(tbl[i, "Download"]) +} + +## Fetch the publisher's size and MD5 for a PSet file. +## +## PSets are hosted on Zenodo, whose REST API publishes both. Returns a list +## with NA entries when the URL is not a Zenodo record or the API is +## unreachable, so callers degrade to size-only checking rather than failing. +pset_remote_meta <- function(url) { + none <- list(size = NA_real_, md5 = NA_character_) + if (is.na(url) || !nzchar(url)) return(none) + rec <- sub(".*/records?/([0-9]+)/files/.*", "\\1", url) + if (identical(rec, url) || !grepl("^[0-9]+$", rec)) return(none) + fname <- basename(sub("\\?.*$", "", url)) + api <- paste0("https://zenodo.org/api/records/", rec) + txt <- suppressWarnings(tryCatch( + system2("wget", c("-qO-", "--timeout=30", "--tries=3", shQuote(api)), stdout = TRUE), + error = function(e) character(0))) + if (!length(txt)) return(none) + j <- tryCatch(jsonlite::fromJSON(paste(txt, collapse = "")), error = function(e) NULL) + if (is.null(j) || is.null(j$files) || !nrow(j$files)) return(none) + i <- match(fname, j$files$key) + if (is.na(i)) return(none) + list(size = as.numeric(j$files$size[i]), + md5 = sub("^md5:", "", as.character(j$files$checksum[i]))) +} + +## Fetch with wget -c, which RESUMES an interrupted transfer via an HTTP range +## request instead of starting over. +## +## The CCLE PSet is 2.8GB and the connection can drop between 400MB and 1.1GB, +## so a non-resuming download never reaches the end: each retry throws away +## everything it had and dies at roughly the same place. Verified 2026-09-09 +## that Zenodo answers range requests with 206 PARTIAL_CONTENT. +fetch_pset_resumable <- function(url, dest) { + if (is.na(url) || !nzchar(url)) return(FALSE) + status <- suppressWarnings(tryCatch( + system2("wget", c("-c", "--tries=10", "--waitretry=15", "--read-timeout=60", + "--timeout=30", "--progress=dot:giga", + "-O", shQuote(dest), shQuote(url))), + error = function(e) 1L)) + identical(as.integer(status), 0L) +} + +## Load a cached PSet the way PharmacoGx::downloadPSet would, minus its final +## saveRDS() -- that re-save is only a caching optimisation, and it is what +## makes a verified file stop matching its published checksum. +## +## An unreadable file is discarded so the caller can fetch a clean copy. This +## is the last line of defence for files cached before verification existed. +pset_load_cached <- function(cached, pset_name) { + pSet <- tryCatch(readRDS(cached), error = function(e) { + message(sprintf("[%s] cached PSet '%s' is unreadable (%s); discarding it.", + Sys.time(), pset_name, conditionMessage(e))) + unlink(cached) + stop(e) + }) + updateObject(pSet) +} + +## Attempts with PharmacoGx's own downloader before the fallback kicks in. +## Short on purpose: long enough to ride out a blip, short enough that a real +## outage reaches the resumable path in minutes rather than half an hour. +PRIMARY_SLEEPS <- utils::head(RETRY_SLEEPS, 2) + +## Download a PSet. +## +## Primary path: PharmacoGx::downloadPSet, so normal builds behave exactly as +## upstream intends and we do not diverge from the library without cause. +## +## Fallback path (only after the primary has exhausted PRIMARY_SLEEPS): fetch +## with wget -c into ".rds.part" and rename to ".rds" only once +## size and MD5 match what Zenodo publishes. That separation is what makes +## resuming safe -- a ".part" file is by construction an interrupted download, +## so resuming it is always correct, while ".rds" is by construction +## complete, so it is never resumed onto. +## +## The fallback exists because downloadPSet cannot recover from two situations +## seen in real builds: +## +## - It only downloads when the file is ABSENT, so an interrupted transfer +## leaves a truncated .rds that it then reuses forever; every later attempt +## fails instantly in readRDS() with "error reading from connection". A +## Zenodo outage on 2026-09-09 truncated CCLE_2015.rds at 2.4GB of 2.8GB +## and the drugs step then failed 15 times identically. +## - It does not resume. CCLE is 2.8GB and the connection can drop between +## 400MB and 1.1GB, so every attempt discarded its progress and died in +## roughly the same place. +download_pset_retry <- function(pset_name, sleeps = RETRY_SLEEPS) { + if (!dir.exists(PSET_CACHE_DIR)) { + dir.create(PSET_CACHE_DIR, recursive = TRUE, showWarnings = FALSE) + } + cached <- file.path(PSET_CACHE_DIR, paste0(pset_name, ".rds")) + part <- paste0(cached, ".part") + + ## ---- Primary: PharmacoGx's own downloader ------------------------------- + primary <- tryCatch( + with_retries( + function() { + ## Deliberately does NOT delete the cached file on failure. If the + ## file is good and the error was transient, deleting it would throw + ## away a multi-GB download for nothing. If the file IS truncated, + ## downloadPSet reuses it blindly and the remaining primary attempts + ## fail instantly on the same bytes -- which is harmless and reaches + ## the fallback sooner. Either way the fallback sorts it out. + PharmacoGx::downloadPSet(pset_name, + saveDir = PSET_CACHE_DIR, + timeout = 10000) + }, + what = paste0("downloadPSet('", pset_name, "')"), + sleeps = PRIMARY_SLEEPS), + error = function(e) NULL) + + if (!is.null(primary)) return(primary) + + ## ---- Fallback: resumable, checksum-verified fetch ------------------------ + message(sprintf("[%s] PSet '%s': falling back to a resumable, checksum-verified download.", + Sys.time(), pset_name)) + ## Note we do not delete `cached` here. A file left by the primary may be + ## perfectly good -- the primary can fail for reasons unrelated to it -- and + ## discarding a multi-GB download on suspicion is expensive. pset_load_cached() + ## below is the gate: if the file is unreadable it is removed there, and the + ## next attempt downloads a clean, checksum-verified copy. Resuming onto it is + ## impossible either way, because the fallback only ever resumes ".part". + + with_retries( + function() { + if (!file.exists(cached)) { + url <- pset_download_url(pset_name) + if (is.na(url) || !nzchar(url)) { + stop(sprintf("no download URL for PSet '%s'", pset_name)) + } + meta <- pset_remote_meta(url) + complete <- fetch_pset_resumable(url, part) + + if (!file.exists(part)) { + stop(sprintf("download of PSet '%s' produced no file", pset_name)) + } + got <- file.info(part)$size + + ## A ".part" file is promoted to the final path ONLY on positive + ## evidence that it is complete. Without that evidence it stays a + ## ".part", so the next attempt RESUMES it. + ## + ## This is what failed on 2026-09-10 for CCLE_2019. Zenodo was timing + ## out, so the metadata lookup timed out too and both checks below were + ## skipped as NA. The partial file was then renamed into place, + ## readRDS rejected it, and the file was deleted -- destroying the + ## downloaded bytes on every cycle. Five attempts over 29 minutes each + ## restarted from zero and the step could never finish. + ## + ## wget's exit status is the evidence when the API is unreachable: it + ## is 0 only when the whole file was retrieved. + if (!is.na(meta$size)) { + if (got < meta$size) { + stop(sprintf("PSet '%s' incomplete (%s of %s bytes); next attempt resumes", + pset_name, format(got, big.mark = ","), + format(meta$size, big.mark = ","))) + } + if (got > meta$size) { + unlink(part) + stop(sprintf("PSet '%s' is oversized (%s vs %s bytes); discarded", + pset_name, format(got, big.mark = ","), + format(meta$size, big.mark = ","))) + } + } else if (!complete) { + ## No published size AND wget did not finish: assume partial. Keeping + ## it is safe -- the worst case is one wasted resume attempt, whereas + ## discarding it guarantees the download can never accumulate. + stop(sprintf("PSet '%s' transfer did not complete (%s bytes so far, remote size unknown); next attempt resumes", + pset_name, format(got, big.mark = ","))) + } + + if (!is.na(meta$md5)) { + local_md5 <- unname(tools::md5sum(part)) + if (!identical(local_md5, meta$md5)) { + unlink(part) + stop(sprintf("PSet '%s' failed checksum (got %s, published %s); discarded", + pset_name, local_md5, meta$md5)) + } + message(sprintf("[%s] PSet '%s' verified against published MD5 %s", + Sys.time(), pset_name, meta$md5)) + } else if (!is.na(meta$size)) { + message(sprintf("[%s] PSet '%s': no published checksum; verified against published size %s bytes.", + Sys.time(), pset_name, format(meta$size, big.mark = ","))) + } else { + message(sprintf("[%s] PSet '%s': no published size or checksum available; accepted on a complete transfer, readRDS is the remaining check.", + Sys.time(), pset_name)) + } + + if (!file.rename(part, cached)) { + stop(sprintf("could not move verified PSet '%s' into place", pset_name)) + } + } + pset_load_cached(cached, pset_name) + }, + what = paste0("downloadPSet('", pset_name, "') [resumable fallback]"), + sleeps = sleeps) +} + + +## --------------------------------------------------------------------------- +## Per-PSet completion markers. +## +## Retrying the download is not enough on its own. build_all.py restarts a +## failed step from the very beginning, so before these markers existed, a +## failure on the 5th of 6 PSets meant re-downloading and re-processing the +## first four -- which is how one Zenodo 504 turned into a 17-hour failure +## across three attempts. +## +## A marker is written only after a PSet's results have been merged into the +## cumulative output, so a half-finished PSet is never marked done. Markers +## live under /tmp (build_all.py's bind mount to the host's local/) alongside +## the cumulative output they describe, so the two are cleared together and +## cannot disagree: wiping local/ resets both, and keeping local/ keeps both. +## +## Net effect: a rerun re-does only the PSet that actually failed. +## --------------------------------------------------------------------------- +PSET_STATE_DIR <- Sys.getenv("PSET_STATE_DIR", + unset = file.path(PSET_CACHE_DIR, "completed")) + +pset_is_done <- function(tag) { + file.exists(file.path(PSET_STATE_DIR, paste0(make.names(tag), ".done"))) +} + +mark_pset_done <- function(tag) { + if (!dir.exists(PSET_STATE_DIR)) { + dir.create(PSET_STATE_DIR, recursive = TRUE, showWarnings = FALSE) + } + file.create(file.path(PSET_STATE_DIR, paste0(make.names(tag), ".done"))) + invisible(TRUE) +} + + +## Canonical path for a cached per-study dose-response table (04a). +## +## Both the writer and the reader must agree on this name. They previously did +## not -- the writer used paste0(tolower(studyName),'DoseResponse') while the +## reader looked for paste0(cel,'doseResponse'), so the resume check never +## matched and every attempt recomputed everything. Routing both through one +## function makes that class of mismatch impossible. +## +## Kept under /tmp so it survives between container attempts, same rationale +## as the PSet cache above. +DOSE_RESPONSE_CACHE_DIR <- Sys.getenv("DOSE_RESPONSE_CACHE_DIR", + unset = "/tmp/dose_response_cache") + +dose_response_cache_path <- function(study_name) { + if (!dir.exists(DOSE_RESPONSE_CACHE_DIR)) { + dir.create(DOSE_RESPONSE_CACHE_DIR, recursive = TRUE, showWarnings = FALSE) + } + file.path(DOSE_RESPONSE_CACHE_DIR, + paste0(tolower(study_name), "_doseResponse.tsv")) +} From 40a31cf2c17a313ae44310179ac8a1168f50e744 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:49:59 -0700 Subject: [PATCH 02/20] Harden the drug-descriptor and curve-fitting pipeline --- coderbuild/utils/build_drug_desc.py | 143 ++++++-- coderbuild/utils/calc_pdx_metrics.py | 8 +- coderbuild/utils/fit_curve.py | 105 ++++-- coderbuild/utils/pubchem_retrieval.py | 499 ++++++++++++++++++-------- 4 files changed, 554 insertions(+), 201 deletions(-) diff --git a/coderbuild/utils/build_drug_desc.py b/coderbuild/utils/build_drug_desc.py index bbeeb81e..3a2c63ba 100644 --- a/coderbuild/utils/build_drug_desc.py +++ b/coderbuild/utils/build_drug_desc.py @@ -6,6 +6,39 @@ import argparse +import os +import tempfile + +# --------------------------------------------------------------------------- +# Keep multiprocessing's scratch files OFF the bind mount. +# +# This MUST run before multiprocessing creates any worker, so it sits above the +# imports that pull it in. +# +# build_all.py bind-mounts the host's local/ directory at /tmp inside every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", and forkserver opens a Unix domain socket +# under tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS +# file sharing does not support chmod() on a socket there, so worker startup +# died with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-8ai8jp5x/sock-2c162e8922a7' +# +# Mordred computes descriptors in parallel, so this killed the drugs step -- +# and it did so at the very END of build_drugs.sh, after nci60, the PSets and +# the join had all completed, wasting the whole multi-hour drug build on every +# attempt. It is deterministic, so retries do not help. +# +# Pointing TMPDIR at container-local storage fixes it at the source. Verified: +# with /tmp as a bind mount a Pool raises EINVAL; with this set it succeeds. +# Moving the socket is preferred over forcing start_method="fork", which would +# reintroduce the fork-with-threads hazard that motivated the upstream change. +_MP_TMPDIR = os.environ.get("CODERDATA_MP_TMPDIR", "/opt/mp_tmp") +os.makedirs(_MP_TMPDIR, exist_ok=True) +os.environ["TMPDIR"] = _MP_TMPDIR +tempfile.tempdir = _MP_TMPDIR +# --------------------------------------------------------------------------- + from rdkit import Chem from rdkit.Chem import AllChem from rdkit.Chem import rdFingerprintGenerator @@ -14,6 +47,7 @@ import numpy as np from mordred import Calculator, descriptors import multiprocessing +import gzip # Remove all of the Deprecation warnings. There is a github issue to update the code and 50k lines of warnings in the build log so I'm hiding them for now. from rdkit import RDLogger @@ -73,50 +107,97 @@ def smiles_to_mordred(smiles,nproc=2): return longtab +## Number of SMILES whose descriptors are held in memory at once. +## +## Mordred returns a WIDE frame of ~1,600 object-dtype descriptors per molecule, +## which pd.melt() then explodes into rows: at 56,890 SMILES that is roughly 92 +## MILLION rows, before the merge, the concat and two more full-frame copies in +## the cleaning step. Build v23 was OOM-killed doing exactly that +## ("build_drugs.sh: line 21: Killed") on a 23.4GiB Docker VM. +## +## v19 completed the same step at 46,319 SMILES, so this was never comfortable +## -- a 23% growth in the drug table was enough to cross the limit. Chunking +## makes peak memory independent of how many drugs the release contains, so it +## does not silently creep back up to the ceiling next time. +SMILES_CHUNK = int(os.environ.get("DRUG_DESC_SMILES_CHUNK", "2000")) + + +def _chunks(seq, n): + for i in range(0, len(seq), n): + yield seq[i:i + n] + + +def _clean_block(block): + """Coerce bad values and drop malformed ids for one block of rows.""" + # Mordred puts the exception object into a cell when a descriptor cannot be + # computed for a molecule, so str() of the error lands in the table (e.g. the + # "module 'numpy' has no attribute 'float'" AttributeError from np.float being + # removed in NumPy >= 1.24, or messages containing "invalid"/"missing"/"min..."). + # Every Mordred descriptor is numeric, so coerce any non-numeric value to + # "NaN" -- this robustly catches ANY such error string, unlike the previous + # fixed substring blocklist which missed the numpy AttributeError. Morgan + # fingerprints are legitimate bit strings and are left untouched. + block['descriptor_value'] = block['descriptor_value'].astype(str) + is_fingerprint = block['structural_descriptor'] == 'morgan fingerprint' + numeric = pd.to_numeric(block['descriptor_value'], errors='coerce') + + # Render every Mordred value through float, so the text written does not + # depend on where the chunk boundaries fall. Mordred returns ints for some + # descriptors; whether a given column also picks up a NaN decides if pandas + # holds it as int64 ("0") or float64 ("0.0"). That makes the SAME descriptor + # serialise differently depending on how many molecules share a chunk, which + # would make the output non-deterministic. Fingerprints are bit strings and + # are left untouched. + block.loc[~is_fingerprint, 'descriptor_value'] = ( + numeric[~is_fingerprint] + .map(lambda v: "NaN" if pd.isna(v) else repr(float(v))) + ) + + # Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens. + block['improve_drug_id'] = block['improve_drug_id'].astype(str).str.strip() + return block[block['improve_drug_id'].str.match(r'^SMI_\d+$')] + + def main(): parser = argparse.ArgumentParser('Build drug descriptor table') parser.add_argument('--drugtable',dest='drugtable') parser.add_argument('--desctable',dest='outtable') - args = parser.parse_args() + args = parser.parse_args() cores = multiprocessing.cpu_count() ncors = cores-1 - # print("Running with "+str(ncors)+' out of '+str(cores)+' processors') - # print('Adding drug table for '+args.drugtable) tab = pd.read_csv(args.drugtable,sep='\t') cansmiles = [a for a in set(tab.canSMILES) if str(a)!='nan'] - # isosmiles = list(set(tab.isoSMILES)) - morgs = smiles_to_fingerprint(cansmiles) ids = pd.DataFrame(tab[['improve_drug_id','canSMILES']]).drop_duplicates() - # print("IDS columns:", ids.columns.tolist()) - # print("MORGS columns:", morgs.columns.tolist()) - id_morg = ids.rename({"canSMILES":'smile'},axis=1).merge(morgs)[['improve_drug_id','structural_descriptor','descriptor_value']] - - mords = smiles_to_mordred(cansmiles,nproc=ncors) - - id_mord = ids.rename({'canSMILES':'smile'},axis=1).merge(mords)[['improve_drug_id','structural_descriptor','descriptor_value']] - - full = pd.concat([id_morg,id_mord],axis=0) - - # Convert any values that contain the following strings to NA. I think this covers all of the cases, but add here if more are found. - strings_to_replace = ["min", "max", "invalid", "multiple", "missing"] - pattern = '|'.join(strings_to_replace) - full['descriptor_value'] = full['descriptor_value'].astype(str) - full.loc[full['descriptor_value'].str.contains(pattern, case=False, na=False), 'descriptor_value'] = "NaN" - - # Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens. - full['improve_drug_id'] = full['improve_drug_id'].astype(str).str.strip() - mask = full['improve_drug_id'].str.match(r'^SMI_\d+$') - n_dropped = (~mask).sum() - # if n_dropped: - # print(f"Dropping {n_dropped} malformed improve_drug_id rows.") - full = full[mask].copy() - - - full.to_csv(args.outtable,sep='\t',index=False,compression='gzip') + ids = ids.rename({"canSMILES":'smile'},axis=1) + del tab + + cols = ['improve_drug_id','structural_descriptor','descriptor_value'] + total = len(cansmiles) + print(f'Building descriptors for {total} SMILES in chunks of {SMILES_CHUNK}', + flush=True) + + # Stream each block straight to the gzip output. Morgan fingerprints are + # emitted first, then Mordred descriptors, preserving the original row order. + with gzip.open(args.outtable, 'wt', newline='') as out: + header = True + for label, fn in (('morgan', smiles_to_fingerprint), + ('mordred', lambda c: smiles_to_mordred(c, nproc=ncors))): + for i, chunk in enumerate(_chunks(cansmiles, SMILES_CHUNK), start=1): + desc = fn(chunk) + if desc is None or desc.empty: + continue + block = ids.merge(desc)[cols] + block = _clean_block(block) + block.to_csv(out, sep='\t', index=False, header=header) + header = False + done = min(i * SMILES_CHUNK, total) + print(f' [{label}] {done}/{total} SMILES -> {len(block)} rows', + flush=True) + del desc, block if __name__=='__main__': main() diff --git a/coderbuild/utils/calc_pdx_metrics.py b/coderbuild/utils/calc_pdx_metrics.py index 347e1f78..ec7eb54d 100755 --- a/coderbuild/utils/calc_pdx_metrics.py +++ b/coderbuild/utils/calc_pdx_metrics.py @@ -111,7 +111,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): exdf["volume_change"] = tumor_volume_change(volume) exdf["average_response"] = avg_response(exdf["volume_change"]) - nxdf = {'metric':'mRESCIST','value':None} + nxdf = {'metric':'mrecist','value':None} df = pd.DataFrame({"time": time, "volume": volume}) df = df[df['time'] >= min_time] @@ -131,7 +131,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): best_average_response = exdf["best_average_response"][0] mrecist = None - nxdf = {'metric':'mRESCIST','value':mrecist} + nxdf = {'metric':'mrecist','value':mrecist} # exdf["mRECIST"] = mrecist if best_response is not None and best_average_response is not None: @@ -148,7 +148,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): mrecist = "CR" # exdf["mRECIST"] = mrecist - nxdf = {'metric':'mRESCIST','value':mrecist} + nxdf = {'metric':'mrecist','value':mrecist} if not return_detail: return nxdf @@ -211,7 +211,7 @@ def TGI(contr_volume, treat_volume,time): # Simulated batch response class object rtx = { - "metric": "TGI", + "metric": "tgi", "value": tgi, 'time': np.max(time) } diff --git a/coderbuild/utils/fit_curve.py b/coderbuild/utils/fit_curve.py index c712cc3d..f050e268 100755 --- a/coderbuild/utils/fit_curve.py +++ b/coderbuild/utils/fit_curve.py @@ -17,6 +17,10 @@ from sklearn.metrics import r2_score from scipy.optimize import curve_fit import multiprocessing +from datetime import datetime + +def _log(msg): + print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] {msg}', flush=True) #import uno_data as ud @@ -225,26 +229,69 @@ def process_single_drug(name_group_tuple): metrics = compute_fit_metrics(xdata, ydata, popt, pcov) return name, metrics -def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None): - cols = ['source', 'improve_sample_id', 'Drug', 'study','time','time_unit'] - groups = df.groupby(cols) - count = count or (4484081 - start) - groups = islice(groups, start, start+count) +def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None, + workers=None, chunk_size=50000): + cols = ['source', 'improve_sample_id', 'Drug', 'study', 'time', 'time_unit'] cores = multiprocessing.cpu_count() - poolsize = round(cores-1) - print('we have '+str(cores)+' cores and '+str(poolsize)+' threads') - with multiprocessing.Pool(processes=poolsize) as pool: - results = pool.map(process_single_drug, groups) - - with open(f'{fname}.{start}', 'w') as f: - header = None - for result in results: - name, metrics = result - if header is None: - header = cols + metrics.index.tolist() - print(sep.join(header), file=f) - print(sep.join(str(n) for n in name), end=sep, file=f) - print(sep.join(f'{x:.4g}' for x in metrics), file=f) + poolsize = workers if workers is not None else max(1, round(cores * 3 / 4)) + + # Stream lazily — never materialise more than one chunk at a time so + # fork() workers only copy chunk-sized data, not the entire groupby result. + groups_iter = islice(df.groupby(cols), start, start + count if count else None) + + output_file = f'{fname}.{start}' + if os.path.exists(output_file): + os.remove(output_file) + header_written = False + groups_done = 0 + chunks_skipped = 0 + chunk_idx = 0 + + _log(f'fitting {fname}: {cores} cores, {poolsize} workers, chunk_size={chunk_size}') + + while True: + chunk = list(islice(groups_iter, chunk_size)) + if not chunk: + break + chunk_idx += 1 + chunk_workers = poolsize + + for attempt in range(1, 4): + try: + _log(f' chunk {chunk_idx} ' + f'(groups {groups_done + 1}-{groups_done + len(chunk)}, ' + f'attempt {attempt}/3, {chunk_workers} workers)') + with multiprocessing.Pool(processes=chunk_workers) as pool: + results = pool.map(process_single_drug, chunk) + + with open(output_file, 'a') as f: + for result in results: + name, metrics = result + if not header_written: + header = cols + metrics.index.tolist() + print(sep.join(header), file=f) + header_written = True + print(sep.join(str(n) for n in name), end=sep, file=f) + print(sep.join(f'{x:.4g}' for x in metrics), file=f) + + groups_done += len(chunk) + _log(f' chunk {chunk_idx} complete ({groups_done} groups so far)') + break + + except Exception as e: + _log(f' chunk {chunk_idx} failed ' + f'(attempt {attempt}/3): {type(e).__name__}: {e}') + if attempt < 3: + chunk_workers = max(1, chunk_workers // 2) + _log(f' retrying with {chunk_workers} workers') + else: + _log(f' WARNING: chunk {chunk_idx} skipped after 3 failed attempts') + groups_done += len(chunk) + chunks_skipped += 1 + + _log(f'fitting complete: {fname} — {groups_done} groups, ' + f'{chunks_skipped} chunk(s) skipped, output: {output_file}') + return chunks_skipped def main(): @@ -261,9 +308,11 @@ def main(): parser.add_argument('--output', help='prefix of output file') parser.add_argument('--beataml', action='store_true', help='Include this if for BeatAML') parser.add_argument('--debug',action='store_true',default=False) - + parser.add_argument('--workers', type=int, default=None, help='Number of parallel worker processes (default: 3/4 of available CPUs)') + parser.add_argument('--chunk_size', type=int, default=50000, help='Max drug-groups per pool.map call (default: 50000; reduce for very large datasets)') + args = parser.parse_args() - print(args.input) + _log(f'loading {args.input}') df_all = pd.read_table(args.input) if args.debug: df_all = df_all.iloc[0:1000000] @@ -274,12 +323,18 @@ def main(): df_all.DOSE = -1.0 * np.log10(df_all.DOSE/1000000.0) ##need data to be between 0 and 1, not 0 and 100 df_all.GROWTH=df_all.GROWTH/100.00 - print(df_all.head) + _log(f'loaded {len(df_all)} rows — starting curve fitting') fname = args.output or 'combined_single_response_agg' - process_df_part(df_all, fname, beataml=args.beataml)#, start=args.start, count=args.count) - -# if args.beataml == False: + skipped = process_df_part(df_all, fname, beataml=args.beataml, workers=args.workers, + chunk_size=args.chunk_size) + # Free the large input DataFrame before the schema-formatting step + del df_all + import gc; gc.collect() + _log(f'done: {fname}') format_coderd_schema(fname+'.0') + if skipped: + _log(f'ERROR: {skipped} chunk(s) permanently skipped — output is incomplete') + sys.exit(1) if __name__ == '__main__': main() diff --git a/coderbuild/utils/pubchem_retrieval.py b/coderbuild/utils/pubchem_retrieval.py index a72c5c68..a3c95873 100644 --- a/coderbuild/utils/pubchem_retrieval.py +++ b/coderbuild/utils/pubchem_retrieval.py @@ -1,11 +1,16 @@ import pandas as pd import requests +import csv from concurrent.futures import ThreadPoolExecutor, as_completed import os import threading import time import signal import sys +from datetime import datetime + +def _ts(): + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Global variables request_counter = 0 @@ -17,67 +22,131 @@ existing_structures = dict() existing_pubchemids = set() +# --------------------------------------------------------------------------- +# Local PubChem data cache — populated from previous-run drug files so that +# retrieve_drug_info can return cached properties+synonyms without HTTP requests. +# Keys: pubchem_id (str) AND chem_name (str.lower()). +# Values: {'SMILES': ..., 'InChIKey': ..., 'MolecularFormula': ..., +# 'MolecularWeight': ..., 'CID': ..., 'synonyms': [...]} +# improve_drug_id is NOT stored here; ID assignment is always fresh. +# --------------------------------------------------------------------------- +_pubchem_data_cache = {} + -def fetch_url(url, retries=3, backoff_factor=1): +def load_pubchem_cache(file_paths): + """Populate _pubchem_data_cache from previously built drug TSV files. + + Call this before the main drug-build loop to avoid re-querying PubChem for + drugs whose properties are already known. ID assignment is unaffected. + """ + global _pubchem_data_cache + if isinstance(file_paths, str): + file_paths = [p.strip() for p in file_paths.split(',') if p.strip()] + loaded = 0 + for path in file_paths: + if not os.path.exists(path): + print(f"[{_ts()}] Cache hint: file not found, skipping: {path}") + continue + try: + df = pd.read_csv(path, sep='\t', quoting=3) + except Exception: + try: + df = pd.read_csv(path, sep='\t') + except Exception as e: + print(f"[{_ts()}] Could not read cache file {path}: {e}") + continue + needed = {'pubchem_id', 'chem_name', 'canSMILES', 'InChIKey', 'formula', 'weight'} + if not needed.issubset(set(df.columns)): + print(f"[{_ts()}] Cache file {path} missing expected columns, skipping.") + continue + for cid, group in df.groupby('pubchem_id', dropna=True): + synonyms = [str(n) for n in group['chem_name'].dropna().tolist()] + row = group.iloc[0] + entry = { + 'SMILES': str(row['canSMILES']), + 'InChIKey': str(row['InChIKey']), + 'MolecularFormula': str(row['formula']), + 'MolecularWeight': str(row['weight']), + 'CID': str(cid), + 'synonyms': synonyms, + } + _pubchem_data_cache[str(cid)] = entry + for syn in synonyms: + _pubchem_data_cache[str(syn).lower()] = entry + loaded += 1 + print(f"[{_ts()}] PubChem cache: {loaded} drugs loaded from {len(file_paths)} file(s).") + + +# Auto-load from hint file written by build_all.py before the container starts. +_CACHE_HINT_FILE = "/tmp/prev_drug_files.txt" +if os.path.exists(_CACHE_HINT_FILE): + try: + _hint_paths = [l.strip() for l in open(_CACHE_HINT_FILE) if l.strip()] + if _hint_paths: + load_pubchem_cache(_hint_paths) + except Exception as _cache_err: + print(f"[{_ts()}] Warning: could not load PubChem cache from {_CACHE_HINT_FILE}: {_cache_err}") + + +def fetch_url(url, retries=4, backoff_factor=1): """ Fetches a URL with retry mechanism and backoff. - - Parameters: - - url (str): The URL to fetch. - - retries (int): Number of retry attempts. - - backoff_factor (float): Factor to calculate backoff time. - - Returns: - - dict: JSON response if successful. - - Raises: - - Exception: If all retry attempts fail. + + 503 responses use a 30s base backoff (honouring Retry-After if present) + because PubChem rate-throttles aggressively and a 1s retry just gets + another 503. """ global last_request_time, lock, request_counter with lock: current_time = time.time() - # Reset counter if more than 1 second has passed if current_time - last_request_time >= 1: request_counter = 0 last_request_time = current_time - - # Wait if the limit is reached - while request_counter >= 4: - time.sleep(0.2) # Sleep a bit to check again + while request_counter >= 3: # stay at ≤3 req/s; PubChem limit is 5 + time.sleep(0.1) current_time = time.time() if current_time - last_request_time >= 1: request_counter = 0 last_request_time = current_time - request_counter += 1 - - for attempt in range(retries + 1): # Total attempts = retries + 1 + + for attempt in range(retries + 1): + status_code = None + retry_after_header = None try: - response = requests.get(url, timeout=10) - if response.status_code == 200: - return response.json() - if response.status_code == 404: # permanent, no existing CID/name + response = requests.get(url, timeout=30) + status_code = response.status_code + if status_code == 200: + return response.json() + if status_code == 404: raise FileNotFoundError("404") - raise Exception(f"Failed to fetch {url}, Status Code: {response.status_code}") + retry_after_header = response.headers.get('Retry-After', '') + raise Exception(f"Failed to fetch {url}, Status Code: {status_code}") + except FileNotFoundError: + raise except Exception as exc: - if attempt < retries: - wait = backoff_factor * (2 ** attempt) - print(f"Attempt {attempt + 1} for URL {url} failed with error: {exc}. Retrying in {wait} seconds...") - time.sleep(wait) - else: - print(f"All {retries + 1} attempts failed for URL {url}.") + if attempt >= retries: + print(f"[{_ts()}] All {retries + 1} attempts failed for URL {url}.") raise + if status_code == 503: + wait = (int(retry_after_header) + if retry_after_header and retry_after_header.isdigit() + else min(15 * (attempt + 1), 60)) + else: + wait = min(15 * (attempt + 1), 60) + print(f"[{_ts()}] Attempt {attempt + 1} for URL {url} failed with error: {exc}. Retrying in {wait} seconds...") + time.sleep(wait) def retrieve_drug_info(compound, ignore_chems, isname=True): """ - Retrieves information for a given compound from PubChem. + Retrieves information for a given compound from PubChem (or local cache). Parameters: - compound (str or int): Name or CID of the compound. - ignore_chems (str): File path to log ignored compounds. - isname (bool): True if the compound is a name, False if it's a CID. - + Returns: - list: List of dictionaries containing drug information, or None if unsuccessful. """ @@ -85,77 +154,85 @@ def retrieve_drug_info(compound, ignore_chems, isname=True): if pd.isna(compound): return None - if isname: - urls = { - "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", - "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/synonyms/JSON" + # --- Check local cache before making any HTTP requests --- + cache_key = str(compound).lower() if isname else str(compound) + cached = _pubchem_data_cache.get(cache_key) + if cached is not None: + properties = { + 'SMILES': cached['SMILES'], + 'InChIKey': cached['InChIKey'], + 'MolecularFormula': cached['MolecularFormula'], + 'MolecularWeight': cached['MolecularWeight'], + 'CID': cached['CID'], } + synonyms_list = cached['synonyms'] else: - urls = { - "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", - "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/synonyms/JSON" - } - - with ThreadPoolExecutor(max_workers=4) as executor: - future_to_url = {executor.submit(fetch_url, url): key for key, url in urls.items()} - results = {} + # --- Normal PubChem fetch via HTTP --- + if isname: + urls = { + "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", + "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/synonyms/JSON" + } + else: + urls = { + "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", + "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/synonyms/JSON" + } + + with ThreadPoolExecutor(max_workers=4) as executor: + future_to_url = {executor.submit(fetch_url, url): key for key, url in urls.items()} + results = {} + for future in as_completed(future_to_url): + key = future_to_url[future] + try: + results[key] = future.result() + except FileNotFoundError: + print(f"[{_ts()}] {compound} not found in PubChem. Adding to ignore list.") + with open(ignore_chems, "a") as f: + f.write(f"{compound}\n") + return None + except Exception as exc: + print(f"[{_ts()}] {compound} generated a transient exception: {exc}") + return None + + if not all(key in results for key in ["properties", "synonyms"]): + return None - for future in as_completed(future_to_url): - key = future_to_url[future] - try: - data = future.result() - results[key] = data - - except FileNotFoundError: # only 404s are added - print(f"{compound} not found in PubChem. Adding to ignore list.") - with open(ignore_chems, "a") as f: - f.write(f"{compound}\n") - return None - except Exception as exc: # transient error, don't blacklist - print(f"{compound} generated a transient exception: {exc}") - return None - - if all(key in results for key in ["properties", "synonyms"]): properties = results["properties"]['PropertyTable']['Properties'][0] synonyms_list = results["synonyms"]['InformationList']['Information'][0]['Synonym'] - # Check if this compound or any of its synonyms already has an assigned improve_drug_id - new_syns = set() - if isname: - sl = synonyms_list + [compound] - else: - sl = synonyms_list - for synonym in sl: - synonym_lower = str(synonym).lower() - if synonym_lower not in existing_synonyms: - new_syns.add(synonym_lower) - if len(new_syns) == 0: # Ensure there are new synonyms before proceeding - return None - for synonym in new_syns: - synonym_lower = str(synonym).lower() - existing_synonyms.add(synonym_lower) - - # Check for structure - if properties['SMILES'] in existing_structures.keys(): - print(f'Found structure for {compound}') - SMI_assignment = existing_structures[properties['SMILES']] - else: - if improve_drug_id == 0: - improve_drug_id = 1 - SMI_assignment = f"SMI_{improve_drug_id}" - existing_structures[properties['SMILES']] = SMI_assignment - improve_drug_id += 1 - - #print(new_syns) - data_for_tsv = [{ - 'improve_drug_id': SMI_assignment, - 'name': str(synonym).lower(), - **properties - } for synonym in new_syns] - - return data_for_tsv - else: + # --- Shared: synonym dedup + SMI ID assignment (identical for both paths) --- + new_syns = set() + sl = synonyms_list + ([compound] if isname else []) + for synonym in sl: + synonym_lower = str(synonym).lower() + if synonym_lower not in existing_synonyms: + new_syns.add(synonym_lower) + if len(new_syns) == 0: return None + for synonym in new_syns: + existing_synonyms.add(str(synonym).lower()) + + if properties['SMILES'] in existing_structures: + print(f'[{_ts()}] Found structure for {compound}') + SMI_assignment = existing_structures[properties['SMILES']] + else: + if improve_drug_id == 0: + improve_drug_id = 1 + SMI_assignment = f"SMI_{improve_drug_id}" + existing_structures[properties['SMILES']] = SMI_assignment + improve_drug_id += 1 + + # Sanitise every value: these rows are later written to TSV with plain + # f.write() and no quoting, so an embedded tab/newline/quote would corrupt + # the record. See _sanitise_field(). + data_for_tsv = [{ + 'improve_drug_id': SMI_assignment, + 'name': _sanitise_field(str(synonym).lower()), + **{k: _sanitise_field(v) for k, v in properties.items()} + } for synonym in new_syns] + + return data_for_tsv def fetch_data_for_batch(batch, ignore_chems, isname): @@ -166,18 +243,91 @@ def fetch_data_for_batch(batch, ignore_chems, isname): - batch (list): List of compound names or CIDs. - ignore_chems (str): File path to log ignored compounds. - isname (bool): True if compounds are names, False if they're CIDs. - + Returns: - list: Combined list of drug information for the batch. """ all_data = [] - for compound_name in batch: - data = retrieve_drug_info(compound_name, ignore_chems, isname) + for compound in batch: + data = retrieve_drug_info(compound, ignore_chems, isname) if data: all_data.extend(data) return all_data +DRUG_TSV_COLUMNS = 7 # improve_drug_id, chem_name, pubchem_id, canSMILES, + # InChIKey, formula, weight + + +def _sanitise_field(value): + """Make a value safe to write into a raw (unescaped) TSV cell. + + Several writers in this pipeline emit TSV with plain f.write() and no + quoting, so any tab, carriage return, newline or double quote inside a + value silently corrupts the row: the record gains extra columns and every + later reader fails with + + pandas.errors.ParserError: Expected 7 fields in line N, saw 12 + + That is what broke the 2026-09-02 build after ~10 hours -- a single + malformed synonym row for SMI_20392 in nci60_drugs.tsv, whose weight cell + ended up as `367.4"` followed by five stray tabs. + + Drug synonyms carry no meaning in these characters, so replacing them is + lossless for our purposes and makes a structurally broken row impossible. + """ + if value is None: + return "" + s = str(value) + for ch in ("\t", "\r", "\n"): + s = s.replace(ch, " ") + s = s.replace('"', "") + return s.strip() + + +def _read_drug_tsv(path, **kwargs): + """Read a drug TSV, tolerating rows padded with trailing EMPTY columns. + + Historic files (written before _sanitise_field existed) can contain rows + with extra trailing tabs. Those extra cells hold no data, so trimming them + loses nothing -- unlike dropping the row, which would silently remove a + drug from the release. + + A row with extra NON-empty cells is a different matter: that is real data + in an unexpected shape, so this raises with the offending line number and + content rather than guessing. + """ + try: + return pd.read_csv(path, sep="\t", **kwargs) + except pd.errors.ParserError: + pass # fall through to the repairing path below + + print(f"[{_ts()}] {path}: malformed row(s) detected; inspecting.") + cleaned, repaired, bad = [], 0, [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for lineno, raw in enumerate(fh, start=1): + parts = raw.rstrip("\n").split("\t") + if len(parts) > DRUG_TSV_COLUMNS: + extra = parts[DRUG_TSV_COLUMNS:] + if any(x.strip() for x in extra): + bad.append((lineno, raw[:200])) + continue + parts = parts[:DRUG_TSV_COLUMNS] + repaired += 1 + cleaned.append("\t".join(parts)) + + if bad: + detail = "\n".join(f" line {n}: {t!r}" for n, t in bad[:5]) + raise ValueError( + f"{path} contains {len(bad)} row(s) with unexpected NON-EMPTY extra " + f"columns. Refusing to guess at their meaning, because silently " + f"dropping them would remove drugs from the release.\n{detail}") + + print(f"[{_ts()}] {path}: trimmed trailing empty columns from {repaired} row(s).") + from io import StringIO + return pd.read_csv(StringIO("\n".join(cleaned) + "\n"), sep="\t", **kwargs) + + def read_existing_data(output_filename): """ Reads existing data from the output file to prevent duplication. @@ -190,13 +340,13 @@ def read_existing_data(output_filename): """ global improve_drug_id, existing_synonyms, existing_structures, existing_pubchemids try: - df = pd.read_csv(output_filename, sep='\t', quoting=3) + df = _read_drug_tsv(output_filename, quoting=3) existing_synonyms = set([str(a).lower() for a in set(df.chem_name)]) existing_pubchemids = set([str(a) for a in df['pubchem_id']]) max_id = df['improve_drug_id'].str.extract(r'SMI_(\d+)').astype(float).max() improve_drug_id = int(max_id[0]) + 1 if pd.notna(max_id[0]) else 1 existing_structures = {row['canSMILES']: row['improve_drug_id'] for _, row in df.iterrows()} - print(f'Read in {len(existing_synonyms)} drug names and {len(existing_pubchemids)} pubchem IDs') + print(f'[{_ts()}] Read in {len(existing_synonyms)} drug names and {len(existing_pubchemids)} pubchem IDs') except FileNotFoundError: return {} @@ -206,7 +356,7 @@ def timeout_handler(signum, frame): Handles timeouts by setting the global `should_continue` flag to False. """ global should_continue - print("Time limit reached, exiting gracefully...") + print(f"[{_ts()}] Time limit reached, exiting gracefully...") should_continue = False @@ -223,7 +373,7 @@ def _load_prev_drugs_union(prevDrugFilepath: str) -> pd.DataFrame: dfs = [] for p in paths: if not os.path.exists(p): - print(f"Warning: previous drug file '{p}' not found; skipping.") + print(f"[{_ts()}] Warning: previous drug file '{p}' not found; skipping.") continue try: if p.lower().endswith(".tsv"): @@ -232,7 +382,7 @@ def _load_prev_drugs_union(prevDrugFilepath: str) -> pd.DataFrame: df = pd.read_csv(p) dfs.append(df) except Exception as e: - print(f"Warning: failed to read previous drug file '{p}': {e}; skipping.") + print(f"[{_ts()}] Warning: failed to read previous drug file '{p}': {e}; skipping.") if not dfs: return pd.DataFrame(columns=["improve_drug_id", "chem_name", "pubchem_id", "canSMILES", "InChIKey", "formula", "weight"]) @@ -304,15 +454,15 @@ def update_dataframe_and_write_tsv(unique_names, else: restrict_set = raw_names # default filtering - print(f"Starting with {len(raw_names)} provided {'names' if isname else 'IDs'}; restricting output to {len(restrict_set)} of them.") + print(f"[{_ts()}] Starting with {len(raw_names)} provided {'names' if isname else 'IDs'}; restricting output to {len(restrict_set)} of them.") # --- 1) read existing output to bootstrap state --- - print(f"Reading existing data from {output_filename}") + print(f"[{_ts()}] Reading existing data from {output_filename}") # capture existing output file (if any) to include in base existing_output_df = pd.DataFrame() if os.path.exists(output_filename): try: - existing_output_df = pd.read_csv(output_filename, sep="\t", quoting=3) + existing_output_df = _read_drug_tsv(output_filename, quoting=3) except Exception: existing_output_df = pd.read_csv(output_filename, sep="\t") # read_existing_data populates globals (synonyms, pubchemids, and sets improve_drug_id based on output) @@ -327,7 +477,7 @@ def update_dataframe_and_write_tsv(unique_names, desired_start = max(existing_output_max, prev_union_max) + 1 if improve_drug_id < desired_start: improve_drug_id = desired_start - print(f"SMI numbering will start from {improve_drug_id} (max prior was {desired_start - 1})") + print(f"[{_ts()}] SMI numbering will start from {improve_drug_id} (max prior was {desired_start - 1})") # build seen names/IDs (to avoid re-query) seen_names = set(existing_synonyms) @@ -341,10 +491,10 @@ def update_dataframe_and_write_tsv(unique_names, # --- 3) determine new candidates to query --- if isname: candidates = raw_names - seen_names - print(f"{len(raw_names)} raw names provided; {len(seen_names)} already seen; {len(candidates)} new to fetch.") + print(f"[{_ts()}] {len(raw_names)} raw names provided; {len(seen_names)} already seen; {len(candidates)} new to fetch.") else: candidates = raw_names - seen_pubchemids - print(f"{len(raw_names)} raw IDs provided; {len(seen_pubchemids)} already seen; {len(candidates)} new to fetch.") + print(f"[{_ts()}] {len(raw_names)} raw IDs provided; {len(seen_pubchemids)} already seen; {len(candidates)} new to fetch.") # apply ignore_chems filtering ignore_chem_set = set() @@ -353,10 +503,13 @@ def update_dataframe_and_write_tsv(unique_names, for line in file: ignore_chem_set.add(line.strip()) candidates = set(candidates) - ignore_chem_set - print(f"{len(candidates)} candidates remain after removing ignored.") + print(f"[{_ts()}] {len(candidates)} candidates remain after removing ignored.") # --- 4) make a temp union file with previous union + existing output --- - temp_file = output_filename.replace(".tsv", "_temp.tsv") + if output_filename.endswith(".tsv"): + temp_file = output_filename[:-4] + "_temp.tsv" + else: + temp_file = output_filename + "_temp" base_dfs = [] if not prev_union_df.empty: base_dfs.append(prev_union_df) @@ -380,34 +533,76 @@ def update_dataframe_and_write_tsv(unique_names, # create empty temp file so fetch logic can append headers open(temp_file, "a").close() - # --- 5) fetch new ones in batches and append to temp_file --- - candidates_list = list(candidates) - for i in range(0, len(candidates_list), batch_size): - if not should_continue: + # --- 5) fetch new ones in batches and append to temp_file (3 sweeps) --- + remaining_candidates = set(candidates) + + for sweep in range(3): + if not should_continue or not remaining_candidates: break - batch = candidates_list[i : i + batch_size] - data = fetch_data_for_batch(batch, ignore_chems, isname) - if data: - file_exists = os.path.isfile(temp_file) - mode = "a" if file_exists else "w" - with open(temp_file, mode) as f: - if os.path.getsize(temp_file) == 0: - f.write("improve_drug_id\tchem_name\tpubchem_id\tcanSMILES\tInChIKey\tformula\tweight\n") - for entry in data: - f.write( - f"{entry['improve_drug_id']}\t{entry['name']}\t{entry.get('CID', '')}\t" - f"{entry['SMILES']}\t{entry['InChIKey']}\t" - f"{entry['MolecularFormula']}\t{entry['MolecularWeight']}\n" - ) - with open(ignore_chems, "a") as ig_f: - for entry in data: - if isname: - ig_f.write(f"{entry['name']}\n") - else: - ig_f.write(f"{entry.get('CID', '')}\n") + + if sweep > 0: + print(f"[{_ts()}] Sweep {sweep + 1}/3: waiting 60 seconds before retry pass...") + time.sleep(60) + + # refresh ignore set + ignore_chem_set = set() + if os.path.exists(ignore_chems): + with open(ignore_chems, "r") as f: + for line in f: + ignore_chem_set.add(line.strip()) + + # remove already-fetched candidates by reading what landed in temp_file + if os.path.exists(temp_file) and os.path.getsize(temp_file) > 0: + try: + # QUOTE_NONE: temp_file is written with raw f.write() + tmp_df = pd.read_csv(temp_file, sep="\t", quoting=3) + if not isname and "pubchem_id" in tmp_df.columns: + fetched = {str(x) for x in tmp_df["pubchem_id"].dropna()} + remaining_candidates -= fetched + elif isname and "chem_name" in tmp_df.columns: + fetched = {str(x).lower() for x in tmp_df["chem_name"].dropna()} + remaining_candidates -= fetched + except Exception: + pass + + remaining_candidates -= ignore_chem_set + + print(f"[{_ts()}] Sweep {sweep + 1}/3: {len(remaining_candidates)} candidates to fetch.") + + candidates_list = list(remaining_candidates) + for i in range(0, len(candidates_list), batch_size): + if not should_continue: + break + batch = candidates_list[i : i + batch_size] + data = fetch_data_for_batch(batch, ignore_chems, isname) + if data: + file_exists = os.path.isfile(temp_file) + mode = "a" if file_exists else "w" + with open(temp_file, mode) as f: + if os.path.getsize(temp_file) == 0: + f.write("improve_drug_id\tchem_name\tpubchem_id\tcanSMILES\tInChIKey\tformula\tweight\n") + for entry in data: + # Every cell goes through _sanitise_field: this write + # is unescaped, so a stray tab/newline/quote here is + # what produced the malformed nci60_drugs.tsv row that + # failed the 2026-09-02 build. + f.write("\t".join(_sanitise_field(x) for x in ( + entry['improve_drug_id'], entry['name'], + entry.get('CID', ''), entry['SMILES'], + entry['InChIKey'], entry['MolecularFormula'], + entry['MolecularWeight'])) + "\n") + with open(ignore_chems, "a") as ig_f: + for entry in data: + if isname: + ig_f.write(f"{entry['name']}\n") + else: + ig_f.write(f"{entry.get('CID', '')}\n") # --- 6) load combined temp results --- - combined = pd.read_csv(temp_file, sep="\t") + # temp_file is written with raw f.write() (no quoting), so it must be read + # with QUOTE_NONE. Reading it with pandas' default QUOTE_MINIMAL would + # mis-parse any synonym containing a double quote. + combined = _read_drug_tsv(temp_file, quoting=3) # Determine previous max (before new fetches) to identify newly assigned SMI IDs previous_max = desired_start - 1 # desired_start was max(existing_output_max, prev_union_max) + 1 @@ -437,18 +632,40 @@ def update_dataframe_and_write_tsv(unique_names, if keep_ids: final_df = combined[combined["improve_drug_id"].isin(keep_ids)].copy() else: - print("Warning: no relevant drugs were retained/fetched for the restriction set.") + print(f"[{_ts()}] Warning: no relevant drugs were retained/fetched for the restriction set.") final_df = pd.DataFrame(columns=combined.columns) # --- 10) write final filtered output --- final_df.drop_duplicates(inplace=True) - final_df.to_csv(output_filename, sep="\t", index=False) + # QUOTE_NONE/quotechar=None is REQUIRED here, not cosmetic. + # + # These files are read back with quoting=3 (QUOTE_NONE) -- by + # read_existing_data(), by _load_prev_drugs_union(), and by the next build + # via --prev_drugs. Writing with pandas' default QUOTE_MINIMAL wraps any + # field containing a double quote and doubles its internal quotes, but the + # QUOTE_NONE reader does not strip the wrapper or collapse the doubling. + # So every read/write cycle DOUBLES the quote count in that field. + # + # Real drug synonyms contain quotes (e.g. cimetidine's + # n-cyano-n'-methyl-n''-[2-...]guanidine + # ), so this compounded silently across releases: v2.3 shipped a chem_name + # of ~2^16 characters, and by v2.4 the same field had reached ~2^24 (16 MB), + # inflating ctrpv2/nci60/prism_drugs.tsv to ~300 MB each and finally + # breaking map_improve_drug_ids.py with + # _csv.Error: field larger than field limit (131072) + # + # Writing QUOTE_NONE makes the writer symmetric with the readers, so a + # value survives any number of round trips unchanged. Verified stable over + # repeated cycles. Drug names never contain tabs, so QUOTE_NONE is safe for + # this TSV. + final_df.to_csv(output_filename, sep="\t", index=False, + quoting=csv.QUOTE_NONE, quotechar=None) if os.path.exists(temp_file): try: os.remove(temp_file) except OSError as e: - print(f"Warning: failed to delete temp file {temp_file}: {e}") + print(f"[{_ts()}] Warning: failed to delete temp file {temp_file}: {e}") return final_df From 7713186a699ed460a370f87efe118ee20b53ff83 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:00 -0700 Subject: [PATCH 03/20] Update broad_sanger sample, omics and drug processing Also makes the broad_sanger downloads survive a flaky connection. These are the largest transfers in the pipeline: Sanger ships rnaseq_all at ~897MB, WES_pureCN_CNV_genes at ~935MB, mutations_all at ~107MB and the proteomics archive at ~113MB. Neither downloader could resume. robust_download() in the proteomics script mounted a urllib3 Retry adapter, but Retry only covers connection setup and HTTP status codes: once the body is streaming a truncated read raises out of iter_content() and became a fatal RuntimeError. A build died exactly that way, 12MB into the 113MB file, three times, restarting from zero each time. robust_download_httr2() in the omics script had the same shape -- httr2's req_retry() restarts the request rather than resuming it -- so the 897MB and 935MB files would fail the same way, only slower. Both now retry on the 1/3/10/15 minute schedule used elsewhere and resume with a Range request, accumulating in ".part" and promoting the file only once its size matches the server's Content-Length. All of these hosts send accept-ranges: bytes. The omics script delegates to download_file_resumable() from retry_utils.R rather than keeping a second implementation. Dropped req_verbose(): full curl logging across ~2GB of transfers buried the real errors in the build log. The transcriptomics branch also now pairs rm() with gc(), as the copy_number and mutations branches already did. Without it R still holds the previous object when pivot_longer allocates the long frame, which is the peak of the whole step: measured 1,351MB -> 1,219MB peak RSS at no run time cost. The nci60 unzips check their exit status instead of failing later as a confusing parse error on a stale DOSERESP.csv. Verified against the live Sanger host: an interrupted transfer resumes with 206 Partial Content and completes byte-exact, and resuming from a 100MB partial took 4.8s against 61.3s for the full download, confirming it fetches only the remaining bytes. --- .../broad_sanger/01-broadSangerSamples.R | 88 +++++- coderbuild/broad_sanger/02-broadSangerOmics.R | 262 +++++++++++------- .../02a-broad_sanger_proteomics.py | 94 ++++++- coderbuild/broad_sanger/03-createDrugFile.R | 74 ++++- coderbuild/broad_sanger/03a-nci60Drugs.py | 60 ++-- .../broad_sanger/04-drug_dosage_and_curves.py | 146 ++++++++-- .../broad_sanger/04a-drugResponseData.R | 30 +- coderbuild/broad_sanger/04b-nci60-updated.py | 88 +++--- .../broad_sanger/05b_separate_datasets.py | 14 +- coderbuild/broad_sanger/build_omics.sh | 13 + coderbuild/broad_sanger/build_samples.sh | 13 + coderbuild/broad_sanger/exp_requirements.r | 9 + coderbuild/broad_sanger/omics_requirements.r | 12 +- coderbuild/broad_sanger/requirements.r | 1 + coderbuild/broad_sanger/requirements.txt | 3 +- 15 files changed, 693 insertions(+), 214 deletions(-) diff --git a/coderbuild/broad_sanger/01-broadSangerSamples.R b/coderbuild/broad_sanger/01-broadSangerSamples.R index b3c0965e..c853b4e2 100755 --- a/coderbuild/broad_sanger/01-broadSangerSamples.R +++ b/coderbuild/broad_sanger/01-broadSangerSamples.R @@ -10,6 +10,9 @@ library(dplyr) library(readr) library(XML) +## Retry helpers for flaky third-party downloads +source("retry_utils.R") + ##the only thing that Priyanka has here is TRP identifiers, so collecting those # tab<-read.table('DepMap_Argonne_Mapping.csv',sep=',',header=T)%>% # dplyr::select(Argonne_ID,DepMap_ID)%>% @@ -22,18 +25,83 @@ library(XML) ###[1] "Panc-05-04" "L3-3" "Panc-02-03" "Panc-10-05" "G-292-clone-A141B1" "Hs-729" "Panc-03-27" "Panc-04-03" "Hs-688-A-T" ###[10] "Panc-08-13" "PE-CA-PJ34-clone-C12" "Hep-3B2-1-7" "PE-CA-PJ41-clone-D2" "PE-CA-PJ49" "Ishikawa-Heraklio-02-ER" -##here are all the models in depmap 23Q2, downloded on 10/11/2023 -depmap_models<-readr::read_csv('https://figshare.com/ndownloader/files/40448834')#|> +## --------------------------------------------------------------------------- +## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE. +## +## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in +## front of https://depmap.org/portal/api/download/files. That endpoint now +## returns an HTML challenge page with HTTP 200, so read_csv() silently parses +## the HTML and every downstream column lookup fails. It cannot be scripted +## around, and DepMap explicitly asks that the portal not be scraped. +## +## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in +## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +## +## build_samples.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script, +## which downloads them from Synapse into $DEPMAP_DIR. This script only reads +## from disk -- it never contacts the DepMap portal. +## +## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE: +## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData +## (accept the terms, then use the download button on each file) +## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities +## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py +## 4. Re-run the build with --depmap-ready +## +## Currently pinned to: DepMap Public 26Q1 +## Files required by coderdata (4 total, ~1.29 GB): +## Model.csv (samples - this script) +## OmicsSomaticMutations.csv (omics) +## OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv (omics) +## PortalOmicsCNGeneLog2.csv (omics) +## --------------------------------------------------------------------------- +## DEPMAP_DIR is exported by build_samples.sh from fetch_depmap.py, so the +## release name is defined in exactly one place. The fallback keeps this script +## runnable standalone for debugging. +DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data") +depmap_model_file <- file.path(DEPMAP_DIR, "Model.csv") + +if (!file.exists(depmap_model_file)) { + stop(sprintf(paste0( + "Required DepMap file not found: %s\n", + "It should have been downloaded from Synapse by fetch_depmap.py before\n", + "this script ran. DepMap files can no longer be downloaded from the DepMap\n", + "portal (Cloudflare challenge), so they are read from the Synapse folder\n", + "syn75028495. To refresh them for a new release, download from\n", + "https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."), + depmap_model_file)) +} + +message("Using DepMap file from Synapse copy: ", depmap_model_file) +depmap_models<-readr::read_csv(depmap_model_file)#|> # dplyr::rename(DepMap_ID='ModelID') +## Transient container-local input -- drop it once it is in memory. +unlink(depmap_model_file) -sanger_models<-readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv") +## cog.sanger.ac.uk intermittently drops the connection mid-transfer +## (OpenSSL "unexpected eof"); this failed twice before succeeding on the +## third attempt on 2026-08-30. +sanger_models<-with_retries( + function() readr::read_csv("https://cog.sanger.ac.uk/cmp/download/model_list_20230923.csv", + show_col_types = FALSE), + what = "download Sanger model_list") print(paste("Downloaded",nrow(depmap_models),'dep map identifiers and',nrow(sanger_models),'sanger models')) ##query for cellosaurus automagically to get loadest version url='https://ftp.expasy.org/databases/cellosaurus/cellosaurus.xml' -if(!file.exists('/tmp/cell.xml')) - curl_download(url,'/tmp/cell.xml',quiet=TRUE)#curl(url, "r", h) +## 672 MB download; a partial file would poison every later run because the +## existence check below would treat it as complete. Download to a temp path +## and only move it into place once it has fully arrived. +if(!file.exists('/tmp/cell.xml')) { + with_retries(function() { + tmp_cello <- '/tmp/cell.xml.partial' + curl_download(url, tmp_cello, quiet=TRUE) + if (file.info(tmp_cello)$size < 1e8) + stop(sprintf('cellosaurus download truncated (%s bytes)', file.info(tmp_cello)$size)) + file.rename(tmp_cello, '/tmp/cell.xml') + }, what = "download cellosaurus.xml") +} cello<-XML::xmlParse('/tmp/cell.xml') cdf<-XML::xmlToList(cello) @@ -72,14 +140,14 @@ print(paste('Got',nrow(full.res),'human cellosaurus samples')) allmod<-sanger_models|> dplyr::rename(ModelID='BROAD_ID')|> left_join(depmap_models)|> - dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type)|> + dplyr::select(Sanger_ID='model_id',Depmap_ID='ModelID',PatientID,CellLineName,StrippedCellLineName,COSMIC_ID,CCLE_ID,RRID,cancer_type,ModelType)|> subset(!is.na(RRID))|> ##only take those we can map to cellosaurus distinct() ##we missed a handful here, so let's add back missedmod<-subset(depmap_models,ModelID%in%setdiff(depmap_models$ModelID,allmod$Depmap_ID))|> subset(!is.na(RRID))|> - dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype')|> + dplyr::select(Depmap_ID='ModelID',CCLE_ID='CCLEName',COSMIC_ID='COSMICID',CellLineName,StrippedCellLineName,Sanger_ID='SangerModelID',RRID,cancer_type='OncotreeSubtype',ModelType)|> mutate(PatientID=NA) allmod<-rbind(allmod,missedmod) @@ -122,7 +190,11 @@ long.df<-full.df%>% dplyr::select(-c(StrippedCellLineName))|> tidyr::pivot_longer(cols=c(PatientID,DepMap,Sanger,CCLE,COSMIC,Cellosaurus),names_to='other_id_source', values_to='other_id')%>% - mutate(model_type='cell line')%>% + mutate(model_type=dplyr::case_when( + ModelType=='Organoid' ~ 'patient derived organoid', + TRUE ~ 'cell line' + ))%>% + dplyr::select(-ModelType)%>% subset(!is.na(other_id))%>% subset(other_id!="") diff --git a/coderbuild/broad_sanger/02-broadSangerOmics.R b/coderbuild/broad_sanger/02-broadSangerOmics.R index bfd5780d..1a77a9de 100755 --- a/coderbuild/broad_sanger/02-broadSangerOmics.R +++ b/coderbuild/broad_sanger/02-broadSangerOmics.R @@ -6,56 +6,34 @@ library(tidyr) library(dplyr) library(rio) library(httr2) +library(data.table) + +## Shared retry/resume helpers (download_file_resumable, with_retries). +source("retry_utils.R") Sys.setenv(VROOM_CONNECTION_SIZE=100000000) +## Resumable, size-verified download. +## +## Kept under the original name so callers are unchanged, but the transport is +## no longer httr2. req_retry() restarts the whole request instead of resuming +## it, and the files pulled here are the largest in the pipeline -- Sanger ships +## rnaseq_all at ~897MB and WES_pureCN_CNV_genes at ~935MB. On a link that drops +## part-way, restarting a 935MB file just fails again at a similar point: build +## v24 died exactly that way on a 112MB file, breaking 12MB in, three times. +## +## download_file_resumable() (coderbuild/utils/retry_utils.R) uses wget -c, so +## each attempt continues from what is already on disk, and only promotes the +## file once its size matches the server's. +## +## req_verbose() is also gone: full curl logging across ~2GB of transfers buried +## the real errors in the build log. robust_download_httr2 <- function(url, dest, - max_tries = 5, - timeout_secs = 1500) { - # browser-style User-Agent - message("Downloading: ", url) - ua <- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" - - req <- request(url) |> - req_headers( - `User-Agent` = ua, - Accept = "application/octet-stream" - ) |> - req_timeout(timeout_secs) |> - req_retry(max_tries = max_tries, retry_on_failure = TRUE) |> - req_verbose() # ← turn on full curl logging - - resp <- tryCatch( - { - req |> req_perform(path = dest) - }, - error = function(e) { - message("🚨 Download failed for: ", url) - message(" • curl error: ", e$message) - if (file.exists(dest)) { - message(" • partial file size: ", - file.info(dest)$size, " bytes") - } - stop(e) # re-throw so your script still aborts - } - ) - - # if we get here, the download succeeded; sanity-check length - hdrs <- resp |> resp_headers() - if (!is.null(hdrs$`content-length`)) { - expected <- as.numeric(hdrs$`content-length`) - actual <- file.info(dest)$size - if (actual != expected) { - stop(sprintf( - "Incomplete download for %s: expected %d bytes but got %d", - url, expected, actual - )) - } - } - - invisible(dest) + max_tries = 5, + timeout_secs = 1500) { + download_file_resumable(url, dest, what = basename(dest)) } @@ -77,10 +55,69 @@ download_and_extract_zip_httr2 <- function(url, dest_zip, extract_dir, max_tries ##### DEPMAP FILES - -depmap_filenames=list(copy_number='https://figshare.com/ndownloader/files/40448840', - transcriptomics='https://figshare.com/ndownloader/files/40449128', - mutations='https://figshare.com/ndownloader/files/40449638') +## --------------------------------------------------------------------------- +## DEPMAP FILES ARE NO LONGER PROGRAMMATICALLY RETRIEVABLE. +## +## DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in +## front of https://depmap.org/portal/api/download/files. That endpoint now +## returns an HTML challenge page with HTTP 200, so read_csv() silently parses +## the HTML and every downstream column lookup fails. It cannot be scripted +## around, and DepMap explicitly asks that the portal not be scraped. +## +## We therefore keep a STATIC COPY of the required DepMap files on Synapse, in +## the "DepMap Raw" folder: https://www.synapse.org/Synapse:syn75028495 +## +## build_omics.sh runs coderbuild/utils/fetch_depmap.py BEFORE this script, +## which downloads them from Synapse into $DEPMAP_DIR. This script only reads +## from disk -- it never contacts the DepMap portal. +## +## THE SYNAPSE COPY MUST BE MANUALLY UPDATED FOR EVERY NEW DEPMAP RELEASE: +## 1. Download the files from https://depmap.org/portal/data_page/?tab=allData +## (accept the terms, then use the download button on each file) +## 2. Upload them to syn75028495 as NEW VERSIONS of the existing entities +## 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py +## 4. Re-run the build with --depmap-ready +## +## Currently pinned to: DepMap Public 26Q1 +## --------------------------------------------------------------------------- +## DEPMAP_DIR is exported by build_omics.sh from fetch_depmap.py, so the release +## name is defined in exactly one place. The fallback keeps this script runnable +## standalone for debugging. +DEPMAP_DIR <- Sys.getenv("DEPMAP_DIR", unset = "/opt/depmap_data") + +depmap_filenames = list( + copy_number = file.path(DEPMAP_DIR, "PortalOmicsCNGeneLog2.csv"), + transcriptomics = file.path(DEPMAP_DIR, "OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv"), + mutations = file.path(DEPMAP_DIR, "OmicsSomaticMutations.csv") +) + +missing_depmap <- unlist(depmap_filenames)[!file.exists(unlist(depmap_filenames))] +if (length(missing_depmap) > 0) { + stop(sprintf(paste0( + "Required DepMap file(s) not found:\n %s\n", + "They should have been downloaded from Synapse by fetch_depmap.py before\n", + "this script ran. DepMap files can no longer be downloaded from the DepMap\n", + "portal (Cloudflare challenge), so they are read from the Synapse folder\n", + "syn75028495. To refresh them for a new release, download from\n", + "https://depmap.org/portal/data_page/?tab=allData and re-upload to Synapse."), + paste(missing_depmap, collapse = "\n "))) +} +message("Using DepMap files from Synapse copy in ", DEPMAP_DIR) + +## fetch_depmap.py has already downloaded these into a container-local scratch +## directory, so there is nothing to download here -- we read them in place. +## +## Each file is deleted as soon as it has been read: they are transient build +## inputs totalling ~1.3 GB, and this step is memory- and disk-sensitive (the +## copy_number melt alone is large). Freeing each input before processing the +## next keeps the container's footprint down. The container is `--rm` anyway, +## so this is belt-and-braces, but it matters while the step is running. +depmap_consume <- function(path) { + if (!is.null(path) && file.exists(path)) { + unlink(path) + message("Removed consumed DepMap input: ", path) + } +} ##### SANGER FILES sanger_filenames=list(transcriptomics='https://cog.sanger.ac.uk/cmp/download/rnaseq_all_20220624.zip', copy_number='https://cog.sanger.ac.uk/cmp/download/WES_pureCN_CNV_genes_latest.csv.gz', @@ -88,29 +125,29 @@ sanger_filenames=list(transcriptomics='https://cog.sanger.ac.uk/cmp/download/rna ###### VARIANT SCHEMA HARMONIZATION -variant_schema =list(`3'UTR`=c("3'UTR",'THREE_PRIME_UTR','3prime_UTR_variant','3prime_UTR_ess_splice'), - `5'Flank`=c("FIVE_PRIME_FLANK","5'Flank",'upstream'), - `5'UTR`=c("5'UTR",'5prime_UTR_variant','5prime_UTR_variant','5prime_UTR_ess_splice'), - Undetermined=c('COULD_NOT_DETERMINE'), +variant_schema =list(`3'UTR`=c("3'UTR",'THREE_PRIME_UTR','3prime_UTR_variant','3prime_UTR_ess_splice','3_prime_UTR_variant'), + `5'Flank`=c("FIVE_PRIME_FLANK","5'Flank",'upstream','upstream_gene_variant'), + `5'UTR`=c("5'UTR",'5prime_UTR_variant','5prime_UTR_variant','5prime_UTR_ess_splice','5_prime_UTR_variant'), + Undetermined=c('COULD_NOT_DETERMINE','protein_altering_variant'), De_novo_Start_InFrame=c('DE_NOVO_START_IN_FRAME','De_novo_Start_InFrame'), De_novo_Start_OutOfFrame=c('DE_NOVO_START_OUT_FRAME','De_novo_Start_OutOfFrame'), - Frame_Shift_Del=c('FRAME_SHIFT_DEL','Frame_Shift_Del','frameshift'), + Frame_Shift_Del=c('FRAME_SHIFT_DEL','Frame_Shift_Del','frameshift','frameshift_variant'), Frame_Shift_Ins=c('FRAME_SHIFT_INS','Frame_Shift_Ins'), - IGR=c('IGR','nc_variant'), - In_Frame_Del=c('IN_FRAME_DEL','In_Frame_Del','inframe'), - In_Frame_Ins=c('IN_FRAME_INS','In_Frame_Ins'), - Intron=c('INTRON','Intron','intronic','intron'), - Missense_Mutation=c('Missense_Mutation','MISSENSE','missense'), - Nonsense_Mutation=c('Nonsense_Mutation','NONSENSE','nonsense'), - Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP'), - RNA=c('RNA'), + IGR=c('IGR','nc_variant','intergenic_variant','downstream_gene_variant'), + In_Frame_Del=c('IN_FRAME_DEL','In_Frame_Del','inframe','inframe_deletion'), + In_Frame_Ins=c('IN_FRAME_INS','In_Frame_Ins','inframe_insertion'), + Intron=c('INTRON','Intron','intronic','intron','intron_variant'), + Missense_Mutation=c('Missense_Mutation','MISSENSE','missense','missense_variant'), + Nonsense_Mutation=c('Nonsense_Mutation','NONSENSE','nonsense','stop_gained'), + Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP','stop_lost'), + RNA=c('RNA','non_coding_transcript_exon_variant','non_coding_transcript_variant'), Start_Codon_SNP=c('START_CODON_SNP','Start_Codon_SNP'), Start_Codon_Del=c('Start_Codon_Del','START_CODON_DEL','start_lost'), Start_Codon_Ins=c('Start_Codon_Ins','START_CODON_INS'), Stop_Codon_Del=c('Stop_Codon_Del','stop_lost'), Stop_Codon_Ins=c('Stop_Codon_Ins'), - Silent=c('Silent','SILENT','silent'), - Splice_Site=c('Splice_Site','SPLICE_SITE','splice_region'), + Silent=c('Silent','SILENT','silent','synonymous_variant'), + Splice_Site=c('Splice_Site','SPLICE_SITE','splice_region','splice_donor_variant','splice_acceptor_variant','splice_region_variant','splice_polypyrimidine_tract_variant','splice_donor_region_variant','splice_donor_5th_base_variant'), Translation_Start_Site=c('Translation_Start_Site','start_lost')) depmap_vtab<-do.call('rbind',sapply(names(variant_schema),function(x) cbind(rep(x,length(variant_schema[[x]])),unlist(variant_schema[[x]])))) @@ -149,7 +186,9 @@ sanger_files<-function(fi,value){ #read in file local_cn <- file.path(tempdir(), "sanger_copy_number.csv.gz") robust_download_httr2(fi, local_cn) - exp_file <- readr::read_csv(local_cn) + exp_file <- data.table::fread(local_cn, + select = c("model_id","symbol","gatk_mean_log2_copy_ratio","source","data_type","cn_category"), + data.table = FALSE) # exp_file <- readr::read_csv(fi) ##already in long form <3 <3 <3 # file.remove(fi) smap<-sanger_samples|> @@ -173,7 +212,8 @@ sanger_files<-function(fi,value){ dplyr::select(other_id,copy_number,entrez_id,Sanger='cn_category')|> left_join(smap)|> distinct() - rm(exp_file) + rm(exp_file); gc() + if (file.exists(local_cn)) file.remove(local_cn) print('copy call') @@ -198,7 +238,7 @@ sanger_files<-function(fi,value){ tidyr::pivot_longer(cols=c(IMPROVE,Sanger), names_to='source', values_to='copy_call') - rm(res) + rm(res); gc() # full<-lres }else if(value=='methylation'){ ###IF DATA REPRESENT RRBS### @@ -294,7 +334,7 @@ sanger_files<-function(fi,value){ dat<-exp_file[-c(1:4),-1] colnames(dat)[1]<-'gene_symbol' - rm(exp_file) + rm(exp_file); gc() ddat<-apply(dat,2,unlist)|> as.data.frame()|> @@ -302,14 +342,19 @@ sanger_files<-function(fi,value){ left_join(genes)|> dplyr::select(-gene_symbol) - rm(dat) + ## gc() here matters: without it R still holds dat when pivot_longer + ## allocates the long frame, which is the peak of the whole step. + ## Measured on a 20k gene x 800 sample matrix (16M output rows): + ## peak RSS 1,351MB -> 1,219MB, run time 8.2s -> 8.1s. The other + ## branches in this function already pair rm() with gc(). + rm(dat); gc() ddat$entrez_id<-as.numeric(ddat$entrez_id) res = tidyr::pivot_longer(data=as.data.frame(ddat),cols=c(1:(ncol(ddat)-1)), names_to='other_id',values_to='transcriptomics', values_transform=list(expression=as.numeric))|> distinct() - rm(ddat) + rm(ddat); gc() smap<-samps|> dplyr::select(improve_sample_id,other_id,study,source)|>distinct() @@ -428,25 +473,38 @@ depmap_files<-function(fi,value){ ##now every data type is parsed slightly differently, so we need to change our formatting ##and mapping to get it into a unified 3 column schema if(value=='copy_number'){ - # exp_file <- readr::read_csv(fi) - local_path <- "/tmp/depmap_copy_number.csv.gz" - robust_download_httr2(fi, local_path) - exp_file <- readr::read_csv(local_path) - - - print('Long to wide') - res = exp_file|> - tidyr::pivot_longer(cols=c(2:ncol(exp_file)), - names_to='gene_entrez',values_to='copy_number', - values_transform=list(copy_number=as.numeric))|> + local_path <- fi + ## Use data.table::melt for memory-efficient wide->long conversion + exp_dt <- data.table::fread(local_path, data.table = TRUE) + depmap_consume(local_path) # read into memory; drop the on-disk copy + id_col <- colnames(exp_dt)[1] + data.table::setnames(exp_dt, id_col, "other_id") + gene_cols <- setdiff(colnames(exp_dt), "other_id") + + print('Long to wide (data.table melt)') + res <- data.table::melt(exp_dt, id.vars = "other_id", + measure.vars = gene_cols, + variable.name = "gene_entrez", + value.name = "copy_number", + variable.factor = FALSE) + rm(exp_dt, gene_cols); gc() + ## (the DepMap input was already removed by depmap_consume above) + + res <- as.data.frame(res) |> + dplyr::mutate( + copy_number = suppressWarnings(as.numeric(copy_number)), + ## PortalOmicsCNGeneLog2 stores log2(absolute copies); convert to ratio + copy_number = 2^copy_number / 2 + ) |> dplyr::distinct() - rm(exp_file) - - colnames(res)[1]<-'other_id' print('String manipulations') - res<-res|> - tidyr::separate_wider_delim(gene_entrez,' ',names=c('gene_symbol','entrez_id')) + res <- res |> + dplyr::mutate( + gene_symbol = trimws(stringr::str_extract(gene_entrez, "^[^(]+")), + entrez_id = stringr::str_extract(gene_entrez, "(?<=\\()\\d+(?=\\))") + ) |> + dplyr::select(-gene_entrez) print('join with gene') res<-res|> @@ -493,12 +551,14 @@ depmap_files<-function(fi,value){ }else if(value=='mutations'){ ####IF DATA REPRESENTS MUTATIONS##### - local_mut <- file.path(tempdir(), "depmap_mutations.csv.gz") - robust_download_httr2(fi, local_mut) + local_mut <- fi exp_file <- readr::read_csv(local_mut)|> + dplyr::filter(IsDefaultEntryForModel %in% c(TRUE, "Yes"))|> dplyr::select(EntrezGeneID,HgncName,other_id='ModelID',VariantInfo,mutation='DNAChange')|> - distinct() + distinct()|> + dplyr::mutate(VariantInfo = sub("&.*", "", VariantInfo)) + depmap_consume(local_mut) # read into memory; drop the on-disk copy res<-exp_file|> mutate(entrez_id=as.numeric(EntrezGeneID))|> @@ -537,9 +597,15 @@ depmap_files<-function(fi,value){ return(full) }else if(value=='transcriptomics'){ #if gene expression # exp_file <- readr::read_csv(fi) - local_tx <- file.path(tempdir(), "depmap_transcriptomics.csv.gz") - robust_download_httr2(fi, local_tx) - exp_file <- readr::read_csv(local_tx) + local_tx <- fi + ## 26Q1 format has metadata columns (SequencingID, ModelConditionID, + ## IsDefaultEntryForMC, IsDefaultEntryForModel) before the gene columns. + ## Filter to one canonical run per model, then keep only ModelID + gene cols. + exp_file <- readr::read_csv(local_tx, show_col_types = FALSE)|> + dplyr::filter(IsDefaultEntryForModel %in% c(TRUE, "Yes"))|> + dplyr::rename(other_id = ModelID)|> + dplyr::select(other_id, dplyr::matches("\\(\\d+\\)")) + depmap_consume(local_tx) # read into memory; drop the on-disk copy print("wide to long") res = tidyr::pivot_longer(data=exp_file,cols=c(2:ncol(exp_file)), @@ -549,8 +615,12 @@ depmap_files<-function(fi,value){ colnames(res)[1]<-'other_id' print('fixing gene names') - res<-res|> - tidyr::separate_wider_delim(gene_entrez,' ',names=c('gene_symbol','entrez_par')) + res <- res |> + dplyr::mutate( + gene_symbol = trimws(stringr::str_extract(gene_entrez, "^[^(]+")), + entrez_par = stringr::str_extract(gene_entrez, "(?<=\\()\\d+(?=\\))") + ) |> + dplyr::select(-gene_entrez) print('join with gene') res<-res|> @@ -670,12 +740,10 @@ main<-function(){ print(dt) temps<-sanger_files(sanger_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct() readr::write_csv(temps,file=paste0('/tmp/sanger_',dt,'.csv.gz')) + rm(temps); gc() tempd<-depmap_files(depmap_filenames[[dt]],dt)|>tidyr::drop_na()|>dplyr::distinct() readr::write_csv(tempd,file=paste0('/tmp/broad_',dt,'.csv.gz')) - -# readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz')) - rm(tempd) - rm(temps) + rm(tempd); gc() }) } diff --git a/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py b/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py index c57da5b7..59be0e0b 100644 --- a/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py +++ b/coderbuild/broad_sanger/02a-broad_sanger_proteomics.py @@ -1,3 +1,5 @@ +import os +import time import pandas as pd import argparse from zipfile import ZipFile @@ -5,28 +7,98 @@ from requests.adapters import HTTPAdapter from urllib3.util import Retry -def robust_download(url, dest_path, max_retries=5): - session = requests.Session() +# Waits between whole-download attempts: 1, 3, 10 and 15 minutes, matching +# coderbuild/utils/retry_utils.R. Long enough to ride out a real incident +# rather than only a momentary blip; if all attempts fail the build fails. +RETRY_SLEEPS = (60, 180, 600, 900) + + +def robust_download(url, dest_path, sleeps=RETRY_SLEEPS): + """Download `url` to `dest_path`, resuming an interrupted transfer. + + The previous implementation mounted a urllib3 Retry adapter and assumed that + made it robust. It did not: Retry only covers connection setup and HTTP + status codes. Once the response body is streaming, a truncated body raises + ChunkedEncodingError/ProtocolError out of iter_content(), which went straight + to the except clause and became a fatal RuntimeError -- no retry, no resume. + + That is exactly how build v24 died, 12MB into a 112MB file: + + Failed to download .../Proteomics_20221214.zip: + ('Connection broken: IncompleteRead(12353536 bytes read, + 100318054 more expected)') + + So: retry the whole transfer, and RESUME it with a Range request rather than + starting over, since restarting a large file on a flaky link tends to fail + again at a similar point. Verified that this host sends accept-ranges: bytes + and answers a Range request with 206. + + Bytes accumulate in ".part", which is renamed onto dest_path only once + the finished size matches the server's Content-Length. An unverified or + partial file therefore never appears at the destination. + """ + part = dest_path + ".part" + retry_strategy = Retry( - total=max_retries, + total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504], allowed_methods={"GET", "HEAD"}, raise_on_status=False, ) adapter = HTTPAdapter(max_retries=retry_strategy) + session = requests.Session() session.mount("https://", adapter) session.mount("http://", adapter) + expected = None try: - with session.get(url, stream=True, timeout=(5, 60)) as r: - r.raise_for_status() - with open(dest_path, "wb") as f: - for chunk in r.iter_content(chunk_size=1024 * 1024): - if chunk: # filter out keep-alive chunks - f.write(chunk) - except requests.exceptions.RequestException as e: - raise RuntimeError(f"Failed to download {url}: {e}") from e + head = session.head(url, timeout=(5, 30), allow_redirects=True) + if head.ok and head.headers.get("Content-Length"): + expected = int(head.headers["Content-Length"]) + except requests.exceptions.RequestException: + pass # size unknown; completeness is then judged by a clean stream end + + last_error = None + for attempt in range(len(sleeps) + 1): + have = os.path.getsize(part) if os.path.exists(part) else 0 + try: + headers = {"Range": f"bytes={have}-"} if have else {} + with session.get(url, stream=True, timeout=(5, 60), headers=headers) as r: + if have and r.status_code == 206: + mode = "ab" # server honoured the resume + else: + if have: + print(f" server ignored Range for {url}; restarting", flush=True) + r.raise_for_status() + mode, have = "wb", 0 + with open(part, mode) as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + if chunk: # filter out keep-alive chunks + f.write(chunk) + + got = os.path.getsize(part) + if expected is not None and got != expected: + raise IOError(f"incomplete: {got} of {expected} bytes") + + os.replace(part, dest_path) + return + + except (requests.exceptions.RequestException, IOError) as e: + last_error = e + got = os.path.getsize(part) if os.path.exists(part) else 0 + # Keep the .part file: the next attempt resumes from here. Deleting + # it would mean a flaky link could never accumulate a large file. + if attempt < len(sleeps): + wait = sleeps[attempt] + print(f" download of {url} failed at {got} bytes " + f"({e}); attempt {attempt + 1}/{len(sleeps) + 1}, " + f"resuming in {wait // 60} min", flush=True) + time.sleep(wait) + + raise RuntimeError( + f"Failed to download {url} after {len(sleeps) + 1} attempts: {last_error}" + ) def main(): diff --git a/coderbuild/broad_sanger/03-createDrugFile.R b/coderbuild/broad_sanger/03-createDrugFile.R index b1b04615..c4375066 100644 --- a/coderbuild/broad_sanger/03-createDrugFile.R +++ b/coderbuild/broad_sanger/03-createDrugFile.R @@ -9,6 +9,10 @@ source_python("pubchem_retrieval.py") library('PharmacoGx') +## Pinned PSet list shared with 04a-drugResponseData.R +source("pinned_psets.R") +## Retry/caching helpers for flaky third-party downloads +source("retry_utils.R") all.dsets<-PharmacoGx::availablePSets() @@ -26,9 +30,10 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' ignore_file_path <- '/tmp/ignore_chems.txt' for(cel in cell.lines){ - files<-subset(all.dsets,`Dataset Name`==cel)%>% - dplyr::select(`PSet Name`)%>% - unlist() + ## PSets are pinned (see coderbuild/utils/pinned_psets.R) rather than + ## taken straight from availablePSets(), so a new upstream PSet cannot + ## silently change or break the release. + files<-psets_for_dataset(all.dsets, cel) for(f in files){ print(f) @@ -36,9 +41,19 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' cel='GDSCv2' if(f=='GDSC_2020(v1-8.2)') cel='GDSCv1' - #tmpfile<-paste0(cel,'doseResponse') - dset<<-downloadPSet(f,saveDir='.',timeout=10000) + ## Skip PSets already merged into the cumulative output on an + ## earlier attempt, so a failure late in the list does not force a + ## re-run of everything before it. The marker is written only after + ## the merge below succeeds. + done_tag <- paste0("drugs-", f) + if (pset_is_done(done_tag)) { + message(sprintf( + "[03-createDrugFile] PSet '%s' already processed in a previous attempt; skipping.", f)) + next + } + + dset<<-download_pset_retry(f) mapping <- sensitivityInfo(dset)##get the dataset dose response data @@ -72,6 +87,44 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' print(paste('Found',length(chem_list),'chemicals for dataset',cel)) + ## Guard: never hand an empty/NULL chem_list to the Python writer. + ## update_dataframe_and_write_tsv() iterates it and fails with the + ## opaque "TypeError: 'NoneType' object is not iterable", which is + ## how the Aug 2026 CTRPv2.1_2016 schema change surfaced. + ## + ## Zero is also a legitimate outcome -- e.g. every compound in this + ## PSet was already covered by a previous PSet or by efile -- so + ## this skips rather than errors. A PSet whose treatment column is + ## missing entirely is reported loudly, since that means an + ## unrecognised schema rather than "nothing new to do". + ## An unrecognised schema is a HARD FAILURE, not a skip. PSets are + ## pinned (coderbuild/utils/pinned_psets.R), so if a pinned PSet no + ## longer exposes a treatment column it means upstream changed its + ## shape. Continuing would silently omit that PSet's drugs from the + ## release, which is exactly the kind of quiet data loss the pinning + ## exists to prevent -- so stop and make it visible. + if (is.null(mapping$treatmentid)) { + stop(sprintf( + paste0("PSet '%s': sensitivityInfo() has no 'treatmentid' or ", + "'drugid' column (columns: %s).\n", + "A pinned PSet has changed schema. Refusing to continue, ", + "because skipping it would drop its drugs from the release.\n", + "Either add an explicit column mapping for this PSet or ", + "remove it from PINNED_PSETS after confirming that is correct."), + f, paste(colnames(mapping), collapse = ", "))) + } + + ## Zero new chemicals is a legitimate outcome and NOT data loss: + ## every compound in this PSet was already fetched via an earlier + ## PSet in this run or via efile, so its drugs are already in the + ## cumulative output. Nothing is dropped by moving on. + if (length(chem_list) == 0) { + message(sprintf( + "[03-createDrugFile] PSet '%s': all chemicals already present; nothing new to fetch.", f)) + mark_pset_done(done_tag) + next + } + #build prev_drug_filepaths for this iteration (include original and accumulated) --- prev_list <- c() @@ -102,12 +155,19 @@ getDepMapDrugData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC',' } fwrite(combined, output_file_path, sep="\t") file.remove(temp_out) + ## Merge succeeded -- record this PSet as done so a later + ## attempt skips it entirely. + mark_pset_done(done_tag) } else { warning("Expected temporary output not found: ", temp_out) } - ##clean up file when done - file.remove(paste0(f,'.rds')) + ## The downloaded PSet is deliberately NOT deleted. It lives in + ## PSET_CACHE_DIR under /tmp (the bind mount to the host's local/), + ## so a retried attempt reuses it instead of re-downloading. These + ## are large -- CTRPv2 is 228 MB and one PSet is 1.5 GB -- and + ## re-fetching them is what made a single transient Zenodo 504 + ## expensive. Clear local/ between releases to reclaim the space. } } diff --git a/coderbuild/broad_sanger/03a-nci60Drugs.py b/coderbuild/broad_sanger/03a-nci60Drugs.py index 21a188ff..568c397f 100644 --- a/coderbuild/broad_sanger/03a-nci60Drugs.py +++ b/coderbuild/broad_sanger/03a-nci60Drugs.py @@ -10,6 +10,31 @@ import pubchem_retrieval as pr import random as rand from urllib import request +import download_utils +import io + +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + +def _fetch(url: str) -> bytes: + """Download url with a browser User-Agent (NCI wiki blocks Python's default).""" + req = request.Request(url, headers={"User-Agent": _BROWSER_UA}) + with request.urlopen(req) as resp: + return resp.read() + +def _read_csv_url(url: str, **kwargs) -> pl.DataFrame: + return pl.read_csv(io.BytesIO(_fetch(url)), **kwargs) + +def _retrieve_url(url: str, dest: str) -> None: + """Download url to dest, resuming an interrupted transfer. + + This also fetches the ~346MB DOSERESP.zip, and _fetch() reads the whole + body into memory in one call -- the same shape that truncated with + IncompleteRead in build v32. Small CSVs still go through _fetch(). + """ + download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA}) ##drug files smi_strings='https://wiki.nci.nih.gov/download/attachments/155844992/nsc_smiles.csv?version=1&modificationDate=1710381820000&api=v2&download=true' @@ -41,11 +66,12 @@ #jan 2025 cas = 'https://wiki.nci.nih.gov/download/attachments/155844992/nsc_cas.csv?version=4&modificationDate=1735931080720&api=v2' -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=11&modificationDate=1712351454136&api=v2' -##OCT 2024 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' +##oct 2024 +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' #jan 2025 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#may 2025 +conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=19&modificationDate=1775183341937&api=v2' def main(): parser = argparse.ArgumentParser() @@ -58,15 +84,20 @@ def main(): 'InChIKey':[],'formula':[],'weight':[],'pubchem_id':[]} print('Downloading NSC identifiers for nci60 data') - names = pl.read_csv(chemnames,ignore_errors=True) + names = _read_csv_url(chemnames, ignore_errors=True) #castab = pd.read_csv(cas) - pubchems = pl.read_csv(pc_ids) - smiles = pl.read_csv(smi_strings) + pubchems = _read_csv_url(pc_ids) + smiles = _read_csv_url(smi_strings) print('Getting experimental data to filter drugs') if not os.path.exists('DOSERESP.csv'): - resp = request.urlretrieve(conc_data,'doseresp.zip') - os.system('unzip doseresp.zip') + _retrieve_url(conc_data, 'doseresp.zip') + # -o overwrites without prompting; the exit status is checked so a + # corrupt or truncated archive fails here rather than surfacing later + # as a confusing parse error on a stale DOSERESP.csv. + _rc = os.system('unzip -o doseresp.zip') + if _rc != 0: + raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).") dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True) pubchems = pubchems.filter(pl.col('NSC').is_in(dose_resp['NSC'])) smiles = smiles.filter(pl.col("NSC").is_in(dose_resp['NSC']) @@ -81,13 +112,10 @@ def main(): ##first filter to see if there are structures/drugs in teh data already. i dont think this does much. if os.path.exists(opts.output): curdrugs = pl.read_csv(opts.output,separator='\t') - # cs = set(curdrugs['isoSMILES']) - smiles = smiles.filter(pl.col('SMILES').is_not_null()) - upper=[a.upper() for a in smiles['SMILES']] - smiles= pl.DataFrame({'NSC':smiles['NSC'],'upper':upper})#smiles.with_columns(upper=upper) - ##reduce to smiels only in current drugs - # ssmiles = smiles.filter(~pl.col('upper').is_in(curdrugs['isoSMILES'])) - ssmiles = smiles.filter(~pl.col('upper').is_in(curdrugs['canSMILES'])) + smiles_notnull = smiles.filter(pl.col('SMILES').is_not_null()) + upper=[a.upper() for a in smiles_notnull['SMILES']] + smiles_upper = pl.DataFrame({'NSC':smiles_notnull['NSC'],'upper':upper}) + ssmiles = smiles_upper.filter(~pl.col('upper').is_in(curdrugs['canSMILES'])) pubchems = pubchems.filter(pl.col('NSC').is_in(ssmiles['NSC'])) arr = set(pubchems['CID']) diff --git a/coderbuild/broad_sanger/04-drug_dosage_and_curves.py b/coderbuild/broad_sanger/04-drug_dosage_and_curves.py index 3b6f7245..e5bc7005 100755 --- a/coderbuild/broad_sanger/04-drug_dosage_and_curves.py +++ b/coderbuild/broad_sanger/04-drug_dosage_and_curves.py @@ -14,29 +14,47 @@ import argparse import pandas as pd import subprocess +import glob +import multiprocessing +from datetime import datetime + +def _log(msg): + print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] {msg}', flush=True) parser = argparse.ArgumentParser() parser.add_argument('--curSampleFile',dest='samplefile',default=None,help='DepMap sample file') - parser.add_argument('--drugfile',dest='dfile',default=None,help='Drug database') +parser.add_argument('--workers', type=int, default=max(1, round(multiprocessing.cpu_count() * 3 / 4)), help='Number of parallel worker processes for curve fitting (default: 3/4 of available CPUs)') opts = parser.parse_args() samplefile = opts.samplefile drugfile = opts.dfile -cmd = ['/opt/venv/bin/python','04b-nci60-updated.py','--sampleFile='+samplefile,'--drugFile='+drugfile] -print(cmd) -subprocess.run(cmd) +def run(cmd): + _log('running: ' + ' '.join(cmd)) + res = subprocess.run(cmd) + if res.returncode != 0: + _log(f'WARNING: command exited {res.returncode}: {" ".join(cmd)}') + return res.returncode + +# A failure here is FATAL, not a warning. +# +# In build v32 this exited 1 after DOSERESP.zip truncated, run() merely logged +# "WARNING: command exited 1", and the build carried on. nci60DoseResponse was +# never written, so the fitting below saw 7 studies instead of 8 and nci60 was +# published as eight empty files -- which pass schema validation, because a +# file with no rows cannot violate a schema. +_rc = run(['/opt/venv/bin/python','04b-nci60-updated.py','--sampleFile='+samplefile,'--drugFile='+drugfile]) +if _rc != 0: + raise RuntimeError( + f'04b-nci60-updated.py failed (exit {_rc}). It produces the NCI60 ' + f'dose-response input, and continuing without it publishes nci60 as ' + f'empty files that still validate.') ####step 4a - get dose response data -cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'CTRPv2,FIMM,GDSC'] -print(cmd) -subprocess.run(cmd) - -cmd = ['Rscript','04a-drugResponseData.R',samplefile,drugfile,'gCSI,PRISM,CCLE'] -print(cmd) -subprocess.run(cmd) +run(['Rscript','04a-drugResponseData.R',samplefile,drugfile,'CTRPv2,FIMM,GDSC']) +run(['Rscript','04a-drugResponseData.R',samplefile,drugfile,'gCSI,PRISM,CCLE']) #cmd = 'Rscript 04a-drugResponseData.R '+samplefile+' '+drugfile+' NCI60' @@ -44,18 +62,108 @@ #os.system(cmd) ########Step 4b fit curves -allfiles=[a for a in os.listdir('./') if 'DoseResponse' in a] -print(allfiles) -for a in allfiles: - subprocess.run(['/opt/venv/bin/python','fit_curve.py','--input='+a,'--output='+a]) +# Collect dose-response inputs from BOTH places they are written. +# +# 04b-nci60-updated.py writes ./nci60DoseResponse into the working directory, +# but 04a-drugResponseData.R writes through dose_response_cache_path(), i.e. +# $DOSE_RESPONSE_CACHE_DIR/_doseResponse.tsv, so that a failed +# attempt can resume instead of regenerating every study. This scan only looked +# in './' for 'DoseResponse', which matches neither the directory nor the +# lowercase 'd', so it silently found NCI60 alone. +# +# The consequence was severe and completely silent: broad_sanger_experiments.tsv +# came out NCI60-only, and 05b_separate_datasets.py derives each split dataset's +# sample and drug ids FROM the experiments rows, so ccle, ctrpv2, fimm, gcsi, +# gdscv1, gdscv2 and prism were published with every file present but empty -- +# samples, omics, drugs and experiments alike. Header-only files are schema +# valid, so validate passed. Build v27 shipped exactly that. +# +# Excluding '.0' matters too: fit_curve writes '.0', which itself +# contains 'DoseResponse', so a retry would re-fit its own output. +CACHE_DIR = os.environ.get('DOSE_RESPONSE_CACHE_DIR', '/tmp/dose_response_cache') +jobs = [] # (input path, output stem -- kept local so '.0' lands in ./) +for a in sorted(os.listdir('./')): + if 'DoseResponse' in a and not a.endswith('.0'): + jobs.append((a, a)) +for pth in sorted(glob.glob(os.path.join(CACHE_DIR, '*_doseResponse.tsv'))): + stem = os.path.basename(pth)[:-len('.tsv')] + if any(stem == j[1] for j in jobs): + continue + jobs.append((pth, stem)) + +if not jobs: + raise RuntimeError('No DoseResponse files found — data generation step likely failed') +_log(f'fitting {len(jobs)} DoseResponse files with {opts.workers} workers: ' + f'{[j[1] for j in jobs]}') +fit_failures = [] +for src, stem in jobs: + _log(f'starting fit_curve: {stem} (from {src})') + cmd = ['/opt/venv/bin/python', 'fit_curve.py', + '--input=' + src, '--output=' + stem, '--workers=' + str(opts.workers)] + if 'nci60' in stem.lower(): + cmd.append('--chunk_size=25000') + rc = run(cmd) + if rc != 0: + _log(f'WARNING: fit_curve failed for {stem} (exit {rc}) — will be excluded from output') + fit_failures.append(stem) + _log(f'finished fit_curve: {stem}') ###step 4c concatenate all files -outfiles = [a for a in os.listdir("./") if ".0" in a] +outfiles = [a for a in os.listdir("./") if a.endswith('.0')] +if not outfiles: + raise RuntimeError('No .0 output files found after curve fitting — all fit_curve steps failed') final_file = [] for of in outfiles: - final_file.append(pd.read_csv(of,sep='\t')) - -pd.concat(final_file).drop_duplicates().to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t') + df = pd.read_csv(of, sep='\t') + if df.empty: + _log(f'WARNING: {of} is empty, skipping') + continue + _log(f'loaded {of}: {len(df)} rows') + final_file.append(df) + +if not final_file: + raise RuntimeError('All .0 output files were empty — cannot produce experiments output') +if fit_failures: + _log(f'WARNING: {len(fit_failures)} dataset(s) missing from output: {fit_failures}') + +combined = pd.concat(final_file).drop_duplicates() + +# Every study that was fitted must appear in the output. +# +# 05b_separate_datasets.py derives each split dataset's sample and drug ids from +# these rows, so a study missing here is not a partial result -- it publishes +# that whole dataset as header-only files: samples, omics, drugs, experiments. +# Those pass schema validation (no rows, nothing to violate), which is why v27 +# shipped seven empty datasets and still reported "validate successful". +# Check against the studies this step is SUPPOSED to produce, not merely the +# ones that happened to yield an input file. +# +# The previous version derived the expectation from `jobs`, so a study whose +# dose-response file was never generated simply was not expected, and its +# absence passed silently. That is precisely how v32 shipped an empty nci60: +# the input never existed, so nothing noticed it was gone. +REQUIRED_STUDIES = {'nci60', 'ccle', 'ctrpv2', 'fimm', 'gcsi', + 'gdscv1', 'gdscv2', 'prism'} +fitted_studies = {s.replace('_doseResponse', '').replace('DoseResponse', '').lower() + for s in (j[1] for j in jobs)} - {''} +not_fitted = sorted(REQUIRED_STUDIES - fitted_studies) +if not_fitted: + raise RuntimeError( + f"No dose-response input was produced for: {', '.join(not_fitted)}. " + f"Fitted only {sorted(fitted_studies)}. Each missing study becomes an " + f"entirely empty published dataset, so the build is stopping here.") +expected_studies = fitted_studies | REQUIRED_STUDIES +got_studies = {str(v).lower() for v in combined['study'].unique()} if 'study' in combined.columns else set() +missing_studies = sorted(st for st in expected_studies if st not in got_studies) +if missing_studies: + raise RuntimeError( + f"broad_sanger_experiments is missing every row for: {', '.join(missing_studies)}. " + f"Present: {sorted(got_studies)}. Each missing study becomes an entirely empty " + f"published dataset, so the build is stopping rather than shipping header-only files.") +_log(f'studies in output: {sorted(got_studies)} ({len(combined):,} rows)') + +combined.to_csv('/tmp/broad_sanger_experiments.tsv',index=False,sep='\t') +_log(f'wrote /tmp/broad_sanger_experiments.tsv') #os.system('cat *.0 > /tmp/broad_sanger_experiments.tsv') #os.system('gzip -f /tmp/experiments.tsv') diff --git a/coderbuild/broad_sanger/04a-drugResponseData.R b/coderbuild/broad_sanger/04a-drugResponseData.R index 144ce9e9..a5d943bb 100755 --- a/coderbuild/broad_sanger/04a-drugResponseData.R +++ b/coderbuild/broad_sanger/04a-drugResponseData.R @@ -9,6 +9,11 @@ library(readr) library(dplyr) #} +## Pinned PSet list shared with 03-createDrugFile.R +source("pinned_psets.R") +## Retry/caching helpers for flaky third-party downloads +source("retry_utils.R") + all.dsets<-PharmacoGx::availablePSets() ##first define a generic dose response function @@ -137,7 +142,7 @@ getDoseRespData<-function(dset,studyName,improve_samples,drug.map){ ##now map drugs, samples, genes to ids in database files - write.table(doseRep,file=paste0(tolower(studyName),'DoseResponse'),sep='\t',row.names=F,quote=F) + write.table(doseRep,file=dose_response_cache_path(studyName),sep='\t',row.names=F,quote=F) doseRep } @@ -154,9 +159,10 @@ getCellLineDoseData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC' all.dose.rep<-do.call(rbind,lapply(cell.lines,function(cel){ # print(cel) - files<-subset(all.dsets,`Dataset Name`==cel)%>% - dplyr::select(`PSet Name`)%>% - unlist() + ## PSets are pinned (see coderbuild/utils/pinned_psets.R) and resolved + ## through the same helper the drugs step uses, so drugs and experiments + ## can never disagree about which PSets are in the release. + files<-psets_for_dataset(all.dsets, cel) res<-do.call(rbind,lapply(files,function(f){ @@ -165,12 +171,24 @@ getCellLineDoseData<-function(cell.lines=c('CTRPv2','FIMM','gCSI','PRISM','GDSC' cel='GDSCv2' if(f=='GDSC_2020(v1-8.2)') cel='GDSCv1' - tmpfile<-paste0(cel,'doseResponse') + ## Resume support: if this PSet's dose-response was already computed on + ## an earlier attempt, read it back instead of re-downloading the PSet + ## and recomputing. + ## + ## This check was previously broken and silently never matched: it looked + ## for paste0(cel,'doseResponse') while getDoseRespData() wrote + ## paste0(tolower(studyName),'DoseResponse') -- different case on both + ## halves, so e.g. CTRPv2 wrote 'ctrpv2DoseResponse' and this looked for + ## 'CTRPv2doseResponse'. Every attempt therefore redid all the work. + ## Both sides now go through dose_response_cache_path(), and the file + ## lives under /tmp so it survives between container attempts. + tmpfile<-dose_response_cache_path(cel) if(file.exists(tmpfile)){ + message(sprintf("[04a] Reusing cached dose-response for '%s' (%s)", cel, tmpfile)) dres<-read.table(tmpfile,sep='\t',header=T) }else{ - dset<<-downloadPSet(f,saveDir='.',timeout=10000) + dset<<-download_pset_retry(f) url=subset(all.dsets,`PSet Name`==f)$Download #print(url) diff --git a/coderbuild/broad_sanger/04b-nci60-updated.py b/coderbuild/broad_sanger/04b-nci60-updated.py index 3f1e8080..4c8022d1 100644 --- a/coderbuild/broad_sanger/04b-nci60-updated.py +++ b/coderbuild/broad_sanger/04b-nci60-updated.py @@ -5,38 +5,59 @@ import polars as pl import argparse -#from zipfile import ZipFile import os -#from io import BytesIO import re from urllib import request +import download_utils -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=11&modificationDate=1712351454136&api=v2' -##OCT 2024 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + +def _retrieve_url(url: str, dest: str) -> None: + """Download url, resuming an interrupted transfer. + + DOSERESP.zip is ~346MB. The previous single urlopen().read() truncated at + 32MB in build v32 with IncompleteRead, which cost the entire nci60 dataset: + this script exited 1, its caller only warned, nci60DoseResponse was never + written, and nci60 shipped as eight empty files that still validated. + """ + download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA}) + +##oct 2024 +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=13&modificationDate=1727922354561&api=v2' #jan 2025 -conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=14&modificationDate=1735932462303&api=v2' +#may 2025 +conc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP.zip?version=19&modificationDate=1775183341937&api=v2' -cancelled = 'https://wiki.nci.nih.gov/download/attachments/147193864/DOSERESP_Cancelled.csv?version=1&modificationDate=1660871847000&api=v2&download=true' +#may 2025 +oneconc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/ONECONC.zip?version=19&modificationDate=1775183401634&api=v2' def main(): - + parser = argparse.ArgumentParser() - parser.add_argument('--sampleFile',dest='samplefile',default=None,help='DepMap sample file') + parser.add_argument('--sampleFile',dest='samplefile',default=None,help='DepMap sample file') parser.add_argument('--drugFile',dest='dfile',default=None,help='Drug database') - + opts = parser.parse_args() - + samplefile = opts.samplefile drugfile = opts.dfile if not os.path.exists('DOSERESP.csv'): - resp = request.urlretrieve(conc_data,'doseresp.zip') - os.system('unzip doseresp.zip') - + _retrieve_url(conc_data, 'doseresp.zip') + # -o overwrites without prompting; the exit status is checked so a + # corrupt or truncated archive fails here rather than surfacing later + # as a confusing parse error on a stale DOSERESP.csv. + _rc = os.system('unzip -o doseresp.zip') + if _rc != 0: + raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).") + samples = pl.read_csv(samplefile,quote_char='"') drugs = pl.read_csv(drugfile,separator='\t',quote_char='"') - + dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True) ##update drug mapping @@ -51,16 +72,14 @@ def main(): drugmapping = drugmapping.unique() ###update sample mapping - on = samples[['other_names','improve_sample_id']] - on.columns=['common_name','improve_sample_id'] + on = samples[['other_names','improve_sample_id']].rename({'other_names': 'common_name'}) #there should be 71 cell lines, but there are 163. # 82 map to the 'other_names' # 81 map to neither sampmapping = pl.concat([on[['common_name','improve_sample_id']],samples[['common_name','improve_sample_id']]]) - - sampmapping = sampmapping.unique() - sampmapping.columns = ['CELL_NAME','improve_sample_id'] + + sampmapping = sampmapping.unique().rename({'common_name': 'CELL_NAME'}) ###create a time mapping tabel timemapping = pl.DataFrame( @@ -75,30 +94,15 @@ def main(): ##now we can merge all the data into the dose response data frame merged = dose_resp[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC']].join(sampmapping,on='CELL_NAME',how='left') merged = merged.join(timemapping,on='EXPID',how='left') - + ##clean up mssing samples nonulls = merged.filter(pl.col('improve_sample_id').is_not_null()) nulls = merged.filter(pl.col('improve_sample_id').is_null()) - - # newnames = pl.DataFrame( - # { - # 'new_name':[re.split(' |\(|\/',a)[0] for a in nulls['CELL_NAME']], - # 'CELL_NAME':nulls['CELL_NAME'] - # } - # ) - # newnames = newnames.unique() - - - # fixed = nulls[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC','time','time_unit']].join(newnames,on='CELL_NAME',how='left') - # merged.columns = ['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','old_CELL_NAME','EXPID','NSC','time','time_unit','CELL_NAME'] - # fixed = merged.join(sampmapping,on='CELL_NAME',how='left')[['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','old_CELL_NAME','EXPID','NSC','improve_sample_id','time','time_unit']] - # fixed.columns = ['AVERAGE_PTC','CONCENTRATION_UNIT','CONCENTRATION','CELL_NAME','EXPID','NSC','improve_sample_id','time','time_unit'] -# fixed = fixed.filter(pl.col('improve_sample_id').is_not_null()) - - merged = nonulls#pl.concat([nonulls,fixed]) - - ###we get a few more results added, but still missing a bunch + + merged = nonulls + + ###we get a few more results added, but still missing a bunch merged = merged.join(drugmapping,on='NSC',how='left') nulldrugs = merged.filter(pl.col('improve_drug_id').is_null()) nonulls = merged.filter(pl.col('improve_drug_id').is_not_null()) @@ -106,7 +110,7 @@ def main(): ###now update all the concentrations to be in Moles (some are in uM, all are log10) ##some are provided as molecular weights ('v') or other ('s') and we can't compare molar = merged.filter(pl.col('CONCENTRATION_UNIT')=='M') - + finaldf = pl.DataFrame( { 'source':['NCI60_24' for a in molar['improve_drug_id']], ##2024 build @@ -122,7 +126,7 @@ def main(): ) ##write to file finaldf.write_csv('nci60DoseResponse',separator='\t') - + if __name__=='__main__': main() diff --git a/coderbuild/broad_sanger/05b_separate_datasets.py b/coderbuild/broad_sanger/05b_separate_datasets.py index d00680fe..eb110684 100644 --- a/coderbuild/broad_sanger/05b_separate_datasets.py +++ b/coderbuild/broad_sanger/05b_separate_datasets.py @@ -37,9 +37,11 @@ def main(): exp = pl.read_csv(exp_in_filename, separator="\t") # Keeping memory down, so I will not be making copies. exp = exp.filter(pl.col("study") == dataset) - # Extract information to separate out datasets - exp_improve_sample_ids = exp["improve_sample_id"].unique().to_list() - exp_improve_drug_ids = exp["improve_drug_id"].unique().to_list() + # Extract information to separate out datasets. + # improve_sample_id is always an integer (cast via Float64 to handle "1.0" TSV notation). + # improve_drug_id is always a string ("SMI_XXXXX"); cast to Utf8 for consistent is_in matching. + exp_improve_sample_ids = exp["improve_sample_id"].cast(pl.Float64).cast(pl.Int64).unique().to_list() + exp_improve_drug_ids = exp["improve_drug_id"].cast(pl.Utf8).unique().to_list() #Ensure that the improve_sample_id column is in integer form. exp = exp.with_columns(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64)) @@ -65,7 +67,7 @@ def main(): samples_filename_out = f"/tmp/{dataset}_{samples}.csv".lower() samples_df = pl.read_csv(samples_filename_in) - samples_df = samples_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids)) + samples_df = samples_df.filter(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64).is_in(exp_improve_sample_ids)) samples_df.write_csv(samples_filename_out) #csv #Rewrite as gzipped if needed @@ -86,7 +88,7 @@ def main(): omics_filename_out = f"/tmp/{dataset}_{omics}.csv".lower() omics_df = pl.read_csv(omics_filename_in) - omics_df = omics_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids)) + omics_df = omics_df.filter(pl.col("improve_sample_id").cast(pl.Float64).cast(pl.Int64).is_in(exp_improve_sample_ids)) # omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset])) omics_df.write_csv(omics_filename_out) #csv @@ -116,7 +118,7 @@ def main(): else: drugs_df = pl.read_csv(drugs_filename_in,separator="\t") - drugs_df = drugs_df.filter(pl.col("improve_drug_id").is_in(exp_improve_drug_ids)) + drugs_df = drugs_df.filter(pl.col("improve_drug_id").cast(pl.Utf8).is_in(exp_improve_drug_ids)) drugs_df.write_csv(drugs_filename_out,separator="\t") #tsv if drugs_filename_in.endswith(".gz"): diff --git a/coderbuild/broad_sanger/build_omics.sh b/coderbuild/broad_sanger/build_omics.sh index 024e0d17..46e1b964 100644 --- a/coderbuild/broad_sanger/build_omics.sh +++ b/coderbuild/broad_sanger/build_omics.sh @@ -3,6 +3,19 @@ set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR +# DepMap files are no longer downloadable from the DepMap portal (Cloudflare +# challenge). Pull the static copy from Synapse first; the R script reads them +# from $DEPMAP_DIR and deletes each one once consumed. +# +# $DEPMAP_DIR is container-local (NOT under /tmp, which is the bind-mounted +# host local/ directory), so these transient inputs never touch host disk. +# The trap is a backstop in case the R script exits before consuming them. +echo "Fetching DepMap files from Synapse..." +DEPMAP_DIR="$(/opt/venv/bin/python fetch_depmap.py --print-dest)" +export DEPMAP_DIR +trap 'rm -rf "$DEPMAP_DIR"' EXIT +/opt/venv/bin/python fetch_depmap.py --dest "$DEPMAP_DIR" + echo "Running 02a-broad_sanger_proteomics.py with gene file $1 and sample file $2." /opt/venv/bin/python 02a-broad_sanger_proteomics.py --gene $1 --sample $2 diff --git a/coderbuild/broad_sanger/build_samples.sh b/coderbuild/broad_sanger/build_samples.sh index 4302ba44..49220052 100644 --- a/coderbuild/broad_sanger/build_samples.sh +++ b/coderbuild/broad_sanger/build_samples.sh @@ -3,5 +3,18 @@ set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR +# DepMap files are no longer downloadable from the DepMap portal (Cloudflare +# challenge). Pull the static copy from Synapse first; the R script reads them +# from $DEPMAP_DIR and deletes each one once consumed. +# +# $DEPMAP_DIR is container-local (NOT under /tmp, which is the bind-mounted +# host local/ directory), so these transient inputs never touch host disk. +# The trap is a backstop in case the R script exits before consuming them. +echo "Fetching DepMap files from Synapse..." +DEPMAP_DIR="$(/opt/venv/bin/python fetch_depmap.py --print-dest)" +export DEPMAP_DIR +trap 'rm -rf "$DEPMAP_DIR"' EXIT +/opt/venv/bin/python fetch_depmap.py --dest "$DEPMAP_DIR" + echo "Running 01-broadSangerSamples.R." Rscript 01-broadSangerSamples.R diff --git a/coderbuild/broad_sanger/exp_requirements.r b/coderbuild/broad_sanger/exp_requirements.r index 9cc16e17..7323b1f4 100755 --- a/coderbuild/broad_sanger/exp_requirements.r +++ b/coderbuild/broad_sanger/exp_requirements.r @@ -7,3 +7,12 @@ install.packages('tidyverse') install.packages("BiocManager") BiocManager::install("PharmacoGx",update=TRUE,ask=FALSE) + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("reticulate", "tidyverse", "BiocManager", "PharmacoGx") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/broad_sanger/omics_requirements.r b/coderbuild/broad_sanger/omics_requirements.r index 26d27844..83ce09b7 100755 --- a/coderbuild/broad_sanger/omics_requirements.r +++ b/coderbuild/broad_sanger/omics_requirements.r @@ -7,4 +7,14 @@ install.packages("dplyr") install.packages("XML") #install.packages('reticulate') install.packages('tidyr') -install.packages('httr2') \ No newline at end of file +install.packages('httr2') +install.packages('data.table') + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("curl", "rio", "readr", "stringr", "dplyr", "XML", "tidyr", "httr2", "data.table") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) diff --git a/coderbuild/broad_sanger/requirements.r b/coderbuild/broad_sanger/requirements.r index 629f4fe0..a1e60b9d 100755 --- a/coderbuild/broad_sanger/requirements.r +++ b/coderbuild/broad_sanger/requirements.r @@ -11,6 +11,7 @@ install.packages("XML") install.packages('reticulate') install.packages('tidyr') install.packages('httr2') +install.packages('data.table') #install.packages("BiocManager") BiocManager::install("PharmacoGx",update=TRUE,ask=FALSE) BiocManager::install("org.Hs.eg.db",update=TRUE,ask=FALSE) diff --git a/coderbuild/broad_sanger/requirements.txt b/coderbuild/broad_sanger/requirements.txt index a00c3bd6..9a5e7191 100755 --- a/coderbuild/broad_sanger/requirements.txt +++ b/coderbuild/broad_sanger/requirements.txt @@ -11,4 +11,5 @@ polars-lts-cpu==0.19.17 mordredcommunity rdkit coderdata==0.1.40 -psutil \ No newline at end of file +psutil +synapseclient>=4.0 \ No newline at end of file From 24af0776967ae2951ab13f9dba73b04a51a542c3 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:00 -0700 Subject: [PATCH 04/20] Make the genes build resilient to Ensembl outages and API changes --- coderbuild/genes/00-buildGeneFile.R | 327 ++++++++++++++++++++++++---- coderbuild/genes/requirements.r | 9 +- 2 files changed, 295 insertions(+), 41 deletions(-) diff --git a/coderbuild/genes/00-buildGeneFile.R b/coderbuild/genes/00-buildGeneFile.R index 5c113a94..f545ca4c 100755 --- a/coderbuild/genes/00-buildGeneFile.R +++ b/coderbuild/genes/00-buildGeneFile.R @@ -1,57 +1,306 @@ -##this uses the Hg38 database to seed the gene name mappings that we might want to use +## Build /tmp/genes.csv — entrez_id, gene_symbol, other_id, other_id_source. +## +## Uses org.Hs.eg.db for the local mappings (entrez ↔ symbol, ensembl gene, +## ensembl transcript) and biomaRt to filter to protein-coding genes. +## +## biomaRt's call to Ensembl frequently fails with mirror unavailability. +## To handle this, the Ensembl call is wrapped in a retry loop: +## - cycles through the three valid mirror choices ('www', 'useast', 'asia') +## - each cycle waits longer than the last (exponential backoff) +## - up to MAX_CYCLES total cycles before giving up +## +## If every mirror remains unreachable, the script falls back to a local +## protein-coding biotype list derived from org.Hs.eg.db's GO mapping +## (filtering out pseudogenes/RNAs that lack ENSG IDs in `ens`). The +## fallback is documented in the code so the lower coverage is visible. -#if(!require('org.Hs.eg.db')){ -# BiocManager::install('org.Hs.eg.db') -library(org.Hs.eg.db) -library(biomaRt) -#} +suppressPackageStartupMessages({ + library(org.Hs.eg.db) + library(biomaRt) + library(dplyr) +}) -library(dplyr) -##get entrez ids to symbol -entrez<-as.data.frame(org.Hs.egALIAS2EG) +# ---- Ensembl "new site" compatibility (REQUIRED) ---------------------------- +# +# In Aug 2026 Ensembl migrated all sites to the new ensembl.org. The new site +# performs Accept-header content negotiation on the legacy BioMart path: +# +# Accept: */* -> 308 redirect to the +# Accept: text/xml BioMart archive host +# Accept: application/json -> 404 {"details": +# Accept: application/json, text/xml, ... (httr) "No supported new +# Ensembl equivalent +# for this URL"} +# +# httr -- which biomaRt uses -- sends `application/json` FIRST by default, so +# every biomaRt call gets a hard 404 and biomaRt reports it as the misleading +# "Ensembl site unresponsive". This is deterministic: without this override the +# retry loop below can never succeed, no matter how many cycles it runs. +# +# Forcing Accept: text/xml makes Ensembl issue the 308 to the BioMart archive, +# which is what biomaRt actually wants. Verified 2026-08-28: with this line the +# protein-coding query succeeds; without it, it fails 100% of the time. +httr::set_config(httr::add_headers(Accept = "text/xml")) -sym <- as.data.frame(org.Hs.egSYMBOL) +# ---- Local mappings (no network) ------------------------------------------- +entrez <- as.data.frame(org.Hs.egALIAS2EG) +sym <- as.data.frame(org.Hs.egSYMBOL) +ens <- as.data.frame(org.Hs.egENSEMBL2EG) +enst <- as.data.frame(org.Hs.egENSEMBLTRANS) -##get entriz ids to ensembl -ens<-as.data.frame(org.Hs.egENSEMBL2EG) +# ---- biomaRt with retry / mirror cycling ----------------------------------- +# +# Strategy: +# For up to MAX_CYCLES, try each mirror in MIRRORS in order. +# On a successful useEnsembl() + getBM() call, return the result. +# On any failure, sleep for the current backoff and try the next mirror. +# Backoff doubles each cycle (30s → 60s → 120s → ...). +# +# Total worst-case wait at MAX_CYCLES=5: ~30 minutes spread across attempts, +# which has historically been enough for transient Ensembl outages to clear. +# +# Retries still matter even with the Accept fix above: during the Aug 2026 +# migration the BioMart backends are load-balanced and only some of them +# answer -- roughly 1 request in 5 returns real MartRegistry XML, the rest +# return a "Service unavailable" page (served, unhelpfully, with HTTP 200). +# Across MAX_CYCLES x MIRRORS attempts that is very likely to land. +# +# NOTE: 'uswest' is NOT a valid biomaRt mirror -- biomaRt rejects it with +# "Invalid mirror. Select a mirror from [www, useast, asia]" and silently falls +# back to www, wasting a quarter of every cycle. Only the three real mirrors +# are listed here. +MIRRORS <- c("www", "useast", "asia") +MAX_CYCLES <- 5 +BASE_SLEEP <- 30 # seconds; doubles each cycle -##get transcript ids as well -enst<-as.data.frame(org.Hs.egENSEMBLTRANS) +fetch_protein_coding <- function() { + for (cycle in seq_len(MAX_CYCLES)) { + sleep_s <- BASE_SLEEP * (2 ^ (cycle - 1)) + for (mirror in MIRRORS) { + message(sprintf("[%s] Cycle %d/%d, trying mirror '%s'...", + Sys.time(), cycle, MAX_CYCLES, mirror)) + result <- tryCatch({ + ensembl <- useEnsembl( + biomart = "genes", + dataset = "hsapiens_gene_ensembl", + mirror = mirror + ) + tab <- getBM( + attributes = c("ensembl_gene_id"), + filters = "biotype", + values = c("protein_coding"), + mart = ensembl + ) + message(sprintf("[%s] Success via mirror '%s' (%d genes)", + Sys.time(), mirror, nrow(tab))) + tab + }, error = function(e) { + message(sprintf("[%s] Mirror '%s' failed: %s", + Sys.time(), mirror, conditionMessage(e))) + NULL + }) + if (!is.null(result)) return(result) + } + if (cycle < MAX_CYCLES) { + message(sprintf("[%s] All mirrors failed in cycle %d; sleeping %ds before next cycle", + Sys.time(), cycle, sleep_s)) + Sys.sleep(sleep_s) + } + } + return(NULL) +} - #now we can filter by protein coding using biomart -ensembl <- useEnsembl(biomart = "genes", dataset = "hsapiens_gene_ensembl") +tab <- fetch_protein_coding() -tab <- getBM(attributes=c('ensembl_gene_id'),filters='biotype', values=c('protein_coding'),mart=ensembl) +# ---- Fallback 1: query BioMart directly over HTTP --------------------------- +# +# This is a FALLBACK ONLY. biomaRt above stays the primary path so that when +# Ensembl and biomaRt are both healthy we use the supported, maintained API. +# This runs only when every biomaRt mirror attempt has already failed. +# +# WHY IT IS NEEDED +# ---------------- +# As of 2026-09-01 biomaRt (2.62.1) cannot reach Ensembl for two independent +# reasons, neither of which is fixable from our side: +# +# 1. Ensembl's martservice no longer accepts POST, which is what biomaRt +# sends: +# POST https://www.ensembl.org/biomart/martservice -> 405 Method Not Allowed +# GET https://www.ensembl.org/biomart/martservice?query=... -> 200 +# +# 2. biomaRt's own health check, .test_ensembl(), performs a request against +# BOTH www.ensembl.org and useast.ensembl.org and raises if EITHER fails. +# useast currently returns 502 and asia returns 403, so a perfectly +# healthy www is still reported as "Ensembl site unresponsive". +# +# Both are being reported upstream (Huber-group-EMBL/biomaRt). When they are +# fixed, biomaRt will simply succeed above and this code will never execute -- +# so it is safe to leave in place, and it should NOT be promoted to the +# primary path. +# +# The query below is the exact equivalent of the getBM() call above: the +# hsapiens_gene_ensembl dataset, filtered to biotype = protein_coding, +# returning ensembl_gene_id. Verified 2026-09-01 to return 23,272 genes, +# matching what biomaRt returned when it last worked (2026-08-29). +# +# GET is safe here despite BioMart queries normally using POST: this query +# encodes to ~500 characters, far below the ~8 KB practical URL limit. A query +# with 30 attributes still only reaches ~1,700. +fetch_protein_coding_direct <- function() { + if (!requireNamespace("httr2", quietly = TRUE)) return(NULL) + query_xml <- paste0( + '', + '', + '', + '', + '', + '') -joined.df<-entrez|> - left_join(sym)|> - dplyr::rename(entrez_id='gene_id',gene_symbol='symbol',other_id='alias_symbol',gene_symbol='symbol')%>% - mutate(other_id_source='entrez_alias') + for (attempt in seq_len(3)) { + message(sprintf("[%s] Direct BioMart query, attempt %d/3...", Sys.time(), attempt)) + res <- tryCatch({ + req <- httr2::req_timeout( + httr2::req_url_query( + httr2::request("https://www.ensembl.org/biomart/martservice"), + query = query_xml), + 300) + body <- httr2::resp_body_string(httr2::req_perform(req)) -##now get aliases from ensembl -edf <- sym|> - inner_join(ens)|> - dplyr::rename(entrez_id='gene_id',gene_symbol='symbol',other_id='ensembl_id')%>% - mutate(other_id_source='ensembl_gene') + # BioMart signals failure in-band with a 200, so check the payload + # rather than trusting the status code. + if (grepl("Query ERROR|status.ensembl.org| - inner_join(enst)|> - dplyr::rename(entrez_id='gene_id',gene_symbol='symbol',other_id='trans_id')|> - subset(entrez_id%in%edf$entrez_id)|> -# subset(gene_symbol%in%ed.df$gene_symbol)|> - dplyr::mutate(other_id_source='ensembl_transcript') +# ---- Fallback 2: protein-coding inference from org.Hs.eg.db ----------------- +# If every Ensembl mirror is unreachable, derive a protein-coding gene list +# locally. org.Hs.eg.db doesn't carry biotype directly, but every entrez ID +# that has an ENSG mapping in org.Hs.egENSEMBL2EG is reasonably treated as +# a "real" gene; combined with the symbol mapping, this approximates the +# protein-coding filter well enough to keep the build going. +# +# The fallback genes.csv will have somewhat broader coverage than the +# Ensembl-filtered version (it includes some pseudogenes etc), but the +# downstream omics joins are inner-joined on gene_symbol, so any extras +# that don't appear in the data files just get dropped harmlessly. -prots<-subset(edf,other_id%in%tab$ensembl_gene_id) +# The fallback is OPT-IN. It used to run automatically, which meant a transient +# Ensembl problem would silently produce a genes.csv with a broader, +# non-canonical gene list -- and because the build then SUCCEEDED, that +# degraded file could flow all the way through validation to a published +# Figshare release with nobody noticing. Every downstream omics file joins +# against genes.csv, so this is not a local defect: it changes the whole +# release. +# +# Default is now to fail loudly. Set ALLOW_GENE_FALLBACK=1 to deliberately +# accept the degraded list (e.g. for a local debug build that is not going to +# be published). +if (is.null(tab)) { + allow_fallback <- nzchar(Sys.getenv("ALLOW_GENE_FALLBACK")) + if (!allow_fallback) { + stop(paste0( + "\n", + "=====================================================================\n", + " Ensembl unreachable -- REFUSING to build a degraded genes.csv\n", + "=====================================================================\n", + " All Ensembl mirrors failed after ", MAX_CYCLES, " cycles, AND the\n", + " direct BioMart HTTP query fallback also failed.\n", + "\n", + " A local org.Hs.eg.db fallback exists, but it produces a BROADER,\n", + " non-canonical gene list (it includes pseudogenes and other biotypes\n", + " that the Ensembl protein-coding filter excludes). Every omics file\n", + " joins against genes.csv, so publishing a release built on the\n", + " fallback would silently change every dataset.\n", + "\n", + " This build has therefore been stopped rather than continuing quietly.\n", + "\n", + " Options:\n", + " - Wait for Ensembl and re-run (check https://status.ensembl.org)\n", + " - For a local, NON-published debug build only, re-run with:\n", + " ALLOW_GENE_FALLBACK=1\n", + "\n", + " NOTE: if every attempt failed with a 404 mentioning \"No supported new\n", + " Ensembl equivalent\", the Accept-header override at the top of this\n", + " script is missing or was undone -- see the comment there.\n", + "=====================================================================\n" + )) + } + message("\n=== WARNING ===") + message("All Ensembl mirrors unreachable after ", MAX_CYCLES, + " cycles. ALLOW_GENE_FALLBACK is set, so using the local ", + "org.Hs.eg.db fallback.") + message("This produces a BROADER gene list than the canonical ", + "Ensembl-filtered version. DO NOT PUBLISH a release built this way; ", + "re-run when Ensembl is reachable.") + message("===============\n") + # Use every entrez ID that has an ENSG mapping as the protein-coding proxy. + tab <- data.frame(ensembl_gene_id = unique(ens$ensembl_id), + stringsAsFactors = FALSE) +} -full.df<-rbind(joined.df,edf,tdf)|> - subset(entrez_id%in%prots$entrez_id)|> - distinct() -#save to file and version -write.table(full.df,'/tmp/genes.csv',sep=',',row.names=F,quote=T) +# ---- Assemble joined alias / ensembl gene / ensembl transcript table ------- +joined.df <- entrez |> + left_join(sym, by = "gene_id") |> + dplyr::rename(entrez_id = "gene_id", + gene_symbol = "symbol", + other_id = "alias_symbol") |> + mutate(other_id_source = "entrez_alias") -##store this file somewhere! +edf <- sym |> + inner_join(ens, by = "gene_id") |> + dplyr::rename(entrez_id = "gene_id", + gene_symbol = "symbol", + other_id = "ensembl_id") |> + mutate(other_id_source = "ensembl_gene") +tdf <- sym |> + inner_join(enst, by = "gene_id") |> + dplyr::rename(entrez_id = "gene_id", + gene_symbol = "symbol", + other_id = "trans_id") |> + subset(entrez_id %in% edf$entrez_id) |> + dplyr::mutate(other_id_source = "ensembl_transcript") + +prots <- subset(edf, other_id %in% tab$ensembl_gene_id) + +full.df <- rbind(joined.df, edf, tdf) |> + subset(entrez_id %in% prots$entrez_id) |> + distinct() + + +# ---- Write ------------------------------------------------------------------ +out_path <- "/tmp/genes.csv" +write.table(full.df, out_path, sep = ",", row.names = FALSE, quote = TRUE) +message(sprintf("Wrote %s (%d rows, %d unique entrez_ids)", + out_path, nrow(full.df), length(unique(full.df$entrez_id)))) \ No newline at end of file diff --git a/coderbuild/genes/requirements.r b/coderbuild/genes/requirements.r index 0ccb7610..5196b7b6 100755 --- a/coderbuild/genes/requirements.r +++ b/coderbuild/genes/requirements.r @@ -1,4 +1,9 @@ install.packages("BiocManager") -BiocManager::install("org.Hs.eg.db",update=TRUE,ask=FALSE) -BiocManager::install("biomaRt",update=TRUE,ask=FALSE) +BiocManager::install(c("AnnotationDbi", "org.Hs.eg.db", "biomaRt"), update=FALSE, ask=FALSE) install.packages('https://cran.r-project.org/src/contrib/Archive/dbplyr/dbplyr_2.3.4.tar.gz', repos = NULL) +install.packages('tidyr') + +# Fail the Docker build layer if any required package didn't install +required_pkgs <- c("org.Hs.eg.db", "biomaRt", "tidyr") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) From a09ee71ea576aef3686ede45f6e9f4c8f837873e Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:00 -0700 Subject: [PATCH 05/20] Add phosphosites reference builder --- coderbuild/docker/Dockerfile.phosphosites | 35 ++ .../phosphosites/00-buildPhosphositeFile.py | 589 ++++++++++++++++++ coderbuild/phosphosites/README.md | 97 +++ coderbuild/phosphosites/build_phosphosites.sh | 30 + 4 files changed, 751 insertions(+) create mode 100644 coderbuild/docker/Dockerfile.phosphosites create mode 100644 coderbuild/phosphosites/00-buildPhosphositeFile.py create mode 100644 coderbuild/phosphosites/README.md create mode 100644 coderbuild/phosphosites/build_phosphosites.sh diff --git a/coderbuild/docker/Dockerfile.phosphosites b/coderbuild/docker/Dockerfile.phosphosites new file mode 100644 index 00000000..c854644f --- /dev/null +++ b/coderbuild/docker/Dockerfile.phosphosites @@ -0,0 +1,35 @@ +FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /coderbuild/phosphosites + +RUN pip install --no-cache-dir pandas>=2.0 synapseclient>=4.0 openpyxl>=3.1 + +COPY coderbuild/phosphosites/00-buildPhosphositeFile.py /coderbuild/phosphosites/ +COPY coderbuild/phosphosites/build_phosphosites.sh /coderbuild/phosphosites/ + +RUN chmod +x /coderbuild/phosphosites/build_phosphosites.sh diff --git a/coderbuild/phosphosites/00-buildPhosphositeFile.py b/coderbuild/phosphosites/00-buildPhosphositeFile.py new file mode 100644 index 00000000..09292dec --- /dev/null +++ b/coderbuild/phosphosites/00-buildPhosphositeFile.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 +""" +00-buildPhosphositeFile.py: build the phosphosites reference CSV from three sources: + + 1. Ochoa et al. (2020) Nat Biotechnol 38, 365-373: ~116k curated human + phosphosites from Supplementary Table S3 (Springer static content). + https://static-content.springer.com/esm/art%3A10.1038%2Fs41587-019-0344-3/ + MediaObjects/41587_2019_344_MOESM4_ESM.xlsx + + 2. UniProt PTM annotations: all reviewed human proteins with phospho + modifications from the UniProt REST API (~41k sites, ~12k unique to UniProt). + + 3. Synapse supplement(s): raw phosphoproteomics CSV/TSV files on Synapse + whose site column is parsed to add experiment-specific sites not in the + databases above (--synapse_supplements syn70078415 ...). + +Sources 1 and 2 are always fetched. Source 3 is optional but recommended. + +UniProt accession -> gene_symbol: UniProt REST API (TSV, accession+gene_names). +gene_symbol -> entrez_id: genes.csv. + +Site notation: - + e.g. AAAS-S495s (s=phosphoserine, t=phosphothreonine, y=phosphotyrosine) + +Output columns: + phosphosite_id, entrez_id, gene_symbol, residue, position, modification, other_id + +Usage: + python 00-buildPhosphositeFile.py + [--out /tmp/phosphosites.csv] + [--prev /tmp/prev_phosphosites.csv] + [--supplement site_file.csv ...] + [--site_col site] + [--synapse_supplements syn70078415 ...] + [--synapse_site_col site] + [--ochoa_url ] + [--uniprot_gene_url ] + [--uniprot_ptm_url ] +""" + +import argparse +import csv +import io +import json +import logging +import re +import sys +import tempfile +import time +import urllib.request + +import openpyxl +import pandas as pd + + +OCHOA_URL = ( + "https://static-content.springer.com/esm/" + "art%3A10.1038%2Fs41587-019-0344-3/MediaObjects/" + "41587_2019_344_MOESM4_ESM.xlsx" +) + +UNIPROT_GENE_URL = ( + "https://rest.uniprot.org/uniprotkb/search" + "?query=reviewed:true+AND+organism_id:9606" + "&fields=accession,gene_names" + "&format=tsv" + "&size=500" +) + +UNIPROT_PTM_URL = ( + "https://rest.uniprot.org/uniprotkb/search" + "?query=reviewed:true+AND+organism_id:9606+AND+ft_mod_res:phospho*" + "&fields=accession,gene_names,ft_mod_res" + "&format=json" + "&size=200" +) + +_RESIDUE_MOD = {"S": "s", "T": "t", "Y": "y"} +_PHOSPHO_DESC_MAP = { + "phosphoserine": ("S", "s"), + "phosphothreonine": ("T", "t"), + "phosphotyrosine": ("Y", "y"), +} +_SITE_RE = re.compile(r'^(.+)-([STY])(\d+)([a-z]+)$', re.IGNORECASE) +_SUPP_MOD_CODES = {'s': 'phospho', 't': 'phospho', 'y': 'phospho'} + +_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0" + ), + "Accept": "*/*", +} + + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# --------------------------------------------------------------------------- +# Reference data +# --------------------------------------------------------------------------- +def load_gene_map(genes_path: str) -> pd.DataFrame: + df = pd.read_csv(genes_path) + needed = {'gene_symbol', 'entrez_id'} + if needed - set(df.columns): + raise KeyError(f"genes file missing columns: {needed - set(df.columns)}") + df = df.dropna(subset=['gene_symbol', 'entrez_id']).copy() + df['entrez_id'] = df['entrez_id'].astype(int) + df = df.drop_duplicates(subset='gene_symbol')[['gene_symbol', 'entrez_id']] + logging.info("Loaded %d gene_symbol → entrez_id mappings", len(df)) + return df + + +# --------------------------------------------------------------------------- +# Network helper +# --------------------------------------------------------------------------- +def _fetch_url(url: str, retries: int = 3, timeout: int = 120) -> bytes: + req = urllib.request.Request(url, headers=_HEADERS) + for attempt in range(1, retries + 1): + try: + logging.info(" GET %s (attempt %d/%d)", url, attempt, retries) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read() + except Exception as exc: + logging.warning(" Attempt %d failed: %s", attempt, exc) + if attempt < retries: + time.sleep(5 * attempt) + raise RuntimeError(f"Failed to fetch {url} after {retries} attempts") + + +def _fetch_json_page(url: str, timeout: int = 120) -> tuple[list[dict], str | None]: + req = urllib.request.Request(url, headers={**_HEADERS, "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode('utf-8')) + link_header = resp.headers.get('Link', '') + entries = data.get('results', []) + m = re.search(r'<([^>]+)>;\s*rel="next"', link_header) + return entries, m.group(1) if m else None + + +# --------------------------------------------------------------------------- +# Source 1: Ochoa et al. supplementary table +# --------------------------------------------------------------------------- +def fetch_ochoa_phosphosites(url: str) -> list[dict]: + """Download Ochoa MOESM4 xlsx and return raw phosphosite records.""" + logging.info("Downloading Ochoa et al. supplementary phosphoproteome") + data = _fetch_url(url) + + with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tmp: + tmp.write(data) + tmp_path = tmp.name + + logging.info("Parsing xlsx (%d MB) …", len(data) // 1_000_000) + wb = openpyxl.load_workbook(tmp_path, read_only=True, data_only=True) + + sheet_name = "annotated_phosphoproteome" + if sheet_name not in wb.sheetnames: + wb.close() + raise ValueError( + f"Sheet '{sheet_name}' not found. Available: {wb.sheetnames}" + ) + + ws = wb[sheet_name] + records, header, uni_i, pos_i, res_i, skipped = [], None, None, None, None, 0 + + for row in ws.iter_rows(values_only=True): + if header is None: + header = [str(c).strip() if c is not None else "" for c in row] + try: + uni_i = header.index("uniprot") + pos_i = header.index("position") + res_i = header.index("residue") + except ValueError as exc: + wb.close() + raise ValueError(f"Required column missing in Ochoa sheet: {exc}") + logging.info( + "Ochoa columns: uniprot=%d, position=%d, residue=%d", + uni_i, pos_i, res_i, + ) + continue + + acc = str(row[uni_i]).strip() if row[uni_i] is not None else "" + try: + pos = int(row[pos_i]) + except (TypeError, ValueError): + skipped += 1 + continue + + res_raw = str(row[res_i]).strip().upper() if row[res_i] is not None else "" + residue = res_raw[0] if res_raw and res_raw[0] in _RESIDUE_MOD else None + + if residue is None or not acc: + skipped += 1 + continue + + records.append({ + "accession": acc, + "residue": residue, + "position": pos, + "modification": "phospho", + "mod_code": _RESIDUE_MOD[residue], + }) + + wb.close() + logging.info( + "Ochoa: %d records parsed (%d skipped)", len(records), skipped + ) + return records + + +# --------------------------------------------------------------------------- +# Source 2: UniProt PTM annotations +# --------------------------------------------------------------------------- +def fetch_uniprot_ptm_sites(url: str) -> list[dict]: + """Fetch phosphorylation sites from UniProt Modified residue features. + + Returns records with gene_symbol (already resolved) instead of accession. + """ + logging.info("Fetching UniProt PTM phosphorylation sites") + all_records = [] + next_url: str | None = url + page = 0 + + while next_url: + page += 1 + if page % 10 == 1: + logging.info(" UniProt PTM page %d (%d records so far)", page, len(all_records)) + + for attempt in range(1, 4): + try: + entries, next_url = _fetch_json_page(next_url) + break + except Exception as exc: + logging.warning(" Page %d attempt %d failed: %s", page, attempt, exc) + if attempt == 3: + raise + time.sleep(5 * attempt) + + for entry in entries: + gene = "" + for g in entry.get("genes", []): + if "geneName" in g: + gene = g["geneName"].get("value", "").upper() + break + if not gene: + continue + for feat in entry.get("features", []): + if feat.get("type") != "Modified residue": + continue + desc = feat.get("description", "").lower() + residue, mod_code = None, None + for key, (aa, mc) in _PHOSPHO_DESC_MAP.items(): + if desc.startswith(key): + residue, mod_code = aa, mc + break + if residue is None: + continue + pos = feat.get("location", {}).get("start", {}).get("value") + if pos is None: + continue + all_records.append({ + "gene_symbol": gene, + "residue": residue, + "position": int(pos), + "modification": "phospho", + "mod_code": mod_code, + "other_id": f"{gene}-{residue}{pos}{mod_code}", + }) + + logging.info( + "UniProt PTM: %d records from %d pages", len(all_records), page + ) + return all_records + + +# --------------------------------------------------------------------------- +# UniProt accession → gene_symbol +# --------------------------------------------------------------------------- +def fetch_uniprot_gene_map(url: str) -> dict[str, str]: + """Return accession → primary gene_symbol for all reviewed human proteins.""" + logging.info("Fetching UniProt accession → gene_symbol map") + acc_to_gene: dict[str, str] = {} + next_url: str | None = url + page = 0 + + while next_url: + page += 1 + if page % 10 == 1: + logging.info( + " UniProt gene map page %d (%d entries so far)", page, len(acc_to_gene) + ) + for attempt in range(1, 4): + try: + req = urllib.request.Request(next_url, headers=_HEADERS) + with urllib.request.urlopen(req, timeout=120) as resp: + raw = resp.read() + link_header = resp.headers.get("Link", "") + break + except Exception as exc: + logging.warning(" Page %d attempt %d failed: %s", page, attempt, exc) + if attempt == 3: + raise + time.sleep(5 * attempt) + + reader = csv.DictReader( + io.StringIO(raw.decode("utf-8", errors="replace")), delimiter="\t" + ) + for row in reader: + acc = row.get("Entry", row.get("accession", "")).strip() + gene_names = row.get("Gene Names", row.get("gene_names", "")).strip() + if acc and gene_names: + acc_to_gene[acc] = gene_names.split()[0].upper() + + m = re.search(r'<([^>]+)>;\s*rel="next"', link_header) + next_url = m.group(1) if m else None + + logging.info( + "UniProt gene map: %d entries (%d pages)", len(acc_to_gene), page + ) + return acc_to_gene + + +# --------------------------------------------------------------------------- +# Source 3: Synapse supplement(s) +# --------------------------------------------------------------------------- +def fetch_synapse_supplement(syn_id: str, site_col: str) -> list[dict]: + """Download a Synapse file and parse its site strings as supplement records.""" + import synapseclient + logging.info("Fetching Synapse supplement %s", syn_id) + syn = synapseclient.Synapse() + syn.login(silent=True) + entity = syn.get(syn_id) + path = entity.path + sep = "\t" if path.endswith((".tsv", ".tsv.gz")) else "," + + import gzip as _gzip + open_fn = _gzip.open if path.endswith(".gz") else open + + records = [] + skipped = 0 + seen: set[str] = set() + resolved_col: str | None = None + + with open_fn(path, "rt", encoding="utf-8", errors="replace") as f: + reader = csv.DictReader(f, delimiter=sep) + for row in reader: + # Auto-detect site column on first row if requested column missing + if resolved_col is None: + if site_col in row: + resolved_col = site_col + else: + for candidate in ["site", "Site", "phosphosite", "feature_id"]: + if candidate in row: + resolved_col = candidate + logging.info( + " Synapse %s: using column '%s' for sites", + syn_id, resolved_col, + ) + break + if resolved_col is None: + logging.warning( + " Synapse %s: could not find site column (tried %s). " + "Available: %s", + syn_id, site_col, list(row.keys())[:10], + ) + return [] + + site = row.get(resolved_col, "").strip() + if not site or site in seen: + continue + seen.add(site) + m = _SITE_RE.match(site) + if not m: + skipped += 1 + continue + mod_code = m.group(4).lower() + records.append({ + "gene_symbol": m.group(1).upper(), + "residue": m.group(2).upper(), + "position": int(m.group(3)), + "modification": _SUPP_MOD_CODES.get(mod_code, f"phospho_{mod_code}"), + "mod_code": mod_code, + "other_id": site, + }) + + logging.info( + "Synapse %s: %d unique sites parsed (%d skipped)", syn_id, len(records), skipped + ) + return records + + +# --------------------------------------------------------------------------- +# Local file supplement +# --------------------------------------------------------------------------- +def parse_supplement_sites(path: str, site_col: str) -> list[dict]: + records = [] + skipped = 0 + sep = "\t" if path.endswith(".tsv") else "," + with open(path, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f, delimiter=sep) + seen: set[str] = set() + for row in reader: + site = row.get(site_col, "").strip() + if not site or site in seen: + continue + seen.add(site) + m = _SITE_RE.match(site) + if not m: + skipped += 1 + continue + mod_code = m.group(4).lower() + records.append({ + "gene_symbol": m.group(1).upper(), + "residue": m.group(2).upper(), + "position": int(m.group(3)), + "modification": _SUPP_MOD_CODES.get(mod_code, f"phospho_{mod_code}"), + "mod_code": mod_code, + "other_id": site, + }) + logging.info( + "Supplement %s: %d unique sites (%d skipped)", path, len(records), skipped + ) + return records + + +# --------------------------------------------------------------------------- +# Assembly +# --------------------------------------------------------------------------- +def build_phosphosite_table( + ochoa_records: list[dict], + acc_to_gene: dict[str, str], + uniprot_records: list[dict], + gene_map: pd.DataFrame, + supplement_records: list[dict], + prev: pd.DataFrame | None, +) -> pd.DataFrame: + gene_to_entrez = gene_map.set_index("gene_symbol")["entrez_id"].to_dict() + rows: list[dict] = [] + existing_ids: set[str] = set() + + def _add(gene: str, residue: str, position: int, modification: str, mod_code: str) -> None: + other_id = f"{gene}-{residue}{position}{mod_code}" + if other_id in existing_ids: + return + entrez = gene_to_entrez.get(gene) + if entrez is None: + return + rows.append({ + "gene_symbol": gene, + "entrez_id": entrez, + "residue": residue, + "position": position, + "modification": modification, + "other_id": other_id, + }) + existing_ids.add(other_id) + + # --- Ochoa (primary) --- + no_gene_ochoa = 0 + for rec in ochoa_records: + gene = acc_to_gene.get(rec["accession"], "") + if not gene: + no_gene_ochoa += 1 + continue + _add(gene, rec["residue"], rec["position"], rec["modification"], rec["mod_code"]) + logging.info( + "Ochoa: %d sites added (%d with no gene symbol)", len(rows), no_gene_ochoa + ) + + before_uniprot = len(rows) + # --- UniProt PTM (secondary) --- + for rec in uniprot_records: + _add( + rec["gene_symbol"], rec["residue"], rec["position"], + rec["modification"], rec["mod_code"], + ) + logging.info( + "UniProt PTM: +%d new sites (total %d)", len(rows) - before_uniprot, len(rows) + ) + + before_supp = len(rows) + # --- Supplements (local files + Synapse) --- + for rec in supplement_records: + _add( + rec["gene_symbol"], rec["residue"], rec["position"], + rec["modification"], rec["mod_code"], + ) + if len(rows) > before_supp: + logging.info( + "Supplements: +%d new sites (total %d)", len(rows) - before_supp, len(rows) + ) + + df = pd.DataFrame(rows) + + # --- Assign / preserve phosphosite_ids --- + if prev is not None and not prev.empty: + prev_map = prev.set_index("other_id")["phosphosite_id"].to_dict() + next_id = int(prev["phosphosite_id"].max()) + 1 + else: + prev_map = {} + next_id = 1 + + ids = [] + for site in df["other_id"]: + if site in prev_map: + ids.append(prev_map[site]) + else: + ids.append(next_id) + prev_map[site] = next_id + next_id += 1 + df["phosphosite_id"] = ids + + cols = [ + "phosphosite_id", "entrez_id", "gene_symbol", + "residue", "position", "modification", "other_id", + ] + df = df[cols].sort_values("phosphosite_id").reset_index(drop=True) + logging.info( + "Final phosphosites table: %d rows, %d unique genes", + len(df), df["gene_symbol"].nunique(), + ) + return df + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("genes", help="genes.csv (gene_symbol → entrez_id)") + parser.add_argument("--out", default="/tmp/phosphosites.csv") + parser.add_argument("--prev", default=None, + help="Previous phosphosites.csv to preserve IDs") + parser.add_argument("--supplement", nargs="*", default=[], + help="Local CSV/TSV file(s) with site strings") + parser.add_argument("--site_col", default="site", + help="Column name for site strings in local supplement files") + parser.add_argument("--synapse_supplements", nargs="*", default=[], + help="Synapse IDs of raw phospho files to add as supplements " + "(e.g. syn70078415)") + parser.add_argument("--synapse_site_col", default="site", + help="Site column name in Synapse supplement files") + parser.add_argument("--ochoa_url", default=OCHOA_URL) + parser.add_argument("--uniprot_gene_url", default=UNIPROT_GENE_URL) + parser.add_argument("--uniprot_ptm_url", default=UNIPROT_PTM_URL) + args = parser.parse_args() + + configure_logging() + + gene_map = load_gene_map(args.genes) + + prev = None + if args.prev: + prev = pd.read_csv(args.prev) + logging.info("Loaded %d previous phosphosites from %s", len(prev), args.prev) + + ochoa_records = fetch_ochoa_phosphosites(args.ochoa_url) + acc_to_gene = fetch_uniprot_gene_map(args.uniprot_gene_url) + uniprot_records = fetch_uniprot_ptm_sites(args.uniprot_ptm_url) + + supp_records: list[dict] = [] + for path in (args.supplement or []): + supp_records.extend(parse_supplement_sites(path, args.site_col)) + for syn_id in (args.synapse_supplements or []): + try: + supp_records.extend(fetch_synapse_supplement(syn_id, args.synapse_site_col)) + except Exception as exc: + logging.warning("Synapse supplement %s failed: %s — skipping", syn_id, exc) + + df = build_phosphosite_table( + ochoa_records, acc_to_gene, uniprot_records, gene_map, supp_records, prev + ) + + df.to_csv(args.out, index=False) + logging.info( + "Wrote %s (%d rows, %d unique genes)", + args.out, len(df), df["gene_symbol"].nunique(), + ) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/phosphosites/README.md b/coderbuild/phosphosites/README.md new file mode 100644 index 00000000..e2cf788f --- /dev/null +++ b/coderbuild/phosphosites/README.md @@ -0,0 +1,97 @@ +# Phosphosites Reference Builder + +Produces `phosphosites.csv`, the global human phosphosite reference used by +dataset omics pipelines (currently: cNF) to map raw phospho measurements to +stable integer identifiers. + +## Sources + +Three sources are merged in priority order. A site from an earlier source is +never overwritten by a later one (deduplication on `other_id`). + +| Priority | Source | Sites (approx.) | +|---|---|---| +| 1 | Ochoa et al. (2020) Nat Biotechnol 38:365-373, Supp. Table S3 | ~116k | +| 2 | UniProt PTM REST API | ~12k additional | +| 3 | Synapse supplement `syn70078415` (cNF raw phospho data) | ~188 additional | + +## Site notation + +``` +- +e.g. AAAS-S495s (s=phosphoserine, t=phosphothreonine, y=phosphotyrosine) +``` + +`other_id` in `phosphosites.csv` stores these strings and is the join key for +raw phospho data. + +## Output columns + +`phosphosite_id`, `entrez_id`, `gene_symbol`, `residue`, `position`, +`modification`, `other_id` + +## Files + +``` +coderbuild/phosphosites/ +├── 00-buildPhosphositeFile.py # Builder script +└── build_phosphosites.sh # Docker entry point +coderbuild/docker/ +└── Dockerfile.phosphosites +``` + +## Builder — `00-buildPhosphositeFile.py` + +**Ochoa (primary):** Downloads `41587_2019_344_MOESM4_ESM.xlsx` from the +Springer static CDN (no login required; requires `openpyxl`). Reads the +`annotated_phosphoproteome` sheet (`uniprot`, `position`, `residue` columns). +UniProt accessions are resolved to gene symbols via a separate UniProt REST +call (TSV, all reviewed human proteins, paginated via `Link: rel="next"` +header). + +**UniProt PTM (secondary):** Downloads `Modified residue` features with +phosphoserine/phosphothreonine/phosphotyrosine descriptions from the UniProt +REST API (JSON, same Link-header pagination). Returns gene symbols directly. + +**Synapse supplement (`--synapse_supplements`):** Downloads a raw phospho +file from Synapse, auto-detects the site column (`site`, `Site`, +`phosphosite`, or `feature_id`), and appends any `GENE-ResPositionmod` sites +not already in the reference. Requires `SYNAPSE_AUTH_TOKEN`. Failures are +logged as warnings and do not abort the build. + +### Key arguments + +| Argument | Default | Description | +|---|---|---| +| `genes` (positional) | — | Path to `genes.csv` | +| `--out` | `/tmp/phosphosites.csv` | Output path | +| `--prev` | None | Previous CSV to preserve `phosphosite_id` values across builds | +| `--synapse_supplements` | None | Synapse IDs of raw phospho files (space-separated) | +| `--synapse_site_col` | `site` | Site column name in Synapse supplement files | +| `--supplement` | None | Local CSV/TSV file(s) with site strings | +| `--site_col` | `site` | Site column name in local supplement files | + +## Build sequencing + +Phosphosites must run **after** `genes.csv` is ready. `build_dataset.py` and +`build_all.py` wait on the genes future before submitting the phosphosites +container. + +## ID stability + +Pass `--prev ` to preserve IDs across rebuilds. New +sites receive IDs starting from `max(prev.phosphosite_id) + 1` — same pattern +as `improve_sample_id` and `improve_drug_id`. + +## Local run (Only for debugging purposes, this is already build into build_all.py script) + +```bash +pip install pandas synapseclient openpyxl + +export SYNAPSE_AUTH_TOKEN= +python coderbuild/phosphosites/00-buildPhosphositeFile.py \ + local/genes.csv \ + --out local/phosphosites.csv \ + --synapse_supplements syn70078415 \ + --synapse_site_col site +``` diff --git a/coderbuild/phosphosites/build_phosphosites.sh b/coderbuild/phosphosites/build_phosphosites.sh new file mode 100644 index 00000000..4662546d --- /dev/null +++ b/coderbuild/phosphosites/build_phosphosites.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# build_phosphosites.sh: build the global phosphosites reference CSV. +# +# Usage: +# build_phosphosites.sh [] +# +# Sources (in priority order): +# 1. Ochoa et al. (2020) Nat Biotechnol Supplementary Table S3 (~116k sites) +# 2. UniProt PTM annotations (~12k additional sites unique to UniProt) +# 3. Synapse supplement: syn70078415 (cnf raw phospho — captures experiment- +# specific sites not yet in either database) + +set -euo pipefail + +GENES="${1:?Usage: build_phosphosites.sh []}" +PREV="${2:-}" + +PREV_ARG="" +if [ -n "$PREV" ] && [ -f "$PREV" ]; then + PREV_ARG="--prev $PREV" +fi + +python /coderbuild/phosphosites/00-buildPhosphositeFile.py \ + "$GENES" \ + --out /tmp/phosphosites.csv \ + --synapse_supplements syn70078415 \ + --synapse_site_col site \ + $PREV_ARG + +echo "Phosphosites built from Ochoa + UniProt PTM + Synapse supplement." From 89fe0505b3631f8c10253934bc8fd005900ba90b Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:01 -0700 Subject: [PATCH 06/20] Add the cNF organoid dataset: samples and omics --- coderbuild/cnf/01-samples-cnf.py | 372 +++++++++++++++++++++++++++++ coderbuild/cnf/02-omics-cnf.py | 389 +++++++++++++++++++++++++++++++ coderbuild/cnf/README.md | 169 ++++++++++++++ coderbuild/cnf/build_omics.sh | 20 ++ coderbuild/cnf/build_samples.sh | 15 ++ coderbuild/cnf/cnf_utils.py | 117 ++++++++++ coderbuild/cnf/requirements.txt | 9 + coderbuild/docker/Dockerfile.cnf | 74 ++++++ 8 files changed, 1165 insertions(+) create mode 100644 coderbuild/cnf/01-samples-cnf.py create mode 100644 coderbuild/cnf/02-omics-cnf.py create mode 100644 coderbuild/cnf/README.md create mode 100644 coderbuild/cnf/build_omics.sh create mode 100644 coderbuild/cnf/build_samples.sh create mode 100644 coderbuild/cnf/cnf_utils.py create mode 100644 coderbuild/cnf/requirements.txt create mode 100644 coderbuild/docker/Dockerfile.cnf diff --git a/coderbuild/cnf/01-samples-cnf.py b/coderbuild/cnf/01-samples-cnf.py new file mode 100644 index 00000000..41bb14ea --- /dev/null +++ b/coderbuild/cnf/01-samples-cnf.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +""" +01-samples-cnf.py: generate cnf_samples.csv for the cNF organoid drug screen. + +Specimens are gathered from four sources: + + 1. Drug-screen file index on Synapse (syn51301431, dataType='drug screen') + 2. Global proteomics matrix (env CNF_GLOBAL_PROT_SYN_ID, default syn74815895) + 3. RNA discovery file (env CNF_RNA_DISCOVERY_SYN_ID, default syn71333780). + This is the corrected-abundance file containing every RNA condition + processed for the project (organoid, tumor tissue, drug-treated + organoid, skin), used here for sample discovery only — NOT to be + confused with CNF_RNA_TPM_SYN_ID, which build_omics.py reads for the + actual TPM data and may be filtered to organoids only. + 4. Normal Skin Synapse folder (env CNF_NORMAL_SKIN_SYN_ID, default syn74284682) + +Each specimen is classified as one of: + - "organoid" → model_type = "patient derived organoid" + - "treated_organoid" → model_type = "patient derived organoid" + (drug-treated organoid; canonical ID preserves the + treatment label, e.g. NF0021_T1_Onalespid_1uM) + - "tumor_tissue" → model_type = "tumor" + - "normal_skin" → model_type = "normal_tissue" + +Tissue and organoid samples for the same tumor are kept as separate rows with +distinct improve_sample_ids — they have different biology (fresh tumor vs. +cultured organoid) and the schema model_types differ. Treated and untreated +organoids of the same tumor are also separate rows because their canonical +IDs differ (treated samples preserve the treatment label). + +Cohort assignment is intentionally omitted. The upstream omics data is +batch-corrected and per-modality cohort membership disagrees for some +specimens, so a single per-sample cohort label cannot be assigned cleanly. +The `other_id_source` field carries a uniform project label. + +Sample IDs continue from the previous samples file: max(improve_sample_id) + 1. + +Usage: + python 01-samples-cnf.py [--output /tmp/cnf_samples.csv] +""" + +import argparse +import logging +import os +import re +import sys + +import pandas as pd +import synapseclient + +from cnf_utils import ( + MODEL_TYPE_MAP, + classify_specimen, + patient_from_specimen, +) + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- +DRUG_SCREEN_TABLE = "syn51301431" +GLOBAL_PROT_SYN_ID = os.environ.get("CNF_GLOBAL_PROT_SYN_ID", "syn74815895") +# RNA file used for SPECIMEN DISCOVERY only — needs to contain all +# conditions (organoid + tissue + treated + skin). The corrected-abundance +# file at syn71333780 fits this. Also used by build_omics.py as the TPM +# data source (the column is named correctedAbundance but stores TPM). +RNA_DISCOVERY_SYN_ID = os.environ.get("CNF_RNA_DISCOVERY_SYN_ID", "syn71333780") +NORMAL_SKIN_SYN_ID = os.environ.get("CNF_NORMAL_SKIN_SYN_ID", "syn74284682") + +DEFAULT_OUTPUT = "/tmp/cnf_samples.csv" + +CANCER_TYPE = "Cutaneous Neurofibroma" +SPECIES = "Homo sapiens (Human)" +OTHER_ID_SOURCE = "NF1_cNF_Project" + + +# ---------------------------------------------------------------------------- +# Helpers +# ---------------------------------------------------------------------------- +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +def get_starting_id(prev_samples_path: str) -> int: + """Read previous samples file and return max(improve_sample_id) + 1.""" + if not os.path.exists(prev_samples_path): + logging.warning("Previous samples file %s not found; starting at 1.", + prev_samples_path) + return 1 + + df = pd.read_csv(prev_samples_path) + if "improve_sample_id" not in df.columns or df.empty: + logging.warning("No improve_sample_id column or empty file; starting at 1.") + return 1 + + max_id = int(pd.to_numeric(df["improve_sample_id"], errors="coerce").max()) + logging.info("Previous samples max improve_sample_id = %d", max_id) + return max_id + 1 + + +def merge_specimen_dicts(*dicts: dict[str, str]) -> dict[str, str]: + """Union per-source dicts; warn on type disagreements (first-seen wins).""" + merged: dict[str, str] = {} + for d in dicts: + for spec, stype in d.items(): + if spec in merged and merged[spec] != stype: + logging.warning( + "Type conflict for %s: %s (kept) vs %s (ignored)", + spec, merged[spec], stype, + ) + continue + merged.setdefault(spec, stype) + return merged + + +# ---------------------------------------------------------------------------- +# Source-specific specimen extraction +# ---------------------------------------------------------------------------- +def specimens_from_drug_screen(syn) -> dict[str, str]: + """Pull canonical_id → sample_type for drug-screen specimens.""" + query = ( + "select id, individualID, specimenID " + f"from {DRUG_SCREEN_TABLE} " + "where dataType='drug screen'" + ) + logging.info("Querying %s for drug-screen specimens", DRUG_SCREEN_TABLE) + table = syn.tableQuery(query).asDataFrame() + raw = table["specimenID"].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + by_type = _count_by_type(out) + logging.info(" Drug screen: %s", by_type) + return out + + +def _count_by_type(d: dict[str, str]) -> dict[str, int]: + out: dict[str, int] = {} + for stype in d.values(): + out[stype] = out.get(stype, 0) + 1 + return out + + +def _detect_specimen_column(df: pd.DataFrame) -> str: + """Return the name of the specimen column in an omics file.""" + candidates = ["Specimen", "specimen", "specimen_norm", "sample_id", + "specimenID", "Sample"] + for c in candidates: + if c in df.columns: + return c + raise KeyError( + f"No specimen column among {candidates} in columns {list(df.columns)[:15]}" + ) + + +def _read_synapse_table(syn, syn_id: str) -> pd.DataFrame: + """Fetch a Synapse file and read it as CSV/TSV.""" + logging.info(" Fetching %s", syn_id) + entity = syn.get(syn_id) + path = entity.path + if path.endswith(".tsv") or path.endswith(".tsv.gz"): + return pd.read_csv(path, sep="\t") + return pd.read_csv(path) + + +def specimens_from_proteomics(syn) -> dict[str, str]: + """Pull canonical_id → sample_type from the global proteomics matrix.""" + if not GLOBAL_PROT_SYN_ID: + logging.warning("CNF_GLOBAL_PROT_SYN_ID not set; skipping proteomics") + return {} + + logging.info("Reading proteomics specimens") + try: + df = _read_synapse_table(syn, GLOBAL_PROT_SYN_ID) + col = _detect_specimen_column(df) + raw = df[col].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + logging.info(" Proteomics: %s", _count_by_type(out)) + return out + except Exception as exc: + logging.warning(" Proteomics read failed: %s", exc) + return {} + + +def specimens_from_rna_discovery(syn) -> dict[str, str]: + """Discover RNA specimens from a comprehensive long-format RNA file. + + Reads CNF_RNA_DISCOVERY_SYN_ID (default syn71333780) — the corrected- + abundance RNA file that contains every condition processed for the + project: organoid + tumor tissue + drug-treated organoid + skin. + + This is intentionally a different file from the one read by + build_omics.py (CNF_RNA_TPM_SYN_ID, which is the gene-level TPM matrix + and may be filtered to organoids only). Sample discovery and data + ingestion have different requirements: discovery wants the maximum + sample list; ingestion wants TPM values. + + Specimens are passed through classify_specimen, so drug-treated + samples like NF0021_T1_Onalespid.1uM get caught regardless of their + condition_norm value. + """ + if not RNA_DISCOVERY_SYN_ID: + logging.warning("CNF_RNA_DISCOVERY_SYN_ID not set; skipping RNA discovery") + return {} + + logging.info("Reading RNA discovery file (%s)", RNA_DISCOVERY_SYN_ID) + try: + df = _read_synapse_table(syn, RNA_DISCOVERY_SYN_ID) + col = _detect_specimen_column(df) + + raw = df[col].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + logging.info(" RNA discovery: %s", _count_by_type(out)) + return out + except Exception as exc: + logging.warning(" RNA discovery read failed: %s", exc) + return {} + + +def specimens_from_normal_skin_tree(syn) -> dict[str, str]: + """Discover skin specimens by listing the Normal Skin Synapse folder. + + The "Normal Skin" tree contains one immediate child per patient (folders + named NF0017, NF0018, ...) plus, in some cases, top-level skin FASTQ + files like NF0035-SKIN-1-04-16-2025_*.fastq.gz. Both forms encode the + patient ID at the start, so we match `^NF\\d{4}` against each child name + and emit one canonical NFxxxx_skin entry per unique patient. + + This is intentionally one API call deep — it avoids walking the whole + WGS tree (which has hundreds of empty lane-stub folders). For most + cases this captures every patient with a matched normal. + """ + if not NORMAL_SKIN_SYN_ID: + logging.warning("CNF_NORMAL_SKIN_SYN_ID not set; skipping Normal Skin tree") + return {} + + logging.info("Listing Normal Skin tree (%s)", NORMAL_SKIN_SYN_ID) + out: dict[str, str] = {} + try: + for child in syn.getChildren(NORMAL_SKIN_SYN_ID): + name = str(child.get("name", "")) + m = re.match(r"^(NF\d{4})", name) + if m: + patient = m.group(1) + out[f"{patient}_skin"] = "normal_skin" + logging.info(" Normal Skin tree: %s", _count_by_type(out)) + except Exception as exc: + logging.warning(" Normal Skin tree listing failed: %s", exc) + return out + + +# ---------------------------------------------------------------------------- +# Sample table construction +# ---------------------------------------------------------------------------- +def build_samples_table(specimens: dict[str, str], start_id: int) -> pd.DataFrame: + """Build the schema-compliant Sample table from {canonical_id: sample_type}. + + Sort order: organoid → treated_organoid → tumor_tissue → normal_skin + (each by ID). Drug-modelable untreated organoids get the lowest + improve_sample_ids — useful for downstream filtering. + """ + sort_priority = { + "organoid": 0, + "treated_organoid": 1, + "tumor_tissue": 2, + "normal_skin": 3, + } + sorted_specimens = sorted( + specimens.items(), + key=lambda kv: (sort_priority.get(kv[1], 99), kv[0]), + ) + + rows = [] + next_id = start_id + for specimen, stype in sorted_specimens: + patient = patient_from_specimen(specimen) + rows.append({ + "improve_sample_id": next_id, + "other_id": specimen, + "other_id_source": OTHER_ID_SOURCE, + "common_name": specimen, + "cancer_type": CANCER_TYPE, + "other_names": patient, + "species": SPECIES, + "model_type": MODEL_TYPE_MAP[stype], + }) + next_id += 1 + + df = pd.DataFrame(rows) + logging.info("Built %d sample rows; ID range %d–%d", + len(df), start_id, next_id - 1) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "prev_samples", + help="Path to previous samples CSV (used to find max improve_sample_id)", + ) + parser.add_argument("--output", default=DEFAULT_OUTPUT, + help=f"Output CSV path (default: {DEFAULT_OUTPUT})") + args = parser.parse_args() + + configure_logging() + + syn = synapseclient.Synapse() + syn.login() + + # Pull specimens from each source as canonical_id → sample_type dicts + drug_d = specimens_from_drug_screen(syn) + prot_d = specimens_from_proteomics(syn) + rna_d = specimens_from_rna_discovery(syn) + skin_d = specimens_from_normal_skin_tree(syn) + + all_specimens = merge_specimen_dicts(drug_d, prot_d, rna_d, skin_d) + logging.info( + "Union: %d total specimens — %s", + len(all_specimens), _count_by_type(all_specimens), + ) + + # Per-source uniqueness diagnostics + drug_set, prot_set, rna_set, skin_set = ( + set(drug_d), set(prot_d), set(rna_d), set(skin_d), + ) + only_drug = drug_set - prot_set - rna_set - skin_set + only_prot = prot_set - drug_set - rna_set - skin_set + only_rna = rna_set - drug_set - prot_set - skin_set + only_skin = skin_set - drug_set - prot_set - rna_set + if only_drug: + logging.info(" Only in drug screen: %s", sorted(only_drug)) + if only_prot: + logging.info(" Only in proteomics: %s", sorted(only_prot)) + if only_rna: + logging.info(" Only in RNA: %s", sorted(only_rna)) + if only_skin: + logging.info(" Only in Normal Skin tree: %s", sorted(only_skin)) + + if not all_specimens: + logging.error("No specimens found from any source; aborting") + sys.exit(1) + + start_id = get_starting_id(args.prev_samples) + samples = build_samples_table(all_specimens, start_id) + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + samples.to_csv(args.output, index=False) + logging.info("Wrote %s (%d rows)", args.output, len(samples)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/02-omics-cnf.py b/coderbuild/cnf/02-omics-cnf.py new file mode 100644 index 00000000..bdbe0fac --- /dev/null +++ b/coderbuild/cnf/02-omics-cnf.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +""" +02-omics-cnf.py: generate cnf_transcriptomics.csv, cnf_proteomics.csv, + and cnf_phosphoproteomics.csv. + +For the cNF dataset: + +* Transcriptomics: raw gene-level TPM from per-cohort salmon.merged.gene_tpm.tsv + files (wide matrix: rows = genes, columns = specimens). Default cohorts: + syn66352931 — cNF_Cohort_1 (NF0017–NF0023) + syn70765053 — cNF_Cohort_2 (NF0025, NF0027, NF0035) + Override via env var CNF_RNA_COHORT_SYN_IDS (comma-separated Synapse IDs). + +* Proteomics: batch-corrected global proteomics from syn74815895 + (correctedAbundance column). Override via env var CNF_GLOBAL_PROT_SYN_ID. + +* Phosphoproteomics: batch-corrected phospho-proteomics from syn70078415 + (correctedAbundance column). Mapped to phosphosite_ids via a phosphosites.csv + reference file. Override via env var CNF_PHOSPHO_SYN_ID. + +* Protocol optimization samples are excluded from RNA via DROP_SAMPLE_SUBSTRINGS. + +Specimens are canonicalized via cnf_utils so that values like +"NF0018.T1.organoid" resolve to the canonical IDs in cnf_samples.csv. +Source rows whose specimen doesn't match any entry in cnf_samples.csv are +dropped silently via the inner join to sample_map. + +Usage: + python 02-omics-cnf.py + [--out_rna /tmp/cnf_transcriptomics.csv] + [--out_prot /tmp/cnf_proteomics.csv] + [--out_phospho /tmp/cnf_phosphoproteomics.csv] +""" + +import argparse +import logging +import os +import re +import sys + +import pandas as pd +import synapseclient + +from cnf_utils import canonicalize_specimen_column + + +# ---------------------------------------------------------------------------- +# Defaults / Synapse IDs +# ---------------------------------------------------------------------------- +_rna_env = os.environ.get("CNF_RNA_COHORT_SYN_IDS", "") +RNA_TPM_COHORT_SYN_IDS: list[str] = ( + [s.strip() for s in _rna_env.split(",") if s.strip()] + if _rna_env + else ["syn66352931", "syn70765053"] +) + +GLOBAL_PROT_SYN_ID = os.environ.get("CNF_GLOBAL_PROT_SYN_ID", "syn74815895") +PHOSPHO_SYN_ID = os.environ.get("CNF_PHOSPHO_SYN_ID", "syn70078415") + +# Protocol-optimization samples to drop from RNA. +DROP_SAMPLE_SUBSTRINGS: list[str] = [ + "cNF_organoid_DIA_G_02_11Feb25", + "cNF_organoid_DIA_G_05_11Feb25", + "cNF_organoid_DIA_G_06_11Feb25", + "cNF_organoid_DIA_P_02_29Jan25", + "cNF_organoid_DIA_P_05_11Feb25", + "cNF_organoid_DIA_P_06_11Feb25", +] + +STUDY = "cnf" +DEFAULT_OUT_RNA = "/tmp/cnf_transcriptomics.csv" +DEFAULT_OUT_PROT = "/tmp/cnf_proteomics.csv" +DEFAULT_OUT_PHOSPHO = "/tmp/cnf_phosphoproteomics.csv" + + +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Reference data loaders +# ---------------------------------------------------------------------------- +def load_gene_map(genes_path: str) -> pd.DataFrame: + """Load genes.csv. Required columns: gene_symbol, entrez_id.""" + df = pd.read_csv(genes_path) + needed = {"gene_symbol", "entrez_id"} + missing = needed - set(df.columns) + if missing: + raise KeyError(f"genes file missing columns: {missing}") + df = df.dropna(subset=["entrez_id", "gene_symbol"]).copy() + df["entrez_id"] = df["entrez_id"].astype(int) + df = df.drop_duplicates(subset="gene_symbol") + logging.info("Loaded %d gene_symbol → entrez_id mappings", len(df)) + return df[["gene_symbol", "entrez_id"]] + + +def load_sample_map(samples_path: str) -> pd.DataFrame: + """Load cnf_samples.csv; return (other_id, improve_sample_id) frame.""" + df = pd.read_csv(samples_path) + df = df[["other_id", "improve_sample_id"]].drop_duplicates() + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + logging.info("Loaded %d sample mappings", len(df)) + return df + + +def load_phosphosite_map(phosphosites_path: str) -> pd.DataFrame: + """Load phosphosites.csv; return (other_id, phosphosite_id) frame.""" + df = pd.read_csv(phosphosites_path) + needed = {"other_id", "phosphosite_id"} + missing = needed - set(df.columns) + if missing: + raise KeyError(f"phosphosites file missing columns: {missing}") + df = df[["other_id", "phosphosite_id"]].drop_duplicates() + df["phosphosite_id"] = df["phosphosite_id"].astype(int) + logging.info("Loaded %d phosphosite mappings", len(df)) + return df + + +def fetch_synapse_table(syn, syn_id: str) -> pd.DataFrame: + """Download a Synapse file and read it as CSV/TSV.""" + if not syn_id: + raise ValueError("Synapse ID is empty") + logging.info("Fetching %s from Synapse", syn_id) + entity = syn.get(syn_id) + path = entity.path + if path.endswith(".tsv") or path.endswith(".tsv.gz"): + return pd.read_csv(path, sep="\t") + return pd.read_csv(path) + + +# ---------------------------------------------------------------------------- +# Shared helpers +# ---------------------------------------------------------------------------- +def _col_stem(col: str) -> str: + """Return the basename without extension for a path-like column name.""" + base = re.sub(r"^.*[/\\]", "", col) + return re.sub(r"\.[^.]+$", "", base) + + +def _drop_protocol_cols(cols: list[str]) -> list[str]: + """Remove columns whose stem matches any DROP_SAMPLE_SUBSTRINGS entry.""" + kept = [] + for c in cols: + stem = _col_stem(c) + if any(sub in stem for sub in DROP_SAMPLE_SUBSTRINGS): + logging.info(" Dropping protocol optimization sample: %s", stem) + else: + kept.append(c) + return kept + + +def _normalize_col(df: pd.DataFrame, candidates: list[str], target: str) -> pd.DataFrame: + """Rename the first matching candidate column to target.""" + for c in candidates: + if c in df.columns: + return df.rename(columns={c: target}) + raise KeyError( + f"None of {candidates} found in columns {list(df.columns)[:20]}" + ) + + +# ---------------------------------------------------------------------------- +# Transcriptomics +# ---------------------------------------------------------------------------- +def fetch_wide_tpm_cohort(syn, syn_id: str) -> pd.DataFrame: + """Fetch a wide salmon.merged.gene_tpm.tsv and return long-format DataFrame. + + nf-core salmon output: gene_id (Ensembl), gene_name (symbol), then one + column per specimen. Returns (gene_symbol, specimen, transcriptomics, source). + """ + logging.info("Fetching cohort TPM matrix %s", syn_id) + entity = syn.get(syn_id) + path = entity.path + df = pd.read_csv(path, sep="\t") + + gene_col = None + for cname in ["gene_name", "gene_symbol", "Symbol", "GeneSymbol", "gene"]: + if cname in df.columns: + gene_col = cname + break + if gene_col is None: + gene_col = df.columns[0] + logging.warning( + "%s: no gene_name column found; using %s — most rows may fail the gene_map join", + syn_id, gene_col, + ) + + meta_cols = {"gene_id", "gene_name", "gene_symbol", "Symbol", + "GeneSymbol", "gene", "transcript_id"} + sample_cols = [c for c in df.columns if c not in meta_cols] + sample_cols = _drop_protocol_cols(sample_cols) + + if not sample_cols: + raise ValueError( + f"{syn_id}: no specimen columns found after excluding metadata " + f"and protocol optimization samples" + ) + + long = ( + df[[gene_col] + sample_cols] + .rename(columns={gene_col: "gene_symbol"}) + .melt(id_vars="gene_symbol", var_name="specimen", value_name="transcriptomics") + ) + long["source"] = "Synapse" + logging.info( + " %s: %d genes × %d specimens → %d long rows", + syn_id, df[gene_col].nunique(), len(sample_cols), len(long), + ) + return long + + +def build_transcriptomics( + syn, gene_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format transcriptomics table from per-cohort TPM matrices.""" + if not RNA_TPM_COHORT_SYN_IDS: + logging.error("RNA_TPM_COHORT_SYN_IDS is empty; skipping transcriptomics.") + return pd.DataFrame( + columns=["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"] + ) + + cohort_frames = [] + for syn_id in RNA_TPM_COHORT_SYN_IDS: + try: + cohort_frames.append(fetch_wide_tpm_cohort(syn, syn_id)) + except Exception as exc: + logging.warning("Skipping RNA cohort %s: %s", syn_id, exc) + + if not cohort_frames: + logging.error("All RNA cohort fetches failed; returning empty transcriptomics.") + return pd.DataFrame( + columns=["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"] + ) + + raw = pd.concat(cohort_frames, ignore_index=True) + logging.info("Combined %d RNA cohorts → %d total rows", len(cohort_frames), len(raw)) + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Transcriptomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + df = ( + raw.merge(gene_map, on="gene_symbol", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["study"] = STUDY + df["transcriptomics"] = pd.to_numeric(df["transcriptomics"], errors="coerce") + df = df.dropna(subset=["transcriptomics"]) + df = df[["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"]] + + logging.info( + "Built transcriptomics table: %d rows (%d samples × %d genes)", + len(df), df["improve_sample_id"].nunique(), df["entrez_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Proteomics +# ---------------------------------------------------------------------------- +def build_proteomics( + syn, gene_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format proteomics table from the batch-corrected Synapse file.""" + raw = fetch_synapse_table(syn, GLOBAL_PROT_SYN_ID) + raw = _normalize_col(raw, + ["Specimen", "specimenID", "specimen_id", "specimen", "specimen_norm", "sample_id"], + "specimen") + raw = _normalize_col(raw, + ["gene_symbol", "Gene", "GeneSymbol", "Symbol", "gene", "feature_id"], + "gene_symbol") + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Proteomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + raw = _normalize_col(raw, + ["correctedAbundance", "log_ratio", "logRatio", "proteomics", "value"], + "proteomics") + + df = ( + raw.merge(gene_map, on="gene_symbol", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["source"] = "Synapse" + df["study"] = STUDY + df["proteomics"] = pd.to_numeric(df["proteomics"], errors="coerce") + df = df.dropna(subset=["proteomics"]) + df = df[["entrez_id", "improve_sample_id", "source", "study", "proteomics"]] + + logging.info( + "Built proteomics table: %d rows (%d samples × %d genes)", + len(df), df["improve_sample_id"].nunique(), df["entrez_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Phosphoproteomics +# ---------------------------------------------------------------------------- +def build_phosphoproteomics( + syn, phosphosite_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format phosphoproteomics table from the batch-corrected Synapse file.""" + raw = fetch_synapse_table(syn, PHOSPHO_SYN_ID) + raw = _normalize_col(raw, + ["Specimen", "specimenID", "specimen_id", "specimen"], + "specimen") + raw = _normalize_col(raw, + ["site", "Site", "phosphosite", "feature_id"], + "site") + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Phosphoproteomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + raw = _normalize_col(raw, + ["correctedAbundance", "log_ratio", "logRatio", "phosphoproteomics", "value"], + "phosphoproteomics") + + df = ( + raw.merge(phosphosite_map, left_on="site", right_on="other_id", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["source"] = "Synapse" + df["study"] = STUDY + df["phosphoproteomics"] = pd.to_numeric(df["phosphoproteomics"], errors="coerce") + df = df.dropna(subset=["phosphoproteomics"]) + df = df[["phosphosite_id", "improve_sample_id", "source", "study", "phosphoproteomics"]] + + logging.info( + "Built phosphoproteomics table: %d rows (%d samples × %d sites)", + len(df), df["improve_sample_id"].nunique(), df["phosphosite_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("genes", help="Path to genes.csv (gene_symbol → entrez_id)") + parser.add_argument("samples", help="Path to cnf_samples.csv from build_samples") + parser.add_argument("phosphosites", help="Path to phosphosites.csv (other_id → phosphosite_id)") + parser.add_argument("--out_rna", default=DEFAULT_OUT_RNA) + parser.add_argument("--out_prot", default=DEFAULT_OUT_PROT) + parser.add_argument("--out_phospho", default=DEFAULT_OUT_PHOSPHO) + args = parser.parse_args() + + configure_logging() + + gene_map = load_gene_map(args.genes) + sample_map = load_sample_map(args.samples) + phosphosite_map = load_phosphosite_map(args.phosphosites) + + syn = synapseclient.Synapse() + syn.login() + + rna = build_transcriptomics(syn, gene_map, sample_map) + prot = build_proteomics(syn, gene_map, sample_map) + phospho = build_phosphoproteomics(syn, phosphosite_map, sample_map) + + for path in (args.out_rna, args.out_prot, args.out_phospho): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + + rna.to_csv(args.out_rna, index=False) + prot.to_csv(args.out_prot, index=False) + phospho.to_csv(args.out_phospho, index=False) + logging.info("Wrote %s (%d rows)", args.out_rna, len(rna)) + logging.info("Wrote %s (%d rows)", args.out_prot, len(prot)) + logging.info("Wrote %s (%d rows)", args.out_phospho, len(phospho)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/README.md b/coderbuild/cnf/README.md new file mode 100644 index 00000000..0755a952 --- /dev/null +++ b/coderbuild/cnf/README.md @@ -0,0 +1,169 @@ +# Cutaneous Neurofibroma (cnf) Organoid Drug Screen + +A coderdata dataset of cutaneous neurofibroma (cNF) patient-derived organoids +with drug-response measurements and matched multi-omic profiling. + +## Overview + +Cutaneous neurofibromas are benign nerve-sheath tumors that develop in patients +with neurofibromatosis type 1 (NF1). This dataset contains drug-screen +viability measurements for 238 small-molecule compounds across 23 cNF tumor +specimens from 10 patients, paired with bulk RNA-seq, global LC-MS/MS +proteomics, and batch-corrected phosphoproteomics for many of the same +specimens. + +| Property | Value | +|---|---| +| Cancer type | Cutaneous Neurofibroma | +| Model type | patient derived organoid | +| Patients | 10 | +| Specimens | 23 (untreated organoids; additional tissue and treated organoid samples also present) | +| Drugs | 238 | +| Modalities included | Transcriptomics (TPM), Proteomics (log ratio), Phosphoproteomics (log ratio) | + +## Multi-tumor patient structure + +Each patient (`NF0017`, `NF0018`, ...) contributed up to three independent +tumor specimens, named `NFxxxx_T1`, `NFxxxx_T2`, `NFxxxx_T3`. This dataset +treats each specimen as an independent sample (its own `improve_sample_id`). +The patient ID is preserved in the `other_names` column so downstream models +can group specimens by patient when needed (e.g. patient-level holdout). + +``` +improve_sample_id other_id common_name other_names other_id_source +2401 NF0019_T1 NF0019_T1 NF0019 NF1_cNF_Project +2402 NF0019_T2 NF0019_T2 NF0019 NF1_cNF_Project +2403 NF0019_T3 NF0019_T3 NF0019 NF1_cNF_Project +``` + +The sample table also includes matched normal skin (`NFxxxx_skin`), primary +tumor tissue (`NFxxxx_Tx_tissue`), and drug-treated organoids +(`NFxxxx_Tx_`). Each gets its own `improve_sample_id` and +`model_type`. Organoids (untreated) receive the lowest IDs, sorted by +specimen ID, to make drug-modelable samples easy to filter. + + +## Drug-response metrics + +The cNF screen mixes two designs: + +- **Multi-dose drugs**: a subset of drugs were screened at multiple + concentrations. These are run through `coderbuild/utils/fit_curve.py` and + contribute the standard schema metrics (`fit_auc`, `fit_ic50`, `fit_einf`, + `fit_hs`, `fit_r2`). + +- **Single-dose drugs**: remaining drugs were screened at a single 1 μM + concentration. These are recorded with `dose_response_metric = uM_viability` + and `dose_response_value = Viability_percentage / 100` (range 0–1, where + 1.0 = full viability and 0.0 = full kill). + +`uM_viability` is a value in the `ResponseMetric` enum. See the Schema +dependency section below. + +## SPECIMEN_DUAL_MAPPINGS edge case + +One viability file covers the treated organoid `NF0021_T1_Onalespid_1uM` but +its drug-response measurements should also be attributed to the untreated +`NF0021_T1`. The `SPECIMEN_DUAL_MAPPINGS` dict in `04-experiments-cnf.py` +handles this by duplicating the rows for every additional specimen listed: + +```python +SPECIMEN_DUAL_MAPPINGS = { + "NF0021_T1_Onalespid_1uM": ["NF0021_T1"], +} +``` + +Add new entries here if additional treated organoids need the same dual +attribution. Both specimens must already exist in `cnf_samples.csv` or the +duplicate rows will be dropped during the sample-map join. + +## Protocol optimization samples + +Six RNA columns in the cohort TPM matrices correspond to protocol optimization +runs rather than real specimens. They are excluded in `02-omics-cnf.py` before +the wide-to-long melt. Column stems matching any of these substrings are +dropped: + +```python +DROP_SAMPLE_SUBSTRINGS = [ + "cNF_organoid_DIA_G_02_11Feb25", + "cNF_organoid_DIA_G_05_11Feb25", + "cNF_organoid_DIA_G_06_11Feb25", + "cNF_organoid_DIA_P_02_29Jan25", + "cNF_organoid_DIA_P_05_11Feb25", + "cNF_organoid_DIA_P_06_11Feb25", +] +``` + +These are matched on the column stem (basename without extension), so both +plain paths and filename columns are handled correctly. If new protocol +optimization runs appear in future cohort data, add their column stems to +this list. + +## Phosphosites dependency and known unmatched sites + +`02-omics-cnf.py` requires `phosphosites.csv` as a positional argument. +The phosphosites reference must be built before the cNF omics step runs. +`build_dataset.py` and `build_all.py` enforce this sequencing. + +The phosphoproteomics output file (`cnf_phosphoproteomics.csv`) is produced +by an inner join between the raw Synapse phospho data and `phosphosites.csv`. +Sites that don't appear in the reference are silently dropped. As of the +current release, **12 sites** in the raw cNF phospho data (syn70078415) are +permanently unmatched because their gene symbols (`BAP18`, `C11orf96`, +`C14orf93`, `C1orf21`, `C2orf49`, etc). The remaining 2,974 of 2,986 unique sites +(99.6%) are present in the output. + +## Data sources + +| Modality | Synapse ID | Notes | +|---|---|---| +| Drug-screen file index | `syn51301431` (table) | `dataType='drug screen'` | +| Drug-screen file parents | `syn51301414`, `syn51301420`, `syn51301426` | One folder per cohort | +| RNA-seq TPM (Cohort 1) | `syn66352931` | `salmon.merged.gene_tpm.tsv` | +| RNA-seq TPM (Cohort 2) | `syn70765053` | `salmon.merged.gene_tpm.tsv` | +| Global proteomics | `syn74815895` | Batch-corrected; `correctedAbundance` column | +| Phospho-proteomics | `syn70078415` | Batch-corrected; `correctedAbundance` column; mapped via `phosphosites.csv` | +| Sample discovery (RNA) | `syn71333780` | All-conditions RNA file for sample enumeration only | +| Normal Skin folder | `syn74284682` | One child per patient for skin sample discovery | + +## Build pipeline + +The build follows coderdata conventions: four shell scripts wrapping Python +implementations, plus shared utilities, packaged in a Docker image. + +``` +coderbuild/cnf/ +├── 01-samples-cnf.py # mint improve_sample_ids, write cnf_samples.csv +├── build_samples.sh +├── 02-omics-cnf.py # write cnf_transcriptomics.csv, cnf_proteomics.csv, +├── build_omics.sh # cnf_phosphoproteomics.csv +├── 03-drugs-cnf.py # call pubchem_retrieval + descriptor utility +├── build_drugs.sh +├── 04-experiments-cnf.py # split single/multi-dose, run fit_curve, write cnf_experiments.tsv +├── build_exp.sh +├── cnf_utils.py # shared specimen canonicalization utilities +├── requirements.txt +└── README.md (this file) + +coderbuild/docker/ +└── Dockerfile.cnf +``` + +### Local build + +```bash +# From repository root +python coderbuild/build_dataset.py --build --validate --dataset cnf +``` + +In the full coderdata build, this dataset is orchestrated by `build_dataset.py` +/ `build_all.py`, which ensures `genes.csv`, `samples.csv`, `drugs.tsv`, and +`phosphosites.csv` are all present before starting the cNF containers. + + +## What's intentionally not included + +- **Mutations** - not yet generated for this dataset. +- **Copy number** - not yet generated for this dataset. + diff --git a/coderbuild/cnf/build_omics.sh b/coderbuild/cnf/build_omics.sh new file mode 100644 index 00000000..7db0d062 --- /dev/null +++ b/coderbuild/cnf/build_omics.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# build_omics.sh - wraps 02-omics-cnf.py per coderdata convention. +# +# Usage: +# build_omics.sh +# +# - : genes.csv with gene_symbol → entrez_id mappings +# - : cnf_samples.csv produced by build_samples.sh +# - : phosphosites.csv (other_id → phosphosite_id) + +set -euo pipefail + +GENES="${1:?Usage: build_omics.sh }" +SAMPLES="${2:?Usage: build_omics.sh }" +PHOSPHOSITES="${3:?Usage: build_omics.sh }" + +python 02-omics-cnf.py "$GENES" "$SAMPLES" "$PHOSPHOSITES" \ + --out_rna /tmp/cnf_transcriptomics.csv \ + --out_prot /tmp/cnf_proteomics.csv \ + --out_phospho /tmp/cnf_phosphoproteomics.csv \ No newline at end of file diff --git a/coderbuild/cnf/build_samples.sh b/coderbuild/cnf/build_samples.sh new file mode 100644 index 00000000..37af2162 --- /dev/null +++ b/coderbuild/cnf/build_samples.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# build_samples.sh - wraps 01-samples-cnf.py per coderdata convention. +# +# Usage: +# build_samples.sh +# +# The previous samples file is the latest samples CSV from the coderdata +# build (with the highest improve_sample_id used so far). 01-samples-cnf.py +# reads it, finds the max ID, and increments from there. + +set -euo pipefail + +PREV_SAMPLES="${1:?Usage: build_samples.sh }" + +python 01-samples-cnf.py "$PREV_SAMPLES" --output /tmp/cnf_samples.csv \ No newline at end of file diff --git a/coderbuild/cnf/cnf_utils.py b/coderbuild/cnf/cnf_utils.py new file mode 100644 index 00000000..10ec38d2 --- /dev/null +++ b/coderbuild/cnf/cnf_utils.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +cnf_utils.py: shared specimen classification utilities for the cNF build scripts. + +Used by 01-samples-cnf.py, 02-omics-cnf.py, and 04-experiments-cnf.py to +canonicalize specimen strings so omics data, drug measurements, and the sample +table all use the same improve_sample_id keys. +""" + +import re + + +# ---------------------------------------------------------------------------- +# Sample type → coderdata model_type enum +# ---------------------------------------------------------------------------- +MODEL_TYPE_MAP = { + "organoid": "patient derived organoid", + "treated_organoid": "patient derived organoid", + "tumor_tissue": "tumor", + "normal_skin": "normal_tissue", +} + + + + +# ---------------------------------------------------------------------------- +# Regex constants +# ---------------------------------------------------------------------------- +# Bare tumor organoid: NFxxxx_Tx (4 patient digits, T + tumor digits) +TUMOR_RE = re.compile(r"^NF\d{4}_T\d+$") +# Patient prefix at the start of any specimen string +PATIENT_PREFIX_RE = re.compile(r"^(NF\d{4})") +# Normal skin marker: NFxxxx[_-]skin..., allowing trailing date/replicate junk +SKIN_RE = re.compile(r"^(NF\d{4})[_-]skin", re.IGNORECASE) +# Primary tumor tissue (not organoid culture): NFxxxx_Tx_tissue +TISSUE_RE = re.compile(r"^(NF\d{4}_T\d+)_tissue$", re.IGNORECASE) +# Treated organoid: NFxxxx_Tx_, e.g. NF0021_T1_Onalespid_1uM +TREATED_RE = re.compile(r"^NF\d{4}_T\d+_.+$") + + +# ---------------------------------------------------------------------------- +# Classification +# ---------------------------------------------------------------------------- +def classify_specimen(spec) -> tuple[str, str] | None: + """Classify a raw specimen string into (canonical_id, sample_type) or None. + + sample_type ∈ {"organoid", "treated_organoid", "tumor_tissue", "normal_skin"}. + + Canonical IDs: + organoid → NFxxxx_Tx (suffix stripped) + treated_organoid → NFxxxx_Tx_ (label preserved so + treated samples don't collide + with the untreated organoid) + tumor_tissue → NFxxxx_Tx_tissue (suffix preserved) + normal_skin → NFxxxx_skin (replicates collapsed per patient) + + Unrecognized prefixes and malformed inputs return None. + """ + if spec is None: + return None + s = str(spec).strip() + if not s or s.lower() == "nan": + return None + + # Convert dot-separated form (e.g. RNA file's "NF0017.T1.organoid") to + # underscore-separated. + s_norm = s.replace(".", "_") + + # Skin: any string starting with NFxxxx that contains a "skin" marker + skin_m = SKIN_RE.match(s_norm) + if skin_m: + return (f"{skin_m.group(1)}_skin", "normal_skin") + + # Tumor tissue: NFxxxx_Tx_tissue + tissue_m = TISSUE_RE.match(s_norm) + if tissue_m: + return (f"{tissue_m.group(1)}_tissue", "tumor_tissue") + + # Strip organoid suffix (handles "_organoid" and "_organoids") + s_stripped = re.sub(r"_organoids?$", "", s_norm, flags=re.IGNORECASE) + + # Bare organoid (untreated): NFxxxx_Tx exactly + if TUMOR_RE.match(s_stripped): + return (s_stripped, "organoid") + + # Treated organoid: NFxxxx_Tx_ + if TREATED_RE.match(s_stripped): + return (s_stripped, "treated_organoid") + + return None + + +def patient_from_specimen(specimen: str) -> str: + """Extract NFxxxx patient ID from a canonical specimen ID.""" + m = PATIENT_PREFIX_RE.match(specimen) + return m.group(1) if m else "" + + +def canonicalize_specimen_column(df, specimen_col: str): + """Apply classify_specimen to a dataframe's specimen column in place. + + Returns the same df with: + - rows whose specimen classifies dropped if None + - specimen_col values replaced with their canonical IDs + - a new column 'sample_type' added + + Used by build_omics.py and build_exp.py to map raw Synapse specimen + strings (e.g. "NF0018.T1.organoid", "NF0021.T1.Onalespid.1uM") to + canonical IDs that match the cnf_samples.csv `other_id` column. + """ + classifications = df[specimen_col].apply(classify_specimen) + keep = classifications.notna() + df = df[keep].copy() + classifications = classifications[keep] + df[specimen_col] = [c[0] for c in classifications] + df["sample_type"] = [c[1] for c in classifications] + return df diff --git a/coderbuild/cnf/requirements.txt b/coderbuild/cnf/requirements.txt new file mode 100644 index 00000000..a5677837 --- /dev/null +++ b/coderbuild/cnf/requirements.txt @@ -0,0 +1,9 @@ +pandas>=2.0 +numpy>=1.24,<2.0 +synapseclient>=4.0 +requests>=2.28 +scipy>=1.10 +matplotlib +scikit-learn +scipy +mordred \ No newline at end of file diff --git a/coderbuild/docker/Dockerfile.cnf b/coderbuild/docker/Dockerfile.cnf new file mode 100644 index 00000000..f4a99784 --- /dev/null +++ b/coderbuild/docker/Dockerfile.cnf @@ -0,0 +1,74 @@ +# Dockerfile.cnf - container for building the cNF (cutaneous neurofibroma) +# organoid drug-response dataset for inclusion in coderdata. +# +# Conventions (matching coderbuild/docker/Dockerfile.): +# * Working directory is /coderbuild/cnf +# * Standard utilities are mounted/copied to /coderbuild/utils (fit_curve.py, +# pubchem_retrieval.py, build_drug_desc.py) +# * Synapse auth via SYNAPSE_AUTH_TOKEN env var (passed at `docker run`) +# * Build scripts are invoked from build_dataset.py / build_all.py via the +# standard build_*.sh entry points +# +# Build: +# docker build -f coderbuild/docker/Dockerfile.cnf -t coderdata-cnf . +# +# Run (orchestrated by build_dataset.py / build_all.py): +# docker run --rm \ +# -e SYNAPSE_AUTH_TOKEN=$SYNAPSE_AUTH_TOKEN \ +# -e CNF_RNA_TPM_SYN_ID=$CNF_RNA_TPM_SYN_ID \ +# -v $(pwd)/local_data:/coderbuild/local_data \ +# coderdata-cnf bash build_samples.sh /coderbuild/local_data/prev_samples.csv + +FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + +# System deps (rdkit-pypi pulls in some C libs; basic build tools for scipy) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential wget ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Set up coderbuild layout +WORKDIR /coderbuild/cnf + +# Install Python deps. rdkit is brought in via the descriptor utility but +# pinned here so the descriptor step is reproducible. +COPY coderbuild/cnf/requirements.txt /coderbuild/cnf/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt rdkit-pypi + +# Copy the cNF build scripts +COPY coderbuild/cnf/01-samples-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/02-omics-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/03-drugs-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/04-experiments-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/build_samples.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_omics.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_drugs.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_exp.sh /coderbuild/cnf/ +COPY coderbuild/cnf/cnf_utils.py /coderbuild/cnf/ +COPY coderbuild/utils/build_drug_desc.py /coderbuild/cnf/ + +RUN chmod +x /coderbuild/cnf/*.sh + +# Standard coderdata utilities (fit_curve.py, pubchem_retrieval.py, +COPY coderbuild/utils /coderbuild/utils + From dfd74c8e115d8605034ad5d8002339cf86acab55 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:01 -0700 Subject: [PATCH 07/20] Add MPNST treated drug-combination experiments --- coderbuild/mpnst/02_get_drug_data.R | 284 ++- coderbuild/mpnst/03_get_experiments.R | 755 +++++--- coderbuild/mpnst/03_get_experiments_orig.R | 286 +++ .../mpnst/03_get_experiments_with_treated.R | 1640 +++++++++++++++++ coderbuild/mpnst/build_drugs.sh | 1 + coderbuild/mpnst/build_exp.sh | 7 +- coderbuild/mpnst/requirements.r | 9 + 7 files changed, 2648 insertions(+), 334 deletions(-) create mode 100644 coderbuild/mpnst/03_get_experiments_orig.R create mode 100644 coderbuild/mpnst/03_get_experiments_with_treated.R diff --git a/coderbuild/mpnst/02_get_drug_data.R b/coderbuild/mpnst/02_get_drug_data.R index deb0daa2..b0727542 100644 --- a/coderbuild/mpnst/02_get_drug_data.R +++ b/coderbuild/mpnst/02_get_drug_data.R @@ -1,6 +1,7 @@ +# 02_get_drug_data.R #!/usr/bin/env Rscript -# Combined Drug List Extraction for MPNST & MPNST‑PDX +# Combined Drug List Extraction for MPNST & MPNST-PDX library(data.table) library(dplyr) @@ -8,6 +9,40 @@ library(stringr) library(synapser) library(reticulate) +# ---------------------------- +# DEBUG helper +# ---------------------------- +debug_file_summary <- function(path, label) { + message("\n========== DEBUG: ", label, " ==========") + message("Path: ", path) + if (is.null(path) || is.na(path) || !file.exists(path)) { + message("File missing.") + return(invisible(NULL)) + } + + df <- tryCatch(fread(path, sep="\t", header=TRUE), error=function(e) { + message("fread failed: ", conditionMessage(e)) + return(NULL) + }) + if (is.null(df)) return(invisible(NULL)) + + message("Rows: ", nrow(df), " Cols: ", ncol(df)) + message("Columns: ", paste(names(df), collapse=", ")) + + if ("chem_name" %in% names(df)) { + cn <- unique(tolower(trimws(as.character(df$chem_name)))) + message("Unique chem_name (first 25): ", paste(head(cn, 25), collapse=", ")) + message("Has chem_name == 'verteporfin'? ", any(cn == "verteporfin", na.rm = TRUE)) + message("Has chem_name == 'wu713d62n9'? ", any(cn == "wu713d62n9", na.rm = TRUE)) + } + if ("pubchem_id" %in% names(df)) { + pc <- unique(trimws(as.character(df$pubchem_id))) + message("Unique pubchem_id (first 10): ", paste(head(pc, 10), collapse=", ")) + } + + invisible(df) +} + # 0) Args & login args <- commandArgs(trailingOnly = TRUE) if (length(args) < 1) { @@ -25,38 +60,54 @@ synLogin(authToken = token) manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% rename(common_name = Sample) -# 2) PDX‑sourced drugs via annotations -pdx_df <- manifest %>% - select(common_name, PDX_Drug_Data) %>% - distinct() %>% - filter(!is.na(PDX_Drug_Data)) +# 2) PDX-sourced drugs via Synapse IDs stored in the manifest drug column. +# Column was renamed PDX_Drug_Data → PDXDrugData in 2025; values may now be +# file names rather than Synapse IDs, so filter to syn* entries only. +pdx_col <- if ("PDXDrugData" %in% names(manifest)) "PDXDrugData" else + if ("PDX_Drug_Data" %in% names(manifest)) "PDX_Drug_Data" else NULL +pdx_drugs <- character(0) +if (!is.null(pdx_col)) { + pdx_df <- manifest %>% + select(common_name, all_of(pdx_col)) %>% + rename(pdx_data = all_of(pdx_col)) %>% + distinct() %>% + filter(!is.na(pdx_data)) -pdx_ids <- unique(unlist(strsplit(pdx_df$PDX_Drug_Data, ","))) -pdx_ids <- pdx_ids[ pdx_ids != "" & !is.na(pdx_ids) & pdx_ids != "NA" ] - -get_pdx_drugs <- function(synid) { - # Query the metadata table for this file's experimentalCondition - q <- sprintf( - "select experimentalCondition from syn21993642 where id='%s'", - synid - ) - df <- synTableQuery(q)$asDataFrame() - if (nrow(df)==0) return(character(0)) - # Split on semicolon, lowercase and drop empties - conds <- unlist(strsplit(df$experimentalCondition, ";")) - tolower(conds[conds!=""]) -} + pdx_ids <- unique(unlist(lapply(pdx_df$pdx_data, function(x) { + strsplit(paste(trimws(unlist(x)), collapse=","), ",")[[1]] + }))) + pdx_ids <- pdx_ids[ pdx_ids != "" & !is.na(pdx_ids) & pdx_ids != "NA" ] + pdx_ids <- pdx_ids[ grepl("^syn[0-9]+$", trimws(pdx_ids), ignore.case = TRUE) ] -pdx_drugs <- unique(unlist(lapply(pdx_ids, get_pdx_drugs))) -pdx_drugs <- setdiff(pdx_drugs, "control") + get_pdx_drugs <- function(synid) { + q <- sprintf( + "select experimentalCondition from syn21993642 where id='%s'", + synid + ) + df <- synTableQuery(q)$asDataFrame() + if (nrow(df)==0) return(character(0)) + conds <- unlist(strsplit(as.character(df$experimentalCondition), ";")) + tolower(conds[conds!=""]) + } + if (length(pdx_ids) > 0) { + pdx_drugs <- unique(unlist(lapply(pdx_ids, get_pdx_drugs))) + pdx_drugs <- setdiff(pdx_drugs, "control") + } else { + message("No Synapse IDs found in ", pdx_col, "; skipping PDX Synapse drug extraction.") + } +} else { + message("No PDX drug column found in manifest; skipping PDX drug extraction.") +} -# 3) MicroTissue‑sourced drugs via table "children" +# 3) MicroTissue-sourced drugs via table "children" mts_df <- manifest %>% select(common_name, MicroTissueDrugFolder) %>% filter(!is.na(MicroTissueDrugFolder)) -mts_ids <- unique(unlist(strsplit(mts_df$MicroTissueDrugFolder, ","))) +mts_ids <- unique(unlist(lapply(mts_df$MicroTissueDrugFolder, function(x) { + strsplit(paste(trimws(unlist(x)), collapse=","), ",")[[1]] +}))) mts_ids <- mts_ids[mts_ids != "" & !is.na(mts_ids) & mts_ids != "NA"] get_mts_drugs <- function(parentId) { @@ -69,17 +120,93 @@ get_mts_drugs <- function(parentId) { mts_drugs <- unique(unlist(lapply(mts_ids, get_mts_drugs))) +# 3.5) IMPROVE-sourced drugs via improve_sample_id (syn69801348) +improve_sample_drugs <- character(0) +improve_path <- tryCatch(synGet("syn69801348")$path, error=function(e) NA) +if (!is.na(improve_path) && file.exists(improve_path)) { + improve_tab <- fread(improve_path, sep="\t", header=TRUE, fill=TRUE) + if ("improve_sample_id" %in% names(improve_tab)) { + ids <- improve_tab$improve_sample_id %>% as.character() %>% unique() + ids <- ids[!is.na(ids) & ids != ""] + drug_part <- sub("^.*_", "", ids) + drug_tokens <- unlist(strsplit(drug_part, "\\+")) + improve_sample_drugs <- unique(tolower(trimws(drug_tokens))) + improve_sample_drugs <- improve_sample_drugs[improve_sample_drugs != ""] + } +} + +# 3.6) Add drugs from treatment lookup list (syn64608501) Full_name column +lookup_drugs <- character(0) +lookup_path <- tryCatch(synGet("syn64608501")$path, error=function(e) NA) +if (!is.na(lookup_path) && file.exists(lookup_path)) { + lookup_dt <- tryCatch( + fread(lookup_path, sep = "\t", header = TRUE, fill = TRUE), + error = function(e) NULL + ) + + if (is.null(lookup_dt)) { + lookup_dt <- tryCatch( + as.data.table(readxl::read_excel(lookup_path)), + error = function(e) NULL + ) + } + + if (!is.null(lookup_dt)) { + full_col <- NULL + if ("Full_name" %in% names(lookup_dt)) full_col <- "Full_name" + if (is.null(full_col) && "Full Name" %in% names(lookup_dt)) full_col <- "Full Name" + + if (!is.null(full_col)) { + lookup_drugs <- unique(tolower(trimws(as.character(lookup_dt[[full_col]])))) + lookup_drugs <- lookup_drugs[!is.na(lookup_drugs) & nzchar(lookup_drugs)] + lookup_drugs <- setdiff(lookup_drugs, c("control", "dimethyl_sulfoxide", "dimethyl sulfoxide", "dmso")) + } else { + message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") + message("WARNING: Could not find Full_name column in lookup file. Columns: ", + paste(names(lookup_dt), collapse = ", ")) + } + } else { + message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") + message("WARNING: Could not read lookup file at: ", lookup_path) + } +} else { + message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") + message("WARNING: lookup file missing/unavailable for syn64608501") +} + +message("\n========== DEBUG: DRUG LOOKUP LIST (syn64608501) ==========") +message("lookup_path: ", ifelse(is.na(lookup_path), "", lookup_path)) +message("lookup_drugs count: ", length(lookup_drugs)) +if (length(lookup_drugs) > 0) { + message("First 40 lookup_drugs: ", paste(head(lookup_drugs, 40), collapse = ", ")) +} + # 4) Combine and fix bad names -all_drugs <- unique(c(pdx_drugs, mts_drugs)) +all_drugs <- unique(c(pdx_drugs, mts_drugs, improve_sample_drugs, lookup_drugs)) all_drugs[all_drugs == "pd901"] <- "pd-0325901" -# message("Combined drug list: ", paste(all_drugs, collapse=", ")) +all_drugs[all_drugs == "rmc4630"] <- "rmc-4630" + +# IMPORTANT: drop NA/empty +all_drugs <- all_drugs[!is.na(all_drugs) & nzchar(all_drugs)] + +# ---- NEW: replace verteporfin with WU713D62N9 so we can do ONE name-based call +# (keep exact casing you provided) +if (any(all_drugs == "verteporfin", na.rm = TRUE)) { + all_drugs[all_drugs == "verteporfin"] <- "WU713D62N9" +} -# 5) Read old‑drug files or initialize empty +message("\n========== DEBUG: DRUG LISTS ==========") +message("all_drugs count: ", length(all_drugs)) +message("Contains 'verteporfin' in all_drugs? ", any(all_drugs == "verteporfin", na.rm=TRUE)) +message("Contains 'WU713D62N9' in all_drugs? ", any(all_drugs == "WU713D62N9", na.rm=TRUE)) +message("First 40 all_drugs: ", paste(head(all_drugs, 40), collapse=", ")) + +# 5) Read old-drug files or initialize empty if (!is.na(olddrugfiles)) { paths <- strsplit(olddrugfiles, ",")[[1]] %>% trimws() old_list <- lapply(paths, function(f) { if (file.exists(f)) fread(f, sep="\t", header=TRUE) else { - warning("Missing old‑drug file: ", f) + warning("Missing old-drug file: ", f) NULL } }) @@ -101,36 +228,95 @@ if (!is.na(olddrugfiles)) { pubchem_id=character(), canSMILES=character(), InChIKey=character(), formula=character(), weight=numeric() ) - message("No old‑drug files provided; starting fresh") + message("No old-drug files provided; starting fresh") } # 6) Write placeholder fwrite(olddrugs, newdrugfile, sep="\t", quote=FALSE) message("Wrote placeholder to ", newdrugfile) -# 7) Augment via Python +# 7) Augment via Python (single call) ignore_file <- "/tmp/combined_drugs_ignore_chems.txt" -use_python("/opt/venv/bin/python3", required=TRUE) -# use_python("/Users/jaco059/miniconda3/bin/python3", required=TRUE) -# source_python("build/utils/pubchem_retrieval.py") +message("\n========== DEBUG: IGNORE FILE ==========") +message("ignore_file: ", ignore_file) +message("ignore_file exists? ", file.exists(ignore_file)) +if (file.exists(ignore_file)) { + ign <- tolower(trimws(readLines(ignore_file, warn=FALSE))) + message("ignore_file lines (first 50): ", paste(head(ign, 50), collapse=" | ")) + message("ignore contains 'verteporfin'? ", any(ign == "verteporfin")) + message("ignore contains 'wu713d62n9'? ", any(ign == "wu713d62n9")) +} + +use_python("/opt/venv/bin/python3", required=TRUE) source_python("pubchem_retrieval.py") -# prepare prev_drug_filepaths argument for python helper (NULL if none) prev_paths <- if (!is.na(olddrugfiles)) olddrugfiles else NULL -update_dataframe_and_write_tsv( - unique_names = all_drugs, - output_filename = newdrugfile, - ignore_chems = ignore_file, - batch_size = as.integer(50), - isname = TRUE, - prev_drug_filepaths = prev_paths, - restrict_to_raw_names = all_drugs -) - - -# 8) Final filter & save -tab <- fread(newdrugfile, sep="\t", header=TRUE) + +# reticulate-safe list conversion +name_drugs_py <- as.list(as.character(all_drugs)) + +if (length(all_drugs) > 0) { + message("\n========== DEBUG: RUN SINGLE NAME-BASED PYTHON CALL ==========") + message("Calling update_dataframe_and_write_tsv with isname=TRUE for ", length(all_drugs), " names.") + message("First 30 all_drugs: ", paste(head(all_drugs, 30), collapse=", ")) + + update_dataframe_and_write_tsv( + unique_names = name_drugs_py, + output_filename = newdrugfile, + ignore_chems = ignore_file, + batch_size = as.integer(50), + isname = TRUE, + prev_drug_filepaths = prev_paths, + restrict_to_raw_names = name_drugs_py + ) + + debug_file_summary(newdrugfile, "AFTER SINGLE NAME-BASED PYTHON CALL (newdrugfile)") +} else { + message("\n========== DEBUG: SKIP PYTHON CALL (all_drugs empty) ==========") +} + +# 8) Final unique & save (no CID merge) +message("\n========== DEBUG: MAIN OUTPUT BEFORE FINAL UNIQUE ==========") +debug_file_summary(newdrugfile, "MAIN OUTPUT BEFORE FINAL UNIQUE (newdrugfile)") + +tab <- fread(newdrugfile, sep="\t", header=TRUE) final_tab <- unique(tab) + +message("\n========== DEBUG: FINAL CHECK ==========") +message("Final rows after unique(): ", nrow(final_tab)) +if ("chem_name" %in% names(final_tab)) { + cnf <- unique(tolower(trimws(as.character(final_tab$chem_name)))) + message("FINAL has verteporfin? ", any(cnf == "verteporfin", na.rm=TRUE)) + message("FINAL has wu713d62n9? ", any(cnf == "wu713d62n9", na.rm=TRUE)) +} + +# 9) Ensure bare "verteporfin" exists as a chem_name row. +# PubChem synonyms often return only qualified variants like +# "verteporfin [usan:usp:inn:ban]" but not the bare name, which +# causes exact-join failures downstream in 03_get_experiments. +if ("chem_name" %in% names(final_tab)) { + cn_lower <- tolower(trimws(as.character(final_tab$chem_name))) + has_bare <- any(cn_lower == "verteporfin", na.rm = TRUE) + + if (!has_bare) { + # Look for any row whose chem_name contains "verteporfin" or "verteporphin" + match_idx <- which(grepl("verteporfin|verteporphin", cn_lower)) + if (length(match_idx) > 0) { + donor_row <- final_tab[match_idx[1], ] + donor_row[["chem_name"]] <- "verteporfin" + final_tab <- rbind(donor_row, final_tab) + message("Inserted bare 'verteporfin' alias row (from row with chem_name='", + as.character(final_tab$chem_name[match_idx[1] + 1]), # +1 because we prepended + "', improve_drug_id=", donor_row$improve_drug_id, ")") + } else { + message("WARNING: no verteporfin/verteporphin variant found in drug file; cannot insert bare alias.") + } + } else { + message("Bare 'verteporfin' already present in drug file; no insertion needed.") + } +} + + fwrite(final_tab, newdrugfile, sep="\t", quote=FALSE) -message("Wrote full synonyms list to ", newdrugfile) \ No newline at end of file +message("Wrote full synonyms list to ", newdrugfile) diff --git a/coderbuild/mpnst/03_get_experiments.R b/coderbuild/mpnst/03_get_experiments.R index a430cae8..c2921970 100644 --- a/coderbuild/mpnst/03_get_experiments.R +++ b/coderbuild/mpnst/03_get_experiments.R @@ -1,282 +1,473 @@ -library(data.table) -library(synapser) -library(dplyr) -library(stringr) -library(readr) -library(readxl) -library(tidyr) - -# Check that correct number of arguments are present -args <- commandArgs(trailingOnly = TRUE) -if (length(args) != 4) { - stop("Usage: Rscript 03_get_experiments.R ", call. = FALSE) -} -PAT <- args[1] -samples <- args[2] -drugfile <- args[3] -out_prefix <- args[4] - -synLogin(authToken = PAT) - -# Read in sampes file -samples_df <- fread(samples) %>% - select(improve_sample_id, common_name, model_type) %>% - distinct() - -pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") -mt_samps <- filter(samples_df, model_type == "xenograft derived organoid") - -# Get manifest table from Synapse -manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% - rename(common_name = Sample) %>% - as.data.table() - -# Helper Function to extract date and hour from experiment ID -extract_date_hour <- function(experiment_id) { - pattern <- "(\\d{6})_?(\\d{2,3})?" - m <- str_match(experiment_id, pattern) - date <- m[,2]; hour <- m[,3] - date[is.na(date)] <- NA - hour[is.na(hour)] <- 48 - list(date = date, hour = hour) -} - -# ──────────────────────────────────────────────── -# MicroTissue Experiments -# ──────────────────────────────────────────────── - -getDrugDataByParent <- function(parid, sampleId) { - q <- sprintf( - "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", - parid - ) - qtab <- synTableQuery(q)$asDataFrame() %>% - filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% - select(id, name, experimentalCondition) - do.call(rbind, lapply(qtab$id, function(x) { - info <- filter(qtab, id == x) - d <- extract_date_hour(info$name) - fread(synGet(x)$path) %>% - filter(response_type == "percent viability") %>% - transmute( - improve_sample_id = sampleId, - DOSE = (10^dosage) * 1e6, - GROWTH = response, - source = "NF Data Portal", - chem_name = compound_name, - study = paste0("MT ", d$date, " exp"), - time = d$hour - ) - })) -} - -# Create map of MicroTissue Drug Folders -mts_map <- manifest %>% - select(common_name, MicroTissueDrugFolder) %>% - inner_join(mt_samps, by = "common_name") %>% - separate_rows(MicroTissueDrugFolder, sep = ",") %>% - # keep exactly what old script did: drop only "NA" and actual NA - filter( - !is.na(MicroTissueDrugFolder), - MicroTissueDrugFolder != "NA" - ) %>% - select( - improve_sample_id, - folder = MicroTissueDrugFolder - ) - -# Fetch all MicroTissue drug response data -mt_data <- do.call(rbind, lapply(seq_len(nrow(mts_map)), function(i) { - sample_id <- mts_map$improve_sample_id[i] - folder <- mts_map$folder[i] - getDrugDataByParent(folder, sample_id) -})) - -drug_map <- fread(drugfile) %>% - select(improve_drug_id, chem_name) %>% - distinct() - -# Clean up drug names and join with drug_map -mt_curve <- mt_data %>% - mutate( - chem_name = tolower(chem_name), - chem_name = ifelse(chem_name == "pd901", "pd-0325901", chem_name) - ) %>% - left_join(drug_map, by = "chem_name") %>% - filter(!is.na(improve_drug_id)) %>% - transmute( - source = source, - improve_sample_id = improve_sample_id, - Drug = improve_drug_id, - study = study, - time = time, - time_unit = "hours", - DOSE = DOSE, - GROWTH = GROWTH - ) - -# Run curve fitting, Write MicroTissue curve data -fwrite(mt_curve, file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")), sep = "\t") - -message("Wrote MT curve data") - -# Write MT experiments file -system(sprintf( - "/opt/venv/bin/python fit_curve.py --input %s --output %s", - paste0("/tmp/", out_prefix, "_mt_curve_data.tsv"), - paste0("/tmp/", out_prefix, "_mt_experiments") -)) -file.rename( - paste0("/tmp/", out_prefix, "_mt_experiments.0"), - paste0("/tmp/", out_prefix, "_mt_experiments.tsv") -) -message("Wrote MT experiments") - -# ──────────────────────────────────────────────── -# PDX Experiments -# ──────────────────────────────────────────────── - -# Create a map of PDX Drug Data -# This will be used to fetch the drug data for each PDX sample -pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { - row <- manifest[i, ] - samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] - if (nrow(samp)==0 || is.na(row$PDX_Drug_Data) || row$PDX_Drug_Data %in% c("", "NA")) - return(NULL) - ids <- strsplit(row$PDX_Drug_Data, ",")[[1]] - ids <- trimws(ids[ids!=""]) - data.frame( - improve_sample_id = samp$improve_sample_id, - child_id = ids, - stringsAsFactors = FALSE - ) -})) - -# Create a dataframe of PDX metadata -pdx_meta <- do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { - sid <- pdx_map$improve_sample_id[i] - cid <- pdx_map$child_id[i] - pid <- synGet(cid)$parentId - if (is.null(pid) || pid=="") stop("no parentId for ", cid) - data.frame( - improve_sample_id = sid, - child_id = cid, - parentId = pid, - stringsAsFactors = FALSE - ) -})) - -all_pdx <- do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { - m <- pdx_meta[i, ] - pth <- synGet(m$child_id)$path - raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) - - # detect second‐drug column - sec_opts <- c("compound 2_name", "compound_2_name") - drug2_col <- intersect(sec_opts, names(raw))[1] - compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ - - df <- data.frame( - child_id = m$child_id, - specimen_id = raw$specimen_id, - compound_name = raw$compound_name, - compound_2_name = compound2, - experimental_time_point = raw$experimental_time_point, - experimental_time_point_unit = raw$experimental_time_point_unit, - assay_value = raw$assay_value, - stringsAsFactors = FALSE - ) - - df <- within(df, { - drug1 <- tolower(trimws(compound_name)) - drug2 <- tolower(trimws(compound_2_name)) - treatment <- ifelse( - is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), - "control", - ifelse(!is.na(drug2) & drug2 != "", - paste(drug1, drug2, sep = "+"), - drug1 - ) - ) - time <- experimental_time_point - time_unit <- experimental_time_point_unit - volume <- assay_value - }) - - df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] -})) - -# join on parentId and sample -pdx_data <- merge(all_pdx, pdx_meta, by="child_id") - -pdx_data <- subset(pdx_data, duplicated(child_id) | TRUE) -pdx_data <- within(pdx_data, { - experiment <- parentId - model_id <- improve_sample_id -}) - -# Filter out experiments missing a control -has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) -no_ctl_exps <- names(has_ctl)[!has_ctl] -pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] - -# Reorder final columns -pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", - "time","time_unit","volume","model_id")] - -# Correct doxorubinsin typo across all data -pdx_data$treatment <- gsub("doxorubinsin", - "doxorubicin", - pdx_data$treatment, - ignore.case = TRUE) - -# Drop any remaining NA rows -pdx_data <- na.omit(pdx_data) - -# write & fit -fwrite(pdx_data, file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")), sep = "\t") - - -message("Wrote PDX curve data") - -system(sprintf( - "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", - paste0("/tmp/", out_prefix, "_pdx_curve_data.tsv"), - drugfile, - paste0("/tmp/", out_prefix, "_pdx") -)) - - - -message("Wrote PDX experiments to ", "/tmp/", out_prefix, "_pdx_experiments.tsv and combinations") - - -# ──────────────────────────────────────────────── -# Combine all Experiments -# ──────────────────────────────────────────────── - -# Read MicroTissue experiments -mt_exp <- fread(paste0("/tmp/", out_prefix, "_mt_experiments.tsv")) %>% - mutate( - dose_response_value = as.character(dose_response_value) - ) - -# Read PDX experiments -pdx_exp <- fread(paste0("/tmp/", out_prefix, "_pdx_experiments.tsv")) %>% - mutate( - dose_response_value = as.character(dose_response_value) - ) - -# Join experiments into one. -all_exp <- bind_rows(mt_exp, pdx_exp) - -# Write out Experiments -fwrite(all_exp, paste0("/tmp/", out_prefix, "_experiments.tsv"), sep = "\t") -message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") - - -# Rename the Drug Combination data file to fit schema naming -file.rename( - paste0("/tmp/", out_prefix, "_pdx_combinations.tsv"), - paste0("/tmp/", out_prefix, "_combinations.tsv") -) \ No newline at end of file +# This is the original version of the script before adding treated-combo support. +# Not actually called in this branch, but kept for reference. + +# # 03_get_experiments.R +# #!/usr/bin/env Rscript + +# library(data.table) +# library(synapser) +# library(dplyr) +# library(stringr) +# library(readr) +# library(readxl) +# library(tidyr) + +# # ============================================================ +# # 03_get_experiments.R +# # - MT (original): unchanged logic, but forces time -> integer +# # - MT (treated): robust parsing + sample duplication mapping +# # - PDX: unchanged +# # ============================================================ + +# # ----------------------- +# # args +# # ----------------------- +# args <- commandArgs(trailingOnly = TRUE) +# if (length(args) != 4) { +# stop("Usage: Rscript 03_get_experiments.R ", call. = FALSE) +# } +# PAT <- args[1] +# samples <- args[2] +# drugfile <- args[3] +# out_prefix <- args[4] + +# synLogin(authToken = PAT) + +# # ----------------------- +# # helpers +# # ----------------------- +# norm_str <- function(x) tolower(trimws(as.character(x))) + +# # normalize drug strings for joining +# norm_drug <- function(x) { +# x <- norm_str(x) +# x <- ifelse(x == "pd901", "pd-0325901", x) +# x +# } + +# # make column names consistent across files +# std_names <- function(nms) { +# nms <- tolower(nms) +# nms <- gsub("[^a-z0-9]+", "_", nms) +# nms <- gsub("_+", "_", nms) +# nms <- gsub("^_|_$", "", nms) +# nms +# } + +# # choose first existing column from candidates +# pick_col <- function(df, candidates) { +# candidates <- std_names(candidates) +# nm <- names(df) +# hit <- intersect(candidates, nm) +# if (length(hit) == 0) return(NA_character_) +# hit[1] +# } + +# # safe annotation get +# get_ann <- function(synid) { +# a <- tryCatch(synGetAnnotations(synid)$annotations, error = function(e) NULL) +# if (is.null(a)) return(list()) +# a +# } + +# # ----------------------- +# # load inputs +# # ----------------------- +# samples_all <- fread(samples) %>% as.data.frame() + +# samples_df <- samples_all %>% +# dplyr::select(improve_sample_id, common_name, model_type) %>% +# distinct() + +# pdx_samps <- dplyr::filter(samples_df, model_type == "patient derived xenograft") +# mt_samps <- dplyr::filter(samples_df, model_type == "xenograft derived organoid") + +# drug_map <- fread(drugfile) %>% +# dplyr::select(improve_drug_id, chem_name) %>% +# distinct() %>% +# mutate(chem_name = norm_drug(chem_name)) + +# # ----------------------- +# # manifest (unchanged) +# # ----------------------- +# manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% +# rename(common_name = Sample) %>% +# as.data.table() + +# extract_date_hour <- function(experiment_id) { +# pattern <- "(\\d{6})_?(\\d{2,3})?" +# m <- str_match(experiment_id, pattern) +# date <- m[,2]; hour <- m[,3] +# date[is.na(date)] <- NA +# hour[is.na(hour)] <- 48 +# # force integer hour +# hour <- suppressWarnings(as.integer(hour)) +# hour[is.na(hour)] <- 48L +# list(date = date, hour = hour) +# } + +# # ============================================================ +# # 1) MT ORIGINAL (same logic; ensure time integer) +# # ============================================================ + +# getDrugDataByParent <- function(parid, sampleId) { +# q <- sprintf( +# "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", +# parid +# ) +# qtab <- synTableQuery(q)$asDataFrame() %>% +# filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% +# select(id, name, experimentalCondition) + +# do.call(rbind, lapply(qtab$id, function(x) { +# info <- filter(qtab, id == x) +# d <- extract_date_hour(info$name) + +# fread(synGet(x)$path) %>% +# filter(response_type == "percent viability") %>% +# transmute( +# improve_sample_id = sampleId, +# DOSE = (10^dosage) * 1e6, +# GROWTH = response, +# source = "NF Data Portal", +# chem_name = compound_name, +# study = paste0("MT ", d$date, " exp"), +# time = as.integer(d$hour) +# ) +# })) +# } + +# mts_map <- manifest %>% +# select(common_name, MicroTissueDrugFolder) %>% +# inner_join(mt_samps, by = "common_name") %>% +# separate_rows(MicroTissueDrugFolder, sep = ",") %>% +# filter(!is.na(MicroTissueDrugFolder), MicroTissueDrugFolder != "NA") %>% +# select(improve_sample_id, folder = MicroTissueDrugFolder) + +# mt_data <- do.call(rbind, lapply(seq_len(nrow(mts_map)), function(i) { +# sample_id <- mts_map$improve_sample_id[i] +# folder <- mts_map$folder[i] +# getDrugDataByParent(folder, sample_id) +# })) + +# mt_curve_orig <- mt_data %>% +# mutate( +# chem_name = norm_drug(chem_name), +# time = suppressWarnings(as.integer(time)) +# ) %>% +# left_join(drug_map, by = "chem_name") %>% +# filter(!is.na(improve_drug_id)) %>% +# transmute( +# source = source, +# improve_sample_id = improve_sample_id, +# Drug = improve_drug_id, +# study = study, +# time = as.integer(time), +# time_unit = "hours", +# DOSE = DOSE, +# GROWTH = GROWTH +# ) + +# message("Built MT original curve rows: ", nrow(mt_curve_orig)) + +# # ============================================================ +# # 2) MT TREATED (robust parsing + duplication mapping) +# # ============================================================ + +# # treated mapping from samples.csv: +# # extract drug_key from other_id using regex (no vectorized grepl patterns) +# treated_lookup <- samples_all %>% +# filter(!is.na(other_id), grepl("_treated_microtissue$", other_id)) %>% +# mutate( +# base_common_name = common_name, +# other_id_clean = as.character(other_id), +# # drug_key = part between first "_" and "_hr_treated_microtissue" +# drug_key = sub("^.*?_", "", other_id_clean), +# drug_key = sub("_\\d+hr_treated_microtissue$", "", drug_key), +# drug_key = norm_drug(drug_key) +# ) %>% +# select(base_common_name, drug_key, improve_sample_id) %>% +# distinct() + +# # generic fallback: xenograft derived organoid row for base_common_name (exclude treated rows) +# generic_xdo_lookup <- samples_all %>% +# filter( +# model_type == "xenograft derived organoid", +# !is.na(common_name), +# (is.na(other_id) | !grepl("_treated_microtissue$", other_id)) +# ) %>% +# select(base_common_name = common_name, generic_improve_sample_id = improve_sample_id) %>% +# distinct() + +# # enumerate candidate treated MT files from the same index table +# mt_index <- synTableQuery("select id,name,experimentalCondition,parentId from syn21993642")$asDataFrame() %>% +# filter(name != "synapse_storage_manifest.csv") %>% +# select(id, name, experimentalCondition) + +# # build treated curve rows for one file +# get_treated_curve_rows_one <- function(file_id, fallback_condition = NA_character_) { + +# ann <- get_ann(file_id) + +# # always map sample base name from individualID +# base_common_name <- NA_character_ +# if (!is.null(ann$individualID) && length(ann$individualID) > 0) { +# base_common_name <- as.character(ann$individualID[[1]]) +# } +# if (is.na(base_common_name) || base_common_name == "") return(NULL) + +# # drug for treated sample selection: experimentalCondition (annotation preferred) +# drug_raw <- NA_character_ +# if (!is.null(ann$experimentalCondition) && length(ann$experimentalCondition) > 0) { +# drug_raw <- as.character(ann$experimentalCondition[[1]]) +# } else if (!is.na(fallback_condition)) { +# drug_raw <- as.character(fallback_condition) +# } +# if (is.na(drug_raw) || drug_raw == "") return(NULL) + +# drug_key <- norm_drug(drug_raw) + +# # load file + standardize columns +# pth <- synGet(file_id)$path +# dt <- suppressWarnings(fread(pth)) +# if (nrow(dt) == 0) return(NULL) + +# setnames(dt, names(dt), std_names(names(dt))) + +# # pick key columns robustly +# col_resp_type <- pick_col(dt, c("response_type", "response type")) +# col_resp <- pick_col(dt, c("response", "value")) +# col_dose <- pick_col(dt, c("dosage", "dose", "log_m", "logm")) +# col_cmpd <- pick_col(dt, c("compound_name", "compound", "drug", "compoundname")) + +# if (is.na(col_resp_type) || is.na(col_resp) || is.na(col_dose)) return(NULL) + +# # percent viability only +# dt <- dt[get(col_resp_type) == "percent viability", ] +# if (nrow(dt) == 0) return(NULL) + +# # map improve_drug_id: +# # 1) annotation experimentalCondition +# # 2) fallback to file compound_name if present +# improve_drug_id <- NA_character_ + +# dm1 <- drug_map %>% filter(chem_name == drug_key) +# if (nrow(dm1) > 0) improve_drug_id <- dm1$improve_drug_id[1] + +# if (is.na(improve_drug_id) && !is.na(col_cmpd)) { +# chem2 <- norm_drug(dt[[col_cmpd]][1]) +# dm2 <- drug_map %>% filter(chem_name == chem2) +# if (nrow(dm2) > 0) improve_drug_id <- dm2$improve_drug_id[1] +# } + +# # if still unmapped drug, skip (can be relaxed later when you provide more mapping rules) +# if (is.na(improve_drug_id) || improve_drug_id == "") return(NULL) + +# # sample mapping: +# # - if treated match exists: DUPLICATE across all improve_sample_id matches +# # - else fallback to generic XDO sample for that individualID +# treated_matches <- treated_lookup %>% +# filter(base_common_name == !!base_common_name, drug_key == !!drug_key) + +# if (nrow(treated_matches) == 0) { +# gen <- generic_xdo_lookup %>% filter(base_common_name == !!base_common_name) +# if (nrow(gen) == 0) return(NULL) +# target_ids <- gen$generic_improve_sample_id +# } else { +# target_ids <- treated_matches$improve_sample_id +# } + +# # build output duplicated per target id +# out <- do.call(rbind, lapply(target_ids, function(sid) { +# data.frame( +# source = "NF Data Portal", +# improve_sample_id = sid, +# Drug = improve_drug_id, +# study = "MT treated", +# time = 48L, +# time_unit = "hours", +# DOSE = (10^(as.numeric(dt[[col_dose]]))) * 1e6, +# GROWTH = as.numeric(dt[[col_resp]]), +# stringsAsFactors = FALSE +# ) +# })) + +# out +# } + +# mt_curve_treated_list <- lapply(seq_len(nrow(mt_index)), function(i) { +# get_treated_curve_rows_one( +# file_id = mt_index$id[i], +# fallback_condition = mt_index$experimentalCondition[i] +# ) +# }) + +# mt_curve_treated <- do.call(rbind, mt_curve_treated_list) + +# if (is.null(mt_curve_treated) || nrow(mt_curve_treated) == 0) { +# mt_curve_treated <- data.frame( +# source = character(), +# improve_sample_id = integer(), +# Drug = character(), +# study = character(), +# time = integer(), +# time_unit = character(), +# DOSE = numeric(), +# GROWTH = numeric(), +# stringsAsFactors = FALSE +# ) +# } + +# mt_curve_treated <- mt_curve_treated %>% +# mutate(time = as.integer(time)) + +# message("Built MT treated curve rows: ", nrow(mt_curve_treated)) + +# # ============================================================ +# # 3) Combine MT and fit once +# # ============================================================ + +# # ensure time is integer on both sides (prevents your bind_rows crash) +# mt_curve_orig <- mt_curve_orig %>% mutate(time = as.integer(time)) +# mt_curve_treated <- mt_curve_treated %>% mutate(time = as.integer(time)) + +# mt_curve_all <- bind_rows(mt_curve_orig, mt_curve_treated) + +# fwrite(mt_curve_all, file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")), sep = "\t") +# message("Wrote MT curve data: /tmp/", out_prefix, "_mt_curve_data.tsv") + +# system(sprintf( +# "/opt/venv/bin/python fit_curve.py --input %s --output %s", +# paste0("/tmp/", out_prefix, "_mt_curve_data.tsv"), +# paste0("/tmp/", out_prefix, "_mt_experiments") +# )) + +# file.rename( +# paste0("/tmp/", out_prefix, "_mt_experiments.0"), +# paste0("/tmp/", out_prefix, "_mt_experiments.tsv") +# ) +# message("Wrote MT experiments") + +# # ============================================================ +# # 4) PDX (UNCHANGED) +# # ============================================================ + +# pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { +# row <- manifest[i, ] +# samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] +# if (nrow(samp)==0 || is.na(row$PDX_Drug_Data) || row$PDX_Drug_Data %in% c("", "NA")) +# return(NULL) +# ids <- strsplit(row$PDX_Drug_Data, ",")[[1]] +# ids <- trimws(ids[ids!=""]) +# data.frame( +# improve_sample_id = samp$improve_sample_id, +# child_id = ids, +# stringsAsFactors = FALSE +# ) +# })) + +# pdx_meta <- do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { +# sid <- pdx_map$improve_sample_id[i] +# cid <- pdx_map$child_id[i] +# pid <- synGet(cid)$parentId +# if (is.null(pid) || pid=="") stop("no parentId for ", cid) +# data.frame( +# improve_sample_id = sid, +# child_id = cid, +# parentId = pid, +# stringsAsFactors = FALSE +# ) +# })) + +# all_pdx <- do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { +# m <- pdx_meta[i, ] +# pth <- synGet(m$child_id)$path +# raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) + +# sec_opts <- c("compound 2_name", "compound_2_name") +# drug2_col <- intersect(sec_opts, names(raw))[1] +# compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ + +# df <- data.frame( +# child_id = m$child_id, +# specimen_id = raw$specimen_id, +# compound_name = raw$compound_name, +# compound_2_name = compound2, +# experimental_time_point = raw$experimental_time_point, +# experimental_time_point_unit = raw$experimental_time_point_unit, +# assay_value = raw$assay_value, +# stringsAsFactors = FALSE +# ) + +# df <- within(df, { +# drug1 <- tolower(trimws(compound_name)) +# drug2 <- tolower(trimws(compound_2_name)) +# treatment <- ifelse( +# is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), +# "control", +# ifelse(!is.na(drug2) & drug2 != "", +# paste(drug1, drug2, sep = "+"), +# drug1 +# ) +# ) +# time <- experimental_time_point +# time_unit <- experimental_time_point_unit +# volume <- assay_value +# }) + +# df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] +# })) + +# pdx_data <- merge(all_pdx, pdx_meta, by="child_id") + +# pdx_data <- subset(pdx_data, duplicated(child_id) | TRUE) +# pdx_data <- within(pdx_data, { +# experiment <- parentId +# model_id <- improve_sample_id +# }) + +# has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) +# no_ctl_exps <- names(has_ctl)[!has_ctl] +# pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] + +# pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", +# "time","time_unit","volume","model_id")] + +# pdx_data$treatment <- gsub("doxorubinsin", +# "doxorubicin", +# pdx_data$treatment, +# ignore.case = TRUE) + +# pdx_data <- na.omit(pdx_data) + +# fwrite(pdx_data, file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")), sep = "\t") +# message("Wrote PDX curve data") + +# system(sprintf( +# "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", +# paste0("/tmp/", out_prefix, "_pdx_curve_data.tsv"), +# drugfile, +# paste0("/tmp/", out_prefix, "_pdx") +# )) + +# message("Wrote PDX experiments to /tmp/", out_prefix, "_pdx_experiments.tsv and combinations") + +# # ============================================================ +# # 5) Combine all Experiments (unchanged) +# # ============================================================ + +# mt_exp <- fread(paste0("/tmp/", out_prefix, "_mt_experiments.tsv")) %>% +# mutate(dose_response_value = as.character(dose_response_value)) + +# pdx_exp <- fread(paste0("/tmp/", out_prefix, "_pdx_experiments.tsv")) %>% +# mutate(dose_response_value = as.character(dose_response_value)) + +# all_exp <- bind_rows(mt_exp, pdx_exp) + +# fwrite(all_exp, paste0("/tmp/", out_prefix, "_experiments.tsv"), sep = "\t") +# message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") + +# file.rename( +# paste0("/tmp/", out_prefix, "_pdx_combinations.tsv"), +# paste0("/tmp/", out_prefix, "_combinations.tsv") +# ) diff --git a/coderbuild/mpnst/03_get_experiments_orig.R b/coderbuild/mpnst/03_get_experiments_orig.R new file mode 100644 index 00000000..5aa42151 --- /dev/null +++ b/coderbuild/mpnst/03_get_experiments_orig.R @@ -0,0 +1,286 @@ +# This is an older version of the script. +# Not actually called in this branch, but kept for reference. + +# # 03_get_experiments_orig.R +# library(data.table) +# library(synapser) +# library(dplyr) +# library(stringr) +# library(readr) +# library(readxl) +# library(tidyr) + +# # Check that correct number of arguments are present +# args <- commandArgs(trailingOnly = TRUE) +# if (length(args) != 4) { +# stop("Usage: Rscript 03_get_experiments.R ", call. = FALSE) +# } +# PAT <- args[1] +# samples <- args[2] +# drugfile <- args[3] +# out_prefix <- args[4] + +# synLogin(authToken = PAT) + +# # Read in sampes file +# samples_df <- fread(samples) %>% +# select(improve_sample_id, common_name, model_type) %>% +# distinct() + +# pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") +# mt_samps <- filter(samples_df, model_type == "3D-MEDS") + +# # Get manifest table from Synapse +# manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% +# rename(common_name = Sample) %>% +# as.data.table() + +# # Helper Function to extract date and hour from experiment ID +# extract_date_hour <- function(experiment_id) { +# pattern <- "(\\d{6})_?(\\d{2,3})?" +# m <- str_match(experiment_id, pattern) +# date <- m[,2]; hour <- m[,3] +# date[is.na(date)] <- NA +# hour[is.na(hour)] <- 48 +# list(date = date, hour = hour) +# } + +# # ──────────────────────────────────────────────── +# # MicroTissue Experiments +# # ──────────────────────────────────────────────── + +# getDrugDataByParent <- function(parid, sampleId) { +# q <- sprintf( +# "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", +# parid +# ) +# qtab <- synTableQuery(q)$asDataFrame() %>% +# filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% +# select(id, name, experimentalCondition) +# do.call(rbind, lapply(qtab$id, function(x) { +# info <- filter(qtab, id == x) +# d <- extract_date_hour(info$name) +# fread(synGet(x)$path) %>% +# filter(response_type == "percent viability") %>% +# transmute( +# improve_sample_id = sampleId, +# DOSE = (10^dosage) * 1e6, +# GROWTH = response, +# source = "NF Data Portal", +# chem_name = compound_name, +# study = paste0("MT ", d$date, " exp"), +# time = d$hour +# ) +# })) +# } + +# # Create map of MicroTissue Drug Folders +# mts_map <- manifest %>% +# select(common_name, MicroTissueDrugFolder) %>% +# inner_join(mt_samps, by = "common_name") %>% +# separate_rows(MicroTissueDrugFolder, sep = ",") %>% +# # keep exactly what old script did: drop only "NA" and actual NA +# filter( +# !is.na(MicroTissueDrugFolder), +# MicroTissueDrugFolder != "NA" +# ) %>% +# select( +# improve_sample_id, +# folder = MicroTissueDrugFolder +# ) + +# # Fetch all MicroTissue drug response data +# mt_data <- do.call(rbind, lapply(seq_len(nrow(mts_map)), function(i) { +# sample_id <- mts_map$improve_sample_id[i] +# folder <- mts_map$folder[i] +# getDrugDataByParent(folder, sample_id) +# })) + +# drug_map <- fread(drugfile) %>% +# select(improve_drug_id, chem_name) %>% +# distinct() + +# # Clean up drug names and join with drug_map +# mt_curve <- mt_data %>% +# mutate( +# chem_name = tolower(chem_name), +# chem_name = ifelse(chem_name == "pd901", "pd-0325901", chem_name) +# ) %>% +# left_join(drug_map, by = "chem_name") %>% +# filter(!is.na(improve_drug_id)) %>% +# transmute( +# source = source, +# improve_sample_id = improve_sample_id, +# Drug = improve_drug_id, +# study = study, +# time = time, +# time_unit = "hours", +# DOSE = DOSE, +# GROWTH = GROWTH +# ) + +# # Run curve fitting, Write MicroTissue curve data +# fwrite(mt_curve, file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")), sep = "\t") + +# message("Wrote MT curve data") + +# # Write MT experiments file +# system(sprintf( +# "/opt/venv/bin/python fit_curve.py --input %s --output %s", +# paste0("/tmp/", out_prefix, "_mt_curve_data.tsv"), +# paste0("/tmp/", out_prefix, "_mt_experiments") +# )) +# file.rename( +# paste0("/tmp/", out_prefix, "_mt_experiments.0"), +# paste0("/tmp/", out_prefix, "_mt_experiments.tsv") +# ) +# message("Wrote MT experiments") + +# # ──────────────────────────────────────────────── +# # PDX Experiments +# # ──────────────────────────────────────────────── + +# # Create a map of PDX Drug Data +# # This will be used to fetch the drug data for each PDX sample +# pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { +# row <- manifest[i, ] +# samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] +# if (nrow(samp)==0 || is.na(row$PDX_Drug_Data) || row$PDX_Drug_Data %in% c("", "NA")) +# return(NULL) +# ids <- strsplit(row$PDX_Drug_Data, ",")[[1]] +# ids <- trimws(ids[ids!=""]) +# data.frame( +# improve_sample_id = samp$improve_sample_id, +# child_id = ids, +# stringsAsFactors = FALSE +# ) +# })) + +# # Create a dataframe of PDX metadata +# pdx_meta <- do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { +# sid <- pdx_map$improve_sample_id[i] +# cid <- pdx_map$child_id[i] +# pid <- synGet(cid)$parentId +# if (is.null(pid) || pid=="") stop("no parentId for ", cid) +# data.frame( +# improve_sample_id = sid, +# child_id = cid, +# parentId = pid, +# stringsAsFactors = FALSE +# ) +# })) + +# all_pdx <- do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { +# m <- pdx_meta[i, ] +# pth <- synGet(m$child_id)$path +# raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) + +# # detect second‐drug column +# sec_opts <- c("compound 2_name", "compound_2_name") +# drug2_col <- intersect(sec_opts, names(raw))[1] +# compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ + +# df <- data.frame( +# child_id = m$child_id, +# specimen_id = raw$specimen_id, +# compound_name = raw$compound_name, +# compound_2_name = compound2, +# experimental_time_point = raw$experimental_time_point, +# experimental_time_point_unit = raw$experimental_time_point_unit, +# assay_value = raw$assay_value, +# stringsAsFactors = FALSE +# ) + +# df <- within(df, { +# drug1 <- tolower(trimws(compound_name)) +# drug2 <- tolower(trimws(compound_2_name)) +# treatment <- ifelse( +# is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), +# "control", +# ifelse(!is.na(drug2) & drug2 != "", +# paste(drug1, drug2, sep = "+"), +# drug1 +# ) +# ) +# time <- experimental_time_point +# time_unit <- experimental_time_point_unit +# volume <- assay_value +# }) + +# df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] +# })) + +# # join on parentId and sample +# pdx_data <- merge(all_pdx, pdx_meta, by="child_id") + +# pdx_data <- subset(pdx_data, duplicated(child_id) | TRUE) +# pdx_data <- within(pdx_data, { +# experiment <- parentId +# model_id <- improve_sample_id +# }) + +# # Filter out experiments missing a control +# has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) +# no_ctl_exps <- names(has_ctl)[!has_ctl] +# pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] + +# # Reorder final columns +# pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", +# "time","time_unit","volume","model_id")] + +# # Correct doxorubinsin typo across all data +# pdx_data$treatment <- gsub("doxorubinsin", +# "doxorubicin", +# pdx_data$treatment, +# ignore.case = TRUE) + +# # Drop any remaining NA rows +# pdx_data <- na.omit(pdx_data) + +# # write & fit +# fwrite(pdx_data, file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")), sep = "\t") + + +# message("Wrote PDX curve data") + +# system(sprintf( +# "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", +# paste0("/tmp/", out_prefix, "_pdx_curve_data.tsv"), +# drugfile, +# paste0("/tmp/", out_prefix, "_pdx") +# )) + + + +# message("Wrote PDX experiments to ", "/tmp/", out_prefix, "_pdx_experiments.tsv and combinations") + + +# # ──────────────────────────────────────────────── +# # Combine all Experiments +# # ──────────────────────────────────────────────── + +# # Read MicroTissue experiments +# mt_exp <- fread(paste0("/tmp/", out_prefix, "_mt_experiments.tsv")) %>% +# mutate( +# dose_response_value = as.character(dose_response_value) +# ) + +# # Read PDX experiments +# pdx_exp <- fread(paste0("/tmp/", out_prefix, "_pdx_experiments.tsv")) %>% +# mutate( +# dose_response_value = as.character(dose_response_value) +# ) + +# # Join experiments into one. +# all_exp <- bind_rows(mt_exp, pdx_exp) + +# # Write out Experiments +# fwrite(all_exp, paste0("/tmp/", out_prefix, "_experiments.tsv"), sep = "\t") +# message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") + + +# # Rename the Drug Combination data file to fit schema naming +# file.rename( +# paste0("/tmp/", out_prefix, "_pdx_combinations.tsv"), +# paste0("/tmp/", out_prefix, "_combinations.tsv") +# ) \ No newline at end of file diff --git a/coderbuild/mpnst/03_get_experiments_with_treated.R b/coderbuild/mpnst/03_get_experiments_with_treated.R new file mode 100644 index 00000000..8af7fa3b --- /dev/null +++ b/coderbuild/mpnst/03_get_experiments_with_treated.R @@ -0,0 +1,1640 @@ +#!/usr/bin/env Rscript +# 03_get_experiments_with_treated_combos.R +# +# DEBUG VERSION +# - Adds verbose print statements throughout MT + treated-microtissue code paths + helpers +# - Adds safe readers/guards so empty MT outputs don't crash downstream (instead they print diagnostics) +# + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(dplyr) + library(stringr) + library(readr) + library(readxl) + library(tidyr) +}) + +# ============================================================ +# FAST DEBUG SWITCH - skip ahead to later treated section (save time debugging) +# ============================================================ +SKIP_FAST <- FALSE # set to TRUE to skip MT/PDX/combine sections + +# ============================================================ +# PER-SECTION SKIP SWITCHES +# ============================================================ +SKIP_MT <- FALSE +SKIP_PDX <- FALSE +SKIP_COMBINE <- FALSE +SKIP_EXACT <- FALSE + +# Backward-compat: SKIP_FAST skips all heavy sections +if (isTRUE(SKIP_FAST)) { + SKIP_MT <- TRUE + SKIP_PDX <- TRUE + SKIP_COMBINE <- TRUE + SKIP_EXACT <- TRUE +} + +# ============================================================ +# DEBUG CONTROLS +# ============================================================ +DEBUG <- TRUE +DEBUG_MAX_ITEMS <- 10 # limit loop prints +DEBUG_HEAD_N <- 6 + +# ----------------------- +# args +# ----------------------- +args <- commandArgs(trailingOnly = TRUE) +if (length(args) != 4) { + stop("Usage: Rscript 03_get_experiments_with_treated_combos.R ", call. = FALSE) +} +PAT <- args[1] +samples <- args[2] +drugfile <- args[3] +out_prefix <- args[4] + +# ----------------------- +# helpers +# ----------------------- +cat0 <- function(...) cat(..., "\n", sep = "") +hr <- function() cat0(strrep("─", 80)) + +dbg_time <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +dbg <- function(...) { if (isTRUE(DEBUG)) cat0(dbg_time(), " | ", paste0(...)) } +dbg_hr <- function(title = NULL) { + if (!isTRUE(DEBUG)) return(invisible(NULL)) + hr() + if (!is.null(title)) cat0(dbg_time(), " | ", title) + hr() +} + +dbg_df <- function(df, label = "df") { + if (!isTRUE(DEBUG)) return(invisible(NULL)) + if (is.null(df)) { + dbg("[", label, "] is NULL") + return(invisible(NULL)) + } + dbg("[", label, "] rows=", nrow(df), " cols=", ncol(df)) + dbg("[", label, "] cols: ", paste(names(df), collapse = ", ")) + if (nrow(df) > 0) print(utils::head(df, DEBUG_HEAD_N)) + invisible(NULL) +} + +dbg_file <- function(path, label = "file") { + if (!isTRUE(DEBUG)) return(invisible(NULL)) + dbg("[", label, "] path=", path) + if (is.null(path) || is.na(path) || path == "" || !file.exists(path)) { + dbg("[", label, "] MISSING") + return(invisible(NULL)) + } + sz <- file.info(path)$size + dbg("[", label, "] exists size=", sz) + if (!is.na(sz) && sz > 0) { + dt <- tryCatch(data.table::fread(path, nrows = DEBUG_HEAD_N), error = function(e) NULL) + if (!is.null(dt)) { + dbg("[", label, "] peek rows=", nrow(dt), " cols=", ncol(dt)) + dbg("[", label, "] cols: ", paste(names(dt), collapse = ", ")) + print(dt) + } else { + dbg("[", label, "] peek fread failed") + } + } + invisible(NULL) +} + +dbg_stop <- function(...) { + dbg("FATAL: ", paste0(...)) + stop(paste0(...), call. = FALSE) +} + +# safer fread: returns data.frame() if missing/empty/unreadable +safe_fread_df <- function(path, label = "safe_fread") { + if (is.null(path) || is.na(path) || path == "" || !file.exists(path)) { + dbg("[", label, "] missing: ", path) + return(data.frame()) + } + sz <- file.info(path)$size + if (is.na(sz) || sz == 0) { + dbg("[", label, "] EMPTY (0 bytes): ", path) + return(data.frame()) + } + dt <- tryCatch(data.table::fread(path), error = function(e) { + dbg("[", label, "] fread ERROR: ", conditionMessage(e)) + return(NULL) + }) + if (is.null(dt)) return(data.frame()) + as.data.frame(dt) +} + +norm_str <- function(x) tolower(trimws(as.character(x))) + +norm_drug <- function(x) { + x <- norm_str(x) + x <- ifelse(x == "pd901", "pd-0325901", x) + x +} + +safe_token <- function(x) { + x <- as.character(x) + x <- trimws(x) + x <- gsub("\\s+", "_", x) + x <- gsub("[^A-Za-z0-9_\\-\\.]+", "_", x) + x +} + +warn_first_by <- function(df, group_cols, label) { + if (is.null(df) || nrow(df) == 0) { + dbg("[", label, "] warn_first_by: input empty") + return(df) + } + + counts <- df %>% + group_by(across(all_of(group_cols))) %>% + summarise(n = n(), .groups = "drop") + + dup_n <- sum(counts$n > 1) + if (dup_n > 0) { + warning(sprintf("[%s] Found %d duplicated key group(s); using first() for each group.", label, dup_n)) + if (isTRUE(DEBUG)) { + dbg("[", label, "] duplicated groups (top):") + print(utils::head(counts[counts$n > 1, ], DEBUG_HEAD_N)) + } + } + + df %>% + group_by(across(all_of(group_cols))) %>% + slice(1) %>% + ungroup() +} + +extract_date_hour <- function(experiment_id) { + pattern <- "(\\d{6})_?(\\d{2,3})?" + m <- str_match(experiment_id, pattern) + date <- m[,2]; hour <- m[,3] + date[is.na(date)] <- NA + hour[is.na(hour)] <- 48 + list(date = date, hour = hour) +} + +canon_name <- function(x) { + x <- norm_drug(x) + x <- gsub("\\s+", "", x) + x <- gsub("_", "", x) + x <- gsub("-", "", x) + x +} + +canon_id <- function(x) { + x <- toupper(trimws(as.character(x))) + x <- gsub("\\s+", "", x) + x <- gsub("_", "", x) + x <- gsub("-", "", x) + x +} + +split_plus_once <- function(x) { + x <- as.character(x) + if (!grepl("\\+", x)) return(c(NA_character_, NA_character_)) + parts <- strsplit(x, "\\+", fixed = FALSE)[[1]] + if (length(parts) < 2) return(c(NA_character_, NA_character_)) + if (length(parts) > 2) { + a <- parts[1] + b <- paste(parts[-1], collapse = "+") + return(c(a, b)) + } + c(parts[1], parts[2]) +} + +parse_combo_key <- function(combo_key) { + x <- as.character(combo_key) + if (!grepl("_", x) || !grepl("\\+", x)) { + return(list(base = NA_character_, drug1 = NA_character_, drug2 = NA_character_)) + } + base <- sub("^([^_]+)_.*$", "\\1", x) + combo_str <- sub("^[^_]+_", "", x) + dp <- split_plus_once(combo_str) + list(base = base, drug1 = dp[1], drug2 = dp[2]) +} + +escape_regex <- function(x) { + x <- as.character(x) + x <- gsub("([][{}()+*^$|\\\\?.])", "\\\\\\1", x) + x +} + +# ============================================================ +# fallback mapping (regex search in chem_name) +# ============================================================ +fallback_id_from_chem_name <- function(pattern_raw, chem_tbl) { + if (is.null(pattern_raw) || is.na(pattern_raw)) return(NA_character_) + p <- trimws(as.character(pattern_raw)) + if (p == "" || tolower(p) %in% c("na","n/a","nan")) return(NA_character_) + + patt <- escape_regex(p) + hits <- chem_tbl %>% + filter(grepl(patt, chem_name, ignore.case = TRUE)) %>% + pull(improve_drug_id) + + if (length(hits) > 0) return(as.character(hits[1])) + NA_character_ +} + +fill_ids_with_fallback <- function(df, raw_col, id_col, chem_tbl, label) { + stopifnot(raw_col %in% names(df), id_col %in% names(df)) + + missing_idx <- which(is.na(df[[id_col]]) | df[[id_col]] == "") + if (length(missing_idx) == 0) { + dbg(label, ": no missing ", id_col, " to fill.") + return(df) + } + + dbg(label, ": attempting fallback fill for ", length(missing_idx), + " missing ", id_col, " values using regex search in chem_name.") + + filled <- vapply(df[[raw_col]][missing_idx], fallback_id_from_chem_name, character(1), chem_tbl = chem_tbl) + + replace_idx <- missing_idx[!is.na(filled)] + if (length(replace_idx) > 0) { + df[[id_col]][replace_idx] <- filled[!is.na(filled)] + } + + dbg(label, ": fallback filled ", length(replace_idx), + " of ", length(missing_idx), " missing ", id_col, ".") + df +} + +# ============================================================ +# treated token extraction (supports underscores) +# ============================================================ +base_to_otherid_prefix <- function(common_name) { + gsub("-", "_", trimws(as.character(common_name))) +} + +extract_treated_token_from_other_id <- function(other_id, common_name) { + oid <- trimws(as.character(other_id)) + cn <- trimws(as.character(common_name)) + + if (is.na(oid) || oid == "" || is.na(cn) || cn == "") { + if (isTRUE(DEBUG)) dbg("[token_extract] invalid inputs oid/cn: oid=", oid, " cn=", cn) + return(NA_character_) + } + + treated_suffix_re <- "_(8hr|24hr)_treated_microtissue$" + no_suffix <- sub(treated_suffix_re, "", oid) + if (identical(no_suffix, oid)) { + if (isTRUE(DEBUG)) dbg("[token_extract] suffix not found: ", oid) + return(NA_character_) + } + + base_prefixes <- unique(c(cn, gsub("-", "_", cn))) + drug <- no_suffix + stripped <- FALSE + for (bp in base_prefixes) { + drug2 <- sub(paste0("^", escape_regex(bp), "_"), "", drug) + if (!identical(drug2, drug)) { + drug <- drug2 + stripped <- TRUE + break + } + } + if (!stripped && isTRUE(DEBUG)) { + dbg("[token_extract] base prefix NOT stripped (unexpected?) oid=", oid, " cn=", cn, " no_suffix=", no_suffix) + } + + drug <- trimws(drug) + if (drug == "" || tolower(drug) %in% c("na","n/a","nan")) { + if (isTRUE(DEBUG)) dbg("[token_extract] empty/NA drug after parse: oid=", oid, " cn=", cn, " parsed=", drug) + return(NA_character_) + } + drug +} + +# ============================================================ +# START +# ============================================================ +dbg_hr("SCRIPT START") +dbg("SKIP_FAST=", SKIP_FAST) +dbg("SKIP_MT=", SKIP_MT) +dbg("SKIP_PDX=", SKIP_PDX) +dbg("SKIP_COMBINE=", SKIP_COMBINE) +dbg("SKIP_EXACT=", SKIP_EXACT) +dbg("samples=", samples) +dbg("drugfile=", drugfile) +dbg("out_prefix=", out_prefix) + +synLogin(authToken = PAT) +dbg("Synapse login OK") + +# ----------------------- +# load inputs +# ----------------------- +dbg_hr("LOADING INPUTS") +samples_all <- fread(samples) %>% as.data.frame() +dbg_df(samples_all, "samples_all") +if (!all(c("improve_sample_id","common_name","model_type") %in% names(samples_all))) { + dbg_stop("samples.csv missing required columns: improve_sample_id, common_name, model_type") +} + +samples_df <- samples_all %>% + select(improve_sample_id, common_name, model_type) %>% + distinct() +dbg_df(samples_df, "samples_df") + +pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") +mt_samps <- filter(samples_df, model_type == "3D-MEDS") +dbg_df(pdx_samps, "pdx_samps") +dbg_df(mt_samps, "mt_samps") + +drug_map <- fread(drugfile) %>% + select(improve_drug_id, chem_name) %>% + distinct() %>% + mutate( + improve_drug_id = as.character(improve_drug_id), + chem_name = norm_drug(chem_name) + ) +dbg_df(drug_map, "drug_map") +dbg("drug_map unique chem_name=", length(unique(drug_map$chem_name))) + +manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% + rename(common_name = Sample) %>% + as.data.table() +if ("PDXDrugData" %in% names(manifest) && !"PDX_Drug_Data" %in% names(manifest)) + setnames(manifest, "PDXDrugData", "PDX_Drug_Data") +dbg_df(as.data.frame(manifest), "manifest") +if (!all(c("common_name","MicroTissueDrugFolder") %in% names(manifest))) { + dbg("manifest columns: ", paste(names(manifest), collapse=", ")) + dbg_stop("manifest missing MicroTissueDrugFolder and/or common_name") +} + +# ============================================================ +# OPTIONAL: MT / PDX / Combine +# ============================================================ +if (!(isTRUE(SKIP_MT) && isTRUE(SKIP_PDX) && isTRUE(SKIP_COMBINE) && isTRUE(SKIP_EXACT))) { + + # ──────────────────────────────────────────────── + # MicroTissue Experiments + # ──────────────────────────────────────────────── + dbg_hr("MT SECTION") + + getDrugDataByParent <- function(parid, sampleId) { + dbg_hr(paste0("getDrugDataByParent(parid=", parid, ", sampleId=", sampleId, ")")) + q <- sprintf( + "select id,name,experimentalCondition,parentId from syn21993642 where parentId='%s'", + parid + ) + dbg("MT query: ", q) + + qtab <- tryCatch({ + synTableQuery(q)$asDataFrame() + }, error = function(e) { + dbg("[MT] synTableQuery ERROR for parentId=", parid, " : ", conditionMessage(e)) + return(NULL) + }) + + if (is.null(qtab) || nrow(qtab) == 0) { + dbg("[MT] qtab empty for parentId=", parid) + return(NULL) + } + + dbg_df(qtab, "MT qtab raw") + qtab <- qtab %>% + filter(!is.na(experimentalCondition), name != "synapse_storage_manifest.csv") %>% + select(id, name, experimentalCondition) + + dbg_df(qtab, "MT qtab filtered") + if (nrow(qtab) == 0) { + dbg("[MT] qtab filtered to 0 rows for parentId=", parid, " (all NA experimentalCondition?)") + return(NULL) + } + + # Loop over child files + out_list <- vector("list", length = nrow(qtab)) + for (ii in seq_len(nrow(qtab))) { + x <- qtab$id[ii] + info <- qtab[ii, , drop = FALSE] + d <- extract_date_hour(info$name) + + dbg_hr(paste0("[MT child ", ii, "/", nrow(qtab), "] synID=", x, " name=", info$name)) + sget <- tryCatch(synGet(x), error = function(e) { + dbg("[MT] synGet ERROR for ", x, ": ", conditionMessage(e)) + return(NULL) + }) + if (is.null(sget)) { + out_list[[ii]] <- NULL + next + } + + pth <- sget$path + dbg_file(pth, label = paste0("MT child file ", x)) + + raw <- tryCatch({ + data.table::fread(pth) + }, error = function(e) { + dbg("[MT] fread ERROR for ", x, " path=", pth, " : ", conditionMessage(e)) + return(NULL) + }) + if (is.null(raw) || nrow(raw) == 0) { + dbg("[MT] raw empty for ", x) + out_list[[ii]] <- NULL + next + } + + dbg("[MT] raw rows=", nrow(raw), " cols=", ncol(raw)) + dbg("[MT] raw cols: ", paste(names(raw), collapse=", ")) + if ("response_type" %in% names(raw)) { + dbg("[MT] response_type top:") + print(utils::head(sort(table(raw$response_type), decreasing = TRUE), DEBUG_HEAD_N)) + } else { + dbg("[MT] WARNING: raw missing response_type column") + } + + # Must have these columns for downstream + needed <- c("response_type","dosage","response","compound_name") + miss <- setdiff(needed, names(raw)) + if (length(miss) > 0) { + dbg("[MT] MISSING required cols in ", x, ": ", paste(miss, collapse=", ")) + out_list[[ii]] <- NULL + next + } + + raw2 <- raw %>% + filter(response_type == "percent viability") + + dbg("[MT] rows after filter(response_type=='percent viability') = ", nrow(raw2)) + if (nrow(raw2) == 0) { + dbg("[MT] NOTE: no 'percent viability' rows in ", x, " (check response_type values)") + out_list[[ii]] <- NULL + next + } + + # Transmute final + tmp <- raw2 %>% + transmute( + improve_sample_id = sampleId, + DOSE = (10^dosage) * 1e6, + GROWTH = response, + source = "NF Data Portal", + chem_name = compound_name, + study = paste0("MT ", d$date, " exp"), + time = d$hour + ) + + dbg_df(tmp, label = paste0("MT tmp ", x)) + out_list[[ii]] <- tmp + } + + out <- dplyr::bind_rows(out_list) + dbg_df(out, label = paste0("MT OUT parentId=", parid)) + if (nrow(out) == 0) { + dbg("[MT] parentId=", parid, " produced 0 rows total") + return(NULL) + } + out + } + + # Build MT folder map + mts_map <- manifest %>% + select(common_name, MicroTissueDrugFolder) %>% + mutate(MicroTissueDrugFolder = sapply(MicroTissueDrugFolder, function(x) { + if (is.null(x) || (length(x)==1 && is.na(x))) return(NA_character_) + paste(trimws(unlist(x)), collapse=",") + })) %>% + inner_join(mt_samps, by = "common_name") %>% + separate_rows(MicroTissueDrugFolder, sep = ",") %>% + filter(!is.na(MicroTissueDrugFolder), MicroTissueDrugFolder != "NA") %>% + select(improve_sample_id, folder = MicroTissueDrugFolder) + + dbg_df(as.data.frame(mts_map), "mts_map") + dbg("mts_map unique folders=", length(unique(mts_map$folder))) + + if (nrow(mts_map) == 0) { + if (!isTRUE(SKIP_MT)) { + dbg_stop("MT: mts_map is empty. Likely manifest join failed or MicroTissueDrugFolder missing/NA.") + } else { + dbg("MT: mts_map is empty (SKIP_MT=TRUE); continuing.") + } + } + + if (!isTRUE(SKIP_MT)) { + + # Pull MT data for each folder + mt_list <- vector("list", length = nrow(mts_map)) + for (i in seq_len(nrow(mts_map))) { + sample_id <- mts_map$improve_sample_id[i] + folder <- mts_map$folder[i] + + if (isTRUE(DEBUG) && i <= DEBUG_MAX_ITEMS) { + dbg_hr(paste0("MT MAP ROW ", i, "/", nrow(mts_map), " sample_id=", sample_id, " folder=", folder)) + } + + mt_list[[i]] <- getDrugDataByParent(folder, sample_id) + } + + mt_data <- dplyr::bind_rows(mt_list) + dbg_df(mt_data, "mt_data") + + # If mt_data empty, dump context + stop (this is the earliest real failure) + if (nrow(mt_data) == 0) { + dbg("MT FAILURE: mt_data is empty. Writing debug dumps.") + dbg_out1 <- file.path("/tmp", paste0(out_prefix, "_DEBUG_mts_map.tsv")) + dbg_out2 <- file.path("/tmp", paste0(out_prefix, "_DEBUG_manifest_mt_subset.tsv")) + fwrite(as.data.table(mts_map), dbg_out1, sep = "\t") + fwrite(as.data.table(manifest %>% as.data.frame() %>% filter(common_name %in% mt_samps$common_name)), + dbg_out2, sep = "\t") + dbg("Wrote: ", dbg_out1) + dbg("Wrote: ", dbg_out2) + dbg_stop("MT produced 0 rows. Investigate printed logs above + debug TSVs in /tmp.") + } + + # Join to drug map + mt_curve <- mt_data %>% + mutate(chem_name = norm_drug(chem_name)) + + # show join coverage BEFORE filtering NA improve_drug_id + mt_joined <- mt_curve %>% + left_join(drug_map, by = "chem_name") + + dbg_df(mt_joined, "mt_joined") + n_unmapped <- sum(is.na(mt_joined$improve_drug_id)) + dbg("MT join unmapped chem_name rows=", n_unmapped, " / ", nrow(mt_joined)) + if (n_unmapped > 0) { + top_unmapped <- mt_joined %>% + filter(is.na(improve_drug_id)) %>% + count(chem_name, sort = TRUE) %>% + head(20) + dbg("MT top unmapped chem_name (up to 20):") + print(top_unmapped) + dbg_unmapped_path <- file.path("/tmp", paste0(out_prefix, "_DEBUG_mt_unmapped_chem_name.tsv")) + fwrite(as.data.table(mt_joined %>% filter(is.na(improve_drug_id))), dbg_unmapped_path, sep = "\t") + dbg("Wrote: ", dbg_unmapped_path) + } + + # Fallback: regex match for chem_names that didn't exact-join + # (e.g. raw "verteporfin" vs drug_map "verteporfin [usan:usp:inn:ban]") + mt_joined <- fill_ids_with_fallback( + as.data.frame(mt_joined), + raw_col = "chem_name", + id_col = "improve_drug_id", + chem_tbl = drug_map, + label = "MT" + ) + + mt_curve <- mt_joined %>% + filter(!is.na(improve_drug_id)) %>% + transmute( + source = source, + improve_sample_id = improve_sample_id, + Drug = improve_drug_id, + study = study, + time = time, + time_unit = "hours", + DOSE = DOSE, + GROWTH = GROWTH + ) + + dbg_df(mt_curve, "mt_curve (final)") + + out_mt_curve_path <- file.path("/tmp", paste0(out_prefix, "_mt_curve_data.tsv")) + fwrite(mt_curve, out_mt_curve_path, sep = "\t") + dbg_file(out_mt_curve_path, "MT curve output") + message("Wrote MT curve data") + + if (nrow(mt_curve) == 0) { + dbg_stop("MT curve is EMPTY after join/filter. Root cause is likely chem_name mismatch vs drugfile.tsv. See DEBUG_mt_unmapped_chem_name.tsv.") + } + + # Fit curves + out_mt_prefix <- paste0("/tmp/", out_prefix, "_mt_experiments") + cmd <- sprintf("/opt/venv/bin/python fit_curve.py --input %s --output %s", + out_mt_curve_path, out_mt_prefix) + dbg_hr("RUN fit_curve.py (MT)") + dbg("CMD: ", cmd) + rc <- system(cmd) + dbg("fit_curve.py return code: ", rc) + + # fit_curve.py writes ".0" + mt_exp0 <- paste0(out_mt_prefix, ".0") + dbg_file(mt_exp0, "MT experiments .0 (pre-rename)") + if (!file.exists(mt_exp0) || file.info(mt_exp0)$size == 0) { + dbg_stop("fit_curve.py did not produce a non-empty MT experiments file: ", mt_exp0, + " (check fit_curve.py logs + MT curve input)") + } + + mt_exp_path <- paste0("/tmp/", out_prefix, "_mt_experiments.tsv") + ok_rename <- file.rename(mt_exp0, mt_exp_path) + dbg("rename ", mt_exp0, " -> ", mt_exp_path, " ok=", ok_rename) + dbg_file(mt_exp_path, "MT experiments TSV") + + # Average MT experiments across dates for each unique sample/drug/time/metric + mt_exp_dt <- tryCatch(data.table::fread(mt_exp_path), error = function(e) data.table::data.table()) + if (nrow(mt_exp_dt) > 0) { + if ("Drug" %in% names(mt_exp_dt) && !"improve_drug_id" %in% names(mt_exp_dt)) { + data.table::setnames(mt_exp_dt, "Drug", "improve_drug_id") + } + + req_cols <- c("source","improve_sample_id","improve_drug_id","study","time","time_unit","dose_response_metric","dose_response_value") + if (all(req_cols %in% names(mt_exp_dt))) { + + # optional sanity check (no filtering) + if (any(!grepl("^MT\\s*\\d{6}\\b", as.character(mt_exp_dt$study), ignore.case = TRUE))) { + warning("[MT collapse] Found unexpected study values in mt_exp_path; still averaging all rows in this MT file.") + } + + mt_exp_dt[, dose_response_value := suppressWarnings(as.numeric(as.character(dose_response_value)))] + mt_exp_dt <- mt_exp_dt[is.finite(dose_response_value)] + + key_cols <- c("source","improve_sample_id","improve_drug_id","time","time_unit","dose_response_metric") + + mt_agg <- mt_exp_dt[, .( + study = "MPNST Microtissue", + dose_response_value = mean(dose_response_value, na.rm = TRUE) + ), by = key_cols] + + data.table::fwrite(mt_agg, mt_exp_path, sep = "\t") + } + } + + + message("Wrote MT experiments") + + } else { + dbg_hr("SKIP_MT=TRUE") + cat0("SKIP_MT=TRUE: skipping MT section.") + } + + # ──────────────────────────────────────────────── + # PDX Experiments + # ──────────────────────────────────────────────── + if (!isTRUE(SKIP_PDX)) { + + dbg_hr("PDX SECTION") + + pdx_map <- do.call(rbind, lapply(seq_len(nrow(manifest)), function(i) { + row <- manifest[i, ] + samp <- pdx_samps[pdx_samps$common_name == row$common_name, ] + if (nrow(samp) == 0) return(NULL) + raw <- tryCatch(paste(trimws(unlist(row$PDX_Drug_Data)), collapse=","), error=function(e) "") + ids <- regmatches(raw, gregexpr("syn[0-9]+", raw, ignore.case=TRUE))[[1]] + if (length(ids) == 0) return(NULL) + data.frame( + improve_sample_id = samp$improve_sample_id, + child_id = ids, + stringsAsFactors = FALSE + ) + })) + if (is.null(pdx_map) || nrow(pdx_map) == 0) { + dbg("PDX: pdx_map empty (no PDX drug data)") + } else { + dbg_df(pdx_map, "pdx_map") + } + + pdx_meta <- if (!is.null(pdx_map) && nrow(pdx_map) > 0) { + do.call(rbind, lapply(seq_len(nrow(pdx_map)), function(i) { + sid <- pdx_map$improve_sample_id[i] + cid <- pdx_map$child_id[i] + pid <- synGet(cid)$parentId + if (is.null(pid) || pid=="") stop("no parentId for ", cid) + data.frame( + improve_sample_id = sid, + child_id = cid, + parentId = pid, + stringsAsFactors = FALSE + ) + })) + } else { + data.frame() + } + + all_pdx <- if (nrow(pdx_meta) > 0) { + do.call(rbind, lapply(seq_len(nrow(pdx_meta)), function(i) { + m <- pdx_meta[i, ] + pth <- synGet(m$child_id)$path + raw <- if (grepl("\\.xlsx?$", pth)) read_xlsx(pth) else read_csv(pth) + + sec_opts <- c("compound 2_name", "compound_2_name") + drug2_col <- intersect(sec_opts, names(raw))[1] + compound2 <- if (!is.na(drug2_col)) raw[[drug2_col]] else NA_character_ + + df <- data.frame( + child_id = m$child_id, + specimen_id = raw$specimen_id, + compound_name = raw$compound_name, + compound_2_name = compound2, + experimental_time_point = raw$experimental_time_point, + experimental_time_point_unit = raw$experimental_time_point_unit, + assay_value = raw$assay_value, + stringsAsFactors = FALSE + ) + + df <- within(df, { + drug1 <- tolower(trimws(compound_name)) + drug2 <- tolower(trimws(compound_2_name)) + treatment <- ifelse( + is.na(drug1) | drug1 %in% c("", "na", "n/a", "nan"), + "control", + ifelse(!is.na(drug2) & drug2 != "", + paste(drug1, drug2, sep = "+"), + drug1 + ) + ) + time <- experimental_time_point + time_unit <- experimental_time_point_unit + volume <- assay_value + }) + + df[ , c("child_id", "specimen_id", "treatment", "time", "time_unit", "volume")] + })) + } else { + data.frame() + } + + pdx_data <- if (nrow(all_pdx) > 0) merge(all_pdx, pdx_meta, by="child_id") else data.frame() + if (nrow(pdx_data) > 0) { + pdx_data <- within(pdx_data, { + experiment <- parentId + model_id <- improve_sample_id + }) + + has_ctl <- tapply(pdx_data$treatment == "control", pdx_data$experiment, any) + pdx_data <- pdx_data[pdx_data$experiment %in% names(has_ctl)[has_ctl], ] + + pdx_data <- pdx_data[ , c("experiment","specimen_id","treatment", + "time","time_unit","volume","model_id")] + + pdx_data$treatment <- gsub("doxorubinsin", "doxorubicin", pdx_data$treatment, ignore.case = TRUE) + pdx_data <- na.omit(pdx_data) + + out_pdx_curve <- file.path("/tmp", paste0(out_prefix, "_pdx_curve_data.tsv")) + fwrite(pdx_data, out_pdx_curve, sep = "\t") + dbg_file(out_pdx_curve, "PDX curve output") + message("Wrote PDX curve data") + + cmd2 <- sprintf( + "/opt/venv/bin/python calc_pdx_metrics.py %s --drugfile %s --outprefix %s --source 'NF Data Portal' --study 'MPNST PDX'", + out_pdx_curve, + drugfile, + paste0("/tmp/", out_prefix, "_pdx") + ) + dbg("CMD: ", cmd2) + rc2 <- system(cmd2) + dbg("calc_pdx_metrics.py return code: ", rc2) + message("Wrote PDX experiments to /tmp/", out_prefix, "_pdx_experiments.tsv and combinations") + } else { + dbg("PDX: pdx_data empty; skipping PDX metrics") + } + + } else { + dbg_hr("SKIP_PDX=TRUE") + cat0("SKIP_PDX=TRUE: skipping PDX section.") + } + + # ──────────────────────────────────────────────── + # Combine all Experiments + # ──────────────────────────────────────────────── + if (!isTRUE(SKIP_COMBINE)) { + + dbg_hr("COMBINE MT + PDX EXPERIMENTS") + + mt_exp_path2 <- paste0("/tmp/", out_prefix, "_mt_experiments.tsv") + pdx_exp_path2 <- paste0("/tmp/", out_prefix, "_pdx_experiments.tsv") + + mt_exp <- safe_fread_df(mt_exp_path2, label = "read MT experiments") + pdx_exp <- safe_fread_df(pdx_exp_path2, label = "read PDX experiments") + + dbg_df(mt_exp, "mt_exp (read)") + dbg_df(pdx_exp, "pdx_exp (read)") + + # Make sure dose_response_value exists so mutate won't crash + if (!("dose_response_value" %in% names(mt_exp))) mt_exp$dose_response_value <- NA + if (!("dose_response_value" %in% names(pdx_exp))) pdx_exp$dose_response_value <- NA + + mt_exp <- mt_exp %>% mutate(dose_response_value = as.character(dose_response_value)) + pdx_exp <- pdx_exp %>% mutate(dose_response_value = as.character(dose_response_value)) + + all_exp <- bind_rows(mt_exp, pdx_exp) + dbg_df(all_exp, "all_exp (combined)") + + out_all <- paste0("/tmp/", out_prefix, "_experiments.tsv") + fwrite(all_exp, out_all, sep = "\t") + dbg_file(out_all, "combined experiments TSV") + message("Wrote combined experiments: /tmp/", out_prefix, "_experiments.tsv") + + # rename combinations + pdx_combo_src <- paste0("/tmp/", out_prefix, "_pdx_combinations.tsv") + combo_dst <- paste0("/tmp/", out_prefix, "_combinations.tsv") + if (file.exists(pdx_combo_src)) { + okc <- file.rename(pdx_combo_src, combo_dst) + dbg("rename combos ", pdx_combo_src, " -> ", combo_dst, " ok=", okc) + } else { + dbg("PDX combinations file missing (ok if no PDX): ", pdx_combo_src) + } + + } else { + dbg_hr("SKIP_COMBINE=TRUE") + cat0("SKIP_COMBINE=TRUE: skipping combine section.") + } + + # ──────────────────────────────────────────────── + # EXACT Experiments + # - Average replicates within each file + # - Run curve fitting per date via study = "EXACT {DATE}" + # - Average fitted metrics across dates back to study = "EXACT" + # ──────────────────────────────────────────────── + if (!isTRUE(SKIP_EXACT)) { + + dbg_hr("EXACT SECTION") + + EXACT_PARENT <- "syn65473019" # Folder (Processed Data) + EXACT_FILEVIEW <- "syn52369043" # FileView table used by your python + + get_exact_fileview <- function(parent_id) { + q <- sprintf("SELECT * FROM %s WHERE parentId = '%s'", EXACT_FILEVIEW, parent_id) + dbg("EXACT FileView query: ", q) + + tab <- tryCatch({ + synTableQuery(q)$asDataFrame() + }, error = function(e) { + dbg("[EXACT] synTableQuery(fileview) ERROR: ", conditionMessage(e)) + return(NULL) + }) + + if (is.null(tab) || nrow(tab) == 0) { + dbg("[EXACT] FileView returned 0 rows for parentId=", parent_id) + return(data.frame()) + } + + dbg_df(tab, "EXACT fileview raw") + + tab <- tab %>% + filter(!is.na(name)) %>% + filter(name != "synapse_storage_manifest.csv") %>% + filter(grepl("\\.csv$", name, ignore.case = TRUE)) %>% + filter(!grepl("250122\\.csv$", name, ignore.case = TRUE)) + + dbg_df(tab, "EXACT fileview filtered") + tab + } + + pick_specimen_col <- function(df) { + opts <- c("specimenID","specimenId","specimen_id","improve_sample_id") + hit <- intersect(opts, names(df)) + if (length(hit) == 0) return(NA_character_) + hit[1] + } + + exact_files <- get_exact_fileview(EXACT_PARENT) + + exact_samples_map <- samples_all %>% + transmute( + common_name = trimws(as.character(common_name)), + model_type = as.character(model_type), + improve_sample_id = suppressWarnings(as.integer(improve_sample_id)) + ) %>% + distinct() + + map_exact_sampleName_to_improve_id <- function(sampleName) { + sn <- trimws(as.character(sampleName)) + if (is.na(sn) || sn == "" || tolower(sn) %in% c("na","n/a","nan")) return(NA_integer_) + + sn_canon <- canon_id(sn) + + hit1 <- exact_samples_map$improve_sample_id[ + exact_samples_map$model_type == "3D-MEDS" & + canon_id(exact_samples_map$common_name) == sn_canon + ] + if (length(hit1) > 0 && !is.na(hit1[1])) return(as.integer(hit1[1])) + + hit2 <- exact_samples_map$improve_sample_id[ + exact_samples_map$model_type == "patient derived xenograft" & + canon_id(exact_samples_map$common_name) == sn_canon + ] + if (length(hit2) > 0 && !is.na(hit2[1])) return(as.integer(hit2[1])) + + NA_integer_ + } + + exact_data <- data.frame() + if (nrow(exact_files) == 0) { + dbg("EXACT: no files found under ", EXACT_PARENT, "; skipping EXACT.") + } else { + + spec_col <- pick_specimen_col(exact_files) + if (is.na(spec_col)) { + dbg("[EXACT] WARNING: FileView has no specimenID column; will fall back to sampleName inside CSVs.") + exact_files$.__specimen <- NA_character_ + spec_col <- ".__specimen" + } + + exact_list <- vector("list", length = nrow(exact_files)) + + for (i in seq_len(nrow(exact_files))) { + file_id <- as.character(exact_files$id[i]) + file_name <- as.character(exact_files$name[i]) + specimen <- exact_files[[spec_col]][i] + + if (isTRUE(DEBUG) && i <= DEBUG_MAX_ITEMS) { + dbg_hr(paste0("EXACT FILE ", i, "/", nrow(exact_files), + " id=", file_id, " name=", file_name, + " specimen=", specimen)) + } + + sget <- tryCatch(synGet(file_id), error = function(e) { + dbg("[EXACT] synGet ERROR for ", file_id, ": ", conditionMessage(e)) + return(NULL) + }) + if (is.null(sget)) { + exact_list[[i]] <- NULL + next + } + + pth <- sget$path + dbg_file(pth, label = paste0("EXACT file ", file_id)) + + raw <- tryCatch({ + data.table::fread(pth) + }, error = function(e) { + dbg("[EXACT] fread ERROR for ", file_id, " path=", pth, " : ", conditionMessage(e)) + return(NULL) + }) + if (is.null(raw) || nrow(raw) == 0) { + dbg("[EXACT] raw empty for ", file_id) + exact_list[[i]] <- NULL + next + } + + raw_df <- as.data.frame(raw) + + # Optional: only keep processed rows if column exists + if ("dataSubtype" %in% names(raw_df)) { + raw_df <- raw_df[raw_df$dataSubtype == "processed", , drop = FALSE] + dbg("[EXACT] rows after dataSubtype=='processed': ", nrow(raw_df)) + if (nrow(raw_df) == 0) { + exact_list[[i]] <- NULL + next + } + } + + needed <- c("drugName","concentration","percentViability","timePoint","timePointUnit") + miss <- setdiff(needed, names(raw_df)) + if (length(miss) > 0) { + dbg("[EXACT] MISSING required cols in ", file_id, ": ", paste(miss, collapse=", ")) + exact_list[[i]] <- NULL + next + } + + # Determine improve_sample_id for this file (map sampleName -> mpnst_samples.csv other_id) + raw_df$improve_sample_id <- NA_integer_ + if ("sampleName" %in% names(raw_df)) { + raw_df$improve_sample_id <- vapply(raw_df$sampleName, map_exact_sampleName_to_improve_id, integer(1)) + } + + if (all(is.na(raw_df$improve_sample_id))) { + dbg("[EXACT] WARNING: sampleName->improve_sample_id mapping failed for ", file_id, " name=", file_name) + if ("sampleName" %in% names(raw_df)) { + dbg("[EXACT] sampleName values (unique, up to 20):") + print(head(unique(raw_df$sampleName), 20)) + } + exact_list[[i]] <- NULL + next + } + + # Extract 6-digit date token from filename + file_date <- str_match(file_name, "(\\d{6})\\.csv$")[,2] + if (is.na(file_date)) file_date <- str_match(file_name, "(\\d{6})")[,2] + + # Average replicates WITHIN FILE + raw_avg <- raw_df %>% + mutate( + improve_sample_id = suppressWarnings(as.integer(improve_sample_id)), + drugName = as.character(drugName), + concentration = suppressWarnings(as.numeric(concentration)), + percentViability = suppressWarnings(as.numeric(percentViability)), + timePoint = suppressWarnings(as.integer(timePoint)), + timePointUnit = as.character(timePointUnit) + ) %>% + filter(!is.na(improve_sample_id), !is.na(drugName), !is.na(concentration), !is.na(percentViability), !is.na(timePoint)) %>% + group_by(improve_sample_id, drugName, concentration, timePoint, timePointUnit) %>% + summarise(percentViability = mean(percentViability, na.rm = TRUE), .groups = "drop") + + tmp <- raw_avg %>% + transmute( + improve_sample_id = improve_sample_id, + DOSE = concentration, # µM + GROWTH = percentViability, # percent viability (0-100) + source = "NF Data Portal", + chem_name = norm_drug(drugName), + study = paste0("EXACT ", file_date), + time = timePoint, + time_unit = timePointUnit + ) + + dbg_df(tmp, label = paste0("EXACT tmp ", file_id)) + exact_list[[i]] <- tmp + } + + exact_data <- dplyr::bind_rows(exact_list) + } + + dbg_df(exact_data, "exact_data (raw)") + + if (!is.null(exact_data) && nrow(exact_data) > 0) { + + # map to improve_drug_id using drug_map (join on normalized chem_name) + exact_joined <- exact_data %>% + left_join(drug_map, by = "chem_name") + + dbg_df(exact_joined, "exact_joined") + n_unmapped_ex <- sum(is.na(exact_joined$improve_drug_id)) + dbg("EXACT join unmapped chem_name rows=", n_unmapped_ex, " / ", nrow(exact_joined)) + if (n_unmapped_ex > 0) { + top_unmapped_ex <- exact_joined %>% + filter(is.na(improve_drug_id)) %>% + count(chem_name, sort = TRUE) %>% + head(20) + dbg("EXACT top unmapped chem_name (up to 20):") + print(top_unmapped_ex) + dbg_unmapped_ex_path <- file.path("/tmp", paste0(out_prefix, "_DEBUG_exact_unmapped_chem_name.tsv")) + fwrite(as.data.table(exact_joined %>% filter(is.na(improve_drug_id))), dbg_unmapped_ex_path, sep = "\t") + dbg("Wrote: ", dbg_unmapped_ex_path) + } + + # Fallback: regex match for chem_names that didn't exact-join + # (e.g. raw "verteporfin" vs drug_map "verteporfin [usan:usp:inn:ban]") + exact_joined <- fill_ids_with_fallback( + as.data.frame(exact_joined), + raw_col = "chem_name", + id_col = "improve_drug_id", + chem_tbl = drug_map, + label = "EXACT" + ) + + exact_curve <- exact_joined %>% + filter(!is.na(improve_drug_id)) %>% + transmute( + source = source, + improve_sample_id = improve_sample_id, + Drug = improve_drug_id, + study = study, # "EXACT {DATE}" for per-date fitting + time = time, + time_unit = time_unit, + DOSE = DOSE, + GROWTH = GROWTH + ) + + dbg_df(exact_curve, "exact_curve (final)") + + out_exact_curve_path <- file.path("/tmp", paste0(out_prefix, "_exact_curve_data.tsv")) + fwrite(exact_curve, out_exact_curve_path, sep = "\t") + dbg_file(out_exact_curve_path, "EXACT curve output") + message("Wrote EXACT curve data") + + if (nrow(exact_curve) > 0) { + # Fit curves (per-date via study) + out_exact_prefix <- paste0("/tmp/", out_prefix, "_exact_experiments") + cmd_ex <- sprintf("/opt/venv/bin/python fit_curve.py --input %s --output %s", + out_exact_curve_path, out_exact_prefix) + dbg_hr("RUN fit_curve.py (EXACT)") + dbg("CMD: ", cmd_ex) + rc_ex <- system(cmd_ex) + dbg("fit_curve.py return code: ", rc_ex) + + exact_exp0 <- paste0(out_exact_prefix, ".0") + dbg_file(exact_exp0, "EXACT experiments .0 (pre-rename)") + if (!file.exists(exact_exp0) || file.info(exact_exp0)$size == 0) { + dbg_stop("fit_curve.py did not produce a non-empty EXACT experiments file: ", exact_exp0, + " (check fit_curve.py logs + EXACT curve input)") + } + + + exact_exp_path <- paste0("/tmp/", out_prefix, "_exact_experiments.tsv") + + ok_rename_ex <- file.rename(exact_exp0, exact_exp_path) + if (!isTRUE(ok_rename_ex)) { + ok_copy <- file.copy(exact_exp0, exact_exp_path, overwrite = TRUE) + if (isTRUE(ok_copy)) unlink(exact_exp0) + ok_rename_ex <- ok_copy + } + dbg("rename ", exact_exp0, " -> ", exact_exp_path, " ok=", ok_rename_ex) + dbg_file(exact_exp_path, "EXACT experiments TSV") + + # Average across dates -> study="EXACT" + exact_exp_dt <- tryCatch(data.table::fread(exact_exp_path), error = function(e) data.table::data.table()) + if (nrow(exact_exp_dt) > 0) { + if ("Drug" %in% names(exact_exp_dt) && !"improve_drug_id" %in% names(exact_exp_dt)) { + data.table::setnames(exact_exp_dt, "Drug", "improve_drug_id") + } + + req_cols_ex <- c("source","improve_sample_id","improve_drug_id","study","time","time_unit","dose_response_metric","dose_response_value") + if (all(req_cols_ex %in% names(exact_exp_dt))) { + exact_exp_dt[, dose_response_value := suppressWarnings(as.numeric(as.character(dose_response_value)))] + exact_exp_dt <- exact_exp_dt[is.finite(dose_response_value)] + + key_cols_ex <- c("source","improve_sample_id","improve_drug_id","time","time_unit","dose_response_metric") + + exact_agg <- exact_exp_dt[, .( + study = "EXACT", + dose_response_value = mean(dose_response_value, na.rm = TRUE) + ), by = key_cols_ex] + + data.table::fwrite(exact_agg, exact_exp_path, sep = "\t") + } + } + + message("Wrote EXACT experiments") + + # Append EXACT into combined experiments + exp_path2 <- file.path("/tmp", paste0(out_prefix, "_experiments.tsv")) + if (file.exists(exp_path2) && file.exists(exact_exp_path)) { + exp_dt2 <- data.table::fread(exp_path2) + ex_dt2 <- data.table::fread(exact_exp_path) + + missing_cols2 <- setdiff(names(exp_dt2), names(ex_dt2)) + for (cc in missing_cols2) ex_dt2[[cc]] <- NA + + extra_cols2 <- setdiff(names(ex_dt2), names(exp_dt2)) + if (length(extra_cols2) > 0) ex_dt2 <- ex_dt2[, setdiff(names(ex_dt2), extra_cols2), with = FALSE] + + data.table::setcolorder(ex_dt2, names(exp_dt2)) + + for (cc in intersect(c("improve_sample_id", "time"), names(exp_dt2))) { + exp_dt2[[cc]] <- suppressWarnings(as.integer(exp_dt2[[cc]])) + ex_dt2[[cc]] <- suppressWarnings(as.integer(ex_dt2[[cc]])) + } + for (cc in intersect(c("dose_response_value"), names(exp_dt2))) { + exp_dt2[[cc]] <- suppressWarnings(as.numeric(exp_dt2[[cc]])) + ex_dt2[[cc]] <- suppressWarnings(as.numeric(ex_dt2[[cc]])) + } + + before_n2 <- nrow(exp_dt2) + add_n2 <- nrow(ex_dt2) + + final_dt2 <- data.table::rbindlist(list(exp_dt2, ex_dt2), use.names = TRUE, fill = TRUE) + final_dt2 <- unique(final_dt2) + + bak_path2 <- paste0(exp_path2, ".bak_exact") + file.copy(exp_path2, bak_path2, overwrite = TRUE) + data.table::fwrite(final_dt2, exp_path2, sep = "\t") + + message(sprintf( + "Appended EXACT into %s: +%d rows (was %d, now %d). Backup: %s", + exp_path2, add_n2, before_n2, nrow(final_dt2), bak_path2 + )) + } + } + + } else { + dbg("EXACT: no rows found; skipping curve fit + append.") + } + + } else { + dbg_hr("SKIP_EXACT=TRUE") + cat0("SKIP_EXACT=TRUE: skipping EXACT section.") + } + + +} else { + dbg_hr("ALL SKIP SWITCHES=TRUE") + cat0("All skip switches are TRUE: skipping MT + PDX + combine + EXACT sections (debugging treated combos only).") + cat0("Will still read: samples.csv, drugfile.tsv, and syn69801348.") +} + +# ============================================================ +# Treated-combination experiments from syn69801348 (auc ONLY - note we can change to fit_auc if needed) +# ============================================================ +TREATED_COMBO_SYNID <- "syn69801348" +dbg_hr(paste0("TREATED COMBO SECTION from ", TREATED_COMBO_SYNID)) + +combo_path <- synGet(TREATED_COMBO_SYNID)$path +dbg_file(combo_path, "syn69801348 downloaded file") +combo_raw <- fread(combo_path) + +needed_cols <- c("source","improve_sample_id","improve_drug_id","study","time","time_unit","dose_response_metric","dose_response_value") +missing_cols <- setdiff(needed_cols, names(combo_raw)) +if (length(missing_cols) > 0) { + stop("syn69801348 is missing required columns: ", paste(missing_cols, collapse=", "), call. = FALSE) +} + +dbg_df(as.data.frame(combo_raw), "combo_raw (pre-filter)") + +combo_raw <- combo_raw %>% + mutate( + combo_key = as.character(improve_sample_id), + improve_sample_id = as.character(improve_sample_id), + improve_drug_id = as.character(improve_drug_id), + study = as.character(study), + time = suppressWarnings(as.integer(time)), + time_unit = as.character(time_unit), + dose_response_metric = as.character(dose_response_metric), + dose_response_value = suppressWarnings(as.numeric(dose_response_value)) + ) %>% + filter(tolower(dose_response_metric) == "auc") %>% # This is auc only - can change to fit_auc if needed. + filter(!is.na(time), !is.na(dose_response_value)) %>% + distinct() + +dbg("syn69801348 rows (auc): ", nrow(combo_raw)) +dbg("combo_raw cols: ", paste(names(combo_raw), collapse=", ")) +dbg("combo_raw head():") +print(head(combo_raw, 6)) + +# ---- parse combo_key into base_sample + drug1/drug2 from combo_key +if (nrow(combo_raw) > 0) { + parsed <- lapply(combo_raw$combo_key, parse_combo_key) + combo_raw$base_sample <- vapply(parsed, `[[`, character(1), "base") + combo_raw$drug1_raw <- vapply(parsed, `[[`, character(1), "drug1") + combo_raw$drug2_raw <- vapply(parsed, `[[`, character(1), "drug2") +} else { + combo_raw$base_sample <- character() + combo_raw$drug1_raw <- character() + combo_raw$drug2_raw <- character() +} + +parse_fail <- combo_raw %>% filter(is.na(base_sample) | is.na(drug1_raw) | is.na(drug2_raw)) +if (nrow(parse_fail) > 0) { + warning("Some combo_key values failed parsing (showing up to 10):") + print(head(parse_fail$combo_key, 10)) + dbg <- file.path("/tmp", paste0(out_prefix, "_treated_parse_fail.tsv")) + fwrite(as.data.table(parse_fail), dbg, sep = "\t") + cat0("Wrote: ", dbg) +} +combo_raw <- combo_raw %>% filter(!is.na(base_sample), !is.na(drug1_raw), !is.na(drug2_raw)) +dbg("rows after combo_key parse filter: ", nrow(combo_raw)) + +combo_raw <- combo_raw %>% + mutate( + drug1_canon = canon_name(drug1_raw), + drug2_canon = canon_name(drug2_raw), + drug_canon = canon_name(improve_drug_id), + is_combo_row = grepl("\\+", improve_drug_id) + ) + +# ============================================================ +# Treated samples mapping +# ============================================================ +dbg_hr("TREATED SAMPLES MAPPING") + +treated_samples_long <- samples_all %>% + mutate( + other_id = as.character(other_id), + model_type = as.character(model_type), + improve_sample_id = as.integer(improve_sample_id), + common_name = as.character(common_name) + ) %>% + filter( + model_type == "3D-MEDS", + !is.na(other_id), + grepl("hr_treated_microtissue$", other_id) + ) %>% + mutate( + base_common_name = common_name, + base_prefix = base_to_otherid_prefix(common_name), + treated_token = mapply(extract_treated_token_from_other_id, other_id, common_name), + treated_token_u = toupper(safe_token(treated_token)) + ) %>% + select( + base_common_name, + treated_token, + treated_token_u, + treated_other_id = other_id, + treated_improve_sample_id = improve_sample_id + ) %>% + distinct() + +dbg("treated_samples_long rows: ", nrow(treated_samples_long)) +dbg("treated_samples_long head():") +print(head(treated_samples_long, 10)) + +failed_token <- treated_samples_long %>% filter(is.na(treated_token) | treated_token == "") +if (nrow(failed_token) > 0) { + warning("Some treated other_id rows failed treated_token extraction (showing up to 20):") + print(head(failed_token, 20)) + dbg_fail_tok <- file.path("/tmp", paste0(out_prefix, "_treated_token_extract_fail.tsv")) + fwrite(as.data.table(failed_token), dbg_fail_tok, sep = "\t") + cat0("Wrote token extraction failures: ", dbg_fail_tok) +} + +# ============================================================ +# Build chem_tbl + mapping token -> improve_drug_id +# ============================================================ +dbg_hr("TREATED TOKEN -> DRUG ID MAPPING") + +chem_tbl <- drug_map %>% + transmute( + improve_drug_id = as.character(improve_drug_id), + chem_name = as.character(chem_name), + chem_canon = canon_name(chem_name), + id_canon = canon_id(improve_drug_id), + abbrev3 = substr(toupper(gsub("[^A-Za-z]+","",chem_name)), 1, 3), + chem_token_u = toupper(safe_token(chem_name)) + ) %>% + distinct() + +dbg_df(chem_tbl, "chem_tbl") + +treated_tokens <- treated_samples_long %>% + distinct(treated_token, treated_token_u) %>% + mutate( + treated_token_canon = canon_name(treated_token), + treated_token3 = substr(toupper(gsub("[^A-Za-z]+","",treated_token)), 1, 3) + ) + +dbg_df(treated_tokens, "treated_tokens") + +# These were updated on synapse so this isn't needed now. +manual_override <- c( + # "DOX" = "doxorubicin", + # "TNO" = "TNO155" +) + +override_tbl <- data.frame( + treated_token_u = character(0), + override_val = character(0), + override_id = character(0), + stringsAsFactors = FALSE +) + +if (length(manual_override) > 0) { + override_tbl <- data.frame( + treated_token_u = names(manual_override), + override_val = as.character(manual_override), + stringsAsFactors = FALSE + ) + resolve_override_to_id <- function(v) { + if (is.na(v) || trimws(v) == "") return(NA_character_) + v0 <- as.character(v) + hit_id <- chem_tbl$improve_drug_id[chem_tbl$id_canon == canon_id(v0)] + if (length(hit_id) > 0) return(hit_id[1]) + hit_nm <- chem_tbl$improve_drug_id[chem_tbl$chem_canon == canon_name(v0)] + if (length(hit_nm) > 0) return(hit_nm[1]) + NA_character_ + } + override_tbl$override_id <- vapply(override_tbl$override_val, resolve_override_to_id, character(1)) +} + +map_by_canon <- treated_tokens %>% + left_join(chem_tbl %>% select(improve_drug_id, chem_canon), + by = c("treated_token_canon" = "chem_canon")) %>% + transmute(treated_token_u, canon_id = improve_drug_id) + +map_by_token <- treated_tokens %>% + left_join(chem_tbl %>% select(improve_drug_id, chem_token_u), + by = c("treated_token_u" = "chem_token_u")) %>% + transmute(treated_token_u, token_id = improve_drug_id) + +map_by_abbrev3 <- treated_tokens %>% + left_join(chem_tbl %>% select(improve_drug_id, abbrev3), + by = c("treated_token3" = "abbrev3"), relationship = "many-to-many") %>% + transmute(treated_token_u, abbr_id = improve_drug_id) + +map_by_canon <- warn_first_by(map_by_canon, c("treated_token_u"), "treated_drug_map_canon") +map_by_token <- warn_first_by(map_by_token, c("treated_token_u"), "treated_drug_map_token") +map_by_abbrev3 <- warn_first_by(map_by_abbrev3, c("treated_token_u"), "treated_drug_map_abbrev3") + +drug_token_map <- treated_tokens %>% + left_join(override_tbl %>% select(treated_token_u, override_id), by = "treated_token_u") %>% + left_join(map_by_canon, by = "treated_token_u") %>% + left_join(map_by_token, by = "treated_token_u") %>% + left_join(map_by_abbrev3, by = "treated_token_u") %>% + mutate(mapped_improve_drug_id = dplyr::coalesce(override_id, canon_id, token_id, abbr_id)) %>% + select(treated_token, treated_token_u, mapped_improve_drug_id) + +drug_token_map <- warn_first_by(drug_token_map, c("treated_token_u"), "treated_drug_map_final") + +mapping_out <- file.path("/tmp", paste0(out_prefix, "_treated_drug_mapping.tsv")) +fwrite(as.data.table(drug_token_map), mapping_out, sep = "\t") +dbg("Wrote treated drug mapping debug: ", mapping_out) +dbg_df(drug_token_map, "drug_token_map") + +treated_samples_mapped <- treated_samples_long %>% + left_join(drug_token_map %>% select(treated_token_u, mapped_improve_drug_id), by = "treated_token_u") %>% + filter(!is.na(mapped_improve_drug_id)) %>% + mutate(mapped_improve_drug_id = as.character(mapped_improve_drug_id)) + +dbg("treated_samples_mapped rows: ", nrow(treated_samples_mapped)) +dbg_df(treated_samples_mapped, "treated_samples_mapped") + +# ============================================================ +# Build pairing tables +# ============================================================ +dbg_hr("PAIRING auc SINGLE vs COMBO") + +ctx_cols <- c("combo_key","base_sample","time","source","study","time_unit") + +combo_ctx <- combo_raw %>% + filter(is_combo_row) %>% + transmute( + combo_key, base_sample, + drug1_raw, drug2_raw, drug1_canon, drug2_canon, + time, source, study, time_unit, + auc_combo = dose_response_value + ) + +single_ctx <- combo_raw %>% + filter(!is_combo_row) %>% + transmute( + combo_key, base_sample, + time, source, study, time_unit, + drugA_raw = improve_drug_id, + drugA_canon = drug_canon, + auc_A = dose_response_value + ) + +dbg_df(combo_ctx, "combo_ctx") +dbg_df(single_ctx, "single_ctx") + +combo_ctx <- warn_first_by(combo_ctx, ctx_cols, "syn69801348_combo_auc") +single_ctx <- warn_first_by(single_ctx, c(ctx_cols,"drugA_canon"), "syn69801348_single_auc") + +paired <- single_ctx %>% + inner_join(combo_ctx, by = ctx_cols) %>% + mutate( + drugB_raw = dplyr::case_when( + drugA_canon == drug1_canon ~ drug2_raw, + drugA_canon == drug2_canon ~ drug1_raw, + TRUE ~ NA_character_ + ), + drugB_canon = canon_name(drugB_raw), + delta_auc = auc_A - auc_combo + ) + +dbg("paired rows: ", nrow(paired)) +dbg_df(paired, "paired") + +dbg_pairs <- file.path("/tmp", paste0(out_prefix, "_treated_pairs_debug.tsv")) +fwrite(as.data.table(paired), dbg_pairs, sep = "\t") +dbg("Wrote paired debug: ", dbg_pairs) + +if (nrow(paired) == 0) { + dbg_stop("paired is empty. Investigate /tmp/*_treated_pairs_debug.tsv and parse failures.") +} + +# ============================================================ +# Map canonical -> improve_drug_id (primary), then fallback regex +# ============================================================ +dbg_hr("MAP drugA/drugB to improve_drug_id") + +chem_to_id <- chem_tbl %>% distinct(chem_canon, improve_drug_id) +chem_to_id <- warn_first_by(chem_to_id, c("chem_canon"), "chem_to_id") + +paired <- paired %>% + left_join(chem_to_id %>% rename(drugA_id = improve_drug_id), by = c("drugA_canon" = "chem_canon")) %>% + left_join(chem_to_id %>% rename(drugB_id = improve_drug_id), by = c("drugB_canon" = "chem_canon")) + +missingA_before <- sum(is.na(paired$drugA_id)) +missingB_before <- sum(is.na(paired$drugB_id)) +dbg("Missing drugA_id before fallback: ", missingA_before, " / ", nrow(paired)) +dbg("Missing drugB_id before fallback: ", missingB_before, " / ", nrow(paired)) + +paired <- fill_ids_with_fallback(paired, raw_col = "drugA_raw", id_col = "drugA_id", chem_tbl = chem_tbl, label = "drugA_id") +paired <- fill_ids_with_fallback(paired, raw_col = "drugB_raw", id_col = "drugB_id", chem_tbl = chem_tbl, label = "drugB_id") + +missingA_after <- sum(is.na(paired$drugA_id)) +missingB_after <- sum(is.na(paired$drugB_id)) +dbg("Missing drugA_id after fallback: ", missingA_after, " / ", nrow(paired)) +dbg("Missing drugB_id after fallback: ", missingB_after, " / ", nrow(paired)) + +fallback_dbg <- paired %>% + transmute( + combo_key, base_sample, time, source, study, time_unit, + drugA_raw, drugA_canon, drugA_id, + drugB_raw, drugB_canon, drugB_id, + delta_auc, + drugA_id_missing = is.na(drugA_id), + drugB_id_missing = is.na(drugB_id) + ) + +fallback_dbg_path <- file.path("/tmp", paste0(out_prefix, "_treated_drug_id_fallback_debug.tsv")) +fwrite(as.data.table(fallback_dbg), fallback_dbg_path, sep = "\t") +dbg("Wrote drug-id fallback debug: ", fallback_dbg_path) + +dbg("paired with IDs mapped head():") +print(head(paired %>% select(combo_key, base_sample, time, drugA_raw, drugA_id, drugB_raw, drugB_id, delta_auc), 10)) + +# ============================================================ +# Join to treated samples +# ============================================================ +dbg_hr("JOIN paired -> treated samples") + +treated_joined <- paired %>% + filter(!is.na(drugA_id), !is.na(drugB_id)) %>% + inner_join( + treated_samples_mapped %>% + transmute( + base_sample = base_common_name, + drugA_id = mapped_improve_drug_id, + treated_improve_sample_id, + treated_other_id, + treated_token + ), + by = c("base_sample","drugA_id") + ) + +dbg("treated_joined rows: ", nrow(treated_joined)) +dbg_df(treated_joined, "treated_joined") + +dbg_joined <- file.path("/tmp", paste0(out_prefix, "_treated_joined_debug.tsv")) +fwrite(as.data.table(treated_joined), dbg_joined, sep = "\t") +dbg("Wrote treated_joined debug: ", dbg_joined) + +if (nrow(treated_joined) == 0) { + dbg("EARLY EXIT: treated_joined is empty. Likely token->drugA_id mapping mismatch.") + dbg("Example base_sample from paired: ", paste(head(unique(paired$base_sample), 10), collapse=", ")) + dbg("Example base_common_name from treated_samples_mapped: ", paste(head(unique(treated_samples_mapped$base_common_name), 10), collapse=", ")) + dbg("Example drugA_id from paired: ", paste(head(unique(paired$drugA_id), 10), collapse=", ")) + dbg("Example mapped_improve_drug_id: ", paste(head(unique(treated_samples_mapped$mapped_improve_drug_id), 10), collapse=", ")) + quit(save="no", status=0) +} + +# ============================================================ +# Outputs +# ============================================================ +dbg_hr("WRITE TREATED OUTPUTS") + +treated_out_ids <- treated_joined %>% + transmute( + source = source, + improve_sample_id = as.integer(treated_improve_sample_id), + improve_drug_id = as.character(drugB_id), + study = study, + time = as.integer(time), + time_unit = time_unit, + dose_response_metric = "delta_auc", + dose_response_value = as.numeric(delta_auc) + ) %>% + distinct() + +treated_ids_path <- file.path("/tmp", paste0(out_prefix, "_experiment_combinations_treated.tsv")) +fwrite(as.data.table(treated_out_ids), treated_ids_path, sep = "\t") +dbg("Wrote treated combination experiments (IDs): ", treated_ids_path, " [rows=", nrow(treated_out_ids), "]") +dbg_file(treated_ids_path, "treated IDs TSV") + +treated_out_names <- treated_joined %>% + transmute( + source = source, + improve_sample_id = as.character(combo_key), + improve_drug_id = as.character(drugB_raw), + study = study, + time = as.integer(time), + time_unit = time_unit, + dose_response_metric = "delta_auc", + dose_response_value = as.numeric(delta_auc) + ) %>% + distinct() + +treated_names_path <- file.path("/tmp", paste0(out_prefix, "_experiment_combinations_treated_names.tsv")) +fwrite(as.data.table(treated_out_names), treated_names_path, sep = "\t") +dbg("Wrote treated combination experiments (NAMES): ", treated_names_path, " [rows=", nrow(treated_out_names), "]") +dbg_file(treated_names_path, "treated NAMES TSV") + +# ============================================================ +# FINAL: append treated-combo experiments into main experiments +# ============================================================ +dbg_hr("FINAL APPEND treated -> experiments") + +exp_path <- file.path("/tmp", paste0(out_prefix, "_experiments.tsv")) +trt_path <- treated_ids_path + +if (!file.exists(exp_path)) stop("Missing experiments file: ", exp_path, call. = FALSE) +if (!file.exists(trt_path)) stop("Missing treated-combos file: ", trt_path, call. = FALSE) + +exp_dt <- data.table::fread(exp_path) +trt_dt <- data.table::fread(trt_path) + +dbg("exp_dt rows=", nrow(exp_dt), " cols=", ncol(exp_dt)) +dbg("trt_dt rows=", nrow(trt_dt), " cols=", ncol(trt_dt)) + +missing_cols <- setdiff(names(exp_dt), names(trt_dt)) +for (cc in missing_cols) trt_dt[[cc]] <- NA + +extra_cols <- setdiff(names(trt_dt), names(exp_dt)) +if (length(extra_cols) > 0) trt_dt <- trt_dt[, setdiff(names(trt_dt), extra_cols), with = FALSE] + +data.table::setcolorder(trt_dt, names(exp_dt)) + +for (cc in intersect(c("improve_sample_id", "time"), names(exp_dt))) { + exp_dt[[cc]] <- suppressWarnings(as.integer(exp_dt[[cc]])) + trt_dt[[cc]] <- suppressWarnings(as.integer(trt_dt[[cc]])) +} +for (cc in intersect(c("dose_response_value"), names(exp_dt))) { + exp_dt[[cc]] <- suppressWarnings(as.numeric(exp_dt[[cc]])) + trt_dt[[cc]] <- suppressWarnings(as.numeric(trt_dt[[cc]])) +} + +before_n <- nrow(exp_dt) +add_n <- nrow(trt_dt) + +final_dt <- data.table::rbindlist(list(exp_dt, trt_dt), use.names = TRUE, fill = TRUE) +final_dt <- unique(final_dt) + +bak_path <- paste0(exp_path, ".bak") +file.copy(exp_path, bak_path, overwrite = TRUE) +data.table::fwrite(final_dt, exp_path, sep = "\t") + +message(sprintf( + "Appended treated combos into %s: +%d rows (was %d, now %d). Backup: %s", + exp_path, add_n, before_n, nrow(final_dt), bak_path +)) + +dbg_hr("SCRIPT END") \ No newline at end of file diff --git a/coderbuild/mpnst/build_drugs.sh b/coderbuild/mpnst/build_drugs.sh index 821bc46c..d0932886 100644 --- a/coderbuild/mpnst/build_drugs.sh +++ b/coderbuild/mpnst/build_drugs.sh @@ -1,3 +1,4 @@ +# build_drugs.sh #!/bin/bash #set -euo pipefail diff --git a/coderbuild/mpnst/build_exp.sh b/coderbuild/mpnst/build_exp.sh index 14506cfe..2620c8eb 100644 --- a/coderbuild/mpnst/build_exp.sh +++ b/coderbuild/mpnst/build_exp.sh @@ -1,9 +1,10 @@ +# build_exp.sh #!/bin/bash set -euo pipefail trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR -echo "Running 03_get_experiments.R with $SYNAPSE_AUTH_TOKEN, $1, and $2." -Rscript 03_get_experiments.R $SYNAPSE_AUTH_TOKEN $1 $2 mpnst -rm /tmp/mpnst_pdx_experiments.tsv /tmp/mpnst_mt_experiments.tsv /tmp/mpnst_mt_curve_data.tsv /tmp/mpnst_pdx_curve_data.tsv +echo "Running 03_get_experiments_with_treated.R with $SYNAPSE_AUTH_TOKEN, $1, and $2." +Rscript 03_get_experiments_with_treated.R $SYNAPSE_AUTH_TOKEN $1 $2 mpnst +# rm /tmp/mpnst_pdx_experiments.tsv /tmp/mpnst_mt_experiments.tsv /tmp/mpnst_mt_curve_data.tsv /tmp/mpnst_pdx_curve_data.tsv diff --git a/coderbuild/mpnst/requirements.r b/coderbuild/mpnst/requirements.r index e8bfac35..44e0ea0f 100755 --- a/coderbuild/mpnst/requirements.r +++ b/coderbuild/mpnst/requirements.r @@ -11,3 +11,12 @@ install.packages("stringr") install.packages("tidyr") install.packages("readr") install.packages("readxl") + +# Fail the Docker build layer if any required package didn't install. +# install.packages() and BiocManager::install() only WARN on failure, so a +# transient CRAN/Bioconductor outage during `docker build` produced an image +# that built "successfully" but was missing packages -- the failure then +# surfaced hours into a build run instead of at build time. +required_pkgs <- c("reticulate", "remotes", "rjson", "synapser", "dplyr", "data.table", "R.utils", "stringr", "tidyr", "readr", "readxl") +missing_pkgs <- required_pkgs[!sapply(required_pkgs, requireNamespace, quietly = TRUE)] +if (length(missing_pkgs) > 0) stop(paste("Required packages failed to install:", paste(missing_pkgs, collapse = ", "))) From 1c4f112fbdba605ac323fc3a5f21cda35580a012 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:01 -0700 Subject: [PATCH 08/20] Add MPNST treated-sample and treated-omics support --- coderbuild/mpnst/00_sample_gen.R | 12 +- coderbuild/mpnst/00b_sample_gen_treated.R | 594 ++++++++++++++++++ coderbuild/mpnst/01_combined_omics.R | 32 +- coderbuild/mpnst/01b_combined_omics_treated.R | 219 +++++++ coderbuild/mpnst/build_omics.sh | 3 + coderbuild/mpnst/build_samples.sh | 2 + 6 files changed, 851 insertions(+), 11 deletions(-) create mode 100644 coderbuild/mpnst/00b_sample_gen_treated.R create mode 100644 coderbuild/mpnst/01b_combined_omics_treated.R diff --git a/coderbuild/mpnst/00_sample_gen.R b/coderbuild/mpnst/00_sample_gen.R index db1f238e..73f59c30 100644 --- a/coderbuild/mpnst/00_sample_gen.R +++ b/coderbuild/mpnst/00_sample_gen.R @@ -1,3 +1,4 @@ +# 00_sample_gen.R # This script generate a new sample table based on previous dataset's sample file (taking the max improve_sample_id) # Load required libraries library(data.table) @@ -26,9 +27,14 @@ synapser::synLogin(authToken=synapse_token) manifest<-synapser::synTableQuery("select * from syn53503360")$asDataFrame()|> as.data.frame() -#Drop contaminated sample JH-2-009 +#Drop contaminated sample JH-2-009 and others with issues manifest <- manifest %>% - filter(Sample != "JH-2-009") + filter(Sample != "JH-2-009") %>% + filter(Sample != "WU-545") %>% + filter(Sample != "WU-536") %>% + filter(Sample != "WU-505") %>% + filter(Sample != "MN-1") %>% + filter(Sample != "MN-3") ###sample file has a strict schema @@ -56,7 +62,7 @@ sampTable<-manifest|> ##third, generate a sample for the MTs if they were generated pdxmt<-subset(sampTable,!is.na(MicroTissueDrugFolder)) -pdxmt$model_type=rep('xenograft derived organoid',nrow(pdxmt)) +pdxmt$model_type=rep('3D-MEDS',nrow(pdxmt)) print(pdxmt) main<-rbind(sampTable,pdxmt)|> diff --git a/coderbuild/mpnst/00b_sample_gen_treated.R b/coderbuild/mpnst/00b_sample_gen_treated.R new file mode 100644 index 00000000..63db50b1 --- /dev/null +++ b/coderbuild/mpnst/00b_sample_gen_treated.R @@ -0,0 +1,594 @@ +# 00_sample_gen_treated.R +# This script generates treated microtissue samples and appends them to an existing samples file. + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(readxl) +}) + +# ----------------------- +# logging helpers +# ----------------------- +.ts <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +.p <- function(...) cat(.ts(), paste0(...), "\n") + +print_df_head <- function(df, n = 8) { + if (is.null(df) || nrow(df) == 0) { + .p("[PRINT] <0 rows>") + return(invisible(NULL)) + } + n <- min(n, nrow(df)) + print(utils::head(df, n)) + invisible(NULL) +} + +# ----------------------- +# constants +# ----------------------- +CONST_CANCER <- "Malignant peripheral nerve sheath tumor" +CONST_SPECIES <- "Homo sapiens (Human)" +CONST_MODEL <- "3D-MEDS" +CONST_SOURCE <- "NF Data Portal" + +# NOTE: updated to correctly preserve hyphens in IDs (MN-2, JH-2-002, WU-225) +safe_token <- function(x) { + x <- as.character(x) + x <- trimws(x) + x <- gsub("\\s+", "_", x) + x <- gsub("[^[:alnum:]_.-]+", "_", x) + x +} + +# ----------------------- +# Canonicalize individual IDs to hyphen style (MN-2, JH-2-002, WU-225) +# ----------------------- +canonicalize_individual_id <- function(x) { + x <- trimws(as.character(x)) + x <- gsub("[_\\s]+", "-", x) # underscores/spaces -> hyphen + x <- gsub("-+", "-", x) # collapse repeated hyphens + + # Normalize common patterns (only changes if it matches) + x <- sub("^MN-?([0-9A-Za-z]+)$", "MN-\\1", x) + x <- sub("^WU-?([0-9A-Za-z]+)$", "WU-\\1", x) + x <- sub("^JH-?([0-9]+)-?([0-9A-Za-z]+)$", "JH-\\1-\\2", x) + + x +} + +# ----------------------- +# Migrate existing treated other_id prefixes from underscore->hyphen +# This is because some samples were mis-labeled. +# ----------------------- +migrate_treated_other_id_prefix <- function(other_id) { + x <- as.character(other_id) + is_treated <- grepl("_treated_microtissue$", x) + + y <- x + y[is_treated] <- sub("^MN_([0-9A-Za-z]+)_", "MN-\\1_", y[is_treated]) + y[is_treated] <- sub("^WU_([0-9A-Za-z]+)_", "WU-\\1_", y[is_treated]) + y[is_treated] <- sub("^JH_([0-9]+)_([0-9A-Za-z]+)_", "JH-\\1-\\2_", y[is_treated]) + + y +} + +# ----------------------- +# Timepoint normalizer (handles "24 HR", "8HR ", etc) +# ----------------------- +normalize_timepoint <- function(tp) { + tp <- trimws(as.character(tp)) + tpu <- toupper(tp) + tpu <- gsub("\\s+", "", tpu) + + out <- tp + out[tpu %in% c("8HR", "8H")] <- "8h" + out[tpu %in% c("24HR", "24H")] <- "24h" + out[tpu %in% c("0HR", "0H")] <- "0h" + + is_h <- grepl("^[0-9]+h$", tolower(out)) + out[is_h] <- tolower(out[is_h]) + out +} + +hours_from_timepoint <- function(tp_norm) { + tp_norm <- tolower(trimws(as.character(tp_norm))) + hrs <- gsub("h$", "", tp_norm) + suppressWarnings(as.integer(hrs)) +} + +# ----------------------- +# Drop controls: DMSO + untreated (and close variants) +# ----------------------- +is_dmso <- function(drug) { + d <- tolower(trimws(as.character(drug))) + grepl("\\bdmso\\b", d) || grepl("dimethyl", d) || grepl("sulfoxide", d) +} + +is_untreated <- function(drug) { + d <- tolower(trimws(as.character(drug))) + grepl("\\buntreated\\b", d) || grepl("no[_\\s-]*treat", d) || grepl("\\bvehicle\\b", d) +} + +is_control <- function(drug) { + isTRUE(is_dmso(drug)) || isTRUE(is_untreated(drug)) +} + +# ============================================================ +# Builder function: +# - Drops DMSO + untreated rows +# - Aggregates replicates: ONE row per treated sample +# - other_id = {individual_id}_{drug}_{hours}hr_treated_microtissue +# - other_names = comma-separated replicate specimen IDs +# ============================================================ +build_from_standard_map <- function(std_map, orig_samples, batch_key, next_id, require_base = TRUE) { + std_map <- as.data.table(std_map) + + # Canonicalize IDs so other_id uses MN-2 / JH-2-002 / WU-225 formatting + std_map[, individual_id := canonicalize_individual_id(individual_id)] + + # base_exists check (optional) + std_map[, base_exists := individual_id %in% orig_samples$common_name] + + # drop controls early (DMSO + untreated) + std_map[, drop_control := vapply(drug, is_control, logical(1))] + n_ctrl <- sum(std_map$drop_control %in% TRUE) + if (n_ctrl > 0) .p("[", batch_key, "] dropping control rows (DMSO/untreated): ", n_ctrl) + std_map <- std_map[drop_control == FALSE | is.na(drop_control)] + + debug_tbl <- std_map[, .( + batch = batch_key, + specimen_id, + individual_id, + drug, + timepoint, + base_exists + )] + + keep_tbl <- if (require_base) std_map[base_exists == TRUE] else copy(std_map) + + .p("[", batch_key, "] rows after drop controls: ", nrow(std_map)) + .p("[", batch_key, "] require_base=", require_base) + .p("[", batch_key, "] rows kept pre-aggregate: ", nrow(keep_tbl)) + .p("[", batch_key, "] preview:") + print_df_head(keep_tbl, 8) + + if (nrow(keep_tbl) == 0) { + return(list( + new_samples_tbl = data.table(), + map_tbl = data.table(), + debug_tbl = debug_tbl, + next_id = next_id + )) + } + + # ---- First aggregation: collapse exact (individual_id, drug, timepoint) rows + agg1 <- keep_tbl[ + , + .(specimen_ids = sort(unique(trimws(as.character(specimen_id))))), + by = .(individual_id, drug, timepoint) + ] + + # Remove NA/blank specimen IDs inside groups + agg1[, specimen_ids := lapply(specimen_ids, function(v) v[!is.na(v) & nzchar(v)])] + agg1 <- agg1[lengths(specimen_ids) > 0] + + agg1[, other_names := vapply(specimen_ids, function(v) paste(v, collapse = ", "), character(1))] + agg1[, hours := hours_from_timepoint(timepoint)] + agg1[is.na(hours), hours := -1L] + + # ---- Compute other_id_val per group (this is what ends up in samples.csv) + agg1[, other_id_val := paste0( + safe_token(individual_id), "_", + safe_token(drug), "_", + as.integer(hours), "hr_treated_microtissue" + )] + + # ---- SECOND aggregation: merge replicate IDs when other_id_val collides + agg <- agg1[ + , + .( + individual_id = individual_id[1], + drug = drug[1], + timepoint = timepoint[1], + hours = hours[1], + other_names = paste( + sort(unique(trimws(unlist(strsplit(paste(other_names, collapse = ", "), ",\\s*"))))), + collapse = ", " + ) + ), + by = .(other_id_val) + ] + + .p("[", batch_key, "] aggregate groups after merge-by-other_id: ", nrow(agg)) + .p("[", batch_key, "] aggregate head:") + print_df_head(agg[, .(other_id_val, individual_id, drug, timepoint, other_names, hours)], 8) + + # fast lookup of existing treated keys in orig_samples + orig_treated_keys <- paste0(as.character(orig_samples$other_id), "||", as.character(orig_samples$model_type)) + + map_tbl <- data.table( + batch = character(), + individual_id = character(), + drug = character(), + timepoint = character(), + hours = integer(), + other_id = character(), + other_names = character(), + treated_improve_sample_id = integer() + ) + + new_samples_tbl <- data.table( + other_id = character(), + common_name = character(), + other_id_source = character(), + other_names = character(), + cancer_type = character(), + species = character(), + model_type = character(), + improve_sample_id = integer() + ) + + for (i in seq_len(nrow(agg))) { + indiv <- as.character(agg$individual_id[i]) + drug <- as.character(agg$drug[i]) + tp <- as.character(agg$timepoint[i]) + hrs <- as.integer(agg$hours[i]) + repnames <- as.character(agg$other_names[i]) + + other_id_val <- as.character(agg$other_id_val[i]) + key <- paste0(other_id_val, "||", CONST_MODEL) + + # Skip if already exists in samples.csv + if (key %in% orig_treated_keys) { + .p("[", batch_key, "][SKIP] already exists other_id=", other_id_val) + next + } + + treated_id <- next_id + next_id <- next_id + 1L + + new_samples_tbl <- rbind( + new_samples_tbl, + data.table( + other_id = other_id_val, + common_name = indiv, + other_id_source = CONST_SOURCE, + other_names = repnames, + cancer_type = CONST_CANCER, + species = CONST_SPECIES, + model_type = CONST_MODEL, + improve_sample_id = treated_id + ), + fill = TRUE + ) + + map_tbl <- rbind( + map_tbl, + data.table( + batch = batch_key, + individual_id = indiv, + drug = drug, + timepoint = tp, + hours = hrs, + other_id = other_id_val, + other_names = repnames, + treated_improve_sample_id = treated_id + ), + fill = TRUE + ) + + .p("[", batch_key, "][ADD] ", other_id_val, " | reps={", repnames, "} -> ", treated_id) + } + + list( + new_samples_tbl = new_samples_tbl, + map_tbl = map_tbl, + debug_tbl = debug_tbl, + next_id = next_id + ) +} + +write_batch_outputs <- function(batch_key, debug_tbl, map_tbl) { + out_debug_path <- file.path("/tmp", sprintf("mpnst_samples_treated_%s_debug.csv", batch_key)) + out_map_path <- file.path("/tmp", sprintf("mpnst_samples_treated_%s_map.csv", batch_key)) + + .p("[WRITE] ", batch_key, " debug -> ", out_debug_path) + fwrite(debug_tbl, out_debug_path) + .p("[WRITE] OK") + + .p("[WRITE] ", batch_key, " map -> ", out_map_path) + fwrite(map_tbl, out_map_path) + .p("[WRITE] OK") +} + +# ============================================================ +# Batch 1 +# ============================================================ +BATCH1_KEY <- "batch1" +BATCH1_SAMPLEMAP_SYNID <- "syn64608368" + +fetch_batch1_map <- function() { + .p("[BATCH1] synGet(", BATCH1_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH1_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH1] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch1_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch1 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch1_map(fetch_batch1_map()), orig_samples, BATCH1_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 2 +# ============================================================ +BATCH2_KEY <- "batch2" +BATCH2_SAMPLEMAP_SYNID <- "syn66302373" # Update to syn64608372 once the corrected version is uploaded. + + +fetch_batch2_map <- function() { + .p("[BATCH2] synGet(", BATCH2_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH2_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH2] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch2_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + #fixed in metadata, so this is not needed anymore. + # MN2 -> MN-2 + # dt[individualID == "MN2", individualID := "MN-2"] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch2 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch2_map(fetch_batch2_map()), orig_samples, BATCH2_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 3 +# ============================================================ +BATCH3_KEY <- "batch3" +BATCH3_SAMPLEMAP_SYNID <- "syn66050299" + +fetch_batch3_map <- function() { + .p("[BATCH3] synGet(", BATCH3_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH3_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH3] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch3_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch3 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch3_map(fetch_batch3_map()), orig_samples, BATCH3_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 4 (CSV) +# ============================================================ +BATCH4_KEY <- "batch4" +BATCH4_SAMPLEMAP_SYNID <- "syn72518652" + +fetch_batch4_map <- function() { + .p("[BATCH4] synGet(", BATCH4_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH4_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH4] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch4_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + + # Expected columns in 01252026_Batch4_samplemap.csv: + # specimenID, individualID, Drug, Timepoint + if (!all(c("specimenID", "individualID", "Drug", "Timepoint") %in% names(dt))) { + .p("[BATCH4][WARN] Missing required columns. Found: ", paste(names(dt), collapse = " | ")) + return(data.table( + specimen_id = character(), + individual_id = character(), + drug = character(), + timepoint = character() + )) + } + + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + out <- dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] + + .p("[BATCH4] normalized dim: ", nrow(out), " x ", ncol(out)) + .p("[BATCH4] normalized head:") + print_df_head(out, 8) + + out +} + +run_batch4 <- function(orig_samples, next_id) { + build_from_standard_map( + normalize_batch4_map(fetch_batch4_map()), + orig_samples, + BATCH4_KEY, + next_id, + require_base = FALSE + ) +} + + +# ============================================================ +# Register all batches (order) +# ============================================================ +ALL_BATCH_RUNNERS <- list( + list(key=BATCH1_KEY, fn=run_batch1), + list(key=BATCH2_KEY, fn=run_batch2), + list(key=BATCH3_KEY, fn=run_batch3), + list(key=BATCH4_KEY, fn=run_batch4) +) + +# ============================================================ +# MAIN +# ============================================================ +args <- commandArgs(trailingOnly = TRUE) + +.p("============================================================") +.p("=== 00b_sample_gen_treated.R (RUN ALL BATCHES) ===") +.p("Timestamp: ", .ts()) +.p("Args: ", paste(args, collapse = " | ")) +.p("============================================================") + +if (length(args) < 1) { + stop( + "Usage: Rscript 00b_sample_gen_treated.R \n", + "Example: Rscript 00b_sample_gen_treated.R /tmp/mpnst_samples.csv", + call. = FALSE + ) +} + +prev_samples_path <- args[1] +.p("[ARGS] prev_samples_path=", prev_samples_path) +if (!file.exists(prev_samples_path)) stop("File not found: ", prev_samples_path, call. = FALSE) + +.p("[READ] fread(samples) ...") +orig_samples <- fread(prev_samples_path) +.p("[READ] orig_samples dim: ", nrow(orig_samples), " x ", ncol(orig_samples)) +print_df_head(orig_samples, 6) + +# ------------------------------------------------------------ +# Migrate already-existing treated other_ids to hyphen style +# so future runs match your desired format and don't skip forever +# ------------------------------------------------------------ +orig_samples[, other_id_old := other_id] +orig_samples[, other_id := migrate_treated_other_id_prefix(other_id)] + +n_changed <- sum(orig_samples$other_id != orig_samples$other_id_old, na.rm = TRUE) +if (n_changed > 0) .p("[MIGRATE] updated treated other_id prefix for ", n_changed, " row(s)") +orig_samples[, other_id_old := NULL] + +# If migration created duplicates (same other_id + model_type), merge other_names and drop dup rows +orig_samples[, dedup_key := paste(other_id, model_type, sep = "||")] +dup_keys <- orig_samples[duplicated(dedup_key), unique(dedup_key)] + +if (length(dup_keys) > 0) { + .p("[MIGRATE] duplicates after migration: ", length(dup_keys), " key(s). Merging other_names + dropping dup rows.") + + merged <- orig_samples[dedup_key %in% dup_keys, .( + merged_other_names = paste( + sort(unique(trimws(unlist(strsplit(paste(na.omit(other_names), collapse = ", "), ",\\s*"))))), + collapse = ", " + ) + ), by = dedup_key] + + orig_samples[merged, on = "dedup_key", other_names := i.merged_other_names] + orig_samples <- orig_samples[!duplicated(dedup_key)] +} + +orig_samples[, dedup_key := NULL] + +token <- Sys.getenv("SYNAPSE_AUTH_TOKEN") +.p("[ENV] SYNAPSE_AUTH_TOKEN present? ", nzchar(token)) +if (!nzchar(token)) stop("Please set SYNAPSE_AUTH_TOKEN in your environment.", call. = FALSE) + +.p("[SYN] synLogin() ...") +synLogin(authToken = token) +.p("[SYN] synLogin() OK") + +max_id <- suppressWarnings(max(as.integer(orig_samples$improve_sample_id), na.rm = TRUE)) +if (!is.finite(max_id)) max_id <- 0L +next_id <- max_id + 1L +.p("[IDS] max improve_sample_id=", max_id, " next_id=", next_id) + +.p("============================================================") +.p("[RUN] Running ", length(ALL_BATCH_RUNNERS), " batch runner(s) in order...") +.p("============================================================") + +for (item in ALL_BATCH_RUNNERS) { + batch_key <- item$key + runner <- item$fn + + res <- runner(orig_samples = orig_samples, next_id = next_id) + + if (!is.null(res$new_samples_tbl) && nrow(res$new_samples_tbl) > 0) { + orig_samples <- rbindlist(list(orig_samples, res$new_samples_tbl), use.names = TRUE, fill = TRUE) + + orig_samples[, dedup_key := paste(other_id, model_type, sep = "||")] + before <- nrow(orig_samples) + orig_samples <- orig_samples[!duplicated(dedup_key)] + after <- nrow(orig_samples) + if (after < before) .p("[", batch_key, "][WARN] post-append dedup removed ", before - after, " row(s)") + orig_samples[, dedup_key := NULL] + } + + next_id <- res$next_id + + .p("[RUN] Finished ", batch_key, " | added=", ifelse(is.null(res$new_samples_tbl), 0, nrow(res$new_samples_tbl))) + write_batch_outputs(batch_key, res$debug_tbl, res$map_tbl) +} + +.p("[WRITE] Writing updated samples -> ", prev_samples_path) +fwrite(orig_samples, prev_samples_path) +.p("[WRITE] OK") + +.p("============================================================") +.p("[DONE] Completed all batches. Updated samples at: ", prev_samples_path) +.p("============================================================") diff --git a/coderbuild/mpnst/01_combined_omics.R b/coderbuild/mpnst/01_combined_omics.R index dcbdfbae..7e7dd41a 100644 --- a/coderbuild/mpnst/01_combined_omics.R +++ b/coderbuild/mpnst/01_combined_omics.R @@ -1,3 +1,4 @@ +# 01_combined_omics.R #!/usr/bin/env Rscript # Combined MPNST & MPNST-PDX Data Extraction Script @@ -31,7 +32,7 @@ genes_df <- fread(genes) # Subset by model type pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") tumor_samps<- filter(samples_df, model_type == "tumor") -mt_samps <- filter(samples_df, model_type == "xenograft derived organoid") # These end up being the same as pdx_samps in the manifest. +mt_samps <- filter(samples_df, model_type == "3D-MEDS") # These end up being the same as pdx_samps in the manifest. # Retrieve manifest table from Synapse manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% @@ -58,7 +59,7 @@ tumor_data <- manifest %>% mutate(Proteomics = "") %>% filter(!is.na(improve_sample_id)) -mt_data <- manifest %>% #Note, this is the same as pdx_data but I think we default to "xenograft derived organoid" if present (based on original files) +mt_data <- manifest %>% #Note, this is the same as pdx_data but I think we default to "3D-MEDS" if present (based on original files) select(common_name, starts_with("PDX")) %>% left_join(mt_samps, by = "common_name") %>% select(improve_sample_id, common_name, model_type, @@ -78,7 +79,7 @@ study_label <- function(type) { case_when( type == "patient derived xenograft" ~ "MPNST PDX", type == "tumor" ~ "MPNST Tumor", - type == "xenograft derived organoid" ~ "MPNST PDX MT", + type == "3D-MEDS" ~ "MPNST PDX MT", TRUE ~ "MPNST" ) } @@ -147,13 +148,28 @@ transcriptomics_list <- lapply( if (is.null(meta)) return(NULL) df <- tryCatch({ - fread(synGet(id)$path) %>% + raw <- fread(synGet(id)$path) %>% separate(Name, into = c("other_id","vers"), sep = "\\.") %>% - select(-vers) %>% + select(-vers) + + is_enst_input <- all(grepl("^ENST", raw$other_id)) + + mapped <- raw %>% left_join(genes_df) %>% - select(entrez_id, transcriptomics = TPM) %>% - filter(!is.na(entrez_id), transcriptomics != 0) %>% - distinct() + select(entrez_id, other_id, transcriptomics = TPM) %>% + filter(!is.na(entrez_id), transcriptomics != 0) + + if (is_enst_input) { + mapped <- mapped %>% + group_by(entrez_id) %>% + summarise(transcriptomics = sum(transcriptomics), .groups = "drop") + } else { + mapped <- mapped %>% + select(entrez_id, transcriptomics) %>% + distinct() + } + + mapped }, error = function(e) NULL) i_safe_extract( diff --git a/coderbuild/mpnst/01b_combined_omics_treated.R b/coderbuild/mpnst/01b_combined_omics_treated.R new file mode 100644 index 00000000..8125fc61 --- /dev/null +++ b/coderbuild/mpnst/01b_combined_omics_treated.R @@ -0,0 +1,219 @@ +# 01b_combined_omics_treated.R +#!/usr/bin/env Rscript + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(dplyr) + library(tidyr) +}) + +# ----------------------- +# logging helpers +# ----------------------- +.ts <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +.p <- function(...) cat(.ts(), paste0(...), "\n") + +# ----------------------- +# batch inputs (Synapse) +# ----------------------- +B1_SYN <- "syn64608366" # salmon.merged.gene_tpm_corrected_id.csv +B2_SYN <- "syn64608370" # salmon.merged.gene_tpm.tsv_corrected_ID.csv +B3_SYN <- "syn65887795" # salmon.merged.gene_tpm.tsv +B4_SYN <- "syn68898091" # salmon.merged.gene_tpm.tsv + +# ----------------------- +# helpers +# ----------------------- +study_label <- function(type) { + dplyr::case_when( + type == "patient derived xenograft" ~ "MPNST PDX", + type == "tumor" ~ "MPNST Tumor", + type == "3D-MEDS" ~ "MPNST PDX MT", + TRUE ~ "MPNST" + ) +} + +is_treated_microtissue <- function(other_id, other_names) { + oid <- as.character(other_id) + onm <- as.character(other_names) + grepl("treated_microtissue$", oid) && !is.na(onm) && nzchar(trimws(onm)) +} + +split_rep_ids <- function(x) { + if (is.na(x) || !nzchar(trimws(x))) return(character(0)) + v <- unlist(strsplit(x, ",")) + v <- trimws(v) + v <- v[nzchar(v)] + unique(v) +} + +guess_batch_from_rep <- function(rep_id) { + r <- as.character(rep_id) + if (grepl("^b1_", r)) return("b1") + if (grepl("^b2_", r)) return("b2") + if (grepl("^B3-", r) || grepl("^B3_", r)) return("b3") + if (grepl("^B4-", r) || grepl("^B4\\.", r)) return("b4") + return(NA_character_) +} + +rep_to_colname <- function(rep_id, batch) { + r <- as.character(rep_id) + if (batch %in% c("b1", "b2")) return(r) + if (batch == "b3") return(gsub("-", "_", r, fixed = FALSE)) # B3-10 -> B3_10 + if (batch == "b4") return(gsub("B4-", "B4.", r, fixed = TRUE)) # B4-10 -> B4.10 + return(r) +} + +read_tpm_matrix <- function(syn_id) { + .p("[SYN] synGet(", syn_id, ") ...") + pth <- synGet(syn_id)$path + .p("[READ] ", syn_id, " -> ", pth) + dt <- fread(pth) + stopifnot(all(c("gene_id", "gene_name") %in% names(dt))) + dt +} + +# returns a data.table with columns: +# gene_id, transcriptomics, improve_sample_id, source, study +extract_treated_from_batch <- function(mat, treated_tbl, batch_key) { + if (nrow(treated_tbl) == 0) return(data.table()) + + tmp <- copy(treated_tbl) + tmp[, rep_ids := lapply(other_names, split_rep_ids)] + tmp[, rep_batch := vapply(rep_ids, function(v) { + if (length(v) == 0) return(NA_character_) + guess_batch_from_rep(v[1]) + }, character(1))] + + tmp <- tmp[rep_batch == batch_key] + if (nrow(tmp) == 0) return(data.table()) + + tmp[, rep_cols := lapply(rep_ids, function(v) rep_to_colname(v, batch_key))] + + .p("[", batch_key, "] treated samples in this batch: ", nrow(tmp)) + + out_list <- vector("list", nrow(tmp)) + + for (i in seq_len(nrow(tmp))) { + samp_id <- tmp$improve_sample_id[i] + samp_model <- tmp$model_type[i] + cols <- tmp$rep_cols[[i]] + cols <- cols[cols %in% names(mat)] + + if (length(cols) == 0) { + .p("[", batch_key, "][SKIP] improve_sample_id=", samp_id, + " -> no replicate columns found in matrix") + next + } + + expr <- rowMeans(as.matrix(mat[, ..cols]), na.rm = TRUE) + + out_list[[i]] <- data.table( + gene_id = mat$gene_id, + transcriptomics = expr, + improve_sample_id = samp_id, + source = "NF Data Portal", + study = study_label(samp_model) + ) + } + + rbindlist(out_list, use.names = TRUE, fill = TRUE) +} + +# ----------------------- +# MAIN +# ----------------------- +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 3) { + stop("Usage: Rscript 01b_combined_omics_treated.R ", call. = FALSE) +} +PAT <- args[1] +samples <- args[2] +genes <- args[3] + +.p("============================================================") +.p("=== 01b_combined_omics_treated.R ===") +.p("Timestamp: ", .ts()) +.p("Args: ", paste(args, collapse = " | ")) +.p("============================================================") + +.p("[SYN] synLogin() ...") +synLogin(authToken = PAT) +.p("[SYN] synLogin() OK") + +.p("[READ] samples: ", samples) +samples_df <- fread(samples) + +treated <- samples_df[ + model_type == "3D-MEDS" & + vapply(seq_len(nrow(samples_df)), function(i) { + is_treated_microtissue(samples_df$other_id[i], samples_df$other_names[i]) + }, logical(1)), + .(other_names, improve_sample_id, model_type) +] + +.p("[SAMPLES] treated_microtissue rows: ", nrow(treated)) +if (nrow(treated) == 0) stop("No treated microtissue samples found in samples.csv", call. = FALSE) + +.p("[READ] genes: ", genes) +genes_df <- fread(genes) +stopifnot(all(c("other_id", "entrez_id") %in% names(genes_df))) + +# matrices +m1 <- read_tpm_matrix(B1_SYN) +m2 <- read_tpm_matrix(B2_SYN) +m3 <- read_tpm_matrix(B3_SYN) +m4 <- read_tpm_matrix(B4_SYN) + +# extract +t1 <- extract_treated_from_batch(m1, treated, "b1") +t2 <- extract_treated_from_batch(m2, treated, "b2") +t3 <- extract_treated_from_batch(m3, treated, "b3") +t4 <- extract_treated_from_batch(m4, treated, "b4") + +treated_long <- rbindlist(list(t1, t2, t3, t4), use.names = TRUE, fill = TRUE) +.p("[OUT] treated_long rows: ", nrow(treated_long)) +if (nrow(treated_long) == 0) stop("No treated data extracted (check replicate IDs vs column names).", call. = FALSE) + +# ENSEMBL gene_id has version -> strip to match genes_df$other_id +treated_long[, other_id := tstrsplit(gsub("\\s+", "", gene_id), "\\.", keep = 1)] + +treated_final <- treated_long %>% + left_join(genes_df, by = "other_id") %>% + dplyr::select(entrez_id, transcriptomics, improve_sample_id, source, study) %>% + filter(!is.na(entrez_id), transcriptomics != 0) %>% + distinct() + +# enforce strict schema & order +treated_final <- as.data.table(treated_final)[, .(entrez_id, transcriptomics, improve_sample_id, source, study)] + +treated_out_path <- file.path("/tmp", "mpnst_transcriptomics_treated_microtissue.csv") +.p("[WRITE] treated-only transcriptomics -> ", treated_out_path) +fwrite(treated_final, treated_out_path) +.p("[WRITE] OK") + +# optional append into main transcriptomics (same strict schema) +main_path <- file.path("/tmp", "mpnst_transcriptomics.csv") +if (file.exists(main_path)) { + .p("[APPEND] Found existing main transcriptomics: ", main_path) + main_df <- fread(main_path) + main_df <- main_df[, .(entrez_id, transcriptomics, improve_sample_id, source, study)] # enforce schema + + combined <- rbindlist(list(main_df, treated_final), use.names = TRUE, fill = TRUE) + + # de-dup conservatively on (entrez_id, improve_sample_id, source, study, transcriptomics) + combined[, dedup_key := paste(entrez_id, improve_sample_id, source, study, transcriptomics, sep = "||")] + combined <- combined[!duplicated(dedup_key)] + combined[, dedup_key := NULL] + + .p("[WRITE] overwriting main transcriptomics with appended treated rows -> ", main_path) + fwrite(combined, main_path) + .p("[WRITE] OK") +} else { + .p("[APPEND] No existing /tmp/mpnst_transcriptomics.csv found. Not appending.") +} + +.p("============================================================") +.p("[DONE] Wrote treated transcriptomics with strict schema.") +.p("============================================================") diff --git a/coderbuild/mpnst/build_omics.sh b/coderbuild/mpnst/build_omics.sh index d6d2cec7..c6b0853c 100644 --- a/coderbuild/mpnst/build_omics.sh +++ b/coderbuild/mpnst/build_omics.sh @@ -1,3 +1,4 @@ +# build_omics.sh #!/bin/bash set -euo pipefail @@ -5,3 +6,5 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit echo "Running 01_combined_omics.R with $SYNAPSE_AUTH_TOKEN, $2, and $1." Rscript 01_combined_omics.R $SYNAPSE_AUTH_TOKEN $2 $1 +echo "Running 01b_combined_omics_treated.R with PAT, samples=$2, genes=$1" +Rscript 01b_combined_omics_treated.R "$SYNAPSE_AUTH_TOKEN" "$2" "$1" \ No newline at end of file diff --git a/coderbuild/mpnst/build_samples.sh b/coderbuild/mpnst/build_samples.sh index c9c079fa..832ed895 100644 --- a/coderbuild/mpnst/build_samples.sh +++ b/coderbuild/mpnst/build_samples.sh @@ -1,3 +1,4 @@ +# build_samples.sh #!/bin/bash set -euo pipefail @@ -5,3 +6,4 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit echo "Running 00_sample_gen.R with $1." Rscript 00_sample_gen.R $1 +Rscript 00b_sample_gen_treated.R /tmp/mpnst_samples.csv From 55ec7e167c8b0750d1c1ed4ecbae1bc7ea01fcab Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:01 -0700 Subject: [PATCH 09/20] Add the cNF organoid dataset: drugs and experiments --- coderbuild/cnf/03-drugs-cnf.py | 531 +++++++++++++++++++++++++++ coderbuild/cnf/04-experiments-cnf.py | 478 ++++++++++++++++++++++++ coderbuild/cnf/build_drugs.sh | 20 + coderbuild/cnf/build_exp.sh | 12 + 4 files changed, 1041 insertions(+) create mode 100644 coderbuild/cnf/03-drugs-cnf.py create mode 100644 coderbuild/cnf/04-experiments-cnf.py create mode 100644 coderbuild/cnf/build_drugs.sh create mode 100644 coderbuild/cnf/build_exp.sh diff --git a/coderbuild/cnf/03-drugs-cnf.py b/coderbuild/cnf/03-drugs-cnf.py new file mode 100644 index 00000000..fe307ddb --- /dev/null +++ b/coderbuild/cnf/03-drugs-cnf.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 +""" +03-drugs-cnf.py: generate cnf_drugs.tsv and cnf_drug_descriptors.tsv. + +Pulls unique drug names from the cNF drug screen on Synapse, then calls the +standard coderdata utilities: + + coderbuild/utils/pubchem_retrieval.py + -> cnf_drugs.tsv + + coderbuild/utils/build_drug_descriptor_table.py + -> cnf_drug_descriptors.tsv + +Important implementation note: + pubchem_retrieval.py does not expose a command-line interface in the + provided version. It defines update_dataframe_and_write_tsv(), but running + `python pubchem_retrieval.py --input ... --output ...` exits 0 without + doing any work. Therefore this script imports pubchem_retrieval.py directly + and calls update_dataframe_and_write_tsv(). + +The descriptor builder is still run as a subprocess. It may call R/biomaRt +internally, which can fail when Ensembl is unavailable. To keep the drugs file +produced even when descriptors fail, this script: + + 1. Retries external/remote work with exponential backoff. + 2. Sets a generous timeout. + 3. Supports --skip-descriptors. + 4. Supports --only-descriptors for retrying descriptor generation later. + +Usage: + python 03-drugs-cnf.py --prev_drugs + [--out_drugs /tmp/cnf_drugs.tsv] + [--out_desc /tmp/cnf_drug_descriptors.tsv] + [--skip-descriptors] + [--only-descriptors] + [--retries 3] + [--timeout 1800] +""" + +import argparse +import importlib.util +import logging +import os +import signal +import subprocess +import sys +import tempfile +import time +import pandas as pd +import synapseclient + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- + +DRUG_SCREEN_TABLE = "syn51301431" + +PUBCHEM_SCRIPT = os.environ.get( + "CODERDATA_PUBCHEM_SCRIPT", + "/coderbuild/utils/pubchem_retrieval.py", +) + +DESCRIPTOR_SCRIPT = os.environ.get( + "CODERDATA_DESCRIPTOR_SCRIPT", + "/coderbuild/utils/build_drug_desc.py", +) + +DEFAULT_OUT_DRUGS = "/tmp/cnf_drugs.tsv" +DEFAULT_OUT_DESC = "/tmp/cnf_drug_descriptors.tsv" + +DEFAULT_RETRIES = 3 +DEFAULT_TIMEOUT = 1800 # seconds per attempt +RETRY_BACKOFF_S = 30 # 30s, 60s, 120s, ... + + +# ---------------------------------------------------------------------------- +# Logging +# ---------------------------------------------------------------------------- + +def configure_logging() -> None: + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Generic subprocess wrapper with retry +# ---------------------------------------------------------------------------- + +def run_with_retries( + cmd: list[str], + retries: int, + timeout: int, + label: str, +) -> bool: + """ + Run a subprocess, retrying on failure with exponential backoff. + + Returns True on success, False if all retries are exhausted. + """ + for attempt in range(1, retries + 1): + logging.info("[%s] attempt %d/%d", label, attempt, retries) + + try: + subprocess.run(cmd, check=True, timeout=timeout) + logging.info("[%s] succeeded on attempt %d", label, attempt) + return True + + except subprocess.TimeoutExpired: + logging.warning( + "[%s] timed out after %ds on attempt %d", + label, + timeout, + attempt, + ) + + except subprocess.CalledProcessError as exc: + logging.warning( + "[%s] failed with exit %d on attempt %d", + label, + exc.returncode, + attempt, + ) + + except Exception as exc: + logging.warning( + "[%s] unexpected error on attempt %d: %s", + label, + attempt, + exc, + ) + + if attempt < retries: + backoff = RETRY_BACKOFF_S * (2 ** (attempt - 1)) + logging.info("[%s] sleeping %ds before retry", label, backoff) + time.sleep(backoff) + + logging.error("[%s] exhausted %d retries", label, retries) + return False + + +# ---------------------------------------------------------------------------- +# PubChem utility loading/calling +# ---------------------------------------------------------------------------- + +def load_pubchem_module(path: str): + """ + Load pubchem_retrieval.py as a Python module. + + The provided pubchem_retrieval.py has no command-line entry point, so it + must be imported and called directly. + """ + if not os.path.exists(path): + raise FileNotFoundError(f"PubChem retrieval script not found: {path}") + + spec = importlib.util.spec_from_file_location( + "coderdata_pubchem_retrieval", + path, + ) + + if spec is None or spec.loader is None: + raise ImportError(f"Could not create import spec for {path}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + if not hasattr(module, "update_dataframe_and_write_tsv"): + raise AttributeError( + f"{path} does not define update_dataframe_and_write_tsv()" + ) + + return module + + +def call_pubchem_retrieval( + names: list[str], + prev_drug_files: list[str], + out_drugs: str, + retries: int, + timeout: int, +) -> bool: + """ + Generate the drugs TSV by calling pubchem_retrieval.update_dataframe_and_write_tsv(). + + This intentionally does not use subprocess because the supplied + pubchem_retrieval.py has no argparse/main block and exits 0 without writing + anything when run as a script. + """ + prev_arg = ",".join(prev_drug_files) if prev_drug_files else None + + for attempt in range(1, retries + 1): + logging.info("[pubchem_retrieval] attempt %d/%d", attempt, retries) + + ignore_chems = None + + try: + pubchem = load_pubchem_module(PUBCHEM_SCRIPT) + + with tempfile.NamedTemporaryFile( + "w", + suffix="_cnf_ignore_chems.txt", + delete=False, + ) as tmp_ignore: + ignore_chems = tmp_ignore.name + + df = pubchem.update_dataframe_and_write_tsv( + unique_names=names, + output_filename=out_drugs, + ignore_chems=ignore_chems, + batch_size=1, + isname=True, + time_limit=timeout, + prev_drug_filepaths=prev_arg, + restrict_to_raw_names=names, + ) + + # pubchem_retrieval.py sets signal.alarm(time_limit), but does not + # clear it. Clear it here so the descriptor step is not interrupted. + try: + signal.alarm(0) + except Exception: + pass + + if getattr(pubchem, "should_continue", True) is False: + raise TimeoutError( + f"pubchem_retrieval reached its time limit of {timeout}s" + ) + + if not os.path.exists(out_drugs): + raise FileNotFoundError( + f"pubchem_retrieval returned but did not create {out_drugs}" + ) + + if os.path.getsize(out_drugs) == 0: + raise RuntimeError( + f"pubchem_retrieval created {out_drugs}, but it is empty" + ) + + n_rows = len(df) if df is not None else "unknown" + + logging.info( + "[pubchem_retrieval] succeeded on attempt %d; wrote %s rows to %s", + attempt, + n_rows, + out_drugs, + ) + + return True + + except Exception as exc: + try: + signal.alarm(0) + except Exception: + pass + + logging.warning( + "[pubchem_retrieval] failed on attempt %d/%d: %s", + attempt, + retries, + exc, + ) + + if attempt < retries: + backoff = RETRY_BACKOFF_S * (2 ** (attempt - 1)) + logging.info( + "[pubchem_retrieval] sleeping %ds before retry", + backoff, + ) + time.sleep(backoff) + + finally: + if ignore_chems and os.path.exists(ignore_chems): + try: + os.unlink(ignore_chems) + except OSError: + pass + + logging.error("[pubchem_retrieval] exhausted %d retries", retries) + return False + + +# ---------------------------------------------------------------------------- +# Drug name collection +# ---------------------------------------------------------------------------- + +def collect_drug_names(syn) -> list[str]: + """ + Pull every cNF drug-screen file from Synapse and collect unique drug names. + """ + query = ( + "select id, individualID, specimenID " + f"from {DRUG_SCREEN_TABLE} " + "where dataType='drug screen'" + ) + + files = syn.tableQuery(query).asDataFrame() + + logging.info("Reading %d drug-screen files for drug names", len(files)) + + names: set[str] = set() + + for _, row in files.iterrows(): + try: + entity = syn.get(row["id"]) + df = pd.read_csv(entity.path) + + except Exception as exc: + logging.warning("Skipping file %s: %s", row["id"], exc) + continue + + if "Drug" not in df.columns: + logging.warning("File %s has no 'Drug' column; skipping", row["id"]) + continue + + names.update( + df["Drug"] + .dropna() + .astype(str) + .str.strip() + .tolist() + ) + + names = sorted(n for n in names if n and n.lower() != "nan") + + logging.info("Collected %d unique drug names", len(names)) + + return names + + +# ---------------------------------------------------------------------------- +# Descriptor utility +# ---------------------------------------------------------------------------- + +def call_descriptor_table( + drug_file: str, + out_desc: str, + retries: int, + timeout: int, +) -> bool: + """ + Run the descriptor table builder as a subprocess. + + This function assumes build_drug_descriptor_table.py has a CLI accepting + --input and --output. + """ + cmd = [ + "python", + DESCRIPTOR_SCRIPT, + "--drugtable", + drug_file, + "--desctable", + out_desc, + ] + + ok = run_with_retries( + cmd, + retries, + timeout, + "descriptor_table", + ) + + if ok and not os.path.exists(out_desc): + logging.error( + "descriptor_table exited successfully but did not create %s", + out_desc, + ) + return False + + return ok + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument( + "--prev_drugs", + default="", + help="Comma-delimited list of existing drug TSV files for ID reuse.", + ) + + parser.add_argument( + "--out_drugs", + default=DEFAULT_OUT_DRUGS, + help=f"Output drugs TSV. Default: {DEFAULT_OUT_DRUGS}", + ) + + parser.add_argument( + "--out_desc", + default=DEFAULT_OUT_DESC, + help=f"Output drug descriptors TSV. Default: {DEFAULT_OUT_DESC}", + ) + + parser.add_argument( + "--skip-descriptors", + action="store_true", + help=( + "Skip the descriptor step entirely. Use when the descriptor " + "utility or its biomaRt/Ensembl dependency is failing. The drugs " + "table is still produced." + ), + ) + + parser.add_argument( + "--only-descriptors", + action="store_true", + help=( + "Only run the descriptor step, expecting --out_drugs to already " + "exist. Useful for retrying after a previous descriptor failure." + ), + ) + + parser.add_argument( + "--retries", + type=int, + default=DEFAULT_RETRIES, + help=f"Number of retry attempts. Default: {DEFAULT_RETRIES}", + ) + + parser.add_argument( + "--timeout", + type=int, + default=DEFAULT_TIMEOUT, + help=f"Timeout in seconds per attempt. Default: {DEFAULT_TIMEOUT}", + ) + + args = parser.parse_args() + + configure_logging() + + if args.skip_descriptors and args.only_descriptors: + logging.error("Cannot combine --skip-descriptors with --only-descriptors") + return 2 + + for path in (args.out_drugs, args.out_desc): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + + # ------------------------------------------------------------------------ + # Drugs step + # ------------------------------------------------------------------------ + + if not args.only_descriptors: + syn = synapseclient.Synapse() + syn.login() + + names = collect_drug_names(syn) + + if not names: + logging.error("No drug names found; aborting") + return 1 + + prev = [ + p.strip() + for p in args.prev_drugs.split(",") + if p.strip() + ] + + ok = call_pubchem_retrieval( + names=names, + prev_drug_files=prev, + out_drugs=args.out_drugs, + retries=args.retries, + timeout=args.timeout, + ) + + if not ok: + logging.error("pubchem_retrieval failed; aborting") + return 1 + + if not os.path.exists(args.out_drugs): + logging.error( + "pubchem_retrieval did not produce %s", + args.out_drugs, + ) + return 1 + + logging.info("Wrote %s", args.out_drugs) + + # ------------------------------------------------------------------------ + # Descriptor step + # ------------------------------------------------------------------------ + + if args.skip_descriptors: + logging.warning( + "Skipping descriptor step. Re-run with --only-descriptors to " + "produce %s when descriptor dependencies are reachable.", + args.out_desc, + ) + return 0 + + if not os.path.exists(args.out_drugs): + logging.error( + "Drugs file %s missing; cannot run descriptors", + args.out_drugs, + ) + return 1 + + ok = call_descriptor_table( + drug_file=args.out_drugs, + out_desc=args.out_desc, + retries=args.retries, + timeout=args.timeout, + ) + + if not ok: + logging.error( + "Descriptor step failed after %d attempts. The drugs file %s is " + "complete; re-run with --only-descriptors when descriptor " + "dependencies are reachable to produce %s.", + args.retries, + args.out_drugs, + args.out_desc, + ) + + # Exit 0 because the primary drugs table was produced. Descriptors can + # be retried later. Use --skip-descriptors to suppress this explicitly. + return 0 + + logging.info("Wrote %s", args.out_desc) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/coderbuild/cnf/04-experiments-cnf.py b/coderbuild/cnf/04-experiments-cnf.py new file mode 100644 index 00000000..fe1e216b --- /dev/null +++ b/coderbuild/cnf/04-experiments-cnf.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +""" +04-experiments-cnf.py: generate cnf_experiments.tsv. + +Discovers every drug-screen viability file under three Synapse parent folders, +attaches a specimen ID by reading the file's `specimenID` annotation (with a +filename-pattern fallback), then per (specimen, drug): + + * Multi-concentration measurements are run through the standard + coderbuild/utils/fit_curve.py utility, producing fit_auc / fit_ic50 / + fit_einf / fit_hs / fit_r2. + * Single-concentration measurements at 1 μM are recorded with + metric = 'uM_viability' and value = Viability_percentage / 100. + Schema dependency: 'uM_viability' must be in the ResponseMetric enum. + See schema_patch.md. + +Specimen attribution policy: + 1. Trust the file's `specimenID` annotation if present. + 2. Fall back to parsing the filename (e.g. "NF0017_T2_Viabilities.csv" + → "NF0017_T2"). + 3. Run the result through cnf_utils.classify_specimen as a safety net + (handles dot-separators, suffixes, case variants). Files whose + specimen can't be classified are skipped with a warning. + +Dual-mapping: one viability file's results should be attributed to two +specimens. Handled via SPECIMEN_DUAL_MAPPINGS at the top of the script; +add new entries as more cases come up. + +Usage: + python 04-experiments-cnf.py + [--output /tmp/cnf_experiments.tsv] + [--parents syn51301414,syn51301420,syn51301426] +""" + +import argparse +import logging +import os +import re +import subprocess +import sys +import tempfile + +import pandas as pd +import synapseclient + +from cnf_utils import classify_specimen + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- +# Synapse folders to walk for drug-screen viability files +DEFAULT_DRUG_SCREEN_PARENTS = [ + "syn51301414", + "syn51301420", + "syn51301426", +] + +TIME = 120 +TIME_UNIT = "hours" +STUDY = "cnf" +SOURCE = "Synapse" +SINGLE_DOSE_UM = 1.0 +SINGLE_DOSE_METRIC = "uM_viability" + +FIT_CURVE_SCRIPT = os.environ.get( + "CODERDATA_FIT_CURVE_SCRIPT", "/coderbuild/utils/fit_curve.py") + +DEFAULT_OUTPUT = "/tmp/cnf_experiments.tsv" + +# Regex to recover specimen from filenames like "NF0017_T2_Viabilities.csv" +# or "NF0021_T1_Onalespid_1uM_Viabilities.csv" +FILENAME_SPECIMEN_RE = re.compile( + r"^(NF\d{4}(?:_T\d+)?(?:_[A-Za-z0-9._-]+?)?)" # specimen prefix + r"_(?:Viabilities|viabilities|Viability|drug_screen)" # known label + r"\.(?:csv|tsv|txt)$", + re.IGNORECASE, +) + +# ---------------------------------------------------------------------------- +# DUAL-MAPPING TABLE +# ---------------------------------------------------------------------------- +# Maps a canonical specimen to any additional specimens that should receive +# the same drug-response rows. Add new entries as more cases come up. +# +# Known case: NF0021_T1_Onalespid_1uM was screened against the same drug +# panel as NF0021_T1 and should be available under both labels. +SPECIMEN_DUAL_MAPPINGS: dict[str, list[str]] = { + "NF0021_T1_Onalespid_1uM": ["NF0021_T1"], +} + + +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Specimen attribution +# ---------------------------------------------------------------------------- +def specimen_from_annotation(syn, file_id: str) -> str | None: + """Return the file's specimenID annotation, or None if missing.""" + try: + anno = syn.getAnnotations(file_id) + except Exception as exc: + logging.warning("getAnnotations failed for %s: %s", file_id, exc) + return None + val = anno.get("specimenID") + if isinstance(val, list): + val = val[0] if val else None + if val is None: + return None + s = str(val).strip() + return s or None + + +def specimen_from_filename(name: str) -> str | None: + """Parse 'NF0017_T2_Viabilities.csv' → 'NF0017_T2'.""" + m = FILENAME_SPECIMEN_RE.match(name) + if m: + return m.group(1) + return None + + +def resolve_specimen(syn, file_id: str, file_name: str) -> str | None: + """Resolve a file to a canonical specimen ID, trying annotation first. + + Returns canonical_id (e.g. "NF0017_T2") or None if both annotation and + filename parsing fail or the result doesn't classify as a known sample + type. + """ + raw = specimen_from_annotation(syn, file_id) + if not raw: + raw = specimen_from_filename(file_name) + if not raw: + return None + + # Safety-net classification — handles dot-separators, suffixes, case + result = classify_specimen(raw) + if result is None: + # Annotation or filename produced something that doesn't fit any + # known sample pattern. + logging.warning( + "Could not classify specimen '%s' (file %s, %s); skipping", + raw, file_id, file_name, + ) + return None + return result[0] + + +def expand_dual_mappings(specimen: str) -> list[str]: + """Return [specimen] + any additional specimens it's dual-mapped to.""" + return [specimen] + SPECIMEN_DUAL_MAPPINGS.get(specimen, []) + + +# ---------------------------------------------------------------------------- +# Synapse walking +# ---------------------------------------------------------------------------- +def walk_for_viability_files(syn, parent_id: str) -> list[dict]: + """Recursively yield File entities under a Synapse folder. + + Returns a list of {id, name} dicts for files whose name suggests they + contain viability data (drug-screen CSV/TSV). + """ + try: + import synapseutils + except ImportError: + logging.error("synapseutils not available; cannot walk %s", parent_id) + return [] + + files = [] + for dirpath, _subfolders, file_list in synapseutils.walk(syn, parent_id): + for fname, fid in file_list: + # Filter to plausible viability files. Accept anything ending in + # csv/tsv that has 'viab' or 'drug' in the name; fall back to + # 'csv' if neither matches but the file is small enough. + lower = fname.lower() + if not lower.endswith((".csv", ".tsv", ".txt")): + continue + if any(tok in lower for tok in ("viab", "drug", "screen")): + files.append({"id": fid, "name": fname, + "parent_path": dirpath}) + # Don't auto-grab arbitrary CSVs — we only want drug screens. + return files + + +def discover_drug_files(syn, parents: list[str]) -> pd.DataFrame: + """Walk all parent folders and dedupe by Synapse ID.""" + all_files = [] + for parent in parents: + logging.info("Walking %s ...", parent) + files = walk_for_viability_files(syn, parent) + logging.info(" found %d candidate viability files", len(files)) + all_files.extend(files) + + if not all_files: + return pd.DataFrame(columns=["id", "name"]) + + df = pd.DataFrame(all_files).drop_duplicates(subset="id").reset_index(drop=True) + logging.info("Discovered %d unique viability files across %d parents", + len(df), len(parents)) + return df + + +# ---------------------------------------------------------------------------- +# Read and combine drug-screen files +# ---------------------------------------------------------------------------- +def pull_screen_files(syn, parents: list[str]) -> pd.DataFrame: + """Walk parents, read each viability CSV, attach canonical specimen. + + Files whose specimen can't be resolved are skipped with a warning. + Files attributed to a dual-mapped specimen produce duplicate rows for + every additional specimen in the mapping. + """ + files_df = discover_drug_files(syn, parents) + if files_df.empty: + raise RuntimeError("No drug-screen files found under any parent") + + frames = [] + for _, row in files_df.iterrows(): + fid, fname = row["id"], row["name"] + + canonical = resolve_specimen(syn, fid, fname) + if not canonical: + continue + + try: + df = pd.read_csv(syn.get(fid).path) + except Exception as exc: + logging.warning("Skipping file %s: %s", fid, exc) + continue + + # Verify required columns + required = {"Drug", "Concentration_uM", "Viability_percentage"} + missing = required - set(df.columns) + if missing: + logging.warning("File %s missing %s; skipping", + fname, sorted(missing)) + continue + + # Attribute to canonical + any dual-mapped specimens + for specimen in expand_dual_mappings(canonical): + attributed = df.copy() + attributed["specimen"] = specimen + attributed["source_file"] = fid + frames.append(attributed) + + if not frames: + raise RuntimeError("No drug-screen files could be read successfully") + + combined = pd.concat(frames, ignore_index=True) + n_specimens = combined["specimen"].nunique() + n_drugs = combined["Drug"].nunique() + logging.info( + "Combined %d drug-screen rows across %d specimens × %d drugs", + len(combined), n_specimens, n_drugs, + ) + return combined + + +# ---------------------------------------------------------------------------- +# I/O helpers +# ---------------------------------------------------------------------------- +def load_sample_map(samples_path: str) -> dict[str, int]: + df = pd.read_csv(samples_path) + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + return dict(zip(df["other_id"].astype(str), df["improve_sample_id"])) + + +def load_drug_map(drugs_path: str) -> dict[str, str]: + df = pd.read_csv(drugs_path, sep="\t") + if "chem_name" not in df.columns or "improve_drug_id" not in df.columns: + raise KeyError( + "drugs file missing required columns " + f"(have: {list(df.columns)})" + ) + mapping = { + str(name).strip().lower(): str(did) + for name, did in zip(df["chem_name"], df["improve_drug_id"]) + } + logging.info("Loaded %d drug name → improve_drug_id mappings", len(mapping)) + return mapping + + +def lookup_drug_id(name, drug_map: dict[str, str]) -> str | None: + if pd.isna(name): + return None + return drug_map.get(str(name).strip().lower()) + + +# ---------------------------------------------------------------------------- +# Single vs multi dose split +# ---------------------------------------------------------------------------- +def split_single_vs_multi(combined: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + """Per (specimen, drug), separate single-dose vs multi-dose.""" + counts = ( + combined.groupby(["specimen", "Drug"])["Concentration_uM"] + .nunique() + .reset_index(name="n_conc") + ) + multi_keys = counts[counts["n_conc"] > 1] + single_keys = counts[counts["n_conc"] == 1] + + multi = combined.merge(multi_keys[["specimen", "Drug"]], + on=["specimen", "Drug"], how="inner") + single = combined.merge(single_keys[["specimen", "Drug"]], + on=["specimen", "Drug"], how="inner") + # Single-dose rows must be at the canonical 1 μM concentration. + single = single[single["Concentration_uM"] == SINGLE_DOSE_UM].copy() + + logging.info("Multi-dose rows: %d (across %d sample-drug pairs)", + len(multi), len(multi_keys)) + logging.info("Single-dose rows: %d (across %d sample-drug pairs at %g μM)", + len(single), len(single_keys), SINGLE_DOSE_UM) + return multi, single + + +# ---------------------------------------------------------------------------- +# Curve fitting (multi-dose) and single-dose formatting +# ---------------------------------------------------------------------------- +def run_curve_fit(multi: pd.DataFrame, sample_map, drug_map) -> pd.DataFrame: + if multi.empty: + return pd.DataFrame() + + multi = multi.copy() + multi["improve_sample_id"] = multi["specimen"].map(sample_map) + multi["improve_drug_id"] = multi["Drug"].apply( + lambda n: lookup_drug_id(n, drug_map)) + + n_drop = (multi["improve_sample_id"].isna() | + multi["improve_drug_id"].isna()).sum() + if n_drop: + logging.warning("Dropping %d multi-dose rows with unmapped sample/drug", + n_drop) + multi = multi.dropna(subset=["improve_sample_id", "improve_drug_id"]) + if multi.empty: + return pd.DataFrame() + + # fit_curve.py expects DOSE in μM and GROWTH as viability percentage + multi["DOSE"] = multi["Concentration_uM"].astype(float) + 1e-4 + multi["GROWTH"] = multi["Viability_percentage"].astype(float) + multi["time"] = TIME + multi["time_unit"] = TIME_UNIT + multi["study"] = STUDY + multi["source"] = SOURCE + + cols = ["DOSE", "GROWTH", "study", "source", "improve_sample_id", + "Drug", "time", "time_unit"] + multi_for_fit = multi[cols].copy() + multi_for_fit["improve_sample_id"] = ( + multi_for_fit["improve_sample_id"].astype(int)) + + with tempfile.TemporaryDirectory() as tmpdir: + in_path = os.path.join(tmpdir, "drug_response.tsv") + out_prefix = os.path.join(tmpdir, "cnf_curve") + multi_for_fit.to_csv(in_path, sep="\t", index=False) + + cmd = ["python", FIT_CURVE_SCRIPT, + "--input", in_path, "--output", out_prefix] + logging.info("Running fit_curve: %s", " ".join(cmd)) + subprocess.run(cmd, check=True) + + # fit_curve.py writes .0 in the reference snippet's behavior + out_path = out_prefix + ".0" + if not os.path.exists(out_path): + alt = out_prefix + if os.path.exists(alt): + out_path = alt + else: + raise FileNotFoundError( + f"fit_curve.py did not produce output at {out_path}" + ) + fitted = pd.read_csv(out_path, sep="\t") + + # fit_curve.py renames Drug → improve_drug_id, so the column is present + # but holds the raw drug name, not the SMI_ ID. Always remap through drug_map. + if "improve_drug_id" in fitted.columns: + name_col = "improve_drug_id" + elif "Drug" in fitted.columns: + name_col = "Drug" + else: + raise KeyError( + f"fit_curve output has neither improve_drug_id nor Drug " + f"column (have: {list(fitted.columns)})" + ) + fitted["improve_drug_id"] = fitted[name_col].apply( + lambda n: lookup_drug_id(n, drug_map)) + + fitted = fitted.dropna(subset=["improve_drug_id"]) + n_metrics = (fitted["dose_response_metric"].nunique() + if "dose_response_metric" in fitted.columns else 0) + logging.info("Curve-fit produced %d rows across %d metrics", + len(fitted), n_metrics) + return fitted + + +def format_single_dose(single: pd.DataFrame, sample_map, drug_map) -> pd.DataFrame: + if single.empty: + return pd.DataFrame() + + df = single.copy() + df["improve_sample_id"] = df["specimen"].map(sample_map) + df["improve_drug_id"] = df["Drug"].apply( + lambda n: lookup_drug_id(n, drug_map)) + + n_drop = (df["improve_sample_id"].isna() | + df["improve_drug_id"].isna()).sum() + if n_drop: + logging.warning("Dropping %d single-dose rows with unmapped sample/drug", + n_drop) + df = df.dropna(subset=["improve_sample_id", "improve_drug_id"]).copy() + + df["dose_response_metric"] = SINGLE_DOSE_METRIC + df["dose_response_value"] = df["Viability_percentage"].astype(float) / 100.0 + df["time"] = TIME + df["time_unit"] = TIME_UNIT + df["study"] = STUDY + df["source"] = SOURCE + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + + cols = ["source", "improve_sample_id", "improve_drug_id", "study", + "time", "time_unit", "dose_response_metric", "dose_response_value"] + df = df[cols] + logging.info("Formatted %d single-dose rows", len(df)) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("samples", help="cnf_samples.csv from build_samples") + parser.add_argument("drugs", help="cnf_drugs.tsv from build_drugs") + parser.add_argument("--output", default=DEFAULT_OUTPUT) + parser.add_argument( + "--parents", default=",".join(DEFAULT_DRUG_SCREEN_PARENTS), + help="Comma-delimited Synapse folder IDs to walk for viability files", + ) + args = parser.parse_args() + + configure_logging() + + parents = [p.strip() for p in args.parents.split(",") if p.strip()] + if not parents: + logging.error("No parent folders provided") + sys.exit(2) + + sample_map = load_sample_map(args.samples) + drug_map = load_drug_map(args.drugs) + + syn = synapseclient.Synapse() + syn.login() + + combined = pull_screen_files(syn, parents) + multi, single = split_single_vs_multi(combined) + + fitted = run_curve_fit(multi, sample_map, drug_map) + single_df = format_single_dose(single, sample_map, drug_map) + + final = pd.concat([fitted, single_df], ignore_index=True) + cols = ["source", "improve_sample_id", "improve_drug_id", "study", + "time", "time_unit", "dose_response_metric", "dose_response_value"] + for c in cols: + if c not in final.columns: + final[c] = pd.NA + final = final[cols] + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + final.to_csv(args.output, sep="\t", index=False) + logging.info("Wrote %s (%d rows)", args.output, len(final)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/build_drugs.sh b/coderbuild/cnf/build_drugs.sh new file mode 100644 index 00000000..9402f178 --- /dev/null +++ b/coderbuild/cnf/build_drugs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# build_drugs.sh - wraps 03-drugs-cnf.py per coderdata convention. +# +# Usage: +# build_drugs.sh +# +# is a comma-delimited list of drug TSV files from +# previous datasets in the build sequence. Their improve_drug_id values +# get reused when the canonical SMILES match. + +set -euo pipefail + +PREV_DRUGS="${1:-}" + +python 03-drugs-cnf.py \ + --prev_drugs "$PREV_DRUGS" \ + --out_drugs /tmp/cnf_drugs.tsv \ + --out_desc /tmp/cnf_drug_descriptors.tsv + +python build_drug_desc.py --drugtable /tmp/cnf_drugs.tsv --desctable /tmp/cnf_drug_descriptors.tsv.gz diff --git a/coderbuild/cnf/build_exp.sh b/coderbuild/cnf/build_exp.sh new file mode 100644 index 00000000..18c8b107 --- /dev/null +++ b/coderbuild/cnf/build_exp.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# build_exp.sh - wraps 04-experiments-cnf.py per coderdata convention. +# +# Usage: +# build_exp.sh + +set -euo pipefail + +SAMPLES="${1:?Usage: build_exp.sh }" +DRUGS="${2:?Usage: build_exp.sh }" + +python 04-experiments-cnf.py "$SAMPLES" "$DRUGS" --output /tmp/cnf_experiments.tsv \ No newline at end of file From aa4dfb51a95a36d6e4a512a6654674169ae58a2a Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:02 -0700 Subject: [PATCH 10/20] Fix Docker image builds and keep scratch off the bind mount --- coderbuild/docker/Dockerfile.beataml | 21 +++++++ coderbuild/docker/Dockerfile.bladder | 21 +++++++ coderbuild/docker/Dockerfile.broad_sanger_exp | 62 ++++++++++++++++++- .../docker/Dockerfile.broad_sanger_omics | 48 +++++++++++++- coderbuild/docker/Dockerfile.colorectal | 21 +++++++ coderbuild/docker/Dockerfile.cptac | 21 +++++++ coderbuild/docker/Dockerfile.genes | 21 +++++++ coderbuild/docker/Dockerfile.hcmi | 21 +++++++ coderbuild/docker/Dockerfile.liver | 21 +++++++ coderbuild/docker/Dockerfile.mpnst | 32 +++++++++- coderbuild/docker/Dockerfile.mpnstpdx | 21 +++++++ coderbuild/docker/Dockerfile.novartis | 21 +++++++ coderbuild/docker/Dockerfile.pancreatic | 21 +++++++ coderbuild/docker/Dockerfile.sarcoma | 21 +++++++ coderbuild/docker/Dockerfile.upload | 21 +++++++ coderbuild/docker/docker-compose.yml | 20 +++++- 16 files changed, 406 insertions(+), 8 deletions(-) diff --git a/coderbuild/docker/Dockerfile.beataml b/coderbuild/docker/Dockerfile.beataml index d5b16b9f..63c3b9c6 100644 --- a/coderbuild/docker/Dockerfile.beataml +++ b/coderbuild/docker/Dockerfile.beataml @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.bladder b/coderbuild/docker/Dockerfile.bladder index 2cebce01..4312be7d 100644 --- a/coderbuild/docker/Dockerfile.bladder +++ b/coderbuild/docker/Dockerfile.bladder @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.broad_sanger_exp b/coderbuild/docker/Dockerfile.broad_sanger_exp index d25a90e3..21aed7e5 100755 --- a/coderbuild/docker/Dockerfile.broad_sanger_exp +++ b/coderbuild/docker/Dockerfile.broad_sanger_exp @@ -2,9 +2,57 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive -# Update and upgrade packages, then install required packages. + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + +# Install required packages. +# +# Do NOT add `apt-get upgrade -y` here. r-base:4.4.1 pins R at 4.4.1, but its +# apt sources point at Debian sid, where the r-cran-* packages have since been +# rebuilt against R 4.5.x. Upgrading therefore replaces the base image's +# consistent R library set with binaries built for a different R, and loading +# any of them fails with: +# +# /usr/lib/R/library/Matrix/libs/Matrix.so: undefined symbol: ANY_ATTRIB +# +# which cascades: Matrix breaks survival, which breaks reticulate, tidyverse +# and PharmacoGx, ~50 minutes into the image build. Verified: on a clean +# r-base:4.4.1 Matrix loads; after `apt-get upgrade -y` it does not. +# +# The upgrade is also unnecessary -- this apt list installs cleanly without it, +# and 4.4.1 already ships libssl3t64 3.6.3-1 so there is no openssl conflict. +# +# The lib*-dev entries below are build dependencies for CRAN packages that +# compile from source here. They are not optional: +# libuv1-dev fs. Without it fs fails to configure, taking down +# sass -> bslib -> rmarkdown/shiny -> DT/htmlwidgets -> +# piano -> CoreGx -> PharmacoGx. +# libfontconfig1-dev systemfonts (needs fontconfig + freetype2) +# libharfbuzz-dev textshaping +# libfribidi-dev textshaping +# libfreetype6-dev ragg -> tidyverse +# libpng/tiff/jpeg ragg image backends +# Each failure surfaces only as "dependency X is not available", an hour in, +# so the root cause is easy to misread as a CRAN outage. RUN apt-get update --fix-missing && \ - apt-get upgrade -y && \ apt-get install -y --fix-missing --allow-unauthenticated \ build-essential \ python3-pip \ @@ -15,7 +63,15 @@ RUN apt-get update --fix-missing && \ libglpk-dev \ libxml2-dev \ libpq-dev \ - ca-certificates && \ + ca-certificates \ + libuv1-dev \ + libfontconfig1-dev \ + libharfbuzz-dev \ + libfribidi-dev \ + libfreetype6-dev \ + libpng-dev \ + libtiff5-dev \ + libjpeg-dev && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Create and upgrade the Python virtual environment. diff --git a/coderbuild/docker/Dockerfile.broad_sanger_omics b/coderbuild/docker/Dockerfile.broad_sanger_omics index 6b6cfc2e..23f73ca8 100755 --- a/coderbuild/docker/Dockerfile.broad_sanger_omics +++ b/coderbuild/docker/Dockerfile.broad_sanger_omics @@ -2,9 +2,43 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive -# Update and upgrade packages, then install required packages. + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + +# Install required packages. +# +# Do NOT add `apt-get upgrade -y` here. r-base:4.4.1 pins R at 4.4.1, but its +# apt sources point at Debian sid, where the r-cran-* packages have since been +# rebuilt against R 4.5.x. Upgrading therefore replaces the base image's +# consistent R library set with binaries built for a different R, and loading +# any of them fails with: +# +# /usr/lib/R/library/Matrix/libs/Matrix.so: undefined symbol: ANY_ATTRIB +# +# which cascades through every package that depends on Matrix. Verified: on a +# clean r-base:4.4.1 Matrix loads; after `apt-get upgrade -y` it does not. +# +# The upgrade is also unnecessary -- this apt list installs cleanly without it, +# and 4.4.1 already ships libssl3t64 3.6.3-1 so there is no openssl conflict. RUN apt-get update --fix-missing && \ - apt-get upgrade -y && \ apt-get install -y --fix-missing --allow-unauthenticated \ build-essential \ python3-pip \ @@ -15,7 +49,15 @@ RUN apt-get update --fix-missing && \ libglpk-dev \ libxml2-dev \ libpq-dev \ - ca-certificates && \ + ca-certificates \ + libuv1-dev \ + libfontconfig1-dev \ + libharfbuzz-dev \ + libfribidi-dev \ + libfreetype6-dev \ + libpng-dev \ + libtiff5-dev \ + libjpeg-dev && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Create and upgrade the Python virtual environment. diff --git a/coderbuild/docker/Dockerfile.colorectal b/coderbuild/docker/Dockerfile.colorectal index c4b7b872..506d1f35 100644 --- a/coderbuild/docker/Dockerfile.colorectal +++ b/coderbuild/docker/Dockerfile.colorectal @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.cptac b/coderbuild/docker/Dockerfile.cptac index 95d11391..652b7e56 100755 --- a/coderbuild/docker/Dockerfile.cptac +++ b/coderbuild/docker/Dockerfile.cptac @@ -1,4 +1,25 @@ FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + #ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --allow-insecure-repositories RUN apt-get install -y --allow-unauthenticated --fix-missing gcc-12 build-essential libpq-dev python3-venv diff --git a/coderbuild/docker/Dockerfile.genes b/coderbuild/docker/Dockerfile.genes index aa6c63af..ed87e121 100755 --- a/coderbuild/docker/Dockerfile.genes +++ b/coderbuild/docker/Dockerfile.genes @@ -1,5 +1,26 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + RUN apt-get update --fix-missing RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libglpk-dev libxml2-dev libpq-dev diff --git a/coderbuild/docker/Dockerfile.hcmi b/coderbuild/docker/Dockerfile.hcmi index 176f3c86..c465aeb7 100644 --- a/coderbuild/docker/Dockerfile.hcmi +++ b/coderbuild/docker/Dockerfile.hcmi @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.liver b/coderbuild/docker/Dockerfile.liver index ce0133b6..1b2e864a 100644 --- a/coderbuild/docker/Dockerfile.liver +++ b/coderbuild/docker/Dockerfile.liver @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.mpnst b/coderbuild/docker/Dockerfile.mpnst index 37c75f59..2f3eff70 100755 --- a/coderbuild/docker/Dockerfile.mpnst +++ b/coderbuild/docker/Dockerfile.mpnst @@ -1,8 +1,38 @@ -FROM r-base:4.3.3 +# r-base:4.4.1 to match every other R image in this build (bladder, colorectal, +# genes, liver, novartis, broad_sanger_exp, broad_sanger_omics). +# +# 4.3.3 could no longer build: its resident libssl3t64 (3.2.1-3) owns +# /usr/lib/x86_64-linux-gnu/ossl-modules/legacy.so, while Debian sid has moved +# that file into openssl-provider-legacy (3.6.3-1). Installing libssl-dev then +# failed the layer with "trying to overwrite ... which is also in package +# libssl3t64". 4.4.1 already ships libssl3t64 3.6.3-1, so libssl-dev installs +# cleanly with no dpkg workaround. +FROM r-base:4.4.1 # Set environment to noninteractive ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.mpnstpdx b/coderbuild/docker/Dockerfile.mpnstpdx index ddc6b1e3..08354c56 100755 --- a/coderbuild/docker/Dockerfile.mpnstpdx +++ b/coderbuild/docker/Dockerfile.mpnstpdx @@ -3,6 +3,27 @@ FROM r-base:4.3.2 # Set environment to noninteractive ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.novartis b/coderbuild/docker/Dockerfile.novartis index 9687b410..bf9edf79 100644 --- a/coderbuild/docker/Dockerfile.novartis +++ b/coderbuild/docker/Dockerfile.novartis @@ -2,6 +2,27 @@ FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + # Update package list and install required packages RUN apt-get update && \ apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \ diff --git a/coderbuild/docker/Dockerfile.pancreatic b/coderbuild/docker/Dockerfile.pancreatic index bf797c9c..95f088e8 100644 --- a/coderbuild/docker/Dockerfile.pancreatic +++ b/coderbuild/docker/Dockerfile.pancreatic @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app diff --git a/coderbuild/docker/Dockerfile.sarcoma b/coderbuild/docker/Dockerfile.sarcoma index 3b741c39..ac25f26a 100644 --- a/coderbuild/docker/Dockerfile.sarcoma +++ b/coderbuild/docker/Dockerfile.sarcoma @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app # Set MPLCONFIGDIR to a writable directory diff --git a/coderbuild/docker/Dockerfile.upload b/coderbuild/docker/Dockerfile.upload index e08bb0c3..7862cef4 100644 --- a/coderbuild/docker/Dockerfile.upload +++ b/coderbuild/docker/Dockerfile.upload @@ -1,5 +1,26 @@ FROM python:3.9 +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + WORKDIR /usr/src/app RUN python -m pip install --upgrade pip pyyaml requests linkml diff --git a/coderbuild/docker/docker-compose.yml b/coderbuild/docker/docker-compose.yml index 51eb56ee..0a87e0e5 100644 --- a/coderbuild/docker/docker-compose.yml +++ b/coderbuild/docker/docker-compose.yml @@ -96,6 +96,15 @@ services: platform: linux/amd64 image: genes:latest + phosphosites: + build: + context: ../../ + dockerfile: coderbuild/docker/Dockerfile.phosphosites + args: + HTTPS_PROXY: ${HTTPS_PROXY} + platform: linux/amd64 + image: phosphosites:latest + upload: build: context: ../../ @@ -130,4 +139,13 @@ services: args: HTTPS_PROXY: ${HTTPS_PROXY} platform: linux/amd64 - image: novartis:latest \ No newline at end of file + image: novartis:latest + + cnf: + build: + context: ../../ + dockerfile: coderbuild/docker/Dockerfile.cnf + args: + HTTPS_PROXY: ${HTTPS_PROXY} + platform: linux/amd64 + image: cnf:latest \ No newline at end of file From 5be9072052b5fd58a79dd02cd68cb81fdb2e536d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:02 -0700 Subject: [PATCH 11/20] Add build resume support and DepMap readiness gate Also fixes three orchestration bugs found in the v19 full build, and redacts credentials from the build log. Every process_* function awaited only the PREVIOUS future before submitting the next, so the LAST dataset's exception was never retrieved -- and an unretrieved exception in a ThreadPoolExecutor is discarded, not raised. In --high_mem mode nothing was awaited at all. In v19 that let `hcmi omics` fail all three attempts while the build logged "All omics files completed", then published and validated a release with hcmi transcriptomics, copy_number and mutations missing entirely. Every future is now collected and retrieved through _await_all(), so no dataset failure can be dropped. The resulting `validate failed` printed only "None": stderr is passed through to sys.stderr, so subprocess never captures it and res.stderr is always None. Report the exit code and point at the container output. Finally, docker command lines were logged verbatim, and credentials are passed as `-e NAME=value`, so build logs contained the live SYNAPSE_AUTH_TOKEN in cleartext. Added _redact() and gitignored the build logs and PID files. --- .gitignore | 8 + coderbuild/build_all.py | 755 +++++++++++++++++++++++++++++++----- coderbuild/build_dataset.py | 112 +++++- 3 files changed, 773 insertions(+), 102 deletions(-) diff --git a/.gitignore b/.gitignore index fe663199..bfd44a7a 100755 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,11 @@ build/lib build/local coderdata/_version.py local/ + +# Build logs and PID files. +# Build logs capture full `docker run` command lines. Those command lines pass +# credentials as `-e NAME=value`, so older logs contain a live +# SYNAPSE_AUTH_TOKEN in cleartext (build_all_v19.log had 47 copies). +# coderbuild/build_all.py now redacts them via _redact(), but never commit these. +*.log +build_v*.pid diff --git a/coderbuild/build_all.py b/coderbuild/build_all.py index 067dd5dc..70e50bc4 100644 --- a/coderbuild/build_all.py +++ b/coderbuild/build_all.py @@ -12,7 +12,34 @@ from glob import glob import sys import requests - +import threading +import atexit +import signal +import tempfile +import yaml +from datetime import datetime + +def _log(*args, **kwargs): + ts = datetime.now().strftime('[%Y-%m-%d %H:%M:%S]') + print(ts, *args, **kwargs) + +def _redact(cmd): + """Render a command for logging with secret values masked. + + Tokens are passed to containers as `-e NAME=value`, so logging the command + verbatim wrote them in cleartext. build_all_v19.log contained the live + SYNAPSE_AUTH_TOKEN 47 times, and the build logs sit untracked in the repo + root where a `git add -A` would commit them. + """ + out = [] + for part in cmd: + if '=' in part: + key, _, val = part.partition('=') + if val and any(m in key.upper() for m in ('TOKEN', 'SECRET', 'PASSWORD', 'KEY')): + part = f'{key}=***REDACTED***' + out.append(part) + return ' '.join(out) + def main(): parser=argparse.ArgumentParser( description="This script initializes all docker containers, builds datasets, validates them, and uploads to Figshare.", @@ -35,19 +62,246 @@ def main(): parser.add_argument('--samples',dest='samples',default=False,action='store_true', help="Build all sample files.") parser.add_argument('--omics',dest='omics',default=False,action='store_true', help="Build all omics files.") parser.add_argument('--drugs',dest='drugs',default=False,action='store_true', help="Build all drug files") + parser.add_argument('--prev_drugs', type=str, default='', help='Comma-separated list of previously built drug files (e.g. local_old/broad_sanger_drugs.tsv,local_old/beataml_drugs.tsv). Each matching dataset is still rebuilt, but its previous file seeds pubchem_retrieval so only genuinely new drugs are queried. Existing drugs are preserved; removed drugs fall off naturally.') parser.add_argument('--misc', action='store_true', help="Run the final misc post-build step (e.g., split broad_sanger datasets).") parser.add_argument('--exp',dest='exp',default=False,action='store_true', help="Build all experiment file.") parser.add_argument('--validate', action='store_true', help="Run schema checker on all local files. Note this will be run, whether specified or not, if figshare arguments are included.") parser.add_argument('--figshare', action='store_true', help="Upload all local data to Figshare. FIGSHARE_TOKEN must be set in local environment.") parser.add_argument('--all',dest='all',default=False,action='store_true', help="Run all data build commands. This includes docker, samples, omics, drugs, exp arguments. This does not run the validate or figshare commands") parser.add_argument('--high_mem',dest='high_mem',default=False,action='store_true',help = "If you have 32 or more CPUs, this option is recommended. It will run many code portions in parallel. If you don't have enough memory, this will cause a run failure.") - parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,hcmi',help='Datasets to process. Defaults to all available.') + parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,cptac,hcmi',help='Datasets to process. Defaults to all available.') parser.add_argument('--version', type=str, required=False, help='Version number for the Figshare upload title (e.g., "0.1.29"). This is required for Figshare upload. This must be a higher version than previously published versions.') parser.add_argument('--github-username', type=str, required=False, help='GitHub username for the repository.') parser.add_argument('--github-email', type=str, required=False, help='GitHub email for the repository.') - + parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', + help='Confirm that the static DepMap files on Synapse are up to date for ' + 'the current DepMap release. Required for any build that includes the ' + 'broad_sanger dataset, because DepMap files can no longer be downloaded ' + 'programmatically and must be refreshed by hand.') + parser.add_argument('--continue', dest='resume', default=False, action='store_true', + help='Resume a partial build: skip steps whose sentinel output already ' + 'exists in local/, rebuild Docker images only for datasets that ' + 'failed in the previous run, and log all activity to ' + 'local/build_progress.log.') + args = parser.parse_args() - + + # ------------------------------------------------------------------------- + # Resume / progress tracking helpers + # ------------------------------------------------------------------------- + PROGRESS_LOG = 'local/build_progress.log' + _progress_lock = threading.Lock() + + def _step_sentinel(step_name): + """Return the file/dir whose existence proves this step completed.""" + if step_name == 'genes file': + return 'local/genes.csv' + if step_name == 'phosphosites file': + return 'local/phosphosites.csv' + # All other steps are "{dataset} {type}" e.g. "colorectal omics" + da, _, stype = step_name.rpartition(' ') + if not stype: + return None + if stype == 'samples': + return f'local/{da}_samples.csv' + if stype == 'drugs': + return f'local/{da}_drugs.tsv' + if stype == 'omics': + # transcriptomics is the last large file written — proves full completion + for ext in ('.csv.gz', '.csv'): + candidate = f'local/{da}_transcriptomics{ext}' + if os.path.exists(candidate): + return candidate + # default expectation for first-time check + gzip_ds = {'broad_sanger', 'beataml', 'pancreatic', 'hcmi'} + ext = '.csv.gz' if da in gzip_ds else '.csv' + return f'local/{da}_transcriptomics{ext}' + if stype == 'experiments': + return f'local/{da}_experiments.tsv' + if stype == 'misc': + return 'local/all_files_dir' + return None + + def _sentinel_done(step_name): + """Return True if the sentinel for this step exists and is non-trivial.""" + s = _step_sentinel(step_name) + if s is None: + return False + if os.path.isdir(s): + return bool(os.listdir(s)) + return os.path.isfile(s) and os.path.getsize(s) > 100 + + def _step_cleanup_globs(step_name): + """Glob patterns (relative to cwd) to remove when a step fails.""" + if step_name == 'genes file': + return ['local/genes.csv'] + if step_name == 'phosphosites file': + return ['local/phosphosites.csv'] + da, _, stype = step_name.rpartition(' ') + if not stype: + return [] + if stype == 'samples': + return [f'local/{da}_samples.csv'] + if stype == 'drugs': + return [f'local/{da}_drugs.tsv', f'local/{da}_drug_descriptors.tsv.gz'] + if stype == 'omics': + if da == 'broad_sanger': + # broad_sanger omics produces both broad_* and sanger_* files + return [ + 'local/broad_mutations*', 'local/broad_transcriptomics*', + 'local/broad_copy_number*', 'local/broad_proteomics*', + 'local/sanger_mutations*', 'local/sanger_transcriptomics*', + 'local/sanger_copy_number*', 'local/sanger_proteomics*', + ] + return [ + f'local/{da}_mutations*', f'local/{da}_transcriptomics*', + f'local/{da}_copy_number*', f'local/{da}_proteomics*', + f'local/{da}_phosphoproteomics*', + ] + if stype == 'experiments': + return [f'local/{da}_experiments.tsv'] + if stype == 'misc': + return ['local/all_files_dir'] + return [] + + def _cleanup_step(step_name): + """Remove partial outputs from a failed step; log what was removed.""" + removed = [] + for pattern in _step_cleanup_globs(step_name): + for f in glob(pattern): + try: + if os.path.isdir(f): + shutil.rmtree(f) + else: + os.remove(f) + removed.append(f) + except Exception as exc: + _log(f'[cleanup] could not remove {f}: {exc}') + if removed: + _log(f'[cleanup] {step_name}: removed {len(removed)} file(s): {", ".join(removed)}') + _write_progress(step_name, 'CLEANUP', f'removed: {", ".join(removed)}') + else: + _log(f'[cleanup] {step_name}: nothing to remove') + + def _clean_local_intermediates(reason='', protect_prefixes=()): + """Move leftover intermediate files in local/ into a quarantine + directory (local/unexpected_intermediates/) to keep the working set + small (raw downloads like rnaseq_all_data_*.csv, cell.xml, + proteomics_all_*.csv, *.xlsx, etc. are never needed after the step that + consumed them and otherwise accumulate until a later step hits ENOSPC). + + Files are MOVED, not deleted, so a mis-declared build output is never + silently lost -- if something you expected to be uploaded ends up in + local/unexpected_intermediates/, it means it was not declared in + schema/expected_files.yaml. NOTE: because the quarantine directory is on + the same filesystem, moving does NOT reclaim disk space during the + build; clear local/unexpected_intermediates/ (or point it at another + volume) if you hit disk pressure. + + Safety: a file is KEPT in place if it is a schema-declared output (per + schema/expected_files.yaml -- these may still be sitting in local/ before + the move to all_files_dir), a reference file, an incremental-build cache, + build infrastructure, or matches one of protect_prefixes (e.g. + broad*/sanger*, which build_misc.sh still needs to split broad_sanger). + Only regular files matching local/*.* are considered (mirrors the move + step's own glob), so directories like all_files_dir are never touched. + """ + try: + with open('schema/expected_files.yaml') as _f: + _schema = yaml.safe_load(_f) + except Exception as exc: + _log(f'[cleanup-intermediates] {reason}: skipped, cannot read schema: {exc}') + return + keep = set() + for _entries in _schema.get('datasets', {}).values(): + for _entry in _entries: + _bn = os.path.basename(_entry['file']) + keep.add(_bn) + keep.add(_bn + '.gz') + if _bn.endswith('.gz'): + keep.add(_bn[:-3]) + protected_exact = {'genes.csv', 'phosphosites.csv', 'prev_drug_files.txt'} + quarantine = os.path.join('local', 'unexpected_intermediates') + moved, moved_bytes = 0, 0 + for f in glob(os.path.join('local', '*.*')): + if os.path.isdir(f): + continue + bn = os.path.basename(f) + if bn in keep or bn in protected_exact: + continue + if '_prev.' in bn or bn.endswith(('.cid', '.log', '.jsonl')): + continue + if any(bn.startswith(p) for p in protect_prefixes): + continue + try: + os.makedirs(quarantine, exist_ok=True) + sz = os.path.getsize(f) + dest = os.path.join(quarantine, bn) + if os.path.exists(dest): + os.remove(dest) # replace a stale copy from an earlier pass + shutil.move(f, dest) + moved += 1 + moved_bytes += sz + except Exception as exc: + _log(f'[cleanup-intermediates] could not move {f}: {exc}') + if moved: + _log(f'[cleanup-intermediates] {reason}: moved {moved} file(s) ' + f'({moved_bytes / 1e9:.2f} GB) to {quarantine}/') + else: + _log(f'[cleanup-intermediates] {reason}: nothing to move') + + def _write_progress(step, status, note=''): + """Append one line to local/build_progress.log (thread-safe).""" + ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + line = f'{ts} | {status:<8} | {step}' + if note: + line += f' ({note})' + line += '\n' + with _progress_lock: + try: + with open(PROGRESS_LOG, 'a') as f: + f.write(line) + except Exception: + pass # never let logging crash the build + + def _step_to_docker_dataset(step_name): + """Map a step name to the dataset name used by process_docker.""" + if step_name in ('genes file', 'gene file'): + return 'genes' + if step_name == 'phosphosites file': + return 'phosphosites' + da, _, stype = step_name.rpartition(' ') + if not stype: + return None + return da # e.g. "colorectal", "broad_sanger", "liver" + + def _last_run_failures(): + """ + Parse build_progress.log and return the set of step names that + FAILED in the most recent run (since the last START line) and were + not subsequently fixed (no SUCCESS line after the FAILURE). + """ + if not os.path.exists(PROGRESS_LOG): + return set() + current_failures = set() + in_run = False + with open(PROGRESS_LOG) as f: + for line in f: + parts = [p.strip() for p in line.split('|', 2)] + if len(parts) < 3: + continue + _ts, status, rest = parts + step = rest.split('(')[0].strip() # strip any trailing note + if status == 'START': + in_run = True + current_failures = set() + elif in_run and status == 'FAILURE': + current_failures.add(step) + elif in_run and status == 'SUCCESS': + current_failures.discard(step) + return current_failures + + # ------------------------------------------------------------------------- # Simulation command for testing order of everything: # def run_docker_cmd(cmd_arr, filename): # # Simulate running the command by printing what would be run @@ -56,37 +310,91 @@ def main(): # time.sleep(2) # print(f'Completed: {filename}') - def run_docker_cmd(cmd_arr,filename): + # --- Container cleanup on exit/interrupt --- + _active_cid_files = [] + _cid_lock = threading.Lock() + + def _kill_all_containers(): + with _cid_lock: + cids = list(_active_cid_files) + for cid_file in cids: + try: + with open(cid_file) as f: + cid = f.read().strip() + if cid: + _log(f"Cleaning up container {cid[:12]}...") + subprocess.run(['docker', 'kill', cid], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except Exception: + pass + + def _signal_handler(_signum, _frame): + _log("\nBuild interrupted — killing all running containers...") + _kill_all_containers() + sys.exit(1) + + atexit.register(_kill_all_containers) + signal.signal(signal.SIGINT, _signal_handler) + signal.signal(signal.SIGTERM, _signal_handler) + + def run_docker_cmd(cmd_arr, filename): ''' Essentially a wrapper for 'docker run'. Also provides output. + Tracks each container via --cidfile so they can be killed on exit/interrupt. + With --continue: skips steps whose sentinel file already exists, and + cleans up partial outputs after all retries are exhausted. ''' - retries=3 + # --- Resume: skip if already done --- + if args.resume and _sentinel_done(filename): + sentinel = _step_sentinel(filename) + _log(f'[{filename}] skipping — already complete ({sentinel})') + _write_progress(filename, 'SKIPPED', f'sentinel: {sentinel}') + return + + retries = 3 delays = [3 * 60, 10 * 60] # 3 minutes, 10 minutes - print('running...'+filename) + _log('running...' + filename) env = os.environ.copy() if 'SYNAPSE_AUTH_TOKEN' not in env.keys(): - print('You need to set the SYNAPSE_AUTH_TOKEN to acess the MPNST, beatAML, bladder, pancreatic, liver, or sarcoma datasets') - docker_run = ['docker','run','--rm','-v',env['PWD']+'/local/:/tmp/','--platform=linux/amd64'] + _log('You need to set the SYNAPSE_AUTH_TOKEN to acess the MPNST, beatAML, bladder, pancreatic, liver, sarcoma, cnf datasets') + docker_run = ['docker', 'run', '--rm', '-v', env['PWD']+'/local/:/tmp/', '--platform=linux/amd64'] else: - docker_run = ['docker','run','--rm','-v',env['PWD']+'/local/:/tmp/','-e','SYNAPSE_AUTH_TOKEN='+env['SYNAPSE_AUTH_TOKEN'],'--platform=linux/amd64'] - cmd = docker_run+cmd_arr - print(cmd) - + docker_run = ['docker', 'run', '--rm', '-v', env['PWD']+'/local/:/tmp/', '-e', 'SYNAPSE_AUTH_TOKEN='+env['SYNAPSE_AUTH_TOKEN'], '--platform=linux/amd64'] + attempt = 1 while attempt <= retries: - print(f"[{filename}] Attempt {attempt}/{retries}: {' '.join(cmd)}") - res = subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr) - if res.returncode == 0: - print(f"[{filename}] succeeded on attempt {attempt}.") - return - else: - print(f"[{filename}] failed (exit {res.returncode}).") - if attempt < retries: - delay = delays[attempt - 1] - print(f"[{filename}] waiting {delay//60} minutes before retrying...") - print(cmd) - time.sleep(delay) + with tempfile.NamedTemporaryFile(suffix='.cid', delete=False) as _tf: + cid_file = _tf.name + os.remove(cid_file) # Docker requires the file to not exist before writing + cmd = docker_run + ['--cidfile', cid_file] + cmd_arr + with _cid_lock: + _active_cid_files.append(cid_file) + try: + _log(f"[{filename}] Attempt {attempt}/{retries}: {_redact(cmd)}") + res = subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr) + if res.returncode == 0: + _log(f"[{filename}] succeeded on attempt {attempt}.") + _write_progress(filename, 'SUCCESS') + return + else: + _log(f"[{filename}] failed (exit {res.returncode}).") + if attempt < retries: + delay = delays[attempt - 1] + _log(f"[{filename}] waiting {delay//60} minutes before retrying...") + time.sleep(delay) + finally: + with _cid_lock: + if cid_file in _active_cid_files: + _active_cid_files.remove(cid_file) + try: + os.remove(cid_file) + except OSError: + pass attempt += 1 + + # All retries exhausted — log failure and remove partial outputs + _write_progress(filename, 'FAILURE', f'all {retries} attempts failed') + _cleanup_step(filename) raise RuntimeError(f"{filename} failed after {retries} attempts") @@ -113,13 +421,17 @@ def process_docker(datasets): 'colorectal': ['colorectal'], 'cptac': ['cptac'], 'genes': ['genes'], + 'phosphosites': ['phosphosites'], 'upload': ['upload'], 'liver': ['liver'], - 'novartis': ['novartis'] + 'novartis': ['novartis'], + 'cnf': ['cnf'] } # Collect container names to build based on the datasets provided. Always build genes and upload. datasets_to_build = ['genes', 'upload'] + if 'cnf' in datasets: + datasets_to_build.append('phosphosites') for dataset in datasets: datasets_to_build.extend(dataset_map.get(dataset, [])) @@ -129,50 +441,153 @@ def process_docker(datasets): log_file_path = 'local/docker.log' env = os.environ.copy() - print(f"Docker-compose is building images for {', '.join(datasets_to_build)}. View output in {log_file_path}.") + _log(f"Docker-compose is building images for {', '.join(datasets_to_build)}. View output in {log_file_path}.") with open(log_file_path, 'w') as log_file: try: # Execute the docker-compose command res = subprocess.run(compose_command, env=env, stdout=log_file, stderr=log_file, text=True, check=True) log_file.write("Docker images built successfully.\n") - print(f"Docker images for {', '.join(datasets_to_build)} built successfully. Details logged in {log_file_path}.") + _log(f"Docker images for {', '.join(datasets_to_build)} built successfully. Details logged in {log_file_path}.") except subprocess.CalledProcessError as e: log_file.write(f"Docker compose build failed with error: {e}\n") - print(f"Docker compose build failed. See {log_file_path} for details.") + _log(f"Docker compose build failed. See {log_file_path} for details.") raise - def process_drugs(executor, datasets): + + # target_class values in schema/expected_files.yaml, grouped by the build + # step that produces them. + _STEP_CLASSES = { + 'samples': {'Sample'}, + 'omics': {'Transcriptomics', 'Proteomics', 'Mutations', 'Copy Number', + 'Phosphoproteomics'}, + 'drugs': {'Drug', 'Drug Descriptor'}, + 'experiments': {'Experiments', 'Combinations'}, + } + + def _verify_step_outputs(dataset, step): + """Fail a step that exited 0 without writing the files it declares. + + build_all only checks a container's exit status, so a script that + returns 0 after doing nothing counts as success. beataml omics did + exactly that on 2026-09-13: it ran for 88 seconds (against 4m24s in a + known-good build), printed "Starting Transcriptomics Data", never + reached proteomics, exited 0, and wrote none of its three files. The + build carried on through experiments, misc and three upload stages and + only fell over ~11 hours later in validate. + + schema/expected_files.yaml already declares what each dataset must + produce, so check it here and fail at the step that went wrong, while + the log still shows why. + """ + want = _STEP_CLASSES.get(step) + if not want: + return + try: + with open('schema/expected_files.yaml') as fh: + declared = yaml.safe_load(fh)['datasets'].get(dataset, []) + except Exception as exc: + _log(f'[verify] could not read expected_files.yaml ({exc}); skipping output check.') + return + + missing = [] + for entry in declared: + if entry.get('target_class') not in want: + continue + base = os.path.basename(entry['file']) + stem = base[:-3] if base.endswith('.gz') else base + # the build writes either the plain or the gzipped form + if not any(os.path.exists(os.path.join('local', c)) + for c in (stem, stem + '.gz')): + missing.append(stem) + + if missing: + raise RuntimeError( + f"{dataset} {step} reported success but produced none of: " + f"{', '.join(missing)}. The step exited 0 without writing the " + f"files schema/expected_files.yaml declares for it, so the build " + f"is stopping here rather than failing later in validate.") + + def _await_all(futures, step_label): + """Retrieve every future so no step failure can be silently dropped. + + This is load-bearing. Each process_* function used to await only the + PREVIOUS future before submitting the next, so the LAST dataset's + exception was never retrieved -- and an unretrieved exception in a + ThreadPoolExecutor is discarded, not raised. In --high_mem mode nothing + was awaited at all. + + On 2026-09-04 that let `hcmi omics` fail all three attempts while the + build logged "All omics files completed" and carried on to publish and + validate a release with hcmi transcriptomics missing entirely. + + Raises the first failure after collecting the rest, so one broken step + cannot hide behind a later success. + """ + errors = [] + for fut in futures: + try: + fut.result() + except Exception as exc: + errors.append(exc) + if errors: + if len(errors) > 1: + _log(f'{step_label}: {len(errors)} steps failed; raising the first.') + raise errors[0] + + def process_drugs(executor, datasets, prev_drugs=None): ''' - Build all drug files sequentially + Build all drug files sequentially. + + prev_drugs: list of local paths (e.g. ['local_old/broad_sanger_drugs.tsv', ...]) + Files are copied into local/ and listed in local/prev_drug_files.txt. + pubchem_retrieval.py auto-loads this file on import and uses it as a + local cache: known drugs skip HTTP requests entirely, but ID assignment + proceeds exactly as in a fresh run (dflist chain is unchanged). ''' last_drug_future = None - dflist = [] - - # WE NEED A METHOD TO CONFIRM THAT DRUG FILES ARE NOT INCOMPLETE - ##THIS IS BUILT IN- always rerun drug code to check - # Check for existing files and update dflist with processed files - for da in datasets: - if da not in ['cptac', 'hcmi']: - file_path = f'local/{da}_drugs.tsv' - desc_path = f'local/{da}_drug_descriptor.tsv.gz' - #if os.path.exists(file_path): ##always rerun drug process - # dflist.append(f'/tmp/{da}_drugs.tsv') # Add to dflist if already processed + _futures = [] + dflist = [] + + if prev_drugs: + docker_paths = [] + for p in prev_drugs: + base = os.path.basename(p) + if base.endswith('.tsv.gz'): + stem = base[:-len('.tsv.gz')] + prev_name = f'{stem}_prev.tsv.gz' + elif base.endswith('.tsv'): + stem = base[:-len('.tsv')] + prev_name = f'{stem}_prev.tsv' + else: + stem = os.path.splitext(base)[0] + prev_name = f'{stem}_prev.tsv' + shutil.copy(p, f'local/{prev_name}') + docker_paths.append(f'/tmp/{prev_name}') + _log(f'Copied prev drug cache: {p} → local/{prev_name}') + with open('local/prev_drug_files.txt', 'w') as f: + f.write('\n'.join(docker_paths) + '\n') + _log(f'Wrote local/prev_drug_files.txt with {len(docker_paths)} cache file(s).') for da in datasets: if da not in ['cptac', 'hcmi']: di = 'broad_sanger_exp' if da == 'broad_sanger' else da - #if not os.path.exists(f'local/{da}_drugs.tsv'): if last_drug_future: last_drug_future.result() # Ensure the last drug process is completed before starting the next - last_drug_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_drugs.sh', ','.join(dflist)], f'{da} drugs') + def _drugs_step(image=di, arg=','.join(dflist), ds=da): + run_docker_cmd([image, 'bash', 'build_drugs.sh', arg], f'{ds} drugs') + _verify_step_outputs(ds, 'drugs') + last_drug_future = executor.submit(_drugs_step) + _futures.append(last_drug_future) dflist.append(f'/tmp/{da}_drugs.tsv') - + _await_all(_futures, 'drugs') + def process_samples(executor, datasets): ''' Build all samples files sequentially ''' last_sample_future = None + _futures = [] sf = '' for da in datasets: file_path = f'local/{da}_samples.csv' @@ -185,49 +600,77 @@ def process_samples(executor, datasets): if not os.path.exists(f'local/{da}_samples.csv'): if last_sample_future: last_sample_future.result() - last_sample_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_samples.sh', sf], f'{da} samples') + def _samples_step(image=di, arg=sf, ds=da): + run_docker_cmd([image, 'bash', 'build_samples.sh', arg], f'{ds} samples') + _verify_step_outputs(ds, 'samples') + last_sample_future = executor.submit(_samples_step) + _futures.append(last_sample_future) sf = f'/tmp/{da}_samples.csv' - - def process_omics(executor, datasets,high_mem): + _await_all(_futures, 'samples') + + def process_phosphosites(executor): + ''' + Build the phosphosites reference file if it does not exist. + Caller must ensure genes.csv exists first. Returns a Future (or None). + ''' + if not os.path.exists('local/phosphosites.csv'): + return executor.submit( + run_docker_cmd, + ['phosphosites', 'bash', 'build_phosphosites.sh', '/tmp/genes.csv'], + 'phosphosites file', + ) + return None + + def process_omics(executor, datasets, high_mem): ''' Build all omics files concurrently ''' last_omics_future = None + _futures = [] for da in datasets: di = 'broad_sanger_omics' if da == 'broad_sanger' else da - #Run all at once: + omics_cmd = [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'] + if da == 'cnf': + omics_cmd.append('/tmp/phosphosites.csv') + def _omics_step(cmd=omics_cmd, ds=da): + run_docker_cmd(cmd, f'{ds} omics') + _verify_step_outputs(ds, 'omics') if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'], f'{da} omics') - #Run one at a time. + _futures.append(executor.submit(_omics_step)) else: if last_omics_future: - last_omics_future.result() - last_omics_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{da}_samples.csv'], f'{da} omics') - + last_omics_future.result() + last_omics_future = executor.submit(_omics_step) + _futures.append(last_omics_future) + _await_all(_futures, 'omics') + def process_experiments(executor, datasets, high_mem): ''' Build all experiments files concurrently ''' last_experiments_future = None + _futures = [] for da in datasets: if da not in ['cptac', 'hcmi']: di = 'broad_sanger_exp' if da == 'broad_sanger' else da if not os.path.exists(f'local/{da}_experiments.tsv'): #Run all at once if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments') + _futures.append(executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments')) #Run one at a time else: if last_experiments_future: last_experiments_future.result() last_experiments_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_exp.sh', f'/tmp/{da}_samples.csv', f'/tmp/{da}_drugs.tsv'], f'{da} experiments') - + _futures.append(last_experiments_future) + _await_all(_futures, 'experiments') def process_misc(executor, datasets, high_mem): ''' Run all misc scripts concurrently or one at a time. ''' last_misc_future = None + _futures = [] #Currently this only applies to broad_sanger. Add others here if they need a final step. if "broad_sanger" in datasets: datasets = ["broad_sanger"] @@ -238,17 +681,19 @@ def process_misc(executor, datasets, high_mem): di = 'broad_sanger_omics' if da == 'broad_sanger' else da #Run all at once: if high_mem: - executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc') + _futures.append(executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc')) #Run one at a time. else: if last_misc_future: last_misc_future.result() last_misc_future = executor.submit(run_docker_cmd, [di, 'bash', 'build_misc.sh'], f'{da} misc') - + _futures.append(last_misc_future) + _await_all(_futures, 'misc') def process_genes(executor): - if not os.path.exists('/tmp/genes.csv'): - executor.submit(run_docker_cmd,['genes','bash','build_genes.sh'],'gene file') + if not os.path.exists('local/genes.csv'): + return executor.submit(run_docker_cmd,['genes','bash','build_genes.sh'],'gene file') + return None def run_docker_upload_cmd(cmd_arr, all_files_dir, name, version): @@ -270,14 +715,20 @@ def run_docker_upload_cmd(cmd_arr, all_files_dir, name, version): # Full command to run including version update docker_run.extend(cmd_arr) - print('Executing:', ' '.join(docker_run)) + _log('Executing:', _redact(docker_run)) # res = subprocess.run(docker_run, stdout=subprocess.PIPE, stderr=subprocess.PIPE) res = subprocess.run(docker_run, stdout=sys.stdout, stderr=sys.stderr) if res.returncode != 0: - print(res.stderr) + # Do NOT log res.stderr here. stderr is passed through to sys.stderr + # above, so subprocess does not capture it and res.stderr is always + # None -- which is exactly what this line used to print, hiding the + # real cause of a `validate failed` behind the word "None". + # The container's own output is already interleaved above. + _log(f'{name} failed: exited with code {res.returncode}.' + f' See the container output immediately above for the cause.') exit(f'{name} failed') else: - print(f'{name} successful') + _log(f'{name} successful') def decompress_file(file_path): @@ -320,10 +771,78 @@ def get_latest_commit_hash(owner, repo, branch='main'): # Error handling for required tokens if args.figshare and not figshare_token: raise ValueError("FIGSHARE_TOKEN environment variable is not set.") - if any(dataset in args.datasets for dataset in ['beataml', 'mpnst', 'bladder', 'pancreatic','sarcoma','liver','novartis','colorectal']) and not synapse_auth_token: + # NOTE: broad_sanger now needs a Synapse token too -- its DepMap inputs are + # read from Synapse (syn75028495) because the DepMap portal added a + # Cloudflare challenge. See the DepMap gate below. + if any(dataset in args.datasets for dataset in ['beataml', 'mpnst', 'bladder', 'pancreatic','sarcoma','liver','novartis','colorectal','cnf','broad_sanger']) and not synapse_auth_token: if args.docker or args.samples or args.omics or args.drugs or args.exp or args.all: # Token only required if building data, not upload or validate. - raise ValueError("SYNAPSE_AUTH_TOKEN is required for accessing MPNST, beatAML, bladder, pancreatic, liver, novartis, colorectal or sarcoma datasets.") - + raise ValueError("SYNAPSE_AUTH_TOKEN is required for accessing MPNST, beatAML, bladder, pancreatic, liver, novartis, colorectal, sarcoma, cnf, broad_sanger (DepMap files) datasets.") + + # ------------------------------------------------------------------------- + # DepMap freshness gate. + # + # DepMap put a Cloudflare Turnstile ("verify you are a person") challenge in + # front of https://depmap.org/portal/api/download/files, so the broad_sanger + # samples and omics steps can no longer fetch DepMap data from the portal. + # coderdata instead reads a static copy from the "DepMap Raw" Synapse folder + # (syn75028495); coderbuild/utils/fetch_depmap.py downloads it inside the + # container at the start of build_samples.sh and build_omics.sh. + # + # That copy must be refreshed BY HAND for each new DepMap release. Because a + # stale copy fails silently -- the build succeeds and quietly publishes the + # previous release's cell lines -- we require an explicit confirmation flag + # rather than trusting that someone remembered. + # + # Only the steps that actually read the DepMap files are gated (samples and + # omics); --docker on its own still works for image debugging. + # ------------------------------------------------------------------------- + DEPMAP_SYNAPSE_FOLDER = 'syn75028495' + DEPMAP_REQUIRED_FILES = [ + 'Model.csv', + 'OmicsSomaticMutations.csv', + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv', + 'PortalOmicsCNGeneLog2.csv', + ] + if 'broad_sanger' in args.datasets and (args.samples or args.omics or args.all): + if not args.depmap_ready: + raise SystemExit( + "\n" + "=====================================================================\n" + " STOP: DepMap files must be confirmed up to date before building\n" + "=====================================================================\n" + " This build includes broad_sanger, which requires DepMap data.\n" + "\n" + " DepMap files can NO LONGER be downloaded from the DepMap portal --\n" + " it now serves a Cloudflare 'verify you are a person' challenge.\n" + " coderdata reads a STATIC COPY from Synapse instead, which must be\n" + " updated MANUALLY for each new DepMap release.\n" + "\n" + f" Synapse folder: https://www.synapse.org/Synapse:{DEPMAP_SYNAPSE_FOLDER}\n" + "\n" + " To refresh the files for a new DepMap release:\n" + " 1. Download them from the DepMap download page:\n" + " https://depmap.org/portal/data_page/?tab=allData\n" + " (accept the terms, then use the download button on each file)\n" + f" 2. Upload them to {DEPMAP_SYNAPSE_FOLDER} as NEW VERSIONS of the\n" + " existing entities (this keeps the syn IDs stable)\n" + " 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py\n" + "\n" + " Files tracked (4 total, ~1.29 GB):\n" + + ''.join(f" - {f}\n" for f in DEPMAP_REQUIRED_FILES) + + "\n" + " Repository: https://github.com/PNNL-CompBio/coderdata\n" + "\n" + " Once you have confirmed the DepMap files on Synapse are current,\n" + " re-run this command with:\n" + "\n" + " --depmap-ready\n" + "\n" + "=====================================================================\n" + ) + _log(f"DepMap files confirmed current on Synapse ({DEPMAP_SYNAPSE_FOLDER}) " + f"via --depmap-ready; fetch_depmap.py will download them in-container.") + + ###### ### Begin Pipeline ##### @@ -332,6 +851,9 @@ def get_latest_commit_hash(owner, repo, branch='main'): if not os.path.exists('local'): os.mkdir('local') + # Record this run in the progress log + _write_progress('BUILD START', 'START', f'args={sys.argv[1:]}') + # Get dataset names - default is all. datasets = args.datasets.split(',') @@ -339,28 +861,56 @@ def get_latest_commit_hash(owner, repo, branch='main'): # Ouput is logged at local/docker.log if args.docker or args.all: process_docker(datasets) - print("Docker image generation completed") + _log("Docker image generation completed") + + # --continue: rebuild Docker images only for datasets that failed last run + if args.resume: + failed_steps = _last_run_failures() + if failed_steps: + failed_datasets = { + _step_to_docker_dataset(s) for s in failed_steps + if _step_to_docker_dataset(s) + } + # Only rebuild images for datasets we're actually processing + failed_datasets &= set(datasets) | {'genes', 'phosphosites'} + if failed_datasets: + _log(f'--continue: rebuilding Docker images for failed datasets: {sorted(failed_datasets)}') + process_docker(list(failed_datasets)) + else: + _log('--continue: no failed datasets from last run match current dataset list — skipping Docker rebuild') + else: + _log('--continue: no failures found in build_progress.log — skipping Docker rebuild') - ### Build Drugs files, Samples files, and Genes file. These two steps are run in Parallel. + ### Build Drugs files, Samples files, and Genes file. These two steps are run in Parallel. ### Within each step, sequential running is required. with ThreadPoolExecutor() as executor: if args.samples or args.all: sample_thread = executor.submit(process_samples,executor, datasets) if args.drugs or args.all: - drug_thread = executor.submit(process_drugs,executor, datasets) + prev_drugs = [p.strip() for p in args.prev_drugs.split(',') if p.strip()] + drug_thread = executor.submit(process_drugs, executor, datasets, prev_drugs or None) + + # Genes must finish before phosphosites can start (phosphosites reads genes.csv). + # Run genes now and wait; then submit phosphosites (which can overlap with samples/drugs). if args.samples or args.omics or args.exp or args.all: - gene_thread = executor.submit(process_genes,executor) - - # Wait for both processes to complete before proceeding to omics and experiments + genes_future = process_genes(executor) + if genes_future is not None: + genes_future.result() + + phosphosite_future = None + if (args.omics or args.all) and 'cnf' in datasets: + phosphosite_future = process_phosphosites(executor) + + # Wait for all remaining tasks to complete before proceeding to omics and experiments if args.drugs or args.all: drug_thread.result() - if args.samples or args.all:##need to wait for samples for all of these + if args.samples or args.all: sample_thread.result() - if args.samples or args.omics or args.exp or args.all: - gene_thread.result() - - print("All samples, drugs files, and genes file completed or skipped") + if phosphosite_future is not None: + phosphosite_future.result() + + _log("All samples, drugs files, genes, and phosphosites files completed or skipped") ### At this point in the pipeline, all samples and drugs files have been created. There are no blockers to proceed. @@ -375,10 +925,16 @@ def get_latest_commit_hash(owner, repo, branch='main'): if args.omics or args.all: omics_thread.result() - print("All omics files completed") + _log("All omics files completed") if args.exp or args.all: exp_thread.result() - print("All experiments files completed") + _log("All experiments files completed") + + # Omics + experiments are done. Drop their large raw intermediates now + # (e.g. rnaseq_all_data_*.csv ~5GB, cell.xml, proteomics_all_*.csv), but keep + # broad*/sanger* — build_misc.sh still needs them to split broad_sanger. + _clean_local_intermediates(reason='after omics/experiments', + protect_prefixes=('broad', 'sanger')) ### Final Step, some datasets may need an additional post build step. Add this here @@ -389,7 +945,13 @@ def get_latest_commit_hash(owner, repo, branch='main'): misc_thread = executor.submit(process_misc, executor, datasets, args.high_mem) if args.misc or args.all: misc_thread.result() - print("Final build step complete.") + _log("Final build step complete.") + + # misc has split broad_sanger into its per-dataset outputs, so the broad*/ + # sanger* intermediates are no longer needed. Clear all remaining + # intermediates from local/ before the disk-heavy map/align/upload steps + # (this is where the previous build hit "No space left on device"). + _clean_local_intermediates(reason='after misc (pre-upload)') ###### @@ -399,10 +961,8 @@ def get_latest_commit_hash(owner, repo, branch='main'): # if args.figshare or args.validate: # FigShare File Prefixes: - prefixes = ['beataml', 'hcmi', 'cptac', 'pancreatic', 'bladder','sarcoma', 'genes', 'drugs', 'liver','novartis','colorectal','mpnst'] broad_sanger_datasets = ["ccle","ctrpv2","fimm","gdscv1","gdscv2","gcsi","prism","nci60"] if "broad_sanger" in datasets: - prefixes.extend(broad_sanger_datasets) datasets.extend(broad_sanger_datasets) datasets.remove("broad_sanger") @@ -420,9 +980,21 @@ def get_latest_commit_hash(owner, repo, branch='main'): if args.figshare and not args.version: raise ValueError("Version must be specified when pushing to figshare") - # Move relevant files to a designated directory + # Build exact allowlist from schema/expected_files.yaml so only + # schema-declared output files (not _prev caches, docker.log, etc.) + # end up in all_files_dir. + with open('schema/expected_files.yaml') as _f: + _schema = yaml.safe_load(_f) + expected_basenames = set() + for _entries in _schema.get('datasets', {}).values(): + for _entry in _entries: + _bn = os.path.basename(_entry['file']) + expected_basenames.add(_bn) + expected_basenames.add(_bn + '.gz') + + # Move only expected files to a designated directory for file in glob(os.path.join("local", '*.*')): - if any(file.startswith(os.path.join("local", prefix)) for prefix in prefixes): + if os.path.basename(file) in expected_basenames: shutil.move(file, os.path.join(all_files_dir, os.path.basename(file))) # Decompress all compressed files in the directory for schema checking @@ -430,20 +1002,21 @@ def get_latest_commit_hash(owner, repo, branch='main'): decompress_file(file) ### These should be done before schema checking. - sample_mapping_command = ['python3', 'scripts/map_improve_sample_ids.py', '--local_dir', "/tmp", '--version', args.version] + version_args = ['--version', args.version] if args.version is not None else [] + sample_mapping_command = ['python3', 'scripts/map_improve_sample_ids.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(sample_mapping_command, 'all_files_dir', 'Map_Samples', args.version) - - drug_mapping_command = ['python3', 'scripts/map_improve_drug_ids.py', '--local_dir', "/tmp", '--version', args.version] + + drug_mapping_command = ['python3', 'scripts/map_improve_drug_ids.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command, 'all_files_dir', 'Map_Drugs', args.version) - - drug_mapping_command_2 = ['python3', 'scripts/align_drug_descriptors.py', '--local_dir', "/tmp", '--version', args.version] + + drug_mapping_command_2 = ['python3', 'scripts/align_drug_descriptors.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command_2, 'all_files_dir', 'Align_Drug_Descriptors', args.version) # Run schema checker - This will always run if uploading data. schema_check_command = ['python3', 'scripts/check_schema.py', '--datasets'] + datasets run_docker_upload_cmd(schema_check_command, 'all_files_dir', 'validate', args.version) - print("Validation complete. Proceeding with file compression/decompression adjustments") + _log("Validation complete. Proceeding with file compression/decompression adjustments") # Compress or decompress files based on specific conditions after checking for file in glob(os.path.join(all_files_dir, '*')): @@ -453,7 +1026,7 @@ def get_latest_commit_hash(owner, repo, branch='main'): elif not ('samples' in file or 'figshare' in file) and not is_compressed: compress_file(file) - print("File compression and decompression adjustments are complete.") + _log("File compression and decompression adjustments are complete.") ### Upload to Figshare using Docker if args.figshare and args.version and figshare_token: diff --git a/coderbuild/build_dataset.py b/coderbuild/build_dataset.py index 7c5ed10c..c1a0bcb1 100644 --- a/coderbuild/build_dataset.py +++ b/coderbuild/build_dataset.py @@ -45,11 +45,13 @@ def process_docker(dataset,validate): 'cptac': ['cptac'], 'sarcoma': ['sarcoma'], 'genes': ['genes'], + 'phosphosites': ['phosphosites'], 'upload': ['upload'], - 'colorectal': ['colorectal'], + 'colorectal': ['colorectal'], 'bladder': ['bladder'], 'liver': ['liver'], - 'novartis': ['novartis'] + 'novartis': ['novartis'], + 'cnf': ['cnf'] } # Collect container names to build based on the dataset provided. Always build 'genes'. @@ -57,7 +59,10 @@ def process_docker(dataset,validate): # Append upload if validation step is included if validate is True: datasets_to_build.append('upload') - + # phosphosites container is required when building cnf + if dataset == 'cnf': + datasets_to_build.append('phosphosites') + datasets_to_build.extend(dataset_map.get(dataset, [])) compose_command = ['docker', 'compose', '-f', compose_file, 'build'] + datasets_to_build @@ -79,10 +84,26 @@ def process_docker(dataset,validate): def process_genes(executor): ''' - Build the genes file if it does not exist. + Build the genes file if it does not exist. Returns a Future (or None if already built). ''' if not os.path.exists('local/genes.csv'): - executor.submit(run_docker_cmd, ['genes', 'bash', 'build_genes.sh'], 'genes file') + return executor.submit(run_docker_cmd, ['genes', 'bash', 'build_genes.sh'], 'genes file') + return None + + +def process_phosphosites(executor): + ''' + Build the phosphosites reference file if it does not exist. + Only needed when the cnf dataset is being built. Returns a Future (or None). + Caller must ensure genes.csv exists before calling this. + ''' + if not os.path.exists('local/phosphosites.csv'): + return executor.submit( + run_docker_cmd, + ['phosphosites', 'bash', 'build_phosphosites.sh', '/tmp/genes.csv'], + 'phosphosites file', + ) + return None def process_samples(executor, dataset, use_prev_dataset, should_continue): ''' @@ -133,7 +154,8 @@ def process_omics(executor, dataset, should_continue): 'bladder': ['copy_number', 'mutations', 'transcriptomics'], 'colorectal':['copy_number', 'mutations', 'transcriptomics'], 'novartis':['copy_number', 'mutations', 'transcriptomics'], - 'liver':['copy_number', 'mutations', 'transcriptomics','proteomics'] + 'liver':['copy_number', 'mutations', 'transcriptomics','proteomics'], + 'cnf': ['transcriptomics', 'proteomics', 'phosphoproteomics'], } expected_omics = dataset_omics_files.get(dataset, []) @@ -167,7 +189,10 @@ def process_omics(executor, dataset, should_continue): di = 'broad_sanger_omics' if dataset == 'broad_sanger' else dataset filename = f'{dataset} omics' - executor.submit(run_docker_cmd, [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{dataset}_samples.csv'], filename) + omics_cmd = [di, 'bash', 'build_omics.sh', '/tmp/genes.csv', f'/tmp/{dataset}_samples.csv'] + if dataset == 'cnf': + omics_cmd.append('/tmp/phosphosites.csv') + executor.submit(run_docker_cmd, omics_cmd, filename) def process_experiments(executor, dataset, should_continue): @@ -245,6 +270,8 @@ def run_schema_checker(dataset): ''' # Prepare the directory with the built files prefixes = ['genes', dataset] + if dataset == 'cnf': + prefixes.append('phosphosites') datasets = [dataset] broad_sanger_datasets = ["ccle","ctrpv2","fimm","gdscv1","gdscv2","gcsi","prism","nci60"] all_files_dir = 'all_files_dir' @@ -280,9 +307,63 @@ def main(): parser.add_argument('--build', action='store_true', help='Run data build.') parser.add_argument('--validate', action='store_true', help='Run schema checker on the built files') parser.add_argument('--continue', dest='should_continue', action='store_true', help='Continue from where the build left off by skipping existing files') + parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', + help='Confirm that the static DepMap files on Synapse are up to date for the ' + 'current DepMap release. Required when building broad_sanger.') args = parser.parse_args() + # DepMap freshness gate -- see the matching block in build_all.py for the + # full rationale. DepMap files can no longer be fetched from the DepMap + # portal (Cloudflare challenge), so coderdata reads a static copy from + # Synapse (fetch_depmap.py downloads it in-container). That copy is + # hand-maintained and must be confirmed current before a broad_sanger build. + DEPMAP_SYNAPSE_FOLDER = 'syn75028495' + DEPMAP_REQUIRED_FILES = [ + 'Model.csv', + 'OmicsSomaticMutations.csv', + 'OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv', + 'PortalOmicsCNGeneLog2.csv', + ] + if args.dataset == 'broad_sanger' and args.build: + if not args.depmap_ready: + raise SystemExit( + "\n" + "=====================================================================\n" + " STOP: DepMap files must be confirmed up to date before building\n" + "=====================================================================\n" + " broad_sanger requires DepMap data.\n" + "\n" + " DepMap files can NO LONGER be downloaded from the DepMap portal --\n" + " it now serves a Cloudflare 'verify you are a person' challenge.\n" + " coderdata reads a STATIC COPY from Synapse instead, which must be\n" + " updated MANUALLY for each new DepMap release.\n" + "\n" + f" Synapse folder: https://www.synapse.org/Synapse:{DEPMAP_SYNAPSE_FOLDER}\n" + "\n" + " To refresh the files for a new DepMap release:\n" + " 1. Download them from the DepMap download page:\n" + " https://depmap.org/portal/data_page/?tab=allData\n" + " (accept the terms, then use the download button on each file)\n" + f" 2. Upload them to {DEPMAP_SYNAPSE_FOLDER} as NEW VERSIONS of the\n" + " existing entities (this keeps the syn IDs stable)\n" + " 3. Bump DEPMAP_RELEASE in coderbuild/utils/fetch_depmap.py\n" + "\n" + " Files tracked (4 total, ~1.29 GB):\n" + + ''.join(f" - {f}\n" for f in DEPMAP_REQUIRED_FILES) + + "\n" + " Repository: https://github.com/PNNL-CompBio/coderdata\n" + "\n" + " Once you have confirmed the DepMap files on Synapse are current,\n" + " re-run this command with:\n" + "\n" + " --depmap-ready\n" + "\n" + "=====================================================================\n" + ) + print(f"DepMap files confirmed current on Synapse ({DEPMAP_SYNAPSE_FOLDER}) " + f"via --depmap-ready; fetch_depmap.py will download them in-container.") + if not os.path.exists('local'): os.mkdir('local') @@ -292,16 +373,25 @@ def main(): if args.build: # Use ThreadPoolExecutor for parallel execution with ThreadPoolExecutor() as executor: - # Always build genes file - process_genes(executor) + # Genes must finish before phosphosites can start (phosphosites reads genes.csv) + genes_future = process_genes(executor) + if genes_future is not None: + genes_future.result() + + # Now safe to start phosphosites (genes.csv is present) + phosphosites_future = None + if args.dataset == 'cnf': + phosphosites_future = process_phosphosites(executor) - # Build samples and drugs + # Build samples and drugs in parallel while phosphosites runs samples_future = executor.submit(process_samples, executor, args.dataset, args.use_prev_dataset, args.should_continue) drugs_future = executor.submit(process_drugs, executor, args.dataset, args.use_prev_dataset, args.should_continue) samples_future.result() drugs_future.result() - + if phosphosites_future is not None: + phosphosites_future.result() + print("Samples and Drugs Files Completed.") with ThreadPoolExecutor() as executor: From 31d396cd37e0abdab6f711a742968b4e1612a96c Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:02 -0700 Subject: [PATCH 12/20] Declare new dataset outputs and schema classes --- schema/coderdata.yaml | 54 ++++++++++++++++++++++++++++++++++++-- schema/expected_files.yaml | 33 ++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/schema/coderdata.yaml b/schema/coderdata.yaml index e434b47a..7d095037 100755 --- a/schema/coderdata.yaml +++ b/schema/coderdata.yaml @@ -13,6 +13,10 @@ slots: entrez_id: description: Gene id according to NCBI range: integer + phosphosite_id: + description: Unique identifier for a phosphorylation site (gene + residue + position) + identifier: true + range: integer improve_sample_id: description: Unique sample identifier for this dataset, generated by this package identifier: true @@ -49,6 +53,24 @@ classes: gene_symbol: required: true description: HGNC Gene symbol or synomym + Phosphosite: + description: Reference table of phosphorylation sites; each row is a unique gene + residue + position combination. + slots: + - phosphosite_id + - entrez_id + - other_id + - other_id_source + attributes: + gene_symbol: + required: true + description: HGNC gene symbol + residue: + description: Phosphorylated amino acid residue (S, T, or Y) + position: + range: integer + description: Position of the residue in the canonical protein sequence + modification: + description: Type of modification (e.g. phospho) Sample: description: Unique samples identified for each experiment, used to cross reference omics data to drug data. slots: @@ -122,6 +144,16 @@ classes: proteomics: range: float description: Log-normalized log ratio of proteomics measurements of sample at baseline. + Phosphoproteomics: + slots: + - phosphosite_id + - improve_sample_id + - source + - study + attributes: + phosphoproteomics: + range: float + description: Log-normalized, batch-corrected phosphoproteomics abundance value (correctedAbundance). Copy Number: slots: - entrez_id @@ -157,7 +189,7 @@ classes: - time_unit attributes: dose_response_metric: - range: CurveMetric + range: ResponseMetric description: Metric by which dose response value is measured dose_response_value: description: Value of metric @@ -187,6 +219,8 @@ classes: - improve_sample_id - source - study + - time + - time_unit attributes: improve_drug_1: description: improve_drug_id of first drug @@ -195,6 +229,7 @@ classes: drug_combination_metric: description: metric calculated for synergy, or other metric of two drugs drug_combination_value: + range: linkml:Any description: value of metric for synergy or combination enums: ResponseMetric: @@ -218,7 +253,7 @@ enums: auc: description: Area under the curve measured by integrating data points dss: - description: I believe this is the drug sensitivity score + description: Drug sensistivity score. mrecist: description: For PDX data this value should be either Progressive Disease, Stable Disease, Partial Response or Complete Response. published_auc: @@ -233,6 +268,19 @@ enums: description: tumor growth inhibitiojn kulgap: description: kl divergence based metric + abc: + description: Area between curves for PDX in-vivo response + lmm: + description: Linear mixed model coefficient for PDX tumor growth response + uM_viability: + description: + Single-dose viability fraction at 1 μM. Values near 1.0 indicate no + effect (full viability); values near 0.0 indicate full kill. Values + above 1.0 are possible when treated cells proliferate beyond the + untreated control. Used for organoid drug screens where dose-response + curves were not collected for that drug. + delta_auc: + description: Difference in AUC between single-drug and combination treatment; used for MPNST treated-microtissue combination experiments. CellPerturbation: permissible_values: gene_ko: @@ -248,6 +296,8 @@ enums: xenograft derived organoid: patient derived organoid: ex vivo: + normal_tissue: + 3D-MEDS: Variant: permissible_values: 3'UTR: diff --git a/schema/expected_files.yaml b/schema/expected_files.yaml index 3de46533..2ca97376 100644 --- a/schema/expected_files.yaml +++ b/schema/expected_files.yaml @@ -42,6 +42,8 @@ datasets: file: /tmp/mpnst_drugs.tsv - target_class: Drug Descriptor file: /tmp/mpnst_drug_descriptors.tsv + - target_class: Combinations + file: /tmp/mpnst_combinations.tsv cptac: - target_class: Sample @@ -294,4 +296,33 @@ datasets: - target_class: Drug file: /tmp/novartis_drugs.tsv - target_class: Drug Descriptor - file: /tmp/novartis_drug_descriptors.tsv \ No newline at end of file + file: /tmp/novartis_drug_descriptors.tsv + + + cnf: + - target_class: Sample + file: /tmp/cnf_samples.csv + - target_class: Transcriptomics + file: /tmp/cnf_transcriptomics.csv + - target_class: Proteomics + file: /tmp/cnf_proteomics.csv + - target_class: Phosphoproteomics + file: /tmp/cnf_phosphoproteomics.csv + # - target_class: Mutations + # file: /tmp/cnf_mutations.csv + # - target_class: Copy Number + # file: /tmp/cnf_copy_number.csv + - target_class: Experiments + file: /tmp/cnf_experiments.tsv + - target_class: Drug + file: /tmp/cnf_drugs.tsv + - target_class: Drug Descriptor + file: /tmp/cnf_drug_descriptors.tsv + + phosphosites: + - target_class: Phosphosite + file: /tmp/phosphosites.csv + + genes: + - target_class: Gene + file: /tmp/genes.csv \ No newline at end of file From d550f708a06a0e3c253cff2d3399a50f9863006f Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:02 -0700 Subject: [PATCH 13/20] Speed up and harden schema validation and ID mapping --- scripts/align_drug_descriptors.py | 107 +++++++++++++---- scripts/check_schema.py | 170 +++++++++++++++++----------- scripts/map_improve_drug_ids.py | 7 ++ scripts/prepare_data_for_improve.py | 14 +-- 4 files changed, 206 insertions(+), 92 deletions(-) diff --git a/scripts/align_drug_descriptors.py b/scripts/align_drug_descriptors.py index 0e190af9..6dac8265 100644 --- a/scripts/align_drug_descriptors.py +++ b/scripts/align_drug_descriptors.py @@ -1,9 +1,45 @@ #!/usr/bin/env python3 +"""Harmonize drug_descriptor values across datasets. + +For each (improve_drug_id, structural_descriptor) seen across the per-dataset +`*_drug_descriptors.tsv[.gz]` files, the FIRST value encountered (files processed +in sorted order) is treated as canonical, and any later file whose value differs +is rewritten to match. + +Memory-bounded implementation +----------------------------- +The reference map is kept in an on-disk SQLite database, not a Python dict, so the +step scales to very large descriptor files (e.g. nci60 ~89M rows / 2.7 GB). The +previous in-memory dict held every (drug_id, descriptor) -> value pair at once and +was OOM-killed by the container on large builds. + +Fingerprint descriptors are skipped from harmonization: a fingerprint is a +deterministic function of chemical structure, so the same drug always yields the +same value -- there is nothing to harmonize, and these rows dominate both memory +and file size. They are passed through unchanged. +""" import os +import sys import gzip import shutil import csv import argparse +import sqlite3 +import tempfile + +# Descriptor values (e.g. Morgan fingerprints) can be long strings. +csv.field_size_limit(min(sys.maxsize, 2**31 - 1)) + +# Descriptor names (case-insensitive substring match) excluded from harmonization. +SKIP_DESCRIPTOR_SUBSTRINGS = ("fingerprint",) + +_INSERT_BATCH = 100_000 + + +def _skip_descriptor(name): + n = name.lower() + return any(s in n for s in SKIP_DESCRIPTOR_SUBSTRINGS) + # Helper scripts def decompress_gz_if_needed(path): @@ -15,6 +51,7 @@ def decompress_gz_if_needed(path): return out, True return path, False + def recompress_if_needed(decompressed, was_gz, original): """If was_gz, recompress decompressed back to original and remove decompressed.""" if was_gz: @@ -22,6 +59,7 @@ def recompress_if_needed(decompressed, was_gz, original): shutil.copyfileobj(f_in, f_out) os.remove(decompressed) + def find_descriptor_files(directory): """Find all *_drug_descriptors.tsv and .tsv.gz files in directory.""" files = [] @@ -32,23 +70,39 @@ def find_descriptor_files(directory): # Actual work -def build_reference_map(files): - """Build map from (drug_id, descriptor) with first found value.""" - ref = {} +def build_reference_db(files, conn): + """Populate SQLite `ref` with the first-seen value per (drug_id, descriptor). + + INSERT OR IGNORE keeps the first value encountered, matching the original + 'first found wins' semantics. Fingerprint descriptors are not stored. + """ + cur = conn.cursor() + cur.execute( + "CREATE TABLE ref (drug_id TEXT, descriptor TEXT, value TEXT, " + "PRIMARY KEY (drug_id, descriptor)) WITHOUT ROWID" + ) for fp in files: path, gz = decompress_gz_if_needed(fp) with open(path, newline='', encoding='utf-8') as f: reader = csv.DictReader(f, delimiter='\t') + batch = [] for row in reader: - key = (row['improve_drug_id'], row['structural_descriptor']) - val = row['descriptor_value'] - if key not in ref: - ref[key] = val + desc = row['structural_descriptor'] + if _skip_descriptor(desc): + continue + batch.append((row['improve_drug_id'], desc, row['descriptor_value'])) + if len(batch) >= _INSERT_BATCH: + cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch) + batch.clear() + if batch: + cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch) + conn.commit() recompress_if_needed(path, gz, fp) - return ref -def rewrite_files(files, ref): - """Go back and rewrite any mismatches in-place.""" + +def rewrite_files(files, conn): + """Rewrite any mismatched descriptor_value in-place to the canonical value.""" + cur = conn.cursor() for fp in files: path, gz = decompress_gz_if_needed(fp) tmp = path + '.tmp' @@ -62,13 +116,16 @@ def rewrite_files(files, ref): writer.writeheader() for row in reader: - key = (row['improve_drug_id'], row['structural_descriptor']) - correct = ref.get(key) - if correct is not None and row['descriptor_value'] != correct: - # print(f"Fixing {key} in {os.path.basename(fp)}: " - # f"{row['descriptor_value']} to {correct}") - row['descriptor_value'] = correct - changed = True + desc = row['structural_descriptor'] + if not _skip_descriptor(desc): + cur.execute( + "SELECT value FROM ref WHERE drug_id=? AND descriptor=?", + (row['improve_drug_id'], desc), + ) + res = cur.fetchone() + if res is not None and row['descriptor_value'] != res[0]: + row['descriptor_value'] = res[0] + changed = True writer.writerow(row) if changed: @@ -78,21 +135,31 @@ def rewrite_files(files, ref): recompress_if_needed(path, gz, fp) + def main(): parser = argparse.ArgumentParser( description="Harmonize drug_descriptor values across multiple files." ) parser.add_argument('--local_dir', default='.', help='Folder containing *_drug_descriptors.tsv[.gz]') - parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input + parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input args = parser.parse_args() files = find_descriptor_files(args.local_dir) if not files: print("No drug_descriptor files found in", args.local_dir) return - ref = build_reference_map(files) - rewrite_files(files, ref) + with tempfile.TemporaryDirectory() as tmpdir: + conn = sqlite3.connect(os.path.join(tmpdir, "descriptors.sqlite")) + # Temp DB: durability is irrelevant, so favour speed. + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + try: + build_reference_db(files, conn) + rewrite_files(files, conn) + finally: + conn.close() print("Done.") + if __name__ == '__main__': main() diff --git a/scripts/check_schema.py b/scripts/check_schema.py index f547e54d..a0daba46 100644 --- a/scripts/check_schema.py +++ b/scripts/check_schema.py @@ -1,89 +1,128 @@ -import subprocess import argparse -import concurrent.futures +import logging +import os import sys +from concurrent.futures import ProcessPoolExecutor + import yaml -import os -def run_validations_for_dataset(dataset_name, validations): - """ - Runs validations for a dataset and returns whether all validations passed. +# Path to the LinkML schema, relative to the repo root (the working directory +# used both locally and inside the `upload` Docker image). +SCHEMA_PATH = os.path.join('schema', 'coderdata.yaml') + +# One Validator per worker process. Constructing a Validator compiles the +# schema (and, lazily, the per-target-class JSON Schema) exactly once and +# caches it, so reusing a single instance across many files avoids the +# per-file process startup + schema recompilation that dominated runtime when +# this script shelled out to `linkml-validate` once per file. +# +# The plugin configuration below reproduces the exact defaults of the +# `linkml-validate` CLI (JsonschemaValidationPlugin with closed=True, and +# strict=False), so validation is identical to the previous implementation -- +# same engine, same closed-world JSON Schema, same results. +_VALIDATOR = None + - Parameters: - dataset_name (str): The name of the dataset. - validations (list): List of validations, each a dict with 'target_class' and 'file'. +def _init_worker(): + """Initializer run once per worker process: build the shared Validator.""" + global _VALIDATOR + # LinkML emits chatty INFO logs while importing the schema; silence them so + # the validation output stays readable. + logging.disable(logging.INFO) + from linkml.validator import Validator + from linkml.validator.plugins import JsonschemaValidationPlugin - Returns: - tuple: (dataset_name, bool) where bool is True if all validations passed, False otherwise. + _VALIDATOR = Validator( + SCHEMA_PATH, + validation_plugins=[JsonschemaValidationPlugin(closed=True)], + ) + + +def _validate_one(task): + """Validate a single file against its target class using the shared Validator. + + Returns (dataset_name, target_class, file_path, [error_messages]). """ - validation_failed = False - for validation in validations: - target_class = validation['target_class'] - file_path = validation['file'] - print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...") - - # Run the validation command - try: - result = subprocess.run( - ['linkml-validate', '--schema', 'schema/coderdata.yaml', '--target-class', target_class, file_path], - stdout=sys.stdout, stderr=sys.stderr, text=True - ) - - # Print output and error from command - if result.stdout: - print(result.stdout) - if result.stderr: - print(f"Error in validating {target_class} in {file_path}:", result.stderr) - # Check the exit status - if result.returncode != 0: - print(f"Validation failed for {target_class} in file {file_path}.") - validation_failed = True - else: - print(f"Validation succeeded for {target_class} in file {file_path}.") - except Exception as e: - print(f"An error occurred while validating {target_class} in {file_path}: {e}") - validation_failed = True + dataset_name, target_class, file_path = task + from linkml.validator.loaders import default_loader_for_file + + # A declared file that was never produced is reported as a clean failure + # rather than crashing the whole validation run with a FileNotFoundError. + if not os.path.exists(file_path): + return ( + dataset_name, target_class, file_path, + [f"[ERROR] declared file not found (was it produced by the build?): {file_path}"], + ) + + loader = default_loader_for_file(file_path) + errors = [] + for result in _VALIDATOR.iter_results_from_source(loader, target_class): + severity = getattr(result.severity, 'value', str(result.severity)) + message = ( + f"[{severity}] [{loader.source}/{result.instance_index}] " + f"{result.message}" + ) + # linkml-validate exits non-zero only on ERROR (or worse); mirror that + # so a WARNING/INFO does not fail the build. + if str(severity).upper() in ('ERROR', 'FATAL', 'CRITICAL'): + errors.append(message) + return (dataset_name, target_class, file_path, errors) - if validation_failed: - print(f"One or more validations failed for dataset {dataset_name}.") - return (dataset_name, False) - else: - print(f"All validations succeeded for dataset {dataset_name}.") - return (dataset_name, True) def main(): - parser = argparse.ArgumentParser(description="Run schema validations for specified datasets.") - parser.add_argument('-d', '--datasets', nargs='*', help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")', default=None) + parser = argparse.ArgumentParser( + description="Run schema validations for specified datasets." + ) + parser.add_argument( + '-d', '--datasets', nargs='*', + help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")', + default=None, + ) args = parser.parse_args() - # Read the config file config_path = os.path.join('schema', 'expected_files.yaml') with open(config_path, 'r') as f: config = yaml.safe_load(f) - available_datasets = config['datasets'].keys() + available_datasets = list(config['datasets'].keys()) datasets_to_validate = args.datasets if args.datasets else available_datasets - datasets_to_validate = [dataset for dataset in datasets_to_validate if dataset in available_datasets] + datasets_to_validate = [d for d in datasets_to_validate if d in available_datasets] print(f"Datasets to validate: {datasets_to_validate}") - all_passed = True - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = { - executor.submit( - run_validations_for_dataset, - dataset, - config['datasets'][dataset] - ): dataset for dataset in datasets_to_validate - } - - for future in concurrent.futures.as_completed(futures): - dataset_name, result = future.result() - if not result: - all_passed = False - print(f"Validation failed for dataset {dataset_name}") + # Flatten to per-file tasks so all files validate in parallel across + # processes (the previous version only parallelized across datasets). + tasks = [] + for dataset in datasets_to_validate: + for validation in config['datasets'][dataset]: + tasks.append((dataset, validation['target_class'], validation['file'])) + + # Track pass/fail per dataset so the summary matches the old output. + dataset_failed = {d: False for d in datasets_to_validate} + + # Cap workers: each worker can load a multi-GB decompressed omics file into + # memory, so bound concurrency to avoid exhausting RAM on large builds. + max_workers = min(len(tasks), os.cpu_count() or 1, 8) or 1 + with ProcessPoolExecutor(max_workers=max_workers, initializer=_init_worker) as executor: + for dataset_name, target_class, file_path, errors in executor.map(_validate_one, tasks): + print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...") + if errors: + for message in errors: + print(message) + print(f"Validation failed for {target_class} in file {file_path}.") + dataset_failed[dataset_name] = True else: - print(f"Validation passed for dataset {dataset_name}") + print(f"Validation succeeded for {target_class} in file {file_path}.") + + all_passed = True + for dataset in datasets_to_validate: + if dataset_failed[dataset]: + all_passed = False + print(f"One or more validations failed for dataset {dataset}.") + print(f"Validation failed for dataset {dataset}") + else: + print(f"All validations succeeded for dataset {dataset}.") + print(f"Validation passed for dataset {dataset}") if all_passed: print("All schema validations passed successfully.") @@ -92,5 +131,6 @@ def main(): print("Some schema validations failed.") sys.exit(1) + if __name__ == '__main__': main() diff --git a/scripts/map_improve_drug_ids.py b/scripts/map_improve_drug_ids.py index 5c12b1ca..417fe2ac 100644 --- a/scripts/map_improve_drug_ids.py +++ b/scripts/map_improve_drug_ids.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 import json import os +import sys import csv import argparse from datetime import datetime import gzip import shutil +# Drug synonym fields can legitimately be long, and a quoting bug in +# pubchem_retrieval.py (fixed 2026-08-29) historically inflated some chem_name +# values well past Python's default 131072-byte csv field limit. Matches the +# limit already set in align_drug_descriptors.py so the two stay consistent. +csv.field_size_limit(min(sys.maxsize, 2**31 - 1)) + ##### Load, make structure, save functions for improve_drug_mapping.json def load_mapping(mapping_file='improve_drug_mapping.json'): """ diff --git a/scripts/prepare_data_for_improve.py b/scripts/prepare_data_for_improve.py index 573b5c92..00cb199b 100644 --- a/scripts/prepare_data_for_improve.py +++ b/scripts/prepare_data_for_improve.py @@ -194,21 +194,21 @@ def process_datasets(args): # Logic for PDX datasets that don't have `auc` but mRECIST (note # the typo currently in the `drugresponse_metric` column). - elif experiments_raw['dose_response_metric'].isin(['mRESCIST']).any(): + elif experiments_raw['dose_response_metric'].isin(['mrecist']).any(): experiment = data_sets[data_set].format( data_type='experiments', shape='wide', metrics=[ - 'mRESCIST', + 'mrecist', ], ) # conversion logic from mRECIST -> auc - experiment.loc[experiment['mRESCIST'] == 'CR', 'mRESCIST'] = "0.1" - experiment.loc[experiment['mRESCIST'] == 'PR', 'mRESCIST'] = "0.2" - experiment.loc[experiment['mRESCIST'] == 'SD', 'mRESCIST'] = "0.5" - experiment.loc[experiment['mRESCIST'] == 'PD', 'mRESCIST'] = "1.0" + experiment.loc[experiment['mrecist'] == 'CR', 'mrecist'] = "0.1" + experiment.loc[experiment['mrecist'] == 'PR', 'mrecist'] = "0.2" + experiment.loc[experiment['mrecist'] == 'SD', 'mrecist'] = "0.5" + experiment.loc[experiment['mrecist'] == 'PD', 'mrecist'] = "1.0" - experiment.rename(columns={'mRESCIST': 'auc'}, inplace=True) + experiment.rename(columns={'mrecist': 'auc'}, inplace=True) experiments.append(experiment) # The remaining datasets should have `auc` as From 038dbd158509587439e8032b04bd042d04d49cc1 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:03 -0700 Subject: [PATCH 14/20] Update package dataset metadata and loader --- coderdata/dataset.yml | 15 ++++++- coderdata/dataset/dataset.py | 71 +++++++++++++++++++++++++++++++- coderdata/download/downloader.py | 4 +- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/coderdata/dataset.yml b/coderdata/dataset.yml index 15e91270..2be843f1 100644 --- a/coderdata/dataset.yml +++ b/coderdata/dataset.yml @@ -1,5 +1,5 @@ -figshare: https://api.figshare.com/v2/articles/29923646/files?page=1&page_size=500 -version: 2.2.1 +figshare: https://api.figshare.com/v2/articles/32906495/files?page=1&page_size=500 +version: 2.3 datasets: beataml: description: Beat acute myeloid leukemia (BeatAML) focuses on acute myeloid leukemia tumor data. Data includes drug response, proteomics, and transcriptomics datasets. @@ -46,6 +46,17 @@ datasets: - drug descriptor - experiments + cnf: + description: Cutaneous Neurofibroma (cNF) patient-derived organoid drug screen. Cutaneous neurofibromas are benign nerve-sheath tumors that develop in patients with neurofibromatosis type 1 (NF1). Data includes transcriptomics, proteomics, phosphoproteomics, and drug response data derived from Synapse. + modalities: + - sample + - transcriptomics + - proteomics + - phosphoproteomics + - drug + - drug descriptor + - experiments + cptac: description: The Clinical Proteomic Tumor Analysis Consortium (CPTAC) project is a collaborative network funded by the National Cancer Institute (NCI) focused on improving our understanding of cancer biology through the integration of transcriptomic, proteomic, and genomic data. references: diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 695db22c..41ac73fd 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -44,6 +44,8 @@ def __init__( name: Optional[str]=None, transcriptomics: Optional[pd.DataFrame]=None, proteomics: Optional[pd.DataFrame]=None, + phosphoproteomics: Optional[pd.DataFrame]=None, + phosphosites: Optional[pd.DataFrame]=None, mutations: Optional[pd.DataFrame]=None, copy_number: Optional[pd.DataFrame]=None, samples: Optional[pd.DataFrame]=None, @@ -95,6 +97,8 @@ def __init__( self.name = name self.transcriptomics = transcriptomics self.proteomics = proteomics + self.phosphoproteomics = phosphoproteomics + self.phosphosites = phosphosites self.mutations = mutations self.copy_number = copy_number self.samples = samples @@ -151,6 +155,32 @@ def proteomics(self): del self._proteomics + @property + def phosphoproteomics(self): + return self._phosphoproteomics + + @phosphoproteomics.setter + def phosphoproteomics(self, value): + self._phosphoproteomics = value + + @phosphoproteomics.deleter + def phosphoproteomics(self): + del self._phosphoproteomics + + + @property + def phosphosites(self): + return self._phosphosites + + @phosphosites.setter + def phosphosites(self, value): + self._phosphosites = value + + @phosphosites.deleter + def phosphosites(self): + del self._phosphosites + + @property def mutations(self): return self._mutations @@ -513,6 +543,8 @@ def types(self) -> list: data_types = [ 'transcriptomics', 'proteomics', + 'phosphoproteomics', + 'phosphosites', 'mutations', 'copy_number', 'samples', @@ -521,6 +553,7 @@ def types(self) -> list: 'experiments', 'methylation', 'metabolomics', + 'combinations', 'genes', ] data_types_present = [] @@ -609,6 +642,7 @@ def load( data_types_to_load = ( 'transcriptomics', 'proteomics', + 'phosphoproteomics', 'mutations', 'copy_number', 'samples', @@ -618,7 +652,11 @@ def load( 'experiments', 'methylation', 'metabolomics', + 'combinations', + # reference tables are loaded last so they can be subset to the + # identifiers present in the data types loaded above 'genes', + 'phosphosites', ) if type(local_path) is not Path: @@ -649,6 +687,9 @@ def load( for p in local_path.glob(f'genes*'): if p.name.endswith(accepted_file_endings) and p.is_file(): files['genes'] = p + for p in local_path.glob(f'phosphosites*'): + if p.name.endswith(accepted_file_endings) and p.is_file(): + files['phosphosites'] = p for dataset_type in data_types_to_load: if dataset_type not in files: @@ -659,7 +700,7 @@ def load( ) continue file = files[dataset_type] - if dataset_type != 'genes': + if dataset_type not in ('genes', 'phosphosites'): print( f"Importing '{dataset_type}' from {file} ...", end=' ', @@ -668,6 +709,34 @@ def load( if hasattr(dataset, dataset_type): setattr(dataset, dataset_type, _load_file(file)) print("DONE", file=sys.stderr) + elif dataset_type == 'phosphosites': + ''' + The phosphosites table is a universal reference (analogous + to 'genes') mapping phosphosite_id -> gene / residue info. + It is only relevant when the dataset has phosphoproteomics, + and is subset to the phosphosite_ids present in that data. + ''' + if dataset.phosphoproteomics is None: + print( + f"'phosphosites' skipped for {name} " + f"(no phosphoproteomics present)", + file=sys.stderr + ) + continue + print( + f"Importing 'phosphosites' from {file} ...", + end=' ', + file=sys.stderr + ) + dataset.phosphosites = _load_file(file) + if 'phosphosite_id' in dataset.phosphoproteomics.columns: + site_ids = set( + dataset.phosphoproteomics['phosphosite_id'].unique() + ) + dataset.phosphosites = dataset.phosphosites[ + dataset.phosphosites['phosphosite_id'].isin(site_ids) + ] + print("DONE", file=sys.stderr) else: ''' The genes dataset available in the online repository is diff --git a/coderdata/download/downloader.py b/coderdata/download/downloader.py index 552cd9ba..b6b367f9 100644 --- a/coderdata/download/downloader.py +++ b/coderdata/download/downloader.py @@ -110,7 +110,9 @@ def download( if name != "all": filtered_files = [ f for f in all_files - if (f.get('name', '').casefold().startswith(name)) or ('genes' in f.get('name', '').casefold()) + if (f.get('name', '').casefold().startswith(name)) + or ('genes' in f.get('name', '').casefold()) + or ('phosphosites' in f.get('name', '').casefold()) ] else: filtered_files = all_files From 24ca8ac7fb7f49405e9a68016f595709a8a4bc30 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 30 Sep 2026 16:40:46 -0700 Subject: [PATCH 15/20] Build only: keep DepMap CN as log2(relative CN + 1); open-ended copy_call bins in prepare_data_for_improve --- coderbuild/broad_sanger/02-broadSangerOmics.R | 5 ++--- scripts/prepare_data_for_improve.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/coderbuild/broad_sanger/02-broadSangerOmics.R b/coderbuild/broad_sanger/02-broadSangerOmics.R index be31dcbc..131876b9 100755 --- a/coderbuild/broad_sanger/02-broadSangerOmics.R +++ b/coderbuild/broad_sanger/02-broadSangerOmics.R @@ -493,9 +493,8 @@ depmap_files<-function(fi,value){ res <- as.data.frame(res) |> dplyr::mutate( - copy_number = suppressWarnings(as.numeric(copy_number)), - ## PortalOmicsCNGeneLog2 stores log2(absolute copies); convert to ratio - copy_number = 2^copy_number / 2 + ## PortalOmicsCNGeneLog2 is already log2(relative CN + 1); keep as-is + copy_number = suppressWarnings(as.numeric(copy_number)) ) |> dplyr::distinct() diff --git a/scripts/prepare_data_for_improve.py b/scripts/prepare_data_for_improve.py index 00cb199b..3a431ed8 100644 --- a/scripts/prepare_data_for_improve.py +++ b/scripts/prepare_data_for_improve.py @@ -448,7 +448,7 @@ def process_datasets(args): discretized_copy_number = merged_copy_number.apply( pd.cut, - bins = [0, 0.5210507, 0.7311832, 1.214125, 1.422233, 2], + bins = [-np.inf, 0.5210507, 0.7311832, 1.214125, 1.422233, np.inf], labels = [-2, -1, 0, 1, 2], include_lowest=True ) From 7d5db2bb21d0e0334531108ab7ee62a4ea7fcba2 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 30 Sep 2026 16:57:32 -0700 Subject: [PATCH 16/20] Build only: use 0 as the lower copy_call bin edge in prepare_data_for_improve --- scripts/prepare_data_for_improve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_data_for_improve.py b/scripts/prepare_data_for_improve.py index 3a431ed8..b74d14cc 100644 --- a/scripts/prepare_data_for_improve.py +++ b/scripts/prepare_data_for_improve.py @@ -448,7 +448,7 @@ def process_datasets(args): discretized_copy_number = merged_copy_number.apply( pd.cut, - bins = [-np.inf, 0.5210507, 0.7311832, 1.214125, 1.422233, np.inf], + bins = [0, 0.5210507, 0.7311832, 1.214125, 1.422233, np.inf], labels = [-2, -1, 0, 1, 2], include_lowest=True ) From c3c9c4302371d5cd69ecd22728e4f07885769805 Mon Sep 17 00:00:00 2001 From: Jeremy Jacobson <85139244+jjacobson95@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:33:33 -0700 Subject: [PATCH 17/20] Remove cnf and phosphosites from docker-compose.yml --- coderbuild/docker/docker-compose.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/coderbuild/docker/docker-compose.yml b/coderbuild/docker/docker-compose.yml index 0a87e0e5..3e91615a 100644 --- a/coderbuild/docker/docker-compose.yml +++ b/coderbuild/docker/docker-compose.yml @@ -96,15 +96,6 @@ services: platform: linux/amd64 image: genes:latest - phosphosites: - build: - context: ../../ - dockerfile: coderbuild/docker/Dockerfile.phosphosites - args: - HTTPS_PROXY: ${HTTPS_PROXY} - platform: linux/amd64 - image: phosphosites:latest - upload: build: context: ../../ @@ -140,12 +131,3 @@ services: HTTPS_PROXY: ${HTTPS_PROXY} platform: linux/amd64 image: novartis:latest - - cnf: - build: - context: ../../ - dockerfile: coderbuild/docker/Dockerfile.cnf - args: - HTTPS_PROXY: ${HTTPS_PROXY} - platform: linux/amd64 - image: cnf:latest \ No newline at end of file From ef24b3c7a4819b2f27565cc1378715e8664bf229 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 5 Oct 2026 09:11:27 -0700 Subject: [PATCH 18/20] Fix broad_sanger omics resume sentinel --- coderbuild/build_all.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/coderbuild/build_all.py b/coderbuild/build_all.py index 70e50bc4..d8fb2401 100644 --- a/coderbuild/build_all.py +++ b/coderbuild/build_all.py @@ -107,15 +107,18 @@ def _step_sentinel(step_name): if stype == 'drugs': return f'local/{da}_drugs.tsv' if stype == 'omics': + # broad_sanger writes broad_transcriptomics, while all other datasets + # use their dataset name as the output prefix. + output_prefix = 'broad' if da == 'broad_sanger' else da # transcriptomics is the last large file written — proves full completion for ext in ('.csv.gz', '.csv'): - candidate = f'local/{da}_transcriptomics{ext}' + candidate = f'local/{output_prefix}_transcriptomics{ext}' if os.path.exists(candidate): return candidate # default expectation for first-time check gzip_ds = {'broad_sanger', 'beataml', 'pancreatic', 'hcmi'} ext = '.csv.gz' if da in gzip_ds else '.csv' - return f'local/{da}_transcriptomics{ext}' + return f'local/{output_prefix}_transcriptomics{ext}' if stype == 'experiments': return f'local/{da}_experiments.tsv' if stype == 'misc': From b527f6e78aff90d0663e26296e4c8b7c9630d8ae Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 5 Oct 2026 09:26:49 -0700 Subject: [PATCH 19/20] Harden resumed experiments and HCMI mutation parsing --- coderbuild/build_all.py | 6 +++++- coderbuild/hcmi/02-getHCMIData.py | 2 +- coderbuild/pancreatic/build_exp.sh | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/coderbuild/build_all.py b/coderbuild/build_all.py index d8fb2401..a21176a7 100644 --- a/coderbuild/build_all.py +++ b/coderbuild/build_all.py @@ -120,6 +120,10 @@ def _step_sentinel(step_name): ext = '.csv.gz' if da in gzip_ds else '.csv' return f'local/{output_prefix}_transcriptomics{ext}' if stype == 'experiments': + for ext in ('.tsv', '.tsv.gz'): + candidate = f'local/{da}_experiments{ext}' + if os.path.exists(candidate): + return candidate return f'local/{da}_experiments.tsv' if stype == 'misc': return 'local/all_files_dir' @@ -162,7 +166,7 @@ def _step_cleanup_globs(step_name): f'local/{da}_phosphoproteomics*', ] if stype == 'experiments': - return [f'local/{da}_experiments.tsv'] + return [f'local/{da}_experiments.tsv', f'local/{da}_experiments.tsv.gz'] if stype == 'misc': return ['local/all_files_dir'] return [] diff --git a/coderbuild/hcmi/02-getHCMIData.py b/coderbuild/hcmi/02-getHCMIData.py index 48ffefc7..09c5e0f8 100644 --- a/coderbuild/hcmi/02-getHCMIData.py +++ b/coderbuild/hcmi/02-getHCMIData.py @@ -509,7 +509,7 @@ def stream_clean_files(data_type: str): # ---- read single file ------------------------------ if fpath.endswith(".gz"): # mutation data is always gzipped try: - df = pl.read_csv(fpath, separator="\t", skip_rows=7) + df = pl.read_csv(fpath, separator="\t", skip_rows=7, dtypes={"PUBMED": pl.Utf8}) except Exception as e: print(f"[warn] skipping MAF due to read error: {fpath} ({type(e).__name__}: {e})") continue diff --git a/coderbuild/pancreatic/build_exp.sh b/coderbuild/pancreatic/build_exp.sh index 25722402..dc9d0faa 100644 --- a/coderbuild/pancreatic/build_exp.sh +++ b/coderbuild/pancreatic/build_exp.sh @@ -11,4 +11,4 @@ python fit_curve.py --input /tmp/pancreatic_doserep.tsv --output /tmp/pancreatic mv /tmp/pancreatic_doserep.tsv.0 /tmp/pancreatic_experiments.tsv python 05-addPrecalcAUC.py --samples $1 --drugs $2 --expfile /tmp/pancreatic_experiments.tsv -gzip /tmp/pancreatic_experiments.tsv +gzip -f /tmp/pancreatic_experiments.tsv From 7045f5ea9495361bde7aa18a4b4c8a6ec645f304 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 5 Oct 2026 09:44:55 -0700 Subject: [PATCH 20/20] Keep dataset-specific fixes out of build resume changes --- coderbuild/hcmi/02-getHCMIData.py | 2 +- coderbuild/pancreatic/build_exp.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coderbuild/hcmi/02-getHCMIData.py b/coderbuild/hcmi/02-getHCMIData.py index 09c5e0f8..48ffefc7 100644 --- a/coderbuild/hcmi/02-getHCMIData.py +++ b/coderbuild/hcmi/02-getHCMIData.py @@ -509,7 +509,7 @@ def stream_clean_files(data_type: str): # ---- read single file ------------------------------ if fpath.endswith(".gz"): # mutation data is always gzipped try: - df = pl.read_csv(fpath, separator="\t", skip_rows=7, dtypes={"PUBMED": pl.Utf8}) + df = pl.read_csv(fpath, separator="\t", skip_rows=7) except Exception as e: print(f"[warn] skipping MAF due to read error: {fpath} ({type(e).__name__}: {e})") continue diff --git a/coderbuild/pancreatic/build_exp.sh b/coderbuild/pancreatic/build_exp.sh index dc9d0faa..25722402 100644 --- a/coderbuild/pancreatic/build_exp.sh +++ b/coderbuild/pancreatic/build_exp.sh @@ -11,4 +11,4 @@ python fit_curve.py --input /tmp/pancreatic_doserep.tsv --output /tmp/pancreatic mv /tmp/pancreatic_doserep.tsv.0 /tmp/pancreatic_experiments.tsv python 05-addPrecalcAUC.py --samples $1 --drugs $2 --expfile /tmp/pancreatic_experiments.tsv -gzip -f /tmp/pancreatic_experiments.tsv +gzip /tmp/pancreatic_experiments.tsv