From 1c4f112fbdba605ac323fc3a5f21cda35580a012 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:01 -0700 Subject: [PATCH 1/4] Add MPNST treated-sample and treated-omics support --- coderbuild/mpnst/00_sample_gen.R | 12 +- coderbuild/mpnst/00b_sample_gen_treated.R | 594 ++++++++++++++++++ coderbuild/mpnst/01_combined_omics.R | 32 +- coderbuild/mpnst/01b_combined_omics_treated.R | 219 +++++++ coderbuild/mpnst/build_omics.sh | 3 + coderbuild/mpnst/build_samples.sh | 2 + 6 files changed, 851 insertions(+), 11 deletions(-) create mode 100644 coderbuild/mpnst/00b_sample_gen_treated.R create mode 100644 coderbuild/mpnst/01b_combined_omics_treated.R diff --git a/coderbuild/mpnst/00_sample_gen.R b/coderbuild/mpnst/00_sample_gen.R index db1f238e..73f59c30 100644 --- a/coderbuild/mpnst/00_sample_gen.R +++ b/coderbuild/mpnst/00_sample_gen.R @@ -1,3 +1,4 @@ +# 00_sample_gen.R # This script generate a new sample table based on previous dataset's sample file (taking the max improve_sample_id) # Load required libraries library(data.table) @@ -26,9 +27,14 @@ synapser::synLogin(authToken=synapse_token) manifest<-synapser::synTableQuery("select * from syn53503360")$asDataFrame()|> as.data.frame() -#Drop contaminated sample JH-2-009 +#Drop contaminated sample JH-2-009 and others with issues manifest <- manifest %>% - filter(Sample != "JH-2-009") + filter(Sample != "JH-2-009") %>% + filter(Sample != "WU-545") %>% + filter(Sample != "WU-536") %>% + filter(Sample != "WU-505") %>% + filter(Sample != "MN-1") %>% + filter(Sample != "MN-3") ###sample file has a strict schema @@ -56,7 +62,7 @@ sampTable<-manifest|> ##third, generate a sample for the MTs if they were generated pdxmt<-subset(sampTable,!is.na(MicroTissueDrugFolder)) -pdxmt$model_type=rep('xenograft derived organoid',nrow(pdxmt)) +pdxmt$model_type=rep('3D-MEDS',nrow(pdxmt)) print(pdxmt) main<-rbind(sampTable,pdxmt)|> diff --git a/coderbuild/mpnst/00b_sample_gen_treated.R b/coderbuild/mpnst/00b_sample_gen_treated.R new file mode 100644 index 00000000..63db50b1 --- /dev/null +++ b/coderbuild/mpnst/00b_sample_gen_treated.R @@ -0,0 +1,594 @@ +# 00_sample_gen_treated.R +# This script generates treated microtissue samples and appends them to an existing samples file. + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(readxl) +}) + +# ----------------------- +# logging helpers +# ----------------------- +.ts <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +.p <- function(...) cat(.ts(), paste0(...), "\n") + +print_df_head <- function(df, n = 8) { + if (is.null(df) || nrow(df) == 0) { + .p("[PRINT] <0 rows>") + return(invisible(NULL)) + } + n <- min(n, nrow(df)) + print(utils::head(df, n)) + invisible(NULL) +} + +# ----------------------- +# constants +# ----------------------- +CONST_CANCER <- "Malignant peripheral nerve sheath tumor" +CONST_SPECIES <- "Homo sapiens (Human)" +CONST_MODEL <- "3D-MEDS" +CONST_SOURCE <- "NF Data Portal" + +# NOTE: updated to correctly preserve hyphens in IDs (MN-2, JH-2-002, WU-225) +safe_token <- function(x) { + x <- as.character(x) + x <- trimws(x) + x <- gsub("\\s+", "_", x) + x <- gsub("[^[:alnum:]_.-]+", "_", x) + x +} + +# ----------------------- +# Canonicalize individual IDs to hyphen style (MN-2, JH-2-002, WU-225) +# ----------------------- +canonicalize_individual_id <- function(x) { + x <- trimws(as.character(x)) + x <- gsub("[_\\s]+", "-", x) # underscores/spaces -> hyphen + x <- gsub("-+", "-", x) # collapse repeated hyphens + + # Normalize common patterns (only changes if it matches) + x <- sub("^MN-?([0-9A-Za-z]+)$", "MN-\\1", x) + x <- sub("^WU-?([0-9A-Za-z]+)$", "WU-\\1", x) + x <- sub("^JH-?([0-9]+)-?([0-9A-Za-z]+)$", "JH-\\1-\\2", x) + + x +} + +# ----------------------- +# Migrate existing treated other_id prefixes from underscore->hyphen +# This is because some samples were mis-labeled. +# ----------------------- +migrate_treated_other_id_prefix <- function(other_id) { + x <- as.character(other_id) + is_treated <- grepl("_treated_microtissue$", x) + + y <- x + y[is_treated] <- sub("^MN_([0-9A-Za-z]+)_", "MN-\\1_", y[is_treated]) + y[is_treated] <- sub("^WU_([0-9A-Za-z]+)_", "WU-\\1_", y[is_treated]) + y[is_treated] <- sub("^JH_([0-9]+)_([0-9A-Za-z]+)_", "JH-\\1-\\2_", y[is_treated]) + + y +} + +# ----------------------- +# Timepoint normalizer (handles "24 HR", "8HR ", etc) +# ----------------------- +normalize_timepoint <- function(tp) { + tp <- trimws(as.character(tp)) + tpu <- toupper(tp) + tpu <- gsub("\\s+", "", tpu) + + out <- tp + out[tpu %in% c("8HR", "8H")] <- "8h" + out[tpu %in% c("24HR", "24H")] <- "24h" + out[tpu %in% c("0HR", "0H")] <- "0h" + + is_h <- grepl("^[0-9]+h$", tolower(out)) + out[is_h] <- tolower(out[is_h]) + out +} + +hours_from_timepoint <- function(tp_norm) { + tp_norm <- tolower(trimws(as.character(tp_norm))) + hrs <- gsub("h$", "", tp_norm) + suppressWarnings(as.integer(hrs)) +} + +# ----------------------- +# Drop controls: DMSO + untreated (and close variants) +# ----------------------- +is_dmso <- function(drug) { + d <- tolower(trimws(as.character(drug))) + grepl("\\bdmso\\b", d) || grepl("dimethyl", d) || grepl("sulfoxide", d) +} + +is_untreated <- function(drug) { + d <- tolower(trimws(as.character(drug))) + grepl("\\buntreated\\b", d) || grepl("no[_\\s-]*treat", d) || grepl("\\bvehicle\\b", d) +} + +is_control <- function(drug) { + isTRUE(is_dmso(drug)) || isTRUE(is_untreated(drug)) +} + +# ============================================================ +# Builder function: +# - Drops DMSO + untreated rows +# - Aggregates replicates: ONE row per treated sample +# - other_id = {individual_id}_{drug}_{hours}hr_treated_microtissue +# - other_names = comma-separated replicate specimen IDs +# ============================================================ +build_from_standard_map <- function(std_map, orig_samples, batch_key, next_id, require_base = TRUE) { + std_map <- as.data.table(std_map) + + # Canonicalize IDs so other_id uses MN-2 / JH-2-002 / WU-225 formatting + std_map[, individual_id := canonicalize_individual_id(individual_id)] + + # base_exists check (optional) + std_map[, base_exists := individual_id %in% orig_samples$common_name] + + # drop controls early (DMSO + untreated) + std_map[, drop_control := vapply(drug, is_control, logical(1))] + n_ctrl <- sum(std_map$drop_control %in% TRUE) + if (n_ctrl > 0) .p("[", batch_key, "] dropping control rows (DMSO/untreated): ", n_ctrl) + std_map <- std_map[drop_control == FALSE | is.na(drop_control)] + + debug_tbl <- std_map[, .( + batch = batch_key, + specimen_id, + individual_id, + drug, + timepoint, + base_exists + )] + + keep_tbl <- if (require_base) std_map[base_exists == TRUE] else copy(std_map) + + .p("[", batch_key, "] rows after drop controls: ", nrow(std_map)) + .p("[", batch_key, "] require_base=", require_base) + .p("[", batch_key, "] rows kept pre-aggregate: ", nrow(keep_tbl)) + .p("[", batch_key, "] preview:") + print_df_head(keep_tbl, 8) + + if (nrow(keep_tbl) == 0) { + return(list( + new_samples_tbl = data.table(), + map_tbl = data.table(), + debug_tbl = debug_tbl, + next_id = next_id + )) + } + + # ---- First aggregation: collapse exact (individual_id, drug, timepoint) rows + agg1 <- keep_tbl[ + , + .(specimen_ids = sort(unique(trimws(as.character(specimen_id))))), + by = .(individual_id, drug, timepoint) + ] + + # Remove NA/blank specimen IDs inside groups + agg1[, specimen_ids := lapply(specimen_ids, function(v) v[!is.na(v) & nzchar(v)])] + agg1 <- agg1[lengths(specimen_ids) > 0] + + agg1[, other_names := vapply(specimen_ids, function(v) paste(v, collapse = ", "), character(1))] + agg1[, hours := hours_from_timepoint(timepoint)] + agg1[is.na(hours), hours := -1L] + + # ---- Compute other_id_val per group (this is what ends up in samples.csv) + agg1[, other_id_val := paste0( + safe_token(individual_id), "_", + safe_token(drug), "_", + as.integer(hours), "hr_treated_microtissue" + )] + + # ---- SECOND aggregation: merge replicate IDs when other_id_val collides + agg <- agg1[ + , + .( + individual_id = individual_id[1], + drug = drug[1], + timepoint = timepoint[1], + hours = hours[1], + other_names = paste( + sort(unique(trimws(unlist(strsplit(paste(other_names, collapse = ", "), ",\\s*"))))), + collapse = ", " + ) + ), + by = .(other_id_val) + ] + + .p("[", batch_key, "] aggregate groups after merge-by-other_id: ", nrow(agg)) + .p("[", batch_key, "] aggregate head:") + print_df_head(agg[, .(other_id_val, individual_id, drug, timepoint, other_names, hours)], 8) + + # fast lookup of existing treated keys in orig_samples + orig_treated_keys <- paste0(as.character(orig_samples$other_id), "||", as.character(orig_samples$model_type)) + + map_tbl <- data.table( + batch = character(), + individual_id = character(), + drug = character(), + timepoint = character(), + hours = integer(), + other_id = character(), + other_names = character(), + treated_improve_sample_id = integer() + ) + + new_samples_tbl <- data.table( + other_id = character(), + common_name = character(), + other_id_source = character(), + other_names = character(), + cancer_type = character(), + species = character(), + model_type = character(), + improve_sample_id = integer() + ) + + for (i in seq_len(nrow(agg))) { + indiv <- as.character(agg$individual_id[i]) + drug <- as.character(agg$drug[i]) + tp <- as.character(agg$timepoint[i]) + hrs <- as.integer(agg$hours[i]) + repnames <- as.character(agg$other_names[i]) + + other_id_val <- as.character(agg$other_id_val[i]) + key <- paste0(other_id_val, "||", CONST_MODEL) + + # Skip if already exists in samples.csv + if (key %in% orig_treated_keys) { + .p("[", batch_key, "][SKIP] already exists other_id=", other_id_val) + next + } + + treated_id <- next_id + next_id <- next_id + 1L + + new_samples_tbl <- rbind( + new_samples_tbl, + data.table( + other_id = other_id_val, + common_name = indiv, + other_id_source = CONST_SOURCE, + other_names = repnames, + cancer_type = CONST_CANCER, + species = CONST_SPECIES, + model_type = CONST_MODEL, + improve_sample_id = treated_id + ), + fill = TRUE + ) + + map_tbl <- rbind( + map_tbl, + data.table( + batch = batch_key, + individual_id = indiv, + drug = drug, + timepoint = tp, + hours = hrs, + other_id = other_id_val, + other_names = repnames, + treated_improve_sample_id = treated_id + ), + fill = TRUE + ) + + .p("[", batch_key, "][ADD] ", other_id_val, " | reps={", repnames, "} -> ", treated_id) + } + + list( + new_samples_tbl = new_samples_tbl, + map_tbl = map_tbl, + debug_tbl = debug_tbl, + next_id = next_id + ) +} + +write_batch_outputs <- function(batch_key, debug_tbl, map_tbl) { + out_debug_path <- file.path("/tmp", sprintf("mpnst_samples_treated_%s_debug.csv", batch_key)) + out_map_path <- file.path("/tmp", sprintf("mpnst_samples_treated_%s_map.csv", batch_key)) + + .p("[WRITE] ", batch_key, " debug -> ", out_debug_path) + fwrite(debug_tbl, out_debug_path) + .p("[WRITE] OK") + + .p("[WRITE] ", batch_key, " map -> ", out_map_path) + fwrite(map_tbl, out_map_path) + .p("[WRITE] OK") +} + +# ============================================================ +# Batch 1 +# ============================================================ +BATCH1_KEY <- "batch1" +BATCH1_SAMPLEMAP_SYNID <- "syn64608368" + +fetch_batch1_map <- function() { + .p("[BATCH1] synGet(", BATCH1_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH1_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH1] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch1_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch1 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch1_map(fetch_batch1_map()), orig_samples, BATCH1_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 2 +# ============================================================ +BATCH2_KEY <- "batch2" +BATCH2_SAMPLEMAP_SYNID <- "syn66302373" # Update to syn64608372 once the corrected version is uploaded. + + +fetch_batch2_map <- function() { + .p("[BATCH2] synGet(", BATCH2_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH2_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH2] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch2_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + #fixed in metadata, so this is not needed anymore. + # MN2 -> MN-2 + # dt[individualID == "MN2", individualID := "MN-2"] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch2 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch2_map(fetch_batch2_map()), orig_samples, BATCH2_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 3 +# ============================================================ +BATCH3_KEY <- "batch3" +BATCH3_SAMPLEMAP_SYNID <- "syn66050299" + +fetch_batch3_map <- function() { + .p("[BATCH3] synGet(", BATCH3_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH3_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH3] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch3_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] +} + +run_batch3 <- function(orig_samples, next_id) { + build_from_standard_map(normalize_batch3_map(fetch_batch3_map()), orig_samples, BATCH3_KEY, next_id, require_base = TRUE) +} + +# ============================================================ +# Batch 4 (CSV) +# ============================================================ +BATCH4_KEY <- "batch4" +BATCH4_SAMPLEMAP_SYNID <- "syn72518652" + +fetch_batch4_map <- function() { + .p("[BATCH4] synGet(", BATCH4_SAMPLEMAP_SYNID, ") ...") + pth <- synGet(BATCH4_SAMPLEMAP_SYNID)$path + dt <- as.data.table(fread(pth)) + .p("[BATCH4] map dim: ", nrow(dt), " x ", ncol(dt)) + print_df_head(dt, 6) + dt +} + +normalize_batch4_map <- function(raw_map) { + dt <- copy(as.data.table(raw_map)) + + # Expected columns in 01252026_Batch4_samplemap.csv: + # specimenID, individualID, Drug, Timepoint + if (!all(c("specimenID", "individualID", "Drug", "Timepoint") %in% names(dt))) { + .p("[BATCH4][WARN] Missing required columns. Found: ", paste(names(dt), collapse = " | ")) + return(data.table( + specimen_id = character(), + individual_id = character(), + drug = character(), + timepoint = character() + )) + } + + dt[, specimenID := trimws(as.character(specimenID))] + dt[, individualID := trimws(as.character(individualID))] + dt[, Drug := trimws(as.character(Drug))] + dt[, Timepoint := trimws(as.character(Timepoint))] + + dt <- dt[!is.na(specimenID) & nzchar(specimenID) & !is.na(individualID) & nzchar(individualID)] + + out <- dt[, .( + specimen_id = specimenID, + individual_id = individualID, + drug = Drug, + timepoint = normalize_timepoint(Timepoint) + )] + + .p("[BATCH4] normalized dim: ", nrow(out), " x ", ncol(out)) + .p("[BATCH4] normalized head:") + print_df_head(out, 8) + + out +} + +run_batch4 <- function(orig_samples, next_id) { + build_from_standard_map( + normalize_batch4_map(fetch_batch4_map()), + orig_samples, + BATCH4_KEY, + next_id, + require_base = FALSE + ) +} + + +# ============================================================ +# Register all batches (order) +# ============================================================ +ALL_BATCH_RUNNERS <- list( + list(key=BATCH1_KEY, fn=run_batch1), + list(key=BATCH2_KEY, fn=run_batch2), + list(key=BATCH3_KEY, fn=run_batch3), + list(key=BATCH4_KEY, fn=run_batch4) +) + +# ============================================================ +# MAIN +# ============================================================ +args <- commandArgs(trailingOnly = TRUE) + +.p("============================================================") +.p("=== 00b_sample_gen_treated.R (RUN ALL BATCHES) ===") +.p("Timestamp: ", .ts()) +.p("Args: ", paste(args, collapse = " | ")) +.p("============================================================") + +if (length(args) < 1) { + stop( + "Usage: Rscript 00b_sample_gen_treated.R \n", + "Example: Rscript 00b_sample_gen_treated.R /tmp/mpnst_samples.csv", + call. = FALSE + ) +} + +prev_samples_path <- args[1] +.p("[ARGS] prev_samples_path=", prev_samples_path) +if (!file.exists(prev_samples_path)) stop("File not found: ", prev_samples_path, call. = FALSE) + +.p("[READ] fread(samples) ...") +orig_samples <- fread(prev_samples_path) +.p("[READ] orig_samples dim: ", nrow(orig_samples), " x ", ncol(orig_samples)) +print_df_head(orig_samples, 6) + +# ------------------------------------------------------------ +# Migrate already-existing treated other_ids to hyphen style +# so future runs match your desired format and don't skip forever +# ------------------------------------------------------------ +orig_samples[, other_id_old := other_id] +orig_samples[, other_id := migrate_treated_other_id_prefix(other_id)] + +n_changed <- sum(orig_samples$other_id != orig_samples$other_id_old, na.rm = TRUE) +if (n_changed > 0) .p("[MIGRATE] updated treated other_id prefix for ", n_changed, " row(s)") +orig_samples[, other_id_old := NULL] + +# If migration created duplicates (same other_id + model_type), merge other_names and drop dup rows +orig_samples[, dedup_key := paste(other_id, model_type, sep = "||")] +dup_keys <- orig_samples[duplicated(dedup_key), unique(dedup_key)] + +if (length(dup_keys) > 0) { + .p("[MIGRATE] duplicates after migration: ", length(dup_keys), " key(s). Merging other_names + dropping dup rows.") + + merged <- orig_samples[dedup_key %in% dup_keys, .( + merged_other_names = paste( + sort(unique(trimws(unlist(strsplit(paste(na.omit(other_names), collapse = ", "), ",\\s*"))))), + collapse = ", " + ) + ), by = dedup_key] + + orig_samples[merged, on = "dedup_key", other_names := i.merged_other_names] + orig_samples <- orig_samples[!duplicated(dedup_key)] +} + +orig_samples[, dedup_key := NULL] + +token <- Sys.getenv("SYNAPSE_AUTH_TOKEN") +.p("[ENV] SYNAPSE_AUTH_TOKEN present? ", nzchar(token)) +if (!nzchar(token)) stop("Please set SYNAPSE_AUTH_TOKEN in your environment.", call. = FALSE) + +.p("[SYN] synLogin() ...") +synLogin(authToken = token) +.p("[SYN] synLogin() OK") + +max_id <- suppressWarnings(max(as.integer(orig_samples$improve_sample_id), na.rm = TRUE)) +if (!is.finite(max_id)) max_id <- 0L +next_id <- max_id + 1L +.p("[IDS] max improve_sample_id=", max_id, " next_id=", next_id) + +.p("============================================================") +.p("[RUN] Running ", length(ALL_BATCH_RUNNERS), " batch runner(s) in order...") +.p("============================================================") + +for (item in ALL_BATCH_RUNNERS) { + batch_key <- item$key + runner <- item$fn + + res <- runner(orig_samples = orig_samples, next_id = next_id) + + if (!is.null(res$new_samples_tbl) && nrow(res$new_samples_tbl) > 0) { + orig_samples <- rbindlist(list(orig_samples, res$new_samples_tbl), use.names = TRUE, fill = TRUE) + + orig_samples[, dedup_key := paste(other_id, model_type, sep = "||")] + before <- nrow(orig_samples) + orig_samples <- orig_samples[!duplicated(dedup_key)] + after <- nrow(orig_samples) + if (after < before) .p("[", batch_key, "][WARN] post-append dedup removed ", before - after, " row(s)") + orig_samples[, dedup_key := NULL] + } + + next_id <- res$next_id + + .p("[RUN] Finished ", batch_key, " | added=", ifelse(is.null(res$new_samples_tbl), 0, nrow(res$new_samples_tbl))) + write_batch_outputs(batch_key, res$debug_tbl, res$map_tbl) +} + +.p("[WRITE] Writing updated samples -> ", prev_samples_path) +fwrite(orig_samples, prev_samples_path) +.p("[WRITE] OK") + +.p("============================================================") +.p("[DONE] Completed all batches. Updated samples at: ", prev_samples_path) +.p("============================================================") diff --git a/coderbuild/mpnst/01_combined_omics.R b/coderbuild/mpnst/01_combined_omics.R index dcbdfbae..7e7dd41a 100644 --- a/coderbuild/mpnst/01_combined_omics.R +++ b/coderbuild/mpnst/01_combined_omics.R @@ -1,3 +1,4 @@ +# 01_combined_omics.R #!/usr/bin/env Rscript # Combined MPNST & MPNST-PDX Data Extraction Script @@ -31,7 +32,7 @@ genes_df <- fread(genes) # Subset by model type pdx_samps <- filter(samples_df, model_type == "patient derived xenograft") tumor_samps<- filter(samples_df, model_type == "tumor") -mt_samps <- filter(samples_df, model_type == "xenograft derived organoid") # These end up being the same as pdx_samps in the manifest. +mt_samps <- filter(samples_df, model_type == "3D-MEDS") # These end up being the same as pdx_samps in the manifest. # Retrieve manifest table from Synapse manifest <- synTableQuery("select * from syn53503360")$asDataFrame() %>% @@ -58,7 +59,7 @@ tumor_data <- manifest %>% mutate(Proteomics = "") %>% filter(!is.na(improve_sample_id)) -mt_data <- manifest %>% #Note, this is the same as pdx_data but I think we default to "xenograft derived organoid" if present (based on original files) +mt_data <- manifest %>% #Note, this is the same as pdx_data but I think we default to "3D-MEDS" if present (based on original files) select(common_name, starts_with("PDX")) %>% left_join(mt_samps, by = "common_name") %>% select(improve_sample_id, common_name, model_type, @@ -78,7 +79,7 @@ study_label <- function(type) { case_when( type == "patient derived xenograft" ~ "MPNST PDX", type == "tumor" ~ "MPNST Tumor", - type == "xenograft derived organoid" ~ "MPNST PDX MT", + type == "3D-MEDS" ~ "MPNST PDX MT", TRUE ~ "MPNST" ) } @@ -147,13 +148,28 @@ transcriptomics_list <- lapply( if (is.null(meta)) return(NULL) df <- tryCatch({ - fread(synGet(id)$path) %>% + raw <- fread(synGet(id)$path) %>% separate(Name, into = c("other_id","vers"), sep = "\\.") %>% - select(-vers) %>% + select(-vers) + + is_enst_input <- all(grepl("^ENST", raw$other_id)) + + mapped <- raw %>% left_join(genes_df) %>% - select(entrez_id, transcriptomics = TPM) %>% - filter(!is.na(entrez_id), transcriptomics != 0) %>% - distinct() + select(entrez_id, other_id, transcriptomics = TPM) %>% + filter(!is.na(entrez_id), transcriptomics != 0) + + if (is_enst_input) { + mapped <- mapped %>% + group_by(entrez_id) %>% + summarise(transcriptomics = sum(transcriptomics), .groups = "drop") + } else { + mapped <- mapped %>% + select(entrez_id, transcriptomics) %>% + distinct() + } + + mapped }, error = function(e) NULL) i_safe_extract( diff --git a/coderbuild/mpnst/01b_combined_omics_treated.R b/coderbuild/mpnst/01b_combined_omics_treated.R new file mode 100644 index 00000000..8125fc61 --- /dev/null +++ b/coderbuild/mpnst/01b_combined_omics_treated.R @@ -0,0 +1,219 @@ +# 01b_combined_omics_treated.R +#!/usr/bin/env Rscript + +suppressPackageStartupMessages({ + library(data.table) + library(synapser) + library(dplyr) + library(tidyr) +}) + +# ----------------------- +# logging helpers +# ----------------------- +.ts <- function() format(Sys.time(), "%Y-%m-%d %H:%M:%S") +.p <- function(...) cat(.ts(), paste0(...), "\n") + +# ----------------------- +# batch inputs (Synapse) +# ----------------------- +B1_SYN <- "syn64608366" # salmon.merged.gene_tpm_corrected_id.csv +B2_SYN <- "syn64608370" # salmon.merged.gene_tpm.tsv_corrected_ID.csv +B3_SYN <- "syn65887795" # salmon.merged.gene_tpm.tsv +B4_SYN <- "syn68898091" # salmon.merged.gene_tpm.tsv + +# ----------------------- +# helpers +# ----------------------- +study_label <- function(type) { + dplyr::case_when( + type == "patient derived xenograft" ~ "MPNST PDX", + type == "tumor" ~ "MPNST Tumor", + type == "3D-MEDS" ~ "MPNST PDX MT", + TRUE ~ "MPNST" + ) +} + +is_treated_microtissue <- function(other_id, other_names) { + oid <- as.character(other_id) + onm <- as.character(other_names) + grepl("treated_microtissue$", oid) && !is.na(onm) && nzchar(trimws(onm)) +} + +split_rep_ids <- function(x) { + if (is.na(x) || !nzchar(trimws(x))) return(character(0)) + v <- unlist(strsplit(x, ",")) + v <- trimws(v) + v <- v[nzchar(v)] + unique(v) +} + +guess_batch_from_rep <- function(rep_id) { + r <- as.character(rep_id) + if (grepl("^b1_", r)) return("b1") + if (grepl("^b2_", r)) return("b2") + if (grepl("^B3-", r) || grepl("^B3_", r)) return("b3") + if (grepl("^B4-", r) || grepl("^B4\\.", r)) return("b4") + return(NA_character_) +} + +rep_to_colname <- function(rep_id, batch) { + r <- as.character(rep_id) + if (batch %in% c("b1", "b2")) return(r) + if (batch == "b3") return(gsub("-", "_", r, fixed = FALSE)) # B3-10 -> B3_10 + if (batch == "b4") return(gsub("B4-", "B4.", r, fixed = TRUE)) # B4-10 -> B4.10 + return(r) +} + +read_tpm_matrix <- function(syn_id) { + .p("[SYN] synGet(", syn_id, ") ...") + pth <- synGet(syn_id)$path + .p("[READ] ", syn_id, " -> ", pth) + dt <- fread(pth) + stopifnot(all(c("gene_id", "gene_name") %in% names(dt))) + dt +} + +# returns a data.table with columns: +# gene_id, transcriptomics, improve_sample_id, source, study +extract_treated_from_batch <- function(mat, treated_tbl, batch_key) { + if (nrow(treated_tbl) == 0) return(data.table()) + + tmp <- copy(treated_tbl) + tmp[, rep_ids := lapply(other_names, split_rep_ids)] + tmp[, rep_batch := vapply(rep_ids, function(v) { + if (length(v) == 0) return(NA_character_) + guess_batch_from_rep(v[1]) + }, character(1))] + + tmp <- tmp[rep_batch == batch_key] + if (nrow(tmp) == 0) return(data.table()) + + tmp[, rep_cols := lapply(rep_ids, function(v) rep_to_colname(v, batch_key))] + + .p("[", batch_key, "] treated samples in this batch: ", nrow(tmp)) + + out_list <- vector("list", nrow(tmp)) + + for (i in seq_len(nrow(tmp))) { + samp_id <- tmp$improve_sample_id[i] + samp_model <- tmp$model_type[i] + cols <- tmp$rep_cols[[i]] + cols <- cols[cols %in% names(mat)] + + if (length(cols) == 0) { + .p("[", batch_key, "][SKIP] improve_sample_id=", samp_id, + " -> no replicate columns found in matrix") + next + } + + expr <- rowMeans(as.matrix(mat[, ..cols]), na.rm = TRUE) + + out_list[[i]] <- data.table( + gene_id = mat$gene_id, + transcriptomics = expr, + improve_sample_id = samp_id, + source = "NF Data Portal", + study = study_label(samp_model) + ) + } + + rbindlist(out_list, use.names = TRUE, fill = TRUE) +} + +# ----------------------- +# MAIN +# ----------------------- +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 3) { + stop("Usage: Rscript 01b_combined_omics_treated.R ", call. = FALSE) +} +PAT <- args[1] +samples <- args[2] +genes <- args[3] + +.p("============================================================") +.p("=== 01b_combined_omics_treated.R ===") +.p("Timestamp: ", .ts()) +.p("Args: ", paste(args, collapse = " | ")) +.p("============================================================") + +.p("[SYN] synLogin() ...") +synLogin(authToken = PAT) +.p("[SYN] synLogin() OK") + +.p("[READ] samples: ", samples) +samples_df <- fread(samples) + +treated <- samples_df[ + model_type == "3D-MEDS" & + vapply(seq_len(nrow(samples_df)), function(i) { + is_treated_microtissue(samples_df$other_id[i], samples_df$other_names[i]) + }, logical(1)), + .(other_names, improve_sample_id, model_type) +] + +.p("[SAMPLES] treated_microtissue rows: ", nrow(treated)) +if (nrow(treated) == 0) stop("No treated microtissue samples found in samples.csv", call. = FALSE) + +.p("[READ] genes: ", genes) +genes_df <- fread(genes) +stopifnot(all(c("other_id", "entrez_id") %in% names(genes_df))) + +# matrices +m1 <- read_tpm_matrix(B1_SYN) +m2 <- read_tpm_matrix(B2_SYN) +m3 <- read_tpm_matrix(B3_SYN) +m4 <- read_tpm_matrix(B4_SYN) + +# extract +t1 <- extract_treated_from_batch(m1, treated, "b1") +t2 <- extract_treated_from_batch(m2, treated, "b2") +t3 <- extract_treated_from_batch(m3, treated, "b3") +t4 <- extract_treated_from_batch(m4, treated, "b4") + +treated_long <- rbindlist(list(t1, t2, t3, t4), use.names = TRUE, fill = TRUE) +.p("[OUT] treated_long rows: ", nrow(treated_long)) +if (nrow(treated_long) == 0) stop("No treated data extracted (check replicate IDs vs column names).", call. = FALSE) + +# ENSEMBL gene_id has version -> strip to match genes_df$other_id +treated_long[, other_id := tstrsplit(gsub("\\s+", "", gene_id), "\\.", keep = 1)] + +treated_final <- treated_long %>% + left_join(genes_df, by = "other_id") %>% + dplyr::select(entrez_id, transcriptomics, improve_sample_id, source, study) %>% + filter(!is.na(entrez_id), transcriptomics != 0) %>% + distinct() + +# enforce strict schema & order +treated_final <- as.data.table(treated_final)[, .(entrez_id, transcriptomics, improve_sample_id, source, study)] + +treated_out_path <- file.path("/tmp", "mpnst_transcriptomics_treated_microtissue.csv") +.p("[WRITE] treated-only transcriptomics -> ", treated_out_path) +fwrite(treated_final, treated_out_path) +.p("[WRITE] OK") + +# optional append into main transcriptomics (same strict schema) +main_path <- file.path("/tmp", "mpnst_transcriptomics.csv") +if (file.exists(main_path)) { + .p("[APPEND] Found existing main transcriptomics: ", main_path) + main_df <- fread(main_path) + main_df <- main_df[, .(entrez_id, transcriptomics, improve_sample_id, source, study)] # enforce schema + + combined <- rbindlist(list(main_df, treated_final), use.names = TRUE, fill = TRUE) + + # de-dup conservatively on (entrez_id, improve_sample_id, source, study, transcriptomics) + combined[, dedup_key := paste(entrez_id, improve_sample_id, source, study, transcriptomics, sep = "||")] + combined <- combined[!duplicated(dedup_key)] + combined[, dedup_key := NULL] + + .p("[WRITE] overwriting main transcriptomics with appended treated rows -> ", main_path) + fwrite(combined, main_path) + .p("[WRITE] OK") +} else { + .p("[APPEND] No existing /tmp/mpnst_transcriptomics.csv found. Not appending.") +} + +.p("============================================================") +.p("[DONE] Wrote treated transcriptomics with strict schema.") +.p("============================================================") diff --git a/coderbuild/mpnst/build_omics.sh b/coderbuild/mpnst/build_omics.sh index d6d2cec7..c6b0853c 100644 --- a/coderbuild/mpnst/build_omics.sh +++ b/coderbuild/mpnst/build_omics.sh @@ -1,3 +1,4 @@ +# build_omics.sh #!/bin/bash set -euo pipefail @@ -5,3 +6,5 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit echo "Running 01_combined_omics.R with $SYNAPSE_AUTH_TOKEN, $2, and $1." Rscript 01_combined_omics.R $SYNAPSE_AUTH_TOKEN $2 $1 +echo "Running 01b_combined_omics_treated.R with PAT, samples=$2, genes=$1" +Rscript 01b_combined_omics_treated.R "$SYNAPSE_AUTH_TOKEN" "$2" "$1" \ No newline at end of file diff --git a/coderbuild/mpnst/build_samples.sh b/coderbuild/mpnst/build_samples.sh index c9c079fa..832ed895 100644 --- a/coderbuild/mpnst/build_samples.sh +++ b/coderbuild/mpnst/build_samples.sh @@ -1,3 +1,4 @@ +# build_samples.sh #!/bin/bash set -euo pipefail @@ -5,3 +6,4 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit echo "Running 00_sample_gen.R with $1." Rscript 00_sample_gen.R $1 +Rscript 00b_sample_gen_treated.R /tmp/mpnst_samples.csv From 55ec7e167c8b0750d1c1ed4ecbae1bc7ea01fcab Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:01 -0700 Subject: [PATCH 2/4] Add the cNF organoid dataset: drugs and experiments --- coderbuild/cnf/03-drugs-cnf.py | 531 +++++++++++++++++++++++++++ coderbuild/cnf/04-experiments-cnf.py | 478 ++++++++++++++++++++++++ coderbuild/cnf/build_drugs.sh | 20 + coderbuild/cnf/build_exp.sh | 12 + 4 files changed, 1041 insertions(+) create mode 100644 coderbuild/cnf/03-drugs-cnf.py create mode 100644 coderbuild/cnf/04-experiments-cnf.py create mode 100644 coderbuild/cnf/build_drugs.sh create mode 100644 coderbuild/cnf/build_exp.sh diff --git a/coderbuild/cnf/03-drugs-cnf.py b/coderbuild/cnf/03-drugs-cnf.py new file mode 100644 index 00000000..fe307ddb --- /dev/null +++ b/coderbuild/cnf/03-drugs-cnf.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 +""" +03-drugs-cnf.py: generate cnf_drugs.tsv and cnf_drug_descriptors.tsv. + +Pulls unique drug names from the cNF drug screen on Synapse, then calls the +standard coderdata utilities: + + coderbuild/utils/pubchem_retrieval.py + -> cnf_drugs.tsv + + coderbuild/utils/build_drug_descriptor_table.py + -> cnf_drug_descriptors.tsv + +Important implementation note: + pubchem_retrieval.py does not expose a command-line interface in the + provided version. It defines update_dataframe_and_write_tsv(), but running + `python pubchem_retrieval.py --input ... --output ...` exits 0 without + doing any work. Therefore this script imports pubchem_retrieval.py directly + and calls update_dataframe_and_write_tsv(). + +The descriptor builder is still run as a subprocess. It may call R/biomaRt +internally, which can fail when Ensembl is unavailable. To keep the drugs file +produced even when descriptors fail, this script: + + 1. Retries external/remote work with exponential backoff. + 2. Sets a generous timeout. + 3. Supports --skip-descriptors. + 4. Supports --only-descriptors for retrying descriptor generation later. + +Usage: + python 03-drugs-cnf.py --prev_drugs + [--out_drugs /tmp/cnf_drugs.tsv] + [--out_desc /tmp/cnf_drug_descriptors.tsv] + [--skip-descriptors] + [--only-descriptors] + [--retries 3] + [--timeout 1800] +""" + +import argparse +import importlib.util +import logging +import os +import signal +import subprocess +import sys +import tempfile +import time +import pandas as pd +import synapseclient + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- + +DRUG_SCREEN_TABLE = "syn51301431" + +PUBCHEM_SCRIPT = os.environ.get( + "CODERDATA_PUBCHEM_SCRIPT", + "/coderbuild/utils/pubchem_retrieval.py", +) + +DESCRIPTOR_SCRIPT = os.environ.get( + "CODERDATA_DESCRIPTOR_SCRIPT", + "/coderbuild/utils/build_drug_desc.py", +) + +DEFAULT_OUT_DRUGS = "/tmp/cnf_drugs.tsv" +DEFAULT_OUT_DESC = "/tmp/cnf_drug_descriptors.tsv" + +DEFAULT_RETRIES = 3 +DEFAULT_TIMEOUT = 1800 # seconds per attempt +RETRY_BACKOFF_S = 30 # 30s, 60s, 120s, ... + + +# ---------------------------------------------------------------------------- +# Logging +# ---------------------------------------------------------------------------- + +def configure_logging() -> None: + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Generic subprocess wrapper with retry +# ---------------------------------------------------------------------------- + +def run_with_retries( + cmd: list[str], + retries: int, + timeout: int, + label: str, +) -> bool: + """ + Run a subprocess, retrying on failure with exponential backoff. + + Returns True on success, False if all retries are exhausted. + """ + for attempt in range(1, retries + 1): + logging.info("[%s] attempt %d/%d", label, attempt, retries) + + try: + subprocess.run(cmd, check=True, timeout=timeout) + logging.info("[%s] succeeded on attempt %d", label, attempt) + return True + + except subprocess.TimeoutExpired: + logging.warning( + "[%s] timed out after %ds on attempt %d", + label, + timeout, + attempt, + ) + + except subprocess.CalledProcessError as exc: + logging.warning( + "[%s] failed with exit %d on attempt %d", + label, + exc.returncode, + attempt, + ) + + except Exception as exc: + logging.warning( + "[%s] unexpected error on attempt %d: %s", + label, + attempt, + exc, + ) + + if attempt < retries: + backoff = RETRY_BACKOFF_S * (2 ** (attempt - 1)) + logging.info("[%s] sleeping %ds before retry", label, backoff) + time.sleep(backoff) + + logging.error("[%s] exhausted %d retries", label, retries) + return False + + +# ---------------------------------------------------------------------------- +# PubChem utility loading/calling +# ---------------------------------------------------------------------------- + +def load_pubchem_module(path: str): + """ + Load pubchem_retrieval.py as a Python module. + + The provided pubchem_retrieval.py has no command-line entry point, so it + must be imported and called directly. + """ + if not os.path.exists(path): + raise FileNotFoundError(f"PubChem retrieval script not found: {path}") + + spec = importlib.util.spec_from_file_location( + "coderdata_pubchem_retrieval", + path, + ) + + if spec is None or spec.loader is None: + raise ImportError(f"Could not create import spec for {path}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + if not hasattr(module, "update_dataframe_and_write_tsv"): + raise AttributeError( + f"{path} does not define update_dataframe_and_write_tsv()" + ) + + return module + + +def call_pubchem_retrieval( + names: list[str], + prev_drug_files: list[str], + out_drugs: str, + retries: int, + timeout: int, +) -> bool: + """ + Generate the drugs TSV by calling pubchem_retrieval.update_dataframe_and_write_tsv(). + + This intentionally does not use subprocess because the supplied + pubchem_retrieval.py has no argparse/main block and exits 0 without writing + anything when run as a script. + """ + prev_arg = ",".join(prev_drug_files) if prev_drug_files else None + + for attempt in range(1, retries + 1): + logging.info("[pubchem_retrieval] attempt %d/%d", attempt, retries) + + ignore_chems = None + + try: + pubchem = load_pubchem_module(PUBCHEM_SCRIPT) + + with tempfile.NamedTemporaryFile( + "w", + suffix="_cnf_ignore_chems.txt", + delete=False, + ) as tmp_ignore: + ignore_chems = tmp_ignore.name + + df = pubchem.update_dataframe_and_write_tsv( + unique_names=names, + output_filename=out_drugs, + ignore_chems=ignore_chems, + batch_size=1, + isname=True, + time_limit=timeout, + prev_drug_filepaths=prev_arg, + restrict_to_raw_names=names, + ) + + # pubchem_retrieval.py sets signal.alarm(time_limit), but does not + # clear it. Clear it here so the descriptor step is not interrupted. + try: + signal.alarm(0) + except Exception: + pass + + if getattr(pubchem, "should_continue", True) is False: + raise TimeoutError( + f"pubchem_retrieval reached its time limit of {timeout}s" + ) + + if not os.path.exists(out_drugs): + raise FileNotFoundError( + f"pubchem_retrieval returned but did not create {out_drugs}" + ) + + if os.path.getsize(out_drugs) == 0: + raise RuntimeError( + f"pubchem_retrieval created {out_drugs}, but it is empty" + ) + + n_rows = len(df) if df is not None else "unknown" + + logging.info( + "[pubchem_retrieval] succeeded on attempt %d; wrote %s rows to %s", + attempt, + n_rows, + out_drugs, + ) + + return True + + except Exception as exc: + try: + signal.alarm(0) + except Exception: + pass + + logging.warning( + "[pubchem_retrieval] failed on attempt %d/%d: %s", + attempt, + retries, + exc, + ) + + if attempt < retries: + backoff = RETRY_BACKOFF_S * (2 ** (attempt - 1)) + logging.info( + "[pubchem_retrieval] sleeping %ds before retry", + backoff, + ) + time.sleep(backoff) + + finally: + if ignore_chems and os.path.exists(ignore_chems): + try: + os.unlink(ignore_chems) + except OSError: + pass + + logging.error("[pubchem_retrieval] exhausted %d retries", retries) + return False + + +# ---------------------------------------------------------------------------- +# Drug name collection +# ---------------------------------------------------------------------------- + +def collect_drug_names(syn) -> list[str]: + """ + Pull every cNF drug-screen file from Synapse and collect unique drug names. + """ + query = ( + "select id, individualID, specimenID " + f"from {DRUG_SCREEN_TABLE} " + "where dataType='drug screen'" + ) + + files = syn.tableQuery(query).asDataFrame() + + logging.info("Reading %d drug-screen files for drug names", len(files)) + + names: set[str] = set() + + for _, row in files.iterrows(): + try: + entity = syn.get(row["id"]) + df = pd.read_csv(entity.path) + + except Exception as exc: + logging.warning("Skipping file %s: %s", row["id"], exc) + continue + + if "Drug" not in df.columns: + logging.warning("File %s has no 'Drug' column; skipping", row["id"]) + continue + + names.update( + df["Drug"] + .dropna() + .astype(str) + .str.strip() + .tolist() + ) + + names = sorted(n for n in names if n and n.lower() != "nan") + + logging.info("Collected %d unique drug names", len(names)) + + return names + + +# ---------------------------------------------------------------------------- +# Descriptor utility +# ---------------------------------------------------------------------------- + +def call_descriptor_table( + drug_file: str, + out_desc: str, + retries: int, + timeout: int, +) -> bool: + """ + Run the descriptor table builder as a subprocess. + + This function assumes build_drug_descriptor_table.py has a CLI accepting + --input and --output. + """ + cmd = [ + "python", + DESCRIPTOR_SCRIPT, + "--drugtable", + drug_file, + "--desctable", + out_desc, + ] + + ok = run_with_retries( + cmd, + retries, + timeout, + "descriptor_table", + ) + + if ok and not os.path.exists(out_desc): + logging.error( + "descriptor_table exited successfully but did not create %s", + out_desc, + ) + return False + + return ok + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument( + "--prev_drugs", + default="", + help="Comma-delimited list of existing drug TSV files for ID reuse.", + ) + + parser.add_argument( + "--out_drugs", + default=DEFAULT_OUT_DRUGS, + help=f"Output drugs TSV. Default: {DEFAULT_OUT_DRUGS}", + ) + + parser.add_argument( + "--out_desc", + default=DEFAULT_OUT_DESC, + help=f"Output drug descriptors TSV. Default: {DEFAULT_OUT_DESC}", + ) + + parser.add_argument( + "--skip-descriptors", + action="store_true", + help=( + "Skip the descriptor step entirely. Use when the descriptor " + "utility or its biomaRt/Ensembl dependency is failing. The drugs " + "table is still produced." + ), + ) + + parser.add_argument( + "--only-descriptors", + action="store_true", + help=( + "Only run the descriptor step, expecting --out_drugs to already " + "exist. Useful for retrying after a previous descriptor failure." + ), + ) + + parser.add_argument( + "--retries", + type=int, + default=DEFAULT_RETRIES, + help=f"Number of retry attempts. Default: {DEFAULT_RETRIES}", + ) + + parser.add_argument( + "--timeout", + type=int, + default=DEFAULT_TIMEOUT, + help=f"Timeout in seconds per attempt. Default: {DEFAULT_TIMEOUT}", + ) + + args = parser.parse_args() + + configure_logging() + + if args.skip_descriptors and args.only_descriptors: + logging.error("Cannot combine --skip-descriptors with --only-descriptors") + return 2 + + for path in (args.out_drugs, args.out_desc): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + + # ------------------------------------------------------------------------ + # Drugs step + # ------------------------------------------------------------------------ + + if not args.only_descriptors: + syn = synapseclient.Synapse() + syn.login() + + names = collect_drug_names(syn) + + if not names: + logging.error("No drug names found; aborting") + return 1 + + prev = [ + p.strip() + for p in args.prev_drugs.split(",") + if p.strip() + ] + + ok = call_pubchem_retrieval( + names=names, + prev_drug_files=prev, + out_drugs=args.out_drugs, + retries=args.retries, + timeout=args.timeout, + ) + + if not ok: + logging.error("pubchem_retrieval failed; aborting") + return 1 + + if not os.path.exists(args.out_drugs): + logging.error( + "pubchem_retrieval did not produce %s", + args.out_drugs, + ) + return 1 + + logging.info("Wrote %s", args.out_drugs) + + # ------------------------------------------------------------------------ + # Descriptor step + # ------------------------------------------------------------------------ + + if args.skip_descriptors: + logging.warning( + "Skipping descriptor step. Re-run with --only-descriptors to " + "produce %s when descriptor dependencies are reachable.", + args.out_desc, + ) + return 0 + + if not os.path.exists(args.out_drugs): + logging.error( + "Drugs file %s missing; cannot run descriptors", + args.out_drugs, + ) + return 1 + + ok = call_descriptor_table( + drug_file=args.out_drugs, + out_desc=args.out_desc, + retries=args.retries, + timeout=args.timeout, + ) + + if not ok: + logging.error( + "Descriptor step failed after %d attempts. The drugs file %s is " + "complete; re-run with --only-descriptors when descriptor " + "dependencies are reachable to produce %s.", + args.retries, + args.out_drugs, + args.out_desc, + ) + + # Exit 0 because the primary drugs table was produced. Descriptors can + # be retried later. Use --skip-descriptors to suppress this explicitly. + return 0 + + logging.info("Wrote %s", args.out_desc) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/coderbuild/cnf/04-experiments-cnf.py b/coderbuild/cnf/04-experiments-cnf.py new file mode 100644 index 00000000..fe1e216b --- /dev/null +++ b/coderbuild/cnf/04-experiments-cnf.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +""" +04-experiments-cnf.py: generate cnf_experiments.tsv. + +Discovers every drug-screen viability file under three Synapse parent folders, +attaches a specimen ID by reading the file's `specimenID` annotation (with a +filename-pattern fallback), then per (specimen, drug): + + * Multi-concentration measurements are run through the standard + coderbuild/utils/fit_curve.py utility, producing fit_auc / fit_ic50 / + fit_einf / fit_hs / fit_r2. + * Single-concentration measurements at 1 μM are recorded with + metric = 'uM_viability' and value = Viability_percentage / 100. + Schema dependency: 'uM_viability' must be in the ResponseMetric enum. + See schema_patch.md. + +Specimen attribution policy: + 1. Trust the file's `specimenID` annotation if present. + 2. Fall back to parsing the filename (e.g. "NF0017_T2_Viabilities.csv" + → "NF0017_T2"). + 3. Run the result through cnf_utils.classify_specimen as a safety net + (handles dot-separators, suffixes, case variants). Files whose + specimen can't be classified are skipped with a warning. + +Dual-mapping: one viability file's results should be attributed to two +specimens. Handled via SPECIMEN_DUAL_MAPPINGS at the top of the script; +add new entries as more cases come up. + +Usage: + python 04-experiments-cnf.py + [--output /tmp/cnf_experiments.tsv] + [--parents syn51301414,syn51301420,syn51301426] +""" + +import argparse +import logging +import os +import re +import subprocess +import sys +import tempfile + +import pandas as pd +import synapseclient + +from cnf_utils import classify_specimen + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- +# Synapse folders to walk for drug-screen viability files +DEFAULT_DRUG_SCREEN_PARENTS = [ + "syn51301414", + "syn51301420", + "syn51301426", +] + +TIME = 120 +TIME_UNIT = "hours" +STUDY = "cnf" +SOURCE = "Synapse" +SINGLE_DOSE_UM = 1.0 +SINGLE_DOSE_METRIC = "uM_viability" + +FIT_CURVE_SCRIPT = os.environ.get( + "CODERDATA_FIT_CURVE_SCRIPT", "/coderbuild/utils/fit_curve.py") + +DEFAULT_OUTPUT = "/tmp/cnf_experiments.tsv" + +# Regex to recover specimen from filenames like "NF0017_T2_Viabilities.csv" +# or "NF0021_T1_Onalespid_1uM_Viabilities.csv" +FILENAME_SPECIMEN_RE = re.compile( + r"^(NF\d{4}(?:_T\d+)?(?:_[A-Za-z0-9._-]+?)?)" # specimen prefix + r"_(?:Viabilities|viabilities|Viability|drug_screen)" # known label + r"\.(?:csv|tsv|txt)$", + re.IGNORECASE, +) + +# ---------------------------------------------------------------------------- +# DUAL-MAPPING TABLE +# ---------------------------------------------------------------------------- +# Maps a canonical specimen to any additional specimens that should receive +# the same drug-response rows. Add new entries as more cases come up. +# +# Known case: NF0021_T1_Onalespid_1uM was screened against the same drug +# panel as NF0021_T1 and should be available under both labels. +SPECIMEN_DUAL_MAPPINGS: dict[str, list[str]] = { + "NF0021_T1_Onalespid_1uM": ["NF0021_T1"], +} + + +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Specimen attribution +# ---------------------------------------------------------------------------- +def specimen_from_annotation(syn, file_id: str) -> str | None: + """Return the file's specimenID annotation, or None if missing.""" + try: + anno = syn.getAnnotations(file_id) + except Exception as exc: + logging.warning("getAnnotations failed for %s: %s", file_id, exc) + return None + val = anno.get("specimenID") + if isinstance(val, list): + val = val[0] if val else None + if val is None: + return None + s = str(val).strip() + return s or None + + +def specimen_from_filename(name: str) -> str | None: + """Parse 'NF0017_T2_Viabilities.csv' → 'NF0017_T2'.""" + m = FILENAME_SPECIMEN_RE.match(name) + if m: + return m.group(1) + return None + + +def resolve_specimen(syn, file_id: str, file_name: str) -> str | None: + """Resolve a file to a canonical specimen ID, trying annotation first. + + Returns canonical_id (e.g. "NF0017_T2") or None if both annotation and + filename parsing fail or the result doesn't classify as a known sample + type. + """ + raw = specimen_from_annotation(syn, file_id) + if not raw: + raw = specimen_from_filename(file_name) + if not raw: + return None + + # Safety-net classification — handles dot-separators, suffixes, case + result = classify_specimen(raw) + if result is None: + # Annotation or filename produced something that doesn't fit any + # known sample pattern. + logging.warning( + "Could not classify specimen '%s' (file %s, %s); skipping", + raw, file_id, file_name, + ) + return None + return result[0] + + +def expand_dual_mappings(specimen: str) -> list[str]: + """Return [specimen] + any additional specimens it's dual-mapped to.""" + return [specimen] + SPECIMEN_DUAL_MAPPINGS.get(specimen, []) + + +# ---------------------------------------------------------------------------- +# Synapse walking +# ---------------------------------------------------------------------------- +def walk_for_viability_files(syn, parent_id: str) -> list[dict]: + """Recursively yield File entities under a Synapse folder. + + Returns a list of {id, name} dicts for files whose name suggests they + contain viability data (drug-screen CSV/TSV). + """ + try: + import synapseutils + except ImportError: + logging.error("synapseutils not available; cannot walk %s", parent_id) + return [] + + files = [] + for dirpath, _subfolders, file_list in synapseutils.walk(syn, parent_id): + for fname, fid in file_list: + # Filter to plausible viability files. Accept anything ending in + # csv/tsv that has 'viab' or 'drug' in the name; fall back to + # 'csv' if neither matches but the file is small enough. + lower = fname.lower() + if not lower.endswith((".csv", ".tsv", ".txt")): + continue + if any(tok in lower for tok in ("viab", "drug", "screen")): + files.append({"id": fid, "name": fname, + "parent_path": dirpath}) + # Don't auto-grab arbitrary CSVs — we only want drug screens. + return files + + +def discover_drug_files(syn, parents: list[str]) -> pd.DataFrame: + """Walk all parent folders and dedupe by Synapse ID.""" + all_files = [] + for parent in parents: + logging.info("Walking %s ...", parent) + files = walk_for_viability_files(syn, parent) + logging.info(" found %d candidate viability files", len(files)) + all_files.extend(files) + + if not all_files: + return pd.DataFrame(columns=["id", "name"]) + + df = pd.DataFrame(all_files).drop_duplicates(subset="id").reset_index(drop=True) + logging.info("Discovered %d unique viability files across %d parents", + len(df), len(parents)) + return df + + +# ---------------------------------------------------------------------------- +# Read and combine drug-screen files +# ---------------------------------------------------------------------------- +def pull_screen_files(syn, parents: list[str]) -> pd.DataFrame: + """Walk parents, read each viability CSV, attach canonical specimen. + + Files whose specimen can't be resolved are skipped with a warning. + Files attributed to a dual-mapped specimen produce duplicate rows for + every additional specimen in the mapping. + """ + files_df = discover_drug_files(syn, parents) + if files_df.empty: + raise RuntimeError("No drug-screen files found under any parent") + + frames = [] + for _, row in files_df.iterrows(): + fid, fname = row["id"], row["name"] + + canonical = resolve_specimen(syn, fid, fname) + if not canonical: + continue + + try: + df = pd.read_csv(syn.get(fid).path) + except Exception as exc: + logging.warning("Skipping file %s: %s", fid, exc) + continue + + # Verify required columns + required = {"Drug", "Concentration_uM", "Viability_percentage"} + missing = required - set(df.columns) + if missing: + logging.warning("File %s missing %s; skipping", + fname, sorted(missing)) + continue + + # Attribute to canonical + any dual-mapped specimens + for specimen in expand_dual_mappings(canonical): + attributed = df.copy() + attributed["specimen"] = specimen + attributed["source_file"] = fid + frames.append(attributed) + + if not frames: + raise RuntimeError("No drug-screen files could be read successfully") + + combined = pd.concat(frames, ignore_index=True) + n_specimens = combined["specimen"].nunique() + n_drugs = combined["Drug"].nunique() + logging.info( + "Combined %d drug-screen rows across %d specimens × %d drugs", + len(combined), n_specimens, n_drugs, + ) + return combined + + +# ---------------------------------------------------------------------------- +# I/O helpers +# ---------------------------------------------------------------------------- +def load_sample_map(samples_path: str) -> dict[str, int]: + df = pd.read_csv(samples_path) + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + return dict(zip(df["other_id"].astype(str), df["improve_sample_id"])) + + +def load_drug_map(drugs_path: str) -> dict[str, str]: + df = pd.read_csv(drugs_path, sep="\t") + if "chem_name" not in df.columns or "improve_drug_id" not in df.columns: + raise KeyError( + "drugs file missing required columns " + f"(have: {list(df.columns)})" + ) + mapping = { + str(name).strip().lower(): str(did) + for name, did in zip(df["chem_name"], df["improve_drug_id"]) + } + logging.info("Loaded %d drug name → improve_drug_id mappings", len(mapping)) + return mapping + + +def lookup_drug_id(name, drug_map: dict[str, str]) -> str | None: + if pd.isna(name): + return None + return drug_map.get(str(name).strip().lower()) + + +# ---------------------------------------------------------------------------- +# Single vs multi dose split +# ---------------------------------------------------------------------------- +def split_single_vs_multi(combined: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + """Per (specimen, drug), separate single-dose vs multi-dose.""" + counts = ( + combined.groupby(["specimen", "Drug"])["Concentration_uM"] + .nunique() + .reset_index(name="n_conc") + ) + multi_keys = counts[counts["n_conc"] > 1] + single_keys = counts[counts["n_conc"] == 1] + + multi = combined.merge(multi_keys[["specimen", "Drug"]], + on=["specimen", "Drug"], how="inner") + single = combined.merge(single_keys[["specimen", "Drug"]], + on=["specimen", "Drug"], how="inner") + # Single-dose rows must be at the canonical 1 μM concentration. + single = single[single["Concentration_uM"] == SINGLE_DOSE_UM].copy() + + logging.info("Multi-dose rows: %d (across %d sample-drug pairs)", + len(multi), len(multi_keys)) + logging.info("Single-dose rows: %d (across %d sample-drug pairs at %g μM)", + len(single), len(single_keys), SINGLE_DOSE_UM) + return multi, single + + +# ---------------------------------------------------------------------------- +# Curve fitting (multi-dose) and single-dose formatting +# ---------------------------------------------------------------------------- +def run_curve_fit(multi: pd.DataFrame, sample_map, drug_map) -> pd.DataFrame: + if multi.empty: + return pd.DataFrame() + + multi = multi.copy() + multi["improve_sample_id"] = multi["specimen"].map(sample_map) + multi["improve_drug_id"] = multi["Drug"].apply( + lambda n: lookup_drug_id(n, drug_map)) + + n_drop = (multi["improve_sample_id"].isna() | + multi["improve_drug_id"].isna()).sum() + if n_drop: + logging.warning("Dropping %d multi-dose rows with unmapped sample/drug", + n_drop) + multi = multi.dropna(subset=["improve_sample_id", "improve_drug_id"]) + if multi.empty: + return pd.DataFrame() + + # fit_curve.py expects DOSE in μM and GROWTH as viability percentage + multi["DOSE"] = multi["Concentration_uM"].astype(float) + 1e-4 + multi["GROWTH"] = multi["Viability_percentage"].astype(float) + multi["time"] = TIME + multi["time_unit"] = TIME_UNIT + multi["study"] = STUDY + multi["source"] = SOURCE + + cols = ["DOSE", "GROWTH", "study", "source", "improve_sample_id", + "Drug", "time", "time_unit"] + multi_for_fit = multi[cols].copy() + multi_for_fit["improve_sample_id"] = ( + multi_for_fit["improve_sample_id"].astype(int)) + + with tempfile.TemporaryDirectory() as tmpdir: + in_path = os.path.join(tmpdir, "drug_response.tsv") + out_prefix = os.path.join(tmpdir, "cnf_curve") + multi_for_fit.to_csv(in_path, sep="\t", index=False) + + cmd = ["python", FIT_CURVE_SCRIPT, + "--input", in_path, "--output", out_prefix] + logging.info("Running fit_curve: %s", " ".join(cmd)) + subprocess.run(cmd, check=True) + + # fit_curve.py writes .0 in the reference snippet's behavior + out_path = out_prefix + ".0" + if not os.path.exists(out_path): + alt = out_prefix + if os.path.exists(alt): + out_path = alt + else: + raise FileNotFoundError( + f"fit_curve.py did not produce output at {out_path}" + ) + fitted = pd.read_csv(out_path, sep="\t") + + # fit_curve.py renames Drug → improve_drug_id, so the column is present + # but holds the raw drug name, not the SMI_ ID. Always remap through drug_map. + if "improve_drug_id" in fitted.columns: + name_col = "improve_drug_id" + elif "Drug" in fitted.columns: + name_col = "Drug" + else: + raise KeyError( + f"fit_curve output has neither improve_drug_id nor Drug " + f"column (have: {list(fitted.columns)})" + ) + fitted["improve_drug_id"] = fitted[name_col].apply( + lambda n: lookup_drug_id(n, drug_map)) + + fitted = fitted.dropna(subset=["improve_drug_id"]) + n_metrics = (fitted["dose_response_metric"].nunique() + if "dose_response_metric" in fitted.columns else 0) + logging.info("Curve-fit produced %d rows across %d metrics", + len(fitted), n_metrics) + return fitted + + +def format_single_dose(single: pd.DataFrame, sample_map, drug_map) -> pd.DataFrame: + if single.empty: + return pd.DataFrame() + + df = single.copy() + df["improve_sample_id"] = df["specimen"].map(sample_map) + df["improve_drug_id"] = df["Drug"].apply( + lambda n: lookup_drug_id(n, drug_map)) + + n_drop = (df["improve_sample_id"].isna() | + df["improve_drug_id"].isna()).sum() + if n_drop: + logging.warning("Dropping %d single-dose rows with unmapped sample/drug", + n_drop) + df = df.dropna(subset=["improve_sample_id", "improve_drug_id"]).copy() + + df["dose_response_metric"] = SINGLE_DOSE_METRIC + df["dose_response_value"] = df["Viability_percentage"].astype(float) / 100.0 + df["time"] = TIME + df["time_unit"] = TIME_UNIT + df["study"] = STUDY + df["source"] = SOURCE + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + + cols = ["source", "improve_sample_id", "improve_drug_id", "study", + "time", "time_unit", "dose_response_metric", "dose_response_value"] + df = df[cols] + logging.info("Formatted %d single-dose rows", len(df)) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("samples", help="cnf_samples.csv from build_samples") + parser.add_argument("drugs", help="cnf_drugs.tsv from build_drugs") + parser.add_argument("--output", default=DEFAULT_OUTPUT) + parser.add_argument( + "--parents", default=",".join(DEFAULT_DRUG_SCREEN_PARENTS), + help="Comma-delimited Synapse folder IDs to walk for viability files", + ) + args = parser.parse_args() + + configure_logging() + + parents = [p.strip() for p in args.parents.split(",") if p.strip()] + if not parents: + logging.error("No parent folders provided") + sys.exit(2) + + sample_map = load_sample_map(args.samples) + drug_map = load_drug_map(args.drugs) + + syn = synapseclient.Synapse() + syn.login() + + combined = pull_screen_files(syn, parents) + multi, single = split_single_vs_multi(combined) + + fitted = run_curve_fit(multi, sample_map, drug_map) + single_df = format_single_dose(single, sample_map, drug_map) + + final = pd.concat([fitted, single_df], ignore_index=True) + cols = ["source", "improve_sample_id", "improve_drug_id", "study", + "time", "time_unit", "dose_response_metric", "dose_response_value"] + for c in cols: + if c not in final.columns: + final[c] = pd.NA + final = final[cols] + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + final.to_csv(args.output, sep="\t", index=False) + logging.info("Wrote %s (%d rows)", args.output, len(final)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/build_drugs.sh b/coderbuild/cnf/build_drugs.sh new file mode 100644 index 00000000..9402f178 --- /dev/null +++ b/coderbuild/cnf/build_drugs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# build_drugs.sh - wraps 03-drugs-cnf.py per coderdata convention. +# +# Usage: +# build_drugs.sh +# +# is a comma-delimited list of drug TSV files from +# previous datasets in the build sequence. Their improve_drug_id values +# get reused when the canonical SMILES match. + +set -euo pipefail + +PREV_DRUGS="${1:-}" + +python 03-drugs-cnf.py \ + --prev_drugs "$PREV_DRUGS" \ + --out_drugs /tmp/cnf_drugs.tsv \ + --out_desc /tmp/cnf_drug_descriptors.tsv + +python build_drug_desc.py --drugtable /tmp/cnf_drugs.tsv --desctable /tmp/cnf_drug_descriptors.tsv.gz diff --git a/coderbuild/cnf/build_exp.sh b/coderbuild/cnf/build_exp.sh new file mode 100644 index 00000000..18c8b107 --- /dev/null +++ b/coderbuild/cnf/build_exp.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# build_exp.sh - wraps 04-experiments-cnf.py per coderdata convention. +# +# Usage: +# build_exp.sh + +set -euo pipefail + +SAMPLES="${1:?Usage: build_exp.sh }" +DRUGS="${2:?Usage: build_exp.sh }" + +python 04-experiments-cnf.py "$SAMPLES" "$DRUGS" --output /tmp/cnf_experiments.tsv \ No newline at end of file From 89fe0505b3631f8c10253934bc8fd005900ba90b Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 21 Sep 2026 11:50:01 -0700 Subject: [PATCH 3/4] Add the cNF organoid dataset: samples and omics --- coderbuild/cnf/01-samples-cnf.py | 372 +++++++++++++++++++++++++++++ coderbuild/cnf/02-omics-cnf.py | 389 +++++++++++++++++++++++++++++++ coderbuild/cnf/README.md | 169 ++++++++++++++ coderbuild/cnf/build_omics.sh | 20 ++ coderbuild/cnf/build_samples.sh | 15 ++ coderbuild/cnf/cnf_utils.py | 117 ++++++++++ coderbuild/cnf/requirements.txt | 9 + coderbuild/docker/Dockerfile.cnf | 74 ++++++ 8 files changed, 1165 insertions(+) create mode 100644 coderbuild/cnf/01-samples-cnf.py create mode 100644 coderbuild/cnf/02-omics-cnf.py create mode 100644 coderbuild/cnf/README.md create mode 100644 coderbuild/cnf/build_omics.sh create mode 100644 coderbuild/cnf/build_samples.sh create mode 100644 coderbuild/cnf/cnf_utils.py create mode 100644 coderbuild/cnf/requirements.txt create mode 100644 coderbuild/docker/Dockerfile.cnf diff --git a/coderbuild/cnf/01-samples-cnf.py b/coderbuild/cnf/01-samples-cnf.py new file mode 100644 index 00000000..41bb14ea --- /dev/null +++ b/coderbuild/cnf/01-samples-cnf.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +""" +01-samples-cnf.py: generate cnf_samples.csv for the cNF organoid drug screen. + +Specimens are gathered from four sources: + + 1. Drug-screen file index on Synapse (syn51301431, dataType='drug screen') + 2. Global proteomics matrix (env CNF_GLOBAL_PROT_SYN_ID, default syn74815895) + 3. RNA discovery file (env CNF_RNA_DISCOVERY_SYN_ID, default syn71333780). + This is the corrected-abundance file containing every RNA condition + processed for the project (organoid, tumor tissue, drug-treated + organoid, skin), used here for sample discovery only — NOT to be + confused with CNF_RNA_TPM_SYN_ID, which build_omics.py reads for the + actual TPM data and may be filtered to organoids only. + 4. Normal Skin Synapse folder (env CNF_NORMAL_SKIN_SYN_ID, default syn74284682) + +Each specimen is classified as one of: + - "organoid" → model_type = "patient derived organoid" + - "treated_organoid" → model_type = "patient derived organoid" + (drug-treated organoid; canonical ID preserves the + treatment label, e.g. NF0021_T1_Onalespid_1uM) + - "tumor_tissue" → model_type = "tumor" + - "normal_skin" → model_type = "normal_tissue" + +Tissue and organoid samples for the same tumor are kept as separate rows with +distinct improve_sample_ids — they have different biology (fresh tumor vs. +cultured organoid) and the schema model_types differ. Treated and untreated +organoids of the same tumor are also separate rows because their canonical +IDs differ (treated samples preserve the treatment label). + +Cohort assignment is intentionally omitted. The upstream omics data is +batch-corrected and per-modality cohort membership disagrees for some +specimens, so a single per-sample cohort label cannot be assigned cleanly. +The `other_id_source` field carries a uniform project label. + +Sample IDs continue from the previous samples file: max(improve_sample_id) + 1. + +Usage: + python 01-samples-cnf.py [--output /tmp/cnf_samples.csv] +""" + +import argparse +import logging +import os +import re +import sys + +import pandas as pd +import synapseclient + +from cnf_utils import ( + MODEL_TYPE_MAP, + classify_specimen, + patient_from_specimen, +) + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- +DRUG_SCREEN_TABLE = "syn51301431" +GLOBAL_PROT_SYN_ID = os.environ.get("CNF_GLOBAL_PROT_SYN_ID", "syn74815895") +# RNA file used for SPECIMEN DISCOVERY only — needs to contain all +# conditions (organoid + tissue + treated + skin). The corrected-abundance +# file at syn71333780 fits this. Also used by build_omics.py as the TPM +# data source (the column is named correctedAbundance but stores TPM). +RNA_DISCOVERY_SYN_ID = os.environ.get("CNF_RNA_DISCOVERY_SYN_ID", "syn71333780") +NORMAL_SKIN_SYN_ID = os.environ.get("CNF_NORMAL_SKIN_SYN_ID", "syn74284682") + +DEFAULT_OUTPUT = "/tmp/cnf_samples.csv" + +CANCER_TYPE = "Cutaneous Neurofibroma" +SPECIES = "Homo sapiens (Human)" +OTHER_ID_SOURCE = "NF1_cNF_Project" + + +# ---------------------------------------------------------------------------- +# Helpers +# ---------------------------------------------------------------------------- +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +def get_starting_id(prev_samples_path: str) -> int: + """Read previous samples file and return max(improve_sample_id) + 1.""" + if not os.path.exists(prev_samples_path): + logging.warning("Previous samples file %s not found; starting at 1.", + prev_samples_path) + return 1 + + df = pd.read_csv(prev_samples_path) + if "improve_sample_id" not in df.columns or df.empty: + logging.warning("No improve_sample_id column or empty file; starting at 1.") + return 1 + + max_id = int(pd.to_numeric(df["improve_sample_id"], errors="coerce").max()) + logging.info("Previous samples max improve_sample_id = %d", max_id) + return max_id + 1 + + +def merge_specimen_dicts(*dicts: dict[str, str]) -> dict[str, str]: + """Union per-source dicts; warn on type disagreements (first-seen wins).""" + merged: dict[str, str] = {} + for d in dicts: + for spec, stype in d.items(): + if spec in merged and merged[spec] != stype: + logging.warning( + "Type conflict for %s: %s (kept) vs %s (ignored)", + spec, merged[spec], stype, + ) + continue + merged.setdefault(spec, stype) + return merged + + +# ---------------------------------------------------------------------------- +# Source-specific specimen extraction +# ---------------------------------------------------------------------------- +def specimens_from_drug_screen(syn) -> dict[str, str]: + """Pull canonical_id → sample_type for drug-screen specimens.""" + query = ( + "select id, individualID, specimenID " + f"from {DRUG_SCREEN_TABLE} " + "where dataType='drug screen'" + ) + logging.info("Querying %s for drug-screen specimens", DRUG_SCREEN_TABLE) + table = syn.tableQuery(query).asDataFrame() + raw = table["specimenID"].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + by_type = _count_by_type(out) + logging.info(" Drug screen: %s", by_type) + return out + + +def _count_by_type(d: dict[str, str]) -> dict[str, int]: + out: dict[str, int] = {} + for stype in d.values(): + out[stype] = out.get(stype, 0) + 1 + return out + + +def _detect_specimen_column(df: pd.DataFrame) -> str: + """Return the name of the specimen column in an omics file.""" + candidates = ["Specimen", "specimen", "specimen_norm", "sample_id", + "specimenID", "Sample"] + for c in candidates: + if c in df.columns: + return c + raise KeyError( + f"No specimen column among {candidates} in columns {list(df.columns)[:15]}" + ) + + +def _read_synapse_table(syn, syn_id: str) -> pd.DataFrame: + """Fetch a Synapse file and read it as CSV/TSV.""" + logging.info(" Fetching %s", syn_id) + entity = syn.get(syn_id) + path = entity.path + if path.endswith(".tsv") or path.endswith(".tsv.gz"): + return pd.read_csv(path, sep="\t") + return pd.read_csv(path) + + +def specimens_from_proteomics(syn) -> dict[str, str]: + """Pull canonical_id → sample_type from the global proteomics matrix.""" + if not GLOBAL_PROT_SYN_ID: + logging.warning("CNF_GLOBAL_PROT_SYN_ID not set; skipping proteomics") + return {} + + logging.info("Reading proteomics specimens") + try: + df = _read_synapse_table(syn, GLOBAL_PROT_SYN_ID) + col = _detect_specimen_column(df) + raw = df[col].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + logging.info(" Proteomics: %s", _count_by_type(out)) + return out + except Exception as exc: + logging.warning(" Proteomics read failed: %s", exc) + return {} + + +def specimens_from_rna_discovery(syn) -> dict[str, str]: + """Discover RNA specimens from a comprehensive long-format RNA file. + + Reads CNF_RNA_DISCOVERY_SYN_ID (default syn71333780) — the corrected- + abundance RNA file that contains every condition processed for the + project: organoid + tumor tissue + drug-treated organoid + skin. + + This is intentionally a different file from the one read by + build_omics.py (CNF_RNA_TPM_SYN_ID, which is the gene-level TPM matrix + and may be filtered to organoids only). Sample discovery and data + ingestion have different requirements: discovery wants the maximum + sample list; ingestion wants TPM values. + + Specimens are passed through classify_specimen, so drug-treated + samples like NF0021_T1_Onalespid.1uM get caught regardless of their + condition_norm value. + """ + if not RNA_DISCOVERY_SYN_ID: + logging.warning("CNF_RNA_DISCOVERY_SYN_ID not set; skipping RNA discovery") + return {} + + logging.info("Reading RNA discovery file (%s)", RNA_DISCOVERY_SYN_ID) + try: + df = _read_synapse_table(syn, RNA_DISCOVERY_SYN_ID) + col = _detect_specimen_column(df) + + raw = df[col].dropna().astype(str).unique().tolist() + out: dict[str, str] = {} + for r in raw: + result = classify_specimen(r) + if result: + cid, stype = result + out[cid] = stype + logging.info(" RNA discovery: %s", _count_by_type(out)) + return out + except Exception as exc: + logging.warning(" RNA discovery read failed: %s", exc) + return {} + + +def specimens_from_normal_skin_tree(syn) -> dict[str, str]: + """Discover skin specimens by listing the Normal Skin Synapse folder. + + The "Normal Skin" tree contains one immediate child per patient (folders + named NF0017, NF0018, ...) plus, in some cases, top-level skin FASTQ + files like NF0035-SKIN-1-04-16-2025_*.fastq.gz. Both forms encode the + patient ID at the start, so we match `^NF\\d{4}` against each child name + and emit one canonical NFxxxx_skin entry per unique patient. + + This is intentionally one API call deep — it avoids walking the whole + WGS tree (which has hundreds of empty lane-stub folders). For most + cases this captures every patient with a matched normal. + """ + if not NORMAL_SKIN_SYN_ID: + logging.warning("CNF_NORMAL_SKIN_SYN_ID not set; skipping Normal Skin tree") + return {} + + logging.info("Listing Normal Skin tree (%s)", NORMAL_SKIN_SYN_ID) + out: dict[str, str] = {} + try: + for child in syn.getChildren(NORMAL_SKIN_SYN_ID): + name = str(child.get("name", "")) + m = re.match(r"^(NF\d{4})", name) + if m: + patient = m.group(1) + out[f"{patient}_skin"] = "normal_skin" + logging.info(" Normal Skin tree: %s", _count_by_type(out)) + except Exception as exc: + logging.warning(" Normal Skin tree listing failed: %s", exc) + return out + + +# ---------------------------------------------------------------------------- +# Sample table construction +# ---------------------------------------------------------------------------- +def build_samples_table(specimens: dict[str, str], start_id: int) -> pd.DataFrame: + """Build the schema-compliant Sample table from {canonical_id: sample_type}. + + Sort order: organoid → treated_organoid → tumor_tissue → normal_skin + (each by ID). Drug-modelable untreated organoids get the lowest + improve_sample_ids — useful for downstream filtering. + """ + sort_priority = { + "organoid": 0, + "treated_organoid": 1, + "tumor_tissue": 2, + "normal_skin": 3, + } + sorted_specimens = sorted( + specimens.items(), + key=lambda kv: (sort_priority.get(kv[1], 99), kv[0]), + ) + + rows = [] + next_id = start_id + for specimen, stype in sorted_specimens: + patient = patient_from_specimen(specimen) + rows.append({ + "improve_sample_id": next_id, + "other_id": specimen, + "other_id_source": OTHER_ID_SOURCE, + "common_name": specimen, + "cancer_type": CANCER_TYPE, + "other_names": patient, + "species": SPECIES, + "model_type": MODEL_TYPE_MAP[stype], + }) + next_id += 1 + + df = pd.DataFrame(rows) + logging.info("Built %d sample rows; ID range %d–%d", + len(df), start_id, next_id - 1) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "prev_samples", + help="Path to previous samples CSV (used to find max improve_sample_id)", + ) + parser.add_argument("--output", default=DEFAULT_OUTPUT, + help=f"Output CSV path (default: {DEFAULT_OUTPUT})") + args = parser.parse_args() + + configure_logging() + + syn = synapseclient.Synapse() + syn.login() + + # Pull specimens from each source as canonical_id → sample_type dicts + drug_d = specimens_from_drug_screen(syn) + prot_d = specimens_from_proteomics(syn) + rna_d = specimens_from_rna_discovery(syn) + skin_d = specimens_from_normal_skin_tree(syn) + + all_specimens = merge_specimen_dicts(drug_d, prot_d, rna_d, skin_d) + logging.info( + "Union: %d total specimens — %s", + len(all_specimens), _count_by_type(all_specimens), + ) + + # Per-source uniqueness diagnostics + drug_set, prot_set, rna_set, skin_set = ( + set(drug_d), set(prot_d), set(rna_d), set(skin_d), + ) + only_drug = drug_set - prot_set - rna_set - skin_set + only_prot = prot_set - drug_set - rna_set - skin_set + only_rna = rna_set - drug_set - prot_set - skin_set + only_skin = skin_set - drug_set - prot_set - rna_set + if only_drug: + logging.info(" Only in drug screen: %s", sorted(only_drug)) + if only_prot: + logging.info(" Only in proteomics: %s", sorted(only_prot)) + if only_rna: + logging.info(" Only in RNA: %s", sorted(only_rna)) + if only_skin: + logging.info(" Only in Normal Skin tree: %s", sorted(only_skin)) + + if not all_specimens: + logging.error("No specimens found from any source; aborting") + sys.exit(1) + + start_id = get_starting_id(args.prev_samples) + samples = build_samples_table(all_specimens, start_id) + + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + samples.to_csv(args.output, index=False) + logging.info("Wrote %s (%d rows)", args.output, len(samples)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/02-omics-cnf.py b/coderbuild/cnf/02-omics-cnf.py new file mode 100644 index 00000000..bdbe0fac --- /dev/null +++ b/coderbuild/cnf/02-omics-cnf.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +""" +02-omics-cnf.py: generate cnf_transcriptomics.csv, cnf_proteomics.csv, + and cnf_phosphoproteomics.csv. + +For the cNF dataset: + +* Transcriptomics: raw gene-level TPM from per-cohort salmon.merged.gene_tpm.tsv + files (wide matrix: rows = genes, columns = specimens). Default cohorts: + syn66352931 — cNF_Cohort_1 (NF0017–NF0023) + syn70765053 — cNF_Cohort_2 (NF0025, NF0027, NF0035) + Override via env var CNF_RNA_COHORT_SYN_IDS (comma-separated Synapse IDs). + +* Proteomics: batch-corrected global proteomics from syn74815895 + (correctedAbundance column). Override via env var CNF_GLOBAL_PROT_SYN_ID. + +* Phosphoproteomics: batch-corrected phospho-proteomics from syn70078415 + (correctedAbundance column). Mapped to phosphosite_ids via a phosphosites.csv + reference file. Override via env var CNF_PHOSPHO_SYN_ID. + +* Protocol optimization samples are excluded from RNA via DROP_SAMPLE_SUBSTRINGS. + +Specimens are canonicalized via cnf_utils so that values like +"NF0018.T1.organoid" resolve to the canonical IDs in cnf_samples.csv. +Source rows whose specimen doesn't match any entry in cnf_samples.csv are +dropped silently via the inner join to sample_map. + +Usage: + python 02-omics-cnf.py + [--out_rna /tmp/cnf_transcriptomics.csv] + [--out_prot /tmp/cnf_proteomics.csv] + [--out_phospho /tmp/cnf_phosphoproteomics.csv] +""" + +import argparse +import logging +import os +import re +import sys + +import pandas as pd +import synapseclient + +from cnf_utils import canonicalize_specimen_column + + +# ---------------------------------------------------------------------------- +# Defaults / Synapse IDs +# ---------------------------------------------------------------------------- +_rna_env = os.environ.get("CNF_RNA_COHORT_SYN_IDS", "") +RNA_TPM_COHORT_SYN_IDS: list[str] = ( + [s.strip() for s in _rna_env.split(",") if s.strip()] + if _rna_env + else ["syn66352931", "syn70765053"] +) + +GLOBAL_PROT_SYN_ID = os.environ.get("CNF_GLOBAL_PROT_SYN_ID", "syn74815895") +PHOSPHO_SYN_ID = os.environ.get("CNF_PHOSPHO_SYN_ID", "syn70078415") + +# Protocol-optimization samples to drop from RNA. +DROP_SAMPLE_SUBSTRINGS: list[str] = [ + "cNF_organoid_DIA_G_02_11Feb25", + "cNF_organoid_DIA_G_05_11Feb25", + "cNF_organoid_DIA_G_06_11Feb25", + "cNF_organoid_DIA_P_02_29Jan25", + "cNF_organoid_DIA_P_05_11Feb25", + "cNF_organoid_DIA_P_06_11Feb25", +] + +STUDY = "cnf" +DEFAULT_OUT_RNA = "/tmp/cnf_transcriptomics.csv" +DEFAULT_OUT_PROT = "/tmp/cnf_proteomics.csv" +DEFAULT_OUT_PHOSPHO = "/tmp/cnf_phosphoproteomics.csv" + + +def configure_logging(): + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + ) + + +# ---------------------------------------------------------------------------- +# Reference data loaders +# ---------------------------------------------------------------------------- +def load_gene_map(genes_path: str) -> pd.DataFrame: + """Load genes.csv. Required columns: gene_symbol, entrez_id.""" + df = pd.read_csv(genes_path) + needed = {"gene_symbol", "entrez_id"} + missing = needed - set(df.columns) + if missing: + raise KeyError(f"genes file missing columns: {missing}") + df = df.dropna(subset=["entrez_id", "gene_symbol"]).copy() + df["entrez_id"] = df["entrez_id"].astype(int) + df = df.drop_duplicates(subset="gene_symbol") + logging.info("Loaded %d gene_symbol → entrez_id mappings", len(df)) + return df[["gene_symbol", "entrez_id"]] + + +def load_sample_map(samples_path: str) -> pd.DataFrame: + """Load cnf_samples.csv; return (other_id, improve_sample_id) frame.""" + df = pd.read_csv(samples_path) + df = df[["other_id", "improve_sample_id"]].drop_duplicates() + df["improve_sample_id"] = df["improve_sample_id"].astype(int) + logging.info("Loaded %d sample mappings", len(df)) + return df + + +def load_phosphosite_map(phosphosites_path: str) -> pd.DataFrame: + """Load phosphosites.csv; return (other_id, phosphosite_id) frame.""" + df = pd.read_csv(phosphosites_path) + needed = {"other_id", "phosphosite_id"} + missing = needed - set(df.columns) + if missing: + raise KeyError(f"phosphosites file missing columns: {missing}") + df = df[["other_id", "phosphosite_id"]].drop_duplicates() + df["phosphosite_id"] = df["phosphosite_id"].astype(int) + logging.info("Loaded %d phosphosite mappings", len(df)) + return df + + +def fetch_synapse_table(syn, syn_id: str) -> pd.DataFrame: + """Download a Synapse file and read it as CSV/TSV.""" + if not syn_id: + raise ValueError("Synapse ID is empty") + logging.info("Fetching %s from Synapse", syn_id) + entity = syn.get(syn_id) + path = entity.path + if path.endswith(".tsv") or path.endswith(".tsv.gz"): + return pd.read_csv(path, sep="\t") + return pd.read_csv(path) + + +# ---------------------------------------------------------------------------- +# Shared helpers +# ---------------------------------------------------------------------------- +def _col_stem(col: str) -> str: + """Return the basename without extension for a path-like column name.""" + base = re.sub(r"^.*[/\\]", "", col) + return re.sub(r"\.[^.]+$", "", base) + + +def _drop_protocol_cols(cols: list[str]) -> list[str]: + """Remove columns whose stem matches any DROP_SAMPLE_SUBSTRINGS entry.""" + kept = [] + for c in cols: + stem = _col_stem(c) + if any(sub in stem for sub in DROP_SAMPLE_SUBSTRINGS): + logging.info(" Dropping protocol optimization sample: %s", stem) + else: + kept.append(c) + return kept + + +def _normalize_col(df: pd.DataFrame, candidates: list[str], target: str) -> pd.DataFrame: + """Rename the first matching candidate column to target.""" + for c in candidates: + if c in df.columns: + return df.rename(columns={c: target}) + raise KeyError( + f"None of {candidates} found in columns {list(df.columns)[:20]}" + ) + + +# ---------------------------------------------------------------------------- +# Transcriptomics +# ---------------------------------------------------------------------------- +def fetch_wide_tpm_cohort(syn, syn_id: str) -> pd.DataFrame: + """Fetch a wide salmon.merged.gene_tpm.tsv and return long-format DataFrame. + + nf-core salmon output: gene_id (Ensembl), gene_name (symbol), then one + column per specimen. Returns (gene_symbol, specimen, transcriptomics, source). + """ + logging.info("Fetching cohort TPM matrix %s", syn_id) + entity = syn.get(syn_id) + path = entity.path + df = pd.read_csv(path, sep="\t") + + gene_col = None + for cname in ["gene_name", "gene_symbol", "Symbol", "GeneSymbol", "gene"]: + if cname in df.columns: + gene_col = cname + break + if gene_col is None: + gene_col = df.columns[0] + logging.warning( + "%s: no gene_name column found; using %s — most rows may fail the gene_map join", + syn_id, gene_col, + ) + + meta_cols = {"gene_id", "gene_name", "gene_symbol", "Symbol", + "GeneSymbol", "gene", "transcript_id"} + sample_cols = [c for c in df.columns if c not in meta_cols] + sample_cols = _drop_protocol_cols(sample_cols) + + if not sample_cols: + raise ValueError( + f"{syn_id}: no specimen columns found after excluding metadata " + f"and protocol optimization samples" + ) + + long = ( + df[[gene_col] + sample_cols] + .rename(columns={gene_col: "gene_symbol"}) + .melt(id_vars="gene_symbol", var_name="specimen", value_name="transcriptomics") + ) + long["source"] = "Synapse" + logging.info( + " %s: %d genes × %d specimens → %d long rows", + syn_id, df[gene_col].nunique(), len(sample_cols), len(long), + ) + return long + + +def build_transcriptomics( + syn, gene_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format transcriptomics table from per-cohort TPM matrices.""" + if not RNA_TPM_COHORT_SYN_IDS: + logging.error("RNA_TPM_COHORT_SYN_IDS is empty; skipping transcriptomics.") + return pd.DataFrame( + columns=["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"] + ) + + cohort_frames = [] + for syn_id in RNA_TPM_COHORT_SYN_IDS: + try: + cohort_frames.append(fetch_wide_tpm_cohort(syn, syn_id)) + except Exception as exc: + logging.warning("Skipping RNA cohort %s: %s", syn_id, exc) + + if not cohort_frames: + logging.error("All RNA cohort fetches failed; returning empty transcriptomics.") + return pd.DataFrame( + columns=["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"] + ) + + raw = pd.concat(cohort_frames, ignore_index=True) + logging.info("Combined %d RNA cohorts → %d total rows", len(cohort_frames), len(raw)) + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Transcriptomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + df = ( + raw.merge(gene_map, on="gene_symbol", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["study"] = STUDY + df["transcriptomics"] = pd.to_numeric(df["transcriptomics"], errors="coerce") + df = df.dropna(subset=["transcriptomics"]) + df = df[["entrez_id", "improve_sample_id", "source", "study", "transcriptomics"]] + + logging.info( + "Built transcriptomics table: %d rows (%d samples × %d genes)", + len(df), df["improve_sample_id"].nunique(), df["entrez_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Proteomics +# ---------------------------------------------------------------------------- +def build_proteomics( + syn, gene_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format proteomics table from the batch-corrected Synapse file.""" + raw = fetch_synapse_table(syn, GLOBAL_PROT_SYN_ID) + raw = _normalize_col(raw, + ["Specimen", "specimenID", "specimen_id", "specimen", "specimen_norm", "sample_id"], + "specimen") + raw = _normalize_col(raw, + ["gene_symbol", "Gene", "GeneSymbol", "Symbol", "gene", "feature_id"], + "gene_symbol") + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Proteomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + raw = _normalize_col(raw, + ["correctedAbundance", "log_ratio", "logRatio", "proteomics", "value"], + "proteomics") + + df = ( + raw.merge(gene_map, on="gene_symbol", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["source"] = "Synapse" + df["study"] = STUDY + df["proteomics"] = pd.to_numeric(df["proteomics"], errors="coerce") + df = df.dropna(subset=["proteomics"]) + df = df[["entrez_id", "improve_sample_id", "source", "study", "proteomics"]] + + logging.info( + "Built proteomics table: %d rows (%d samples × %d genes)", + len(df), df["improve_sample_id"].nunique(), df["entrez_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Phosphoproteomics +# ---------------------------------------------------------------------------- +def build_phosphoproteomics( + syn, phosphosite_map: pd.DataFrame, sample_map: pd.DataFrame +) -> pd.DataFrame: + """Build the long-format phosphoproteomics table from the batch-corrected Synapse file.""" + raw = fetch_synapse_table(syn, PHOSPHO_SYN_ID) + raw = _normalize_col(raw, + ["Specimen", "specimenID", "specimen_id", "specimen"], + "specimen") + raw = _normalize_col(raw, + ["site", "Site", "phosphosite", "feature_id"], + "site") + + n_before = len(raw) + raw = canonicalize_specimen_column(raw, "specimen") + logging.info( + "Phosphoproteomics: canonicalized %d → %d rows (%d dropped as unrecognized specimens)", + n_before, len(raw), n_before - len(raw), + ) + + raw = _normalize_col(raw, + ["correctedAbundance", "log_ratio", "logRatio", "phosphoproteomics", "value"], + "phosphoproteomics") + + df = ( + raw.merge(phosphosite_map, left_on="site", right_on="other_id", how="inner") + .merge(sample_map, left_on="specimen", right_on="other_id", how="inner") + ) + df["source"] = "Synapse" + df["study"] = STUDY + df["phosphoproteomics"] = pd.to_numeric(df["phosphoproteomics"], errors="coerce") + df = df.dropna(subset=["phosphoproteomics"]) + df = df[["phosphosite_id", "improve_sample_id", "source", "study", "phosphoproteomics"]] + + logging.info( + "Built phosphoproteomics table: %d rows (%d samples × %d sites)", + len(df), df["improve_sample_id"].nunique(), df["phosphosite_id"].nunique(), + ) + return df + + +# ---------------------------------------------------------------------------- +# Main +# ---------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("genes", help="Path to genes.csv (gene_symbol → entrez_id)") + parser.add_argument("samples", help="Path to cnf_samples.csv from build_samples") + parser.add_argument("phosphosites", help="Path to phosphosites.csv (other_id → phosphosite_id)") + parser.add_argument("--out_rna", default=DEFAULT_OUT_RNA) + parser.add_argument("--out_prot", default=DEFAULT_OUT_PROT) + parser.add_argument("--out_phospho", default=DEFAULT_OUT_PHOSPHO) + args = parser.parse_args() + + configure_logging() + + gene_map = load_gene_map(args.genes) + sample_map = load_sample_map(args.samples) + phosphosite_map = load_phosphosite_map(args.phosphosites) + + syn = synapseclient.Synapse() + syn.login() + + rna = build_transcriptomics(syn, gene_map, sample_map) + prot = build_proteomics(syn, gene_map, sample_map) + phospho = build_phosphoproteomics(syn, phosphosite_map, sample_map) + + for path in (args.out_rna, args.out_prot, args.out_phospho): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + + rna.to_csv(args.out_rna, index=False) + prot.to_csv(args.out_prot, index=False) + phospho.to_csv(args.out_phospho, index=False) + logging.info("Wrote %s (%d rows)", args.out_rna, len(rna)) + logging.info("Wrote %s (%d rows)", args.out_prot, len(prot)) + logging.info("Wrote %s (%d rows)", args.out_phospho, len(phospho)) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/coderbuild/cnf/README.md b/coderbuild/cnf/README.md new file mode 100644 index 00000000..0755a952 --- /dev/null +++ b/coderbuild/cnf/README.md @@ -0,0 +1,169 @@ +# Cutaneous Neurofibroma (cnf) Organoid Drug Screen + +A coderdata dataset of cutaneous neurofibroma (cNF) patient-derived organoids +with drug-response measurements and matched multi-omic profiling. + +## Overview + +Cutaneous neurofibromas are benign nerve-sheath tumors that develop in patients +with neurofibromatosis type 1 (NF1). This dataset contains drug-screen +viability measurements for 238 small-molecule compounds across 23 cNF tumor +specimens from 10 patients, paired with bulk RNA-seq, global LC-MS/MS +proteomics, and batch-corrected phosphoproteomics for many of the same +specimens. + +| Property | Value | +|---|---| +| Cancer type | Cutaneous Neurofibroma | +| Model type | patient derived organoid | +| Patients | 10 | +| Specimens | 23 (untreated organoids; additional tissue and treated organoid samples also present) | +| Drugs | 238 | +| Modalities included | Transcriptomics (TPM), Proteomics (log ratio), Phosphoproteomics (log ratio) | + +## Multi-tumor patient structure + +Each patient (`NF0017`, `NF0018`, ...) contributed up to three independent +tumor specimens, named `NFxxxx_T1`, `NFxxxx_T2`, `NFxxxx_T3`. This dataset +treats each specimen as an independent sample (its own `improve_sample_id`). +The patient ID is preserved in the `other_names` column so downstream models +can group specimens by patient when needed (e.g. patient-level holdout). + +``` +improve_sample_id other_id common_name other_names other_id_source +2401 NF0019_T1 NF0019_T1 NF0019 NF1_cNF_Project +2402 NF0019_T2 NF0019_T2 NF0019 NF1_cNF_Project +2403 NF0019_T3 NF0019_T3 NF0019 NF1_cNF_Project +``` + +The sample table also includes matched normal skin (`NFxxxx_skin`), primary +tumor tissue (`NFxxxx_Tx_tissue`), and drug-treated organoids +(`NFxxxx_Tx_`). Each gets its own `improve_sample_id` and +`model_type`. Organoids (untreated) receive the lowest IDs, sorted by +specimen ID, to make drug-modelable samples easy to filter. + + +## Drug-response metrics + +The cNF screen mixes two designs: + +- **Multi-dose drugs**: a subset of drugs were screened at multiple + concentrations. These are run through `coderbuild/utils/fit_curve.py` and + contribute the standard schema metrics (`fit_auc`, `fit_ic50`, `fit_einf`, + `fit_hs`, `fit_r2`). + +- **Single-dose drugs**: remaining drugs were screened at a single 1 μM + concentration. These are recorded with `dose_response_metric = uM_viability` + and `dose_response_value = Viability_percentage / 100` (range 0–1, where + 1.0 = full viability and 0.0 = full kill). + +`uM_viability` is a value in the `ResponseMetric` enum. See the Schema +dependency section below. + +## SPECIMEN_DUAL_MAPPINGS edge case + +One viability file covers the treated organoid `NF0021_T1_Onalespid_1uM` but +its drug-response measurements should also be attributed to the untreated +`NF0021_T1`. The `SPECIMEN_DUAL_MAPPINGS` dict in `04-experiments-cnf.py` +handles this by duplicating the rows for every additional specimen listed: + +```python +SPECIMEN_DUAL_MAPPINGS = { + "NF0021_T1_Onalespid_1uM": ["NF0021_T1"], +} +``` + +Add new entries here if additional treated organoids need the same dual +attribution. Both specimens must already exist in `cnf_samples.csv` or the +duplicate rows will be dropped during the sample-map join. + +## Protocol optimization samples + +Six RNA columns in the cohort TPM matrices correspond to protocol optimization +runs rather than real specimens. They are excluded in `02-omics-cnf.py` before +the wide-to-long melt. Column stems matching any of these substrings are +dropped: + +```python +DROP_SAMPLE_SUBSTRINGS = [ + "cNF_organoid_DIA_G_02_11Feb25", + "cNF_organoid_DIA_G_05_11Feb25", + "cNF_organoid_DIA_G_06_11Feb25", + "cNF_organoid_DIA_P_02_29Jan25", + "cNF_organoid_DIA_P_05_11Feb25", + "cNF_organoid_DIA_P_06_11Feb25", +] +``` + +These are matched on the column stem (basename without extension), so both +plain paths and filename columns are handled correctly. If new protocol +optimization runs appear in future cohort data, add their column stems to +this list. + +## Phosphosites dependency and known unmatched sites + +`02-omics-cnf.py` requires `phosphosites.csv` as a positional argument. +The phosphosites reference must be built before the cNF omics step runs. +`build_dataset.py` and `build_all.py` enforce this sequencing. + +The phosphoproteomics output file (`cnf_phosphoproteomics.csv`) is produced +by an inner join between the raw Synapse phospho data and `phosphosites.csv`. +Sites that don't appear in the reference are silently dropped. As of the +current release, **12 sites** in the raw cNF phospho data (syn70078415) are +permanently unmatched because their gene symbols (`BAP18`, `C11orf96`, +`C14orf93`, `C1orf21`, `C2orf49`, etc). The remaining 2,974 of 2,986 unique sites +(99.6%) are present in the output. + +## Data sources + +| Modality | Synapse ID | Notes | +|---|---|---| +| Drug-screen file index | `syn51301431` (table) | `dataType='drug screen'` | +| Drug-screen file parents | `syn51301414`, `syn51301420`, `syn51301426` | One folder per cohort | +| RNA-seq TPM (Cohort 1) | `syn66352931` | `salmon.merged.gene_tpm.tsv` | +| RNA-seq TPM (Cohort 2) | `syn70765053` | `salmon.merged.gene_tpm.tsv` | +| Global proteomics | `syn74815895` | Batch-corrected; `correctedAbundance` column | +| Phospho-proteomics | `syn70078415` | Batch-corrected; `correctedAbundance` column; mapped via `phosphosites.csv` | +| Sample discovery (RNA) | `syn71333780` | All-conditions RNA file for sample enumeration only | +| Normal Skin folder | `syn74284682` | One child per patient for skin sample discovery | + +## Build pipeline + +The build follows coderdata conventions: four shell scripts wrapping Python +implementations, plus shared utilities, packaged in a Docker image. + +``` +coderbuild/cnf/ +├── 01-samples-cnf.py # mint improve_sample_ids, write cnf_samples.csv +├── build_samples.sh +├── 02-omics-cnf.py # write cnf_transcriptomics.csv, cnf_proteomics.csv, +├── build_omics.sh # cnf_phosphoproteomics.csv +├── 03-drugs-cnf.py # call pubchem_retrieval + descriptor utility +├── build_drugs.sh +├── 04-experiments-cnf.py # split single/multi-dose, run fit_curve, write cnf_experiments.tsv +├── build_exp.sh +├── cnf_utils.py # shared specimen canonicalization utilities +├── requirements.txt +└── README.md (this file) + +coderbuild/docker/ +└── Dockerfile.cnf +``` + +### Local build + +```bash +# From repository root +python coderbuild/build_dataset.py --build --validate --dataset cnf +``` + +In the full coderdata build, this dataset is orchestrated by `build_dataset.py` +/ `build_all.py`, which ensures `genes.csv`, `samples.csv`, `drugs.tsv`, and +`phosphosites.csv` are all present before starting the cNF containers. + + +## What's intentionally not included + +- **Mutations** - not yet generated for this dataset. +- **Copy number** - not yet generated for this dataset. + diff --git a/coderbuild/cnf/build_omics.sh b/coderbuild/cnf/build_omics.sh new file mode 100644 index 00000000..7db0d062 --- /dev/null +++ b/coderbuild/cnf/build_omics.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# build_omics.sh - wraps 02-omics-cnf.py per coderdata convention. +# +# Usage: +# build_omics.sh +# +# - : genes.csv with gene_symbol → entrez_id mappings +# - : cnf_samples.csv produced by build_samples.sh +# - : phosphosites.csv (other_id → phosphosite_id) + +set -euo pipefail + +GENES="${1:?Usage: build_omics.sh }" +SAMPLES="${2:?Usage: build_omics.sh }" +PHOSPHOSITES="${3:?Usage: build_omics.sh }" + +python 02-omics-cnf.py "$GENES" "$SAMPLES" "$PHOSPHOSITES" \ + --out_rna /tmp/cnf_transcriptomics.csv \ + --out_prot /tmp/cnf_proteomics.csv \ + --out_phospho /tmp/cnf_phosphoproteomics.csv \ No newline at end of file diff --git a/coderbuild/cnf/build_samples.sh b/coderbuild/cnf/build_samples.sh new file mode 100644 index 00000000..37af2162 --- /dev/null +++ b/coderbuild/cnf/build_samples.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# build_samples.sh - wraps 01-samples-cnf.py per coderdata convention. +# +# Usage: +# build_samples.sh +# +# The previous samples file is the latest samples CSV from the coderdata +# build (with the highest improve_sample_id used so far). 01-samples-cnf.py +# reads it, finds the max ID, and increments from there. + +set -euo pipefail + +PREV_SAMPLES="${1:?Usage: build_samples.sh }" + +python 01-samples-cnf.py "$PREV_SAMPLES" --output /tmp/cnf_samples.csv \ No newline at end of file diff --git a/coderbuild/cnf/cnf_utils.py b/coderbuild/cnf/cnf_utils.py new file mode 100644 index 00000000..10ec38d2 --- /dev/null +++ b/coderbuild/cnf/cnf_utils.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +cnf_utils.py: shared specimen classification utilities for the cNF build scripts. + +Used by 01-samples-cnf.py, 02-omics-cnf.py, and 04-experiments-cnf.py to +canonicalize specimen strings so omics data, drug measurements, and the sample +table all use the same improve_sample_id keys. +""" + +import re + + +# ---------------------------------------------------------------------------- +# Sample type → coderdata model_type enum +# ---------------------------------------------------------------------------- +MODEL_TYPE_MAP = { + "organoid": "patient derived organoid", + "treated_organoid": "patient derived organoid", + "tumor_tissue": "tumor", + "normal_skin": "normal_tissue", +} + + + + +# ---------------------------------------------------------------------------- +# Regex constants +# ---------------------------------------------------------------------------- +# Bare tumor organoid: NFxxxx_Tx (4 patient digits, T + tumor digits) +TUMOR_RE = re.compile(r"^NF\d{4}_T\d+$") +# Patient prefix at the start of any specimen string +PATIENT_PREFIX_RE = re.compile(r"^(NF\d{4})") +# Normal skin marker: NFxxxx[_-]skin..., allowing trailing date/replicate junk +SKIN_RE = re.compile(r"^(NF\d{4})[_-]skin", re.IGNORECASE) +# Primary tumor tissue (not organoid culture): NFxxxx_Tx_tissue +TISSUE_RE = re.compile(r"^(NF\d{4}_T\d+)_tissue$", re.IGNORECASE) +# Treated organoid: NFxxxx_Tx_, e.g. NF0021_T1_Onalespid_1uM +TREATED_RE = re.compile(r"^NF\d{4}_T\d+_.+$") + + +# ---------------------------------------------------------------------------- +# Classification +# ---------------------------------------------------------------------------- +def classify_specimen(spec) -> tuple[str, str] | None: + """Classify a raw specimen string into (canonical_id, sample_type) or None. + + sample_type ∈ {"organoid", "treated_organoid", "tumor_tissue", "normal_skin"}. + + Canonical IDs: + organoid → NFxxxx_Tx (suffix stripped) + treated_organoid → NFxxxx_Tx_ (label preserved so + treated samples don't collide + with the untreated organoid) + tumor_tissue → NFxxxx_Tx_tissue (suffix preserved) + normal_skin → NFxxxx_skin (replicates collapsed per patient) + + Unrecognized prefixes and malformed inputs return None. + """ + if spec is None: + return None + s = str(spec).strip() + if not s or s.lower() == "nan": + return None + + # Convert dot-separated form (e.g. RNA file's "NF0017.T1.organoid") to + # underscore-separated. + s_norm = s.replace(".", "_") + + # Skin: any string starting with NFxxxx that contains a "skin" marker + skin_m = SKIN_RE.match(s_norm) + if skin_m: + return (f"{skin_m.group(1)}_skin", "normal_skin") + + # Tumor tissue: NFxxxx_Tx_tissue + tissue_m = TISSUE_RE.match(s_norm) + if tissue_m: + return (f"{tissue_m.group(1)}_tissue", "tumor_tissue") + + # Strip organoid suffix (handles "_organoid" and "_organoids") + s_stripped = re.sub(r"_organoids?$", "", s_norm, flags=re.IGNORECASE) + + # Bare organoid (untreated): NFxxxx_Tx exactly + if TUMOR_RE.match(s_stripped): + return (s_stripped, "organoid") + + # Treated organoid: NFxxxx_Tx_ + if TREATED_RE.match(s_stripped): + return (s_stripped, "treated_organoid") + + return None + + +def patient_from_specimen(specimen: str) -> str: + """Extract NFxxxx patient ID from a canonical specimen ID.""" + m = PATIENT_PREFIX_RE.match(specimen) + return m.group(1) if m else "" + + +def canonicalize_specimen_column(df, specimen_col: str): + """Apply classify_specimen to a dataframe's specimen column in place. + + Returns the same df with: + - rows whose specimen classifies dropped if None + - specimen_col values replaced with their canonical IDs + - a new column 'sample_type' added + + Used by build_omics.py and build_exp.py to map raw Synapse specimen + strings (e.g. "NF0018.T1.organoid", "NF0021.T1.Onalespid.1uM") to + canonical IDs that match the cnf_samples.csv `other_id` column. + """ + classifications = df[specimen_col].apply(classify_specimen) + keep = classifications.notna() + df = df[keep].copy() + classifications = classifications[keep] + df[specimen_col] = [c[0] for c in classifications] + df["sample_type"] = [c[1] for c in classifications] + return df diff --git a/coderbuild/cnf/requirements.txt b/coderbuild/cnf/requirements.txt new file mode 100644 index 00000000..a5677837 --- /dev/null +++ b/coderbuild/cnf/requirements.txt @@ -0,0 +1,9 @@ +pandas>=2.0 +numpy>=1.24,<2.0 +synapseclient>=4.0 +requests>=2.28 +scipy>=1.10 +matplotlib +scikit-learn +scipy +mordred \ No newline at end of file diff --git a/coderbuild/docker/Dockerfile.cnf b/coderbuild/docker/Dockerfile.cnf new file mode 100644 index 00000000..f4a99784 --- /dev/null +++ b/coderbuild/docker/Dockerfile.cnf @@ -0,0 +1,74 @@ +# Dockerfile.cnf - container for building the cNF (cutaneous neurofibroma) +# organoid drug-response dataset for inclusion in coderdata. +# +# Conventions (matching coderbuild/docker/Dockerfile.): +# * Working directory is /coderbuild/cnf +# * Standard utilities are mounted/copied to /coderbuild/utils (fit_curve.py, +# pubchem_retrieval.py, build_drug_desc.py) +# * Synapse auth via SYNAPSE_AUTH_TOKEN env var (passed at `docker run`) +# * Build scripts are invoked from build_dataset.py / build_all.py via the +# standard build_*.sh entry points +# +# Build: +# docker build -f coderbuild/docker/Dockerfile.cnf -t coderdata-cnf . +# +# Run (orchestrated by build_dataset.py / build_all.py): +# docker run --rm \ +# -e SYNAPSE_AUTH_TOKEN=$SYNAPSE_AUTH_TOKEN \ +# -e CNF_RNA_TPM_SYN_ID=$CNF_RNA_TPM_SYN_ID \ +# -v $(pwd)/local_data:/coderbuild/local_data \ +# coderdata-cnf bash build_samples.sh /coderbuild/local_data/prev_samples.csv + +FROM python:3.11-slim + +# Keep scratch files OFF the /tmp bind mount. +# +# build_all.py bind-mounts the host's local/ directory at /tmp in every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", which opens a Unix domain socket under +# tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS file +# sharing does not support chmod() on a socket there, so every Pool worker +# fails with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-XXXX/sock-YYYY' +# +# This broke build_drug_desc.py and fit_curve.py on 2026-09-03, each at the end +# of a multi-hour step, and it is deterministic so retries do not help. +# +# TMPDIR only affects tempfile.gettempdir() (Python) and tempdir() (R). Explicit +# /tmp/ paths -- which is how every build OUTPUT is written -- still land +# on the bind mount as before. Verified. +RUN mkdir -p /opt/mp_tmp +ENV TMPDIR=/opt/mp_tmp + + +# System deps (rdkit-pypi pulls in some C libs; basic build tools for scipy) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential wget ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Set up coderbuild layout +WORKDIR /coderbuild/cnf + +# Install Python deps. rdkit is brought in via the descriptor utility but +# pinned here so the descriptor step is reproducible. +COPY coderbuild/cnf/requirements.txt /coderbuild/cnf/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt rdkit-pypi + +# Copy the cNF build scripts +COPY coderbuild/cnf/01-samples-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/02-omics-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/03-drugs-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/04-experiments-cnf.py /coderbuild/cnf/ +COPY coderbuild/cnf/build_samples.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_omics.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_drugs.sh /coderbuild/cnf/ +COPY coderbuild/cnf/build_exp.sh /coderbuild/cnf/ +COPY coderbuild/cnf/cnf_utils.py /coderbuild/cnf/ +COPY coderbuild/utils/build_drug_desc.py /coderbuild/cnf/ + +RUN chmod +x /coderbuild/cnf/*.sh + +# Standard coderdata utilities (fit_curve.py, pubchem_retrieval.py, +COPY coderbuild/utils /coderbuild/utils + From 35ac9815d130421279455aa1d40fe132dc047e12 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Thu, 1 Oct 2026 09:47:39 -0700 Subject: [PATCH 4/4] Add cnf service to docker-compose.yml --- coderbuild/docker/docker-compose.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/coderbuild/docker/docker-compose.yml b/coderbuild/docker/docker-compose.yml index 4cd128af..0a87e0e5 100644 --- a/coderbuild/docker/docker-compose.yml +++ b/coderbuild/docker/docker-compose.yml @@ -139,4 +139,13 @@ services: args: HTTPS_PROXY: ${HTTPS_PROXY} platform: linux/amd64 - image: novartis:latest \ No newline at end of file + image: novartis:latest + + cnf: + build: + context: ../../ + dockerfile: coderbuild/docker/Dockerfile.cnf + args: + HTTPS_PROXY: ${HTTPS_PROXY} + platform: linux/amd64 + image: cnf:latest \ No newline at end of file