From 9ba03f867efa5cd6f1af55f09f547af9d860b80a Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 30 Sep 2026 16:37:33 -0700 Subject: [PATCH 1/4] Harmonize copy_number onto the DepMap log2(relative CN + 1) scale Each builder converted raw copy number differently, so copy_number was stored on different scales while copy_call applied one set of cutoffs (0.521 / 0.731 / 1.214 / 1.422, on DepMap's log2(relative CN + 1) scale) to all of them. Convert every source to that scale so the stored values are comparable and the calls are correct (0/1/2/3/4 copies -> deep del / het loss / diploid / gain / amp). - Sanger (gdscv1/v2), mpnst, cptac, pancreatic, liver: log2 ratio s -> log2(2^s + 1) - bladder, colorectal: segment log2 ratio -> log2(2^s + 1) (was 2*2^(s/2)) - novartis: absolute copies -> log2(c/2 + 1) - hcmi: absolute ASCAT copies -> log2(c/ploidy + 1), using per-sample tumor_ploidy from the GDC ASCAT segment files (2 if unavailable) - pancreatic: replace its own cutoffs with the shared ones - cptac: fix the 1.42233 gain cutoff typo - coderdata: copy_call bins use -inf/inf so values outside [0, 2] are labelled instead of becoming NaN --- .../bladder/01_createBladderOmicsFiles.py | 6 +- coderbuild/broad_sanger/02-broadSangerOmics.R | 3 +- coderbuild/colorectal/02-omics-colorectal.py | 6 +- coderbuild/cptac/getCptacData.py | 16 ++--- coderbuild/hcmi/02-getHCMIData.py | 63 ++++++++++++++++++- coderbuild/liver/02-omics-liver.py | 6 +- coderbuild/mpnst/01_combined_omics.R | 3 +- coderbuild/novartis/02-omics-novartis.py | 7 ++- .../02a-getPancreaticDataFromSynapse.py | 13 ++-- coderdata/dataset/dataset.py | 2 +- 10 files changed, 100 insertions(+), 25 deletions(-) diff --git a/coderbuild/bladder/01_createBladderOmicsFiles.py b/coderbuild/bladder/01_createBladderOmicsFiles.py index 3047c2b2..b5833113 100644 --- a/coderbuild/bladder/01_createBladderOmicsFiles.py +++ b/coderbuild/bladder/01_createBladderOmicsFiles.py @@ -20,7 +20,8 @@ def get_copy_call(a): if math.isnan(a): return float('nan') - a_val = math.log2(float(a)+0.000001) + # a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0 + a_val = float(a) if a_val < 0.5210507: return 'deep del' elif a_val < 0.7311832: @@ -118,7 +119,8 @@ def get_bladder_copynumber(synObject, samples, genes): segfile_df.to_csv("bladder_segfile.csv") subprocess.call (["/usr/bin/Rscript", "--vanilla", "CNV-segfile-annotation.R", "bladder_segfile.csv", "bladder_annotated_segfile.csv"]) copynumber = pd.read_csv("bladder_annotated_segfile.csv") - copynumber['copy_number'] = np.exp2(copynumber['score'].div(2))*2 + # score is the segment log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0 + copynumber['copy_number'] = np.log2(np.exp2(copynumber['score']) + 1) copynumber['copy_call'] = [get_copy_call(a) for a in copynumber['copy_number']] copynumber_with_improveids = copynumber.merge(samples, left_on='ID', right_on = 'other_id', how='left') copynumber_with_correct_colnames = copynumber_with_improveids.rename({"ENTREZID":'entrez_id'}, axis=1) diff --git a/coderbuild/broad_sanger/02-broadSangerOmics.R b/coderbuild/broad_sanger/02-broadSangerOmics.R index bfd5780d..10f248e7 100755 --- a/coderbuild/broad_sanger/02-broadSangerOmics.R +++ b/coderbuild/broad_sanger/02-broadSangerOmics.R @@ -167,7 +167,8 @@ sanger_files<-function(fi,value){ res<-exp_file|> dplyr::select(other_id='model_id',gene_symbol='symbol',gatk_mean_log2_copy_ratio,source,data_type,cn_category)|> - mutate(copy_number=2^gatk_mean_log2_copy_ratio,.keep='all')|> + ## log2 ratio -> DepMap scale log2(relative CN + 1); normal = 1.0 + mutate(copy_number=log2(2^gatk_mean_log2_copy_ratio+1),.keep='all')|> distinct()|> left_join(gmap)|> dplyr::select(other_id,copy_number,entrez_id,Sanger='cn_category')|> diff --git a/coderbuild/colorectal/02-omics-colorectal.py b/coderbuild/colorectal/02-omics-colorectal.py index 8a374995..9639d131 100644 --- a/coderbuild/colorectal/02-omics-colorectal.py +++ b/coderbuild/colorectal/02-omics-colorectal.py @@ -173,7 +173,8 @@ def get_copy_call(a): if math.isnan(a): return float('nan') - a_val = math.log2(float(a)+0.000001) + # a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0 + a_val = float(a) if a_val < 0.5210507: return 'deep del' elif a_val < 0.7311832: @@ -216,7 +217,8 @@ def map_copy_number(copy_number_data, improve_id_data, entrez_data): mapped_cn_df = pd.read_csv("output_copy_num.csv") # do copy_number calculation from score and get copy call column - mapped_cn_df['copy_number'] = np.exp2(mapped_cn_df['score'].div(2))*2 + # score is the segment log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0 + mapped_cn_df['copy_number'] = np.log2(np.exp2(mapped_cn_df['score']) + 1) mapped_cn_df['copy_call'] = [get_copy_call(a) for a in mapped_cn_df['copy_number']] # map ID to improve_ID diff --git a/coderbuild/cptac/getCptacData.py b/coderbuild/cptac/getCptacData.py index 4b48d677..5100b399 100755 --- a/coderbuild/cptac/getCptacData.py +++ b/coderbuild/cptac/getCptacData.py @@ -198,8 +198,10 @@ def formatData(df,dtype,ctype,samp_names,source,genes,samples): mlongdf = mlongdf[['entrez_id','improve_sample_id',dtype]].drop_duplicates() - ##if its copy number we need to include copy number call + ##if its copy number, convert the log2 ratio to the DepMap scale log2(relative CN + 1) + ##(normal = 1.0) and add the copy number call if dtype=='copy_number': ##deep del < 0.5210507 < het loss < 0.7311832 < diploid < 1.214125 < gain < 1.422233 < amp + mlongdf['copy_number'] = np.log2(np.power(2.0, mlongdf['copy_number'].astype(float)) + 1) mlongdf[['copy_call']] = mlongdf[['copy_number']].apply(copy_num) @@ -211,18 +213,18 @@ def formatData(df,dtype,ctype,samp_names,source,genes,samples): def copy_num(arr): ''' - Converts copy number values to categorical calls. + Converts copy number values (DepMap scale, log2(relative CN + 1)) to categorical calls. ''' copy_call=[] for a in arr: - a = 2**float(a) - if float(a) < 0.5210507: + a = float(a) + if a < 0.5210507: b = 'deep del' - elif float(a) < 0.7311832: + elif a < 0.7311832: b = 'het loss' - elif float(a) < 1.214125: + elif a < 1.214125: b = 'diploid' - elif float(a) <1.42233: + elif a < 1.422233: b = 'gain' else: b = 'amp' diff --git a/coderbuild/hcmi/02-getHCMIData.py b/coderbuild/hcmi/02-getHCMIData.py index 48ffefc7..39c6427f 100644 --- a/coderbuild/hcmi/02-getHCMIData.py +++ b/coderbuild/hcmi/02-getHCMIData.py @@ -194,6 +194,46 @@ def fetch_metadata(uuids): return response.json() +def fetch_ascat_ploidy(aliquot_ids, batch=200): + """ + Look up ASCAT tumor ploidy for the given tumor aliquots. + + GDC stores ``tumor_ploidy`` on the ASCAT "Allele-specific Copy Number Segment" + files (not on the gene-level files we download), so query those by aliquot. + + Returns + ------- + dict + tumor aliquot_id -> tumor_ploidy (float). Aliquots with no ploidy are omitted. + """ + ploidy = {} + ids = [a for a in dict.fromkeys(aliquot_ids) if a] + for i in range(0, len(ids), batch): + chunk = ids[i:i + batch] + payload = { + "filters": {"op": "and", "content": [ + {"op": "in", "content": {"field": "data_type", "value": ["Allele-specific Copy Number Segment"]}}, + {"op": "in", "content": {"field": "cases.samples.portions.analytes.aliquots.aliquot_id", "value": chunk}}]}, + "fields": "tumor_ploidy,cases.samples.tissue_type,cases.samples.portions.analytes.aliquots.aliquot_id", + "format": "JSON", + "size": str(len(chunk) * 4) + } + r = requests.post("https://api.gdc.cancer.gov/files", json=payload, timeout=120) + r.raise_for_status() + for h in r.json()["data"]["hits"]: + if h.get("tumor_ploidy") is None: + continue + for c in h.get("cases", []): + for smp in c.get("samples", []): + if smp.get("tissue_type") != "Tumor": + continue + for por in smp.get("portions", []): + for ana in por.get("analytes", []): + for al in ana.get("aliquots", []): + ploidy[al["aliquot_id"]] = float(h["tumor_ploidy"]) + return ploidy + + def drop_controlled_access(newfm, batch=400): """Remove dbGaP controlled-access files from a manifest slice. @@ -645,6 +685,15 @@ def map_and_combine_stream( for h in md_hits], }) + # per-file tumor ploidy, only needed for copy number + ploidy_df = None + if data_type == "copy_number": + ploidy_map = fetch_ascat_ploidy(meta_df["aliquot_id"].to_list()) + ploidy_df = meta_df.select(["file_id", "aliquot_id"]).with_columns( + pl.col("aliquot_id").replace_strict(ploidy_map, default=None, return_dtype=pl.Float64).alias("ploidy") + ).select(["file_id", "ploidy"]) + print(f"ASCAT ploidy found for {ploidy_df['ploidy'].is_not_null().sum()} of {ploidy_df.height} copy number files") + for df in dataframe_iter: if data_type == "transcriptomics": df = ( @@ -665,6 +714,17 @@ def map_and_combine_stream( .select(["entrez_id", "copy_number", "file_id"]) ) + # ASCAT copy_number is an absolute copy count. Divide by the sample's + # tumor ploidy (2 if unknown) and convert to the DepMap scale + # log2(relative CN + 1), where normal = 1.0. + df = ( + df.join(ploidy_df, on="file_id", how="left") + .with_columns( + (pl.col("copy_number") / pl.col("ploidy").fill_null(2.0) + 1) + .log(base=2).alias("copy_number")) + .drop("ploidy") + ) + # build categorical copy_call from the concrete Series, not Expr copy_call_series = copy_num(df["copy_number"]) @@ -745,7 +805,8 @@ def get_copy_call(a): if math.isnan(a): return float('nan') - a_val = math.log2(float(a)+0.000001) + # a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0 + a_val = float(a) if a_val < 0.5210507: return 'deep del' elif a_val < 0.7311832: diff --git a/coderbuild/liver/02-omics-liver.py b/coderbuild/liver/02-omics-liver.py index f2c4a589..1e79874b 100644 --- a/coderbuild/liver/02-omics-liver.py +++ b/coderbuild/liver/02-omics-liver.py @@ -169,7 +169,8 @@ def get_copy_call(a): if math.isnan(a): return float('nan') - a_val = math.log2(float(a)+0.000001) + # a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0 + a_val = float(a) if a_val < 0.5210507: return 'deep del' elif a_val < 0.7311832: @@ -207,7 +208,8 @@ def map_copy_number(copy_number_data, improve_id_data, entrez_data): # do copy_number calculation from score and get copy call column long_cn_df = long_cn_df.rename(columns={0:'other_id'}) long_cn_df = long_cn_df.astype({'value':'float'}) - long_cn_df['copy_number'] = pow(2,long_cn_df['value'])*2 + # input is a log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0 + long_cn_df['copy_number'] = np.log2(np.power(2.0, long_cn_df['value']) + 1) long_cn_df['copy_call'] = [get_copy_call(a) for a in long_cn_df['copy_number']] # map ID to improve_ID diff --git a/coderbuild/mpnst/01_combined_omics.R b/coderbuild/mpnst/01_combined_omics.R index dcbdfbae..4706d0b8 100644 --- a/coderbuild/mpnst/01_combined_omics.R +++ b/coderbuild/mpnst/01_combined_omics.R @@ -218,7 +218,8 @@ cnv_list <- lapply( filter(!is.na(entrez_id)) %>% select(entrez_id, log2) %>% distinct() %>% - mutate(copy_number = 2^log2) %>% + ## log2 ratio -> DepMap scale log2(relative CN + 1); normal = 1.0 + mutate(copy_number = log2(2^log2 + 1)) %>% select(-log2) df <- df_long %>% diff --git a/coderbuild/novartis/02-omics-novartis.py b/coderbuild/novartis/02-omics-novartis.py index ce892a12..b5e8688d 100644 --- a/coderbuild/novartis/02-omics-novartis.py +++ b/coderbuild/novartis/02-omics-novartis.py @@ -17,7 +17,8 @@ def get_copy_call(a): if math.isnan(a): return float('nan') - a_val = math.log2(float(a)+0.000001) + # a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0 + a_val = float(a) if a_val < 0.5210507: return 'deep del' elif a_val < 0.7311832: @@ -115,7 +116,9 @@ def map_copy_number_novPDX(copy_number_data, improve_id_data, entrez_data): # get entrez id's from Sample entrez_cn_df = pd.merge(long_cn_df, entrez_data[['other_id','entrez_id']].drop_duplicates(), how = 'inner', left_on= "Sample", right_on= "other_id") - # get copy call from value column (aka copy number) + # value is absolute copy number (normal = 2); convert to the DepMap scale + # log2(relative CN + 1), normal = 1.0, then get the copy call + entrez_cn_df['value'] = np.log2(entrez_cn_df['value'].astype(float) / 2 + 1) entrez_cn_df['copy_call'] = [get_copy_call(a) for a in entrez_cn_df['value']] # get improve sample id diff --git a/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py b/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py index 06b2bfdc..9fd02171 100644 --- a/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py +++ b/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py @@ -6,7 +6,8 @@ def get_copy_call(a): """ - Helper Function - Determine copy call for a value. + Helper Function - Determine copy call for a copy number value on the + DepMap scale, log2(relative CN + 1), where normal = 1.0. """ if a is None: @@ -15,19 +16,17 @@ def get_copy_call(a): if math.isnan(a): return float('nan') - a_val = a##math.log2(float(a)+0.000001) ###this should not be exponent, should be log!!! 2**float(a) - if a_val < 0.0: #0.5210507: + a_val = float(a) + if a_val < 0.5210507: return 'deep del' elif a_val < 0.7311832: return 'het loss' elif a_val < 1.214125: return 'diploid' - elif a_val < 1.731183: + elif a_val < 1.422233: return 'gain' else: return 'amp' - - return pl.Series([get_copy_call(a) for a in arr]) def parseCNVFile(fpath, sampid, genes): log2data = pd.read_csv(fpath, sep='\t', header=None) @@ -37,6 +36,8 @@ def parseCNVFile(fpath, sampid, genes): newdat['study']='pancreatic' newdat['source']='TiriacEtal' newdat = newdat[['improve_sample_id','entrez_id','copy_number','source','study']] + ## input is a log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0 + newdat['copy_number'] = [math.log2(2**float(a) + 1) for a in newdat['copy_number']] newdat['copy_call'] = [get_copy_call(a) for a in newdat['copy_number']] return newdat diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 695db22c..1af1b925 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -792,7 +792,7 @@ def format( if copy_call: ret = ret.apply( pd.cut, - bins = [0, 0.5210507, 0.7311832, 1.214125, 1.422233, 2], + bins = [-np.inf, 0.5210507, 0.7311832, 1.214125, 1.422233, np.inf], labels = ['deep del', 'het loss', 'diploid', 'gain', 'amp'], include_lowest=True ) From 28bfc0d0e5fa597cfaa342589497fca1e7230fb9 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 30 Sep 2026 16:57:29 -0700 Subject: [PATCH 2/4] Use 0 as the lower copy_call bin edge --- coderdata/dataset/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 1af1b925..8c3bb542 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -792,7 +792,7 @@ def format( if copy_call: ret = ret.apply( pd.cut, - bins = [-np.inf, 0.5210507, 0.7311832, 1.214125, 1.422233, np.inf], + bins = [0, 0.5210507, 0.7311832, 1.214125, 1.422233, np.inf], labels = ['deep del', 'het loss', 'diploid', 'gain', 'amp'], include_lowest=True ) From 15ef47ddf3e7fa3c17dcc96a4e26b6d975f7a6c8 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Fri, 2 Oct 2026 08:30:24 -0700 Subject: [PATCH 3/4] Use Polars 0.19-compatible HCMI ploidy mapping --- coderbuild/hcmi/02-getHCMIData.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderbuild/hcmi/02-getHCMIData.py b/coderbuild/hcmi/02-getHCMIData.py index 39c6427f..ef27d8db 100644 --- a/coderbuild/hcmi/02-getHCMIData.py +++ b/coderbuild/hcmi/02-getHCMIData.py @@ -690,7 +690,7 @@ def map_and_combine_stream( if data_type == "copy_number": ploidy_map = fetch_ascat_ploidy(meta_df["aliquot_id"].to_list()) ploidy_df = meta_df.select(["file_id", "aliquot_id"]).with_columns( - pl.col("aliquot_id").replace_strict(ploidy_map, default=None, return_dtype=pl.Float64).alias("ploidy") + pl.col("aliquot_id").map_dict(ploidy_map, default=None, return_dtype=pl.Float64).alias("ploidy") ).select(["file_id", "ploidy"]) print(f"ASCAT ploidy found for {ploidy_df['ploidy'].is_not_null().sum()} of {ploidy_df.height} copy number files") From 65d54843c80ac95d1b4dd82bfdc033c17fad3db7 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 5 Oct 2026 09:44:58 -0700 Subject: [PATCH 4/4] Preserve HCMI mutations with multiple PubMed IDs --- coderbuild/hcmi/02-getHCMIData.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderbuild/hcmi/02-getHCMIData.py b/coderbuild/hcmi/02-getHCMIData.py index ef27d8db..59ff3b86 100644 --- a/coderbuild/hcmi/02-getHCMIData.py +++ b/coderbuild/hcmi/02-getHCMIData.py @@ -549,7 +549,7 @@ def stream_clean_files(data_type: str): # ---- read single file ------------------------------ if fpath.endswith(".gz"): # mutation data is always gzipped try: - df = pl.read_csv(fpath, separator="\t", skip_rows=7) + df = pl.read_csv(fpath, separator="\t", skip_rows=7, dtypes={"PUBMED": pl.Utf8}) except Exception as e: print(f"[warn] skipping MAF due to read error: {fpath} ({type(e).__name__}: {e})") continue