Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions coderbuild/bladder/01_createBladderOmicsFiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ def get_copy_call(a):
if math.isnan(a):
return float('nan')

a_val = math.log2(float(a)+0.000001)
# a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0
a_val = float(a)
if a_val < 0.5210507:
return 'deep del'
elif a_val < 0.7311832:
Expand Down Expand Up @@ -118,7 +119,8 @@ def get_bladder_copynumber(synObject, samples, genes):
segfile_df.to_csv("bladder_segfile.csv")
subprocess.call (["/usr/bin/Rscript", "--vanilla", "CNV-segfile-annotation.R", "bladder_segfile.csv", "bladder_annotated_segfile.csv"])
copynumber = pd.read_csv("bladder_annotated_segfile.csv")
copynumber['copy_number'] = np.exp2(copynumber['score'].div(2))*2
# score is the segment log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0
copynumber['copy_number'] = np.log2(np.exp2(copynumber['score']) + 1)
copynumber['copy_call'] = [get_copy_call(a) for a in copynumber['copy_number']]
copynumber_with_improveids = copynumber.merge(samples, left_on='ID', right_on = 'other_id', how='left')
copynumber_with_correct_colnames = copynumber_with_improveids.rename({"ENTREZID":'entrez_id'}, axis=1)
Expand Down
3 changes: 2 additions & 1 deletion coderbuild/broad_sanger/02-broadSangerOmics.R
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,8 @@ sanger_files<-function(fi,value){

res<-exp_file|>
dplyr::select(other_id='model_id',gene_symbol='symbol',gatk_mean_log2_copy_ratio,source,data_type,cn_category)|>
mutate(copy_number=2^gatk_mean_log2_copy_ratio,.keep='all')|>
## log2 ratio -> DepMap scale log2(relative CN + 1); normal = 1.0
mutate(copy_number=log2(2^gatk_mean_log2_copy_ratio+1),.keep='all')|>
distinct()|>
left_join(gmap)|>
dplyr::select(other_id,copy_number,entrez_id,Sanger='cn_category')|>
Expand Down
6 changes: 4 additions & 2 deletions coderbuild/colorectal/02-omics-colorectal.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,8 @@ def get_copy_call(a):
if math.isnan(a):
return float('nan')

a_val = math.log2(float(a)+0.000001)
# a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0
a_val = float(a)
if a_val < 0.5210507:
return 'deep del'
elif a_val < 0.7311832:
Expand Down Expand Up @@ -216,7 +217,8 @@ def map_copy_number(copy_number_data, improve_id_data, entrez_data):
mapped_cn_df = pd.read_csv("output_copy_num.csv")

# do copy_number calculation from score and get copy call column
mapped_cn_df['copy_number'] = np.exp2(mapped_cn_df['score'].div(2))*2
# score is the segment log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0
mapped_cn_df['copy_number'] = np.log2(np.exp2(mapped_cn_df['score']) + 1)
mapped_cn_df['copy_call'] = [get_copy_call(a) for a in mapped_cn_df['copy_number']]

# map ID to improve_ID
Expand Down
16 changes: 9 additions & 7 deletions coderbuild/cptac/getCptacData.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,10 @@ def formatData(df,dtype,ctype,samp_names,source,genes,samples):

mlongdf = mlongdf[['entrez_id','improve_sample_id',dtype]].drop_duplicates()

##if its copy number we need to include copy number call
##if its copy number, convert the log2 ratio to the DepMap scale log2(relative CN + 1)
##(normal = 1.0) and add the copy number call
if dtype=='copy_number': ##deep del < 0.5210507 < het loss < 0.7311832 < diploid < 1.214125 < gain < 1.422233 < amp
mlongdf['copy_number'] = np.log2(np.power(2.0, mlongdf['copy_number'].astype(float)) + 1)
mlongdf[['copy_call']] = mlongdf[['copy_number']].apply(copy_num)


Expand All @@ -211,18 +213,18 @@ def formatData(df,dtype,ctype,samp_names,source,genes,samples):

def copy_num(arr):
'''
Converts copy number values to categorical calls.
Converts copy number values (DepMap scale, log2(relative CN + 1)) to categorical calls.
'''
copy_call=[]
for a in arr:
a = 2**float(a)
if float(a) < 0.5210507:
a = float(a)
if a < 0.5210507:
b = 'deep del'
elif float(a) < 0.7311832:
elif a < 0.7311832:
b = 'het loss'
elif float(a) < 1.214125:
elif a < 1.214125:
b = 'diploid'
elif float(a) <1.42233:
elif a < 1.422233:
b = 'gain'
else:
b = 'amp'
Expand Down
65 changes: 63 additions & 2 deletions coderbuild/hcmi/02-getHCMIData.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,46 @@ def fetch_metadata(uuids):
return response.json()


def fetch_ascat_ploidy(aliquot_ids, batch=200):
"""
Look up ASCAT tumor ploidy for the given tumor aliquots.

GDC stores ``tumor_ploidy`` on the ASCAT "Allele-specific Copy Number Segment"
files (not on the gene-level files we download), so query those by aliquot.

Returns
-------
dict
tumor aliquot_id -> tumor_ploidy (float). Aliquots with no ploidy are omitted.
"""
ploidy = {}
ids = [a for a in dict.fromkeys(aliquot_ids) if a]
for i in range(0, len(ids), batch):
chunk = ids[i:i + batch]
payload = {
"filters": {"op": "and", "content": [
{"op": "in", "content": {"field": "data_type", "value": ["Allele-specific Copy Number Segment"]}},
{"op": "in", "content": {"field": "cases.samples.portions.analytes.aliquots.aliquot_id", "value": chunk}}]},
"fields": "tumor_ploidy,cases.samples.tissue_type,cases.samples.portions.analytes.aliquots.aliquot_id",
"format": "JSON",
"size": str(len(chunk) * 4)
}
r = requests.post("https://api.gdc.cancer.gov/files", json=payload, timeout=120)
r.raise_for_status()
for h in r.json()["data"]["hits"]:
if h.get("tumor_ploidy") is None:
continue
for c in h.get("cases", []):
for smp in c.get("samples", []):
if smp.get("tissue_type") != "Tumor":
continue
for por in smp.get("portions", []):
for ana in por.get("analytes", []):
for al in ana.get("aliquots", []):
ploidy[al["aliquot_id"]] = float(h["tumor_ploidy"])
return ploidy


def drop_controlled_access(newfm, batch=400):
"""Remove dbGaP controlled-access files from a manifest slice.

Expand Down Expand Up @@ -509,7 +549,7 @@ def stream_clean_files(data_type: str):
# ---- read single file ------------------------------
if fpath.endswith(".gz"): # mutation data is always gzipped
try:
df = pl.read_csv(fpath, separator="\t", skip_rows=7)
df = pl.read_csv(fpath, separator="\t", skip_rows=7, dtypes={"PUBMED": pl.Utf8})
except Exception as e:
print(f"[warn] skipping MAF due to read error: {fpath} ({type(e).__name__}: {e})")
continue
Expand Down Expand Up @@ -645,6 +685,15 @@ def map_and_combine_stream(
for h in md_hits],
})

# per-file tumor ploidy, only needed for copy number
ploidy_df = None
if data_type == "copy_number":
ploidy_map = fetch_ascat_ploidy(meta_df["aliquot_id"].to_list())
ploidy_df = meta_df.select(["file_id", "aliquot_id"]).with_columns(
pl.col("aliquot_id").map_dict(ploidy_map, default=None, return_dtype=pl.Float64).alias("ploidy")
).select(["file_id", "ploidy"])
print(f"ASCAT ploidy found for {ploidy_df['ploidy'].is_not_null().sum()} of {ploidy_df.height} copy number files")

for df in dataframe_iter:
if data_type == "transcriptomics":
df = (
Expand All @@ -665,6 +714,17 @@ def map_and_combine_stream(
.select(["entrez_id", "copy_number", "file_id"])
)

# ASCAT copy_number is an absolute copy count. Divide by the sample's
# tumor ploidy (2 if unknown) and convert to the DepMap scale
# log2(relative CN + 1), where normal = 1.0.
df = (
df.join(ploidy_df, on="file_id", how="left")
.with_columns(
(pl.col("copy_number") / pl.col("ploidy").fill_null(2.0) + 1)
.log(base=2).alias("copy_number"))
.drop("ploidy")
)

# build categorical copy_call from the concrete Series, not Expr
copy_call_series = copy_num(df["copy_number"])

Expand Down Expand Up @@ -745,7 +805,8 @@ def get_copy_call(a):
if math.isnan(a):
return float('nan')

a_val = math.log2(float(a)+0.000001)
# a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0
a_val = float(a)
if a_val < 0.5210507:
return 'deep del'
elif a_val < 0.7311832:
Expand Down
6 changes: 4 additions & 2 deletions coderbuild/liver/02-omics-liver.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ def get_copy_call(a):
if math.isnan(a):
return float('nan')

a_val = math.log2(float(a)+0.000001)
# a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0
a_val = float(a)
if a_val < 0.5210507:
return 'deep del'
elif a_val < 0.7311832:
Expand Down Expand Up @@ -207,7 +208,8 @@ def map_copy_number(copy_number_data, improve_id_data, entrez_data):
# do copy_number calculation from score and get copy call column
long_cn_df = long_cn_df.rename(columns={0:'other_id'})
long_cn_df = long_cn_df.astype({'value':'float'})
long_cn_df['copy_number'] = pow(2,long_cn_df['value'])*2
# input is a log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0
long_cn_df['copy_number'] = np.log2(np.power(2.0, long_cn_df['value']) + 1)
long_cn_df['copy_call'] = [get_copy_call(a) for a in long_cn_df['copy_number']]

# map ID to improve_ID
Expand Down
3 changes: 2 additions & 1 deletion coderbuild/mpnst/01_combined_omics.R
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,8 @@ cnv_list <- lapply(
filter(!is.na(entrez_id)) %>%
select(entrez_id, log2) %>%
distinct() %>%
mutate(copy_number = 2^log2) %>%
## log2 ratio -> DepMap scale log2(relative CN + 1); normal = 1.0
mutate(copy_number = log2(2^log2 + 1)) %>%
select(-log2)

df <- df_long %>%
Expand Down
7 changes: 5 additions & 2 deletions coderbuild/novartis/02-omics-novartis.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ def get_copy_call(a):
if math.isnan(a):
return float('nan')

a_val = math.log2(float(a)+0.000001)
# a is on the DepMap scale, log2(relative CN + 1), where normal = 1.0
a_val = float(a)
if a_val < 0.5210507:
return 'deep del'
elif a_val < 0.7311832:
Expand Down Expand Up @@ -115,7 +116,9 @@ def map_copy_number_novPDX(copy_number_data, improve_id_data, entrez_data):
# get entrez id's from Sample
entrez_cn_df = pd.merge(long_cn_df, entrez_data[['other_id','entrez_id']].drop_duplicates(), how = 'inner', left_on= "Sample", right_on= "other_id")

# get copy call from value column (aka copy number)
# value is absolute copy number (normal = 2); convert to the DepMap scale
# log2(relative CN + 1), normal = 1.0, then get the copy call
entrez_cn_df['value'] = np.log2(entrez_cn_df['value'].astype(float) / 2 + 1)
entrez_cn_df['copy_call'] = [get_copy_call(a) for a in entrez_cn_df['value']]

# get improve sample id
Expand Down
13 changes: 7 additions & 6 deletions coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@

def get_copy_call(a):
"""
Helper Function - Determine copy call for a value.
Helper Function - Determine copy call for a copy number value on the
DepMap scale, log2(relative CN + 1), where normal = 1.0.
"""

if a is None:
Expand All @@ -15,19 +16,17 @@ def get_copy_call(a):
if math.isnan(a):
return float('nan')

a_val = a##math.log2(float(a)+0.000001) ###this should not be exponent, should be log!!! 2**float(a)
if a_val < 0.0: #0.5210507:
a_val = float(a)
if a_val < 0.5210507:
return 'deep del'
elif a_val < 0.7311832:
return 'het loss'
elif a_val < 1.214125:
return 'diploid'
elif a_val < 1.731183:
elif a_val < 1.422233:
return 'gain'
else:
return 'amp'

return pl.Series([get_copy_call(a) for a in arr])

def parseCNVFile(fpath, sampid, genes):
log2data = pd.read_csv(fpath, sep='\t', header=None)
Expand All @@ -37,6 +36,8 @@ def parseCNVFile(fpath, sampid, genes):
newdat['study']='pancreatic'
newdat['source']='TiriacEtal'
newdat = newdat[['improve_sample_id','entrez_id','copy_number','source','study']]
## input is a log2 ratio; convert to the DepMap scale log2(relative CN + 1), normal = 1.0
newdat['copy_number'] = [math.log2(2**float(a) + 1) for a in newdat['copy_number']]
newdat['copy_call'] = [get_copy_call(a) for a in newdat['copy_number']]
return newdat

Expand Down
2 changes: 1 addition & 1 deletion coderdata/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -792,7 +792,7 @@ def format(
if copy_call:
ret = ret.apply(
pd.cut,
bins = [0, 0.5210507, 0.7311832, 1.214125, 1.422233, 2],
bins = [0, 0.5210507, 0.7311832, 1.214125, 1.422233, np.inf],
labels = ['deep del', 'het loss', 'diploid', 'gain', 'amp'],
include_lowest=True
)
Expand Down