From 1a66d38d9e5a013d6711d478aadc4de7a7a7d2c2 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 11 Nov 2026 17:38:38 -0500 Subject: [PATCH] Use the schema's variant types consistently across datasets Closes #532 --- coderbuild/beatAML/GetBeatAML.py | 15 +++++-- coderbuild/broad_sanger/02-broadSangerOmics.R | 44 +++++++++++++++---- coderbuild/colorectal/02-omics-colorectal.py | 6 +++ coderbuild/cptac/getCptacData.py | 12 +++-- coderbuild/novartis/02-omics-novartis.py | 43 ++++++++++++++---- .../02a-getPancreaticDataFromSynapse.py | 11 +++-- .../sarcoma/01_createSarcomaOmicsFiles.py | 8 +++- 7 files changed, 106 insertions(+), 33 deletions(-) diff --git a/coderbuild/beatAML/GetBeatAML.py b/coderbuild/beatAML/GetBeatAML.py index 08a6d012..df529274 100755 --- a/coderbuild/beatAML/GetBeatAML.py +++ b/coderbuild/beatAML/GetBeatAML.py @@ -212,19 +212,26 @@ def map_and_combine(df, data_type, entrez_map_file, improve_map_file, map_file=N mapped_df["mutation"] = mapped_df["mutation"].astype(str).str.split(":", n=1).str[-1] variant_mapping = { - 'frameshift_variant': 'Frameshift_Variant', 'missense_variant': 'Missense_Mutation', 'stop_gained': 'Nonsense_Mutation', 'inframe_deletion': 'In_Frame_Del', - 'protein_altering_variant': 'Protein_Altering_Variant', 'splice_acceptor_variant': 'Splice_Site', 'splice_donor_variant': 'Splice_Site', - 'start_lost': 'Start_Codon_Del', + 'start_lost': 'Translation_Start_Site', 'inframe_insertion': 'In_Frame_Ins', 'stop_lost': 'Nonstop_Mutation' } - mapped_df['variant_classification'] = mapped_df['variant_classification'].map(variant_mapping) + # Other datasets split these by indel type; HGVS dup/ins add bases, del removes them + # (protein_altering_variant here is in-frame insertions, e.g. c.283_284insGCG) + vc = mapped_df['variant_classification'] + is_ins = mapped_df['mutation'].str.contains("dup|ins") & ~mapped_df['mutation'].str.contains("del") + indel_class = pd.Series(None, index=mapped_df.index, dtype=object) + indel_class[(vc == 'frameshift_variant') & is_ins] = 'Frame_Shift_Ins' + indel_class[(vc == 'frameshift_variant') & ~is_ins] = 'Frame_Shift_Del' + indel_class[(vc == 'protein_altering_variant') & is_ins] = 'In_Frame_Ins' + indel_class[(vc == 'protein_altering_variant') & ~is_ins] = 'In_Frame_Del' + mapped_df['variant_classification'] = vc.map(variant_mapping).fillna(indel_class) elif data_type == "proteomics": mapped_ids['sampleID'] = mapped_ids['sampleID'].str.split('_').apply(lambda x: x[2]) diff --git a/coderbuild/broad_sanger/02-broadSangerOmics.R b/coderbuild/broad_sanger/02-broadSangerOmics.R index 131876b9..f7c6cc1e 100755 --- a/coderbuild/broad_sanger/02-broadSangerOmics.R +++ b/coderbuild/broad_sanger/02-broadSangerOmics.R @@ -139,16 +139,40 @@ variant_schema =list(`3'UTR`=c("3'UTR",'THREE_PRIME_UTR','3prime_UTR_variant','3 Intron=c('INTRON','Intron','intronic','intron','intron_variant'), Missense_Mutation=c('Missense_Mutation','MISSENSE','missense','missense_variant'), Nonsense_Mutation=c('Nonsense_Mutation','NONSENSE','nonsense','stop_gained'), - Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP','stop_lost'), + ## each source value must appear once in this list: a value listed under + ## two classes gives every such mutation two rows (start_lost and + ## stop_lost used to be listed twice) + Nonstop_Mutation=c('Nonstop_Mutation','NONSTOP','stop_lost','Stop_Codon_Del','Stop_Codon_Ins'), RNA=c('RNA','non_coding_transcript_exon_variant','non_coding_transcript_variant'), - Start_Codon_SNP=c('START_CODON_SNP','Start_Codon_SNP'), - Start_Codon_Del=c('Start_Codon_Del','START_CODON_DEL','start_lost'), - Start_Codon_Ins=c('Start_Codon_Ins','START_CODON_INS'), - Stop_Codon_Del=c('Stop_Codon_Del','stop_lost'), - Stop_Codon_Ins=c('Stop_Codon_Ins'), Silent=c('Silent','SILENT','silent','synonymous_variant'), Splice_Site=c('Splice_Site','SPLICE_SITE','splice_region','splice_donor_variant','splice_acceptor_variant','splice_region_variant','splice_polypyrimidine_tract_variant','splice_donor_region_variant','splice_donor_5th_base_variant'), - Translation_Start_Site=c('Translation_Start_Site','start_lost')) + Translation_Start_Site=c('Translation_Start_Site','start_lost','START_CODON_SNP','Start_Codon_SNP', + 'Start_Codon_Del','START_CODON_DEL','Start_Codon_Ins','START_CODON_INS')) + +## protein_altering_variant (and other undetermined) indels in coding sequence get +## their class from the HGVS c. change: inserted/deleted length divisible by 3 +## is in-frame, otherwise a frameshift. Intronic/UTR changes (+, -, * offsets) stay +## Undetermined. +indel_class <- function(mutation) { + m <- sub('^.*:', '', as.character(mutation)) + coding <- grepl('^c\\.[0-9]+(_[0-9]+)?(del|ins|dup)', m) + start <- suppressWarnings(as.numeric(sub('^c\\.([0-9]+).*$', '\\1', m))) + end <- suppressWarnings(as.numeric(ifelse(grepl('^c\\.[0-9]+_[0-9]+', m), sub('^c\\.[0-9]+_([0-9]+).*$', '\\1', m), start))) + span <- end - start + 1 + seqlen <- function(tag) ifelse(grepl(paste0(tag, '[ACGTN]+$'), m), nchar(sub(paste0('^.*', tag, '([ACGTN]+)$'), '\\1', m)), NA) + delins <- grepl('delins', m) + net <- ifelse(delins, seqlen('delins') - span, + ifelse(grepl('del', m), -ifelse(is.na(seqlen('del')), span, seqlen('del')), + ifelse(grepl('dup', m), ifelse(is.na(seqlen('dup')), span, seqlen('dup')), + ifelse(grepl('ins', m), seqlen('ins'), NA)))) + cls <- ifelse(is.na(net) | net == 0, NA, + paste0(ifelse(net %% 3 == 0, 'In_Frame_', 'Frame_Shift_'), ifelse(net > 0, 'Ins', 'Del'))) + ifelse(coding, cls, NA) +} +refine_undetermined <- function(variant_classification, mutation) { + cls <- indel_class(mutation) + ifelse(!is.na(variant_classification) & variant_classification == 'Undetermined' & !is.na(cls), cls, variant_classification) +} depmap_vtab<-do.call('rbind',sapply(names(variant_schema),function(x) cbind(rep(x,length(variant_schema[[x]])),unlist(variant_schema[[x]])))) colnames(depmap_vtab)<-c('variant_classification','VariantInfo') @@ -286,7 +310,8 @@ sanger_files<-function(fi,value){ left_join(smap)|> mutate(study='Sanger')|> dplyr::select(-c(other_id,gene_symbol))|> - left_join(as.data.frame(sanger_vtab)) + left_join(as.data.frame(sanger_vtab))|> + mutate(variant_classification=refine_undetermined(variant_classification, mutation)) ##now many variants are missing??? missing<-res|> @@ -563,7 +588,8 @@ depmap_files<-function(fi,value){ res<-exp_file|> mutate(entrez_id=as.numeric(EntrezGeneID))|> filter(entrez_id %in% genes$entrez_id) |> - left_join(as.data.frame(depmap_vtab)) + left_join(as.data.frame(depmap_vtab))|> + mutate(variant_classification=refine_undetermined(variant_classification, mutation)) ##now many variants are missing??? missing<-res|> diff --git a/coderbuild/colorectal/02-omics-colorectal.py b/coderbuild/colorectal/02-omics-colorectal.py index 9639d131..f7cef104 100644 --- a/coderbuild/colorectal/02-omics-colorectal.py +++ b/coderbuild/colorectal/02-omics-colorectal.py @@ -70,6 +70,12 @@ def map_mutations(mutation_data, improve_id_data, entrez_data): # the data's variant classification matches scheme well, except "Non-coding_Transcript". let's change those to RNA mapped_mutation_data.loc[mapped_mutation_data['Variant_Classification'] == "Non-coding_Transcript",'Variant_Classification'] = "RNA" + # Start_Codon_* / Stop_Codon_* are named Translation_Start_Site / Nonstop_Mutation + # in the other datasets + mapped_mutation_data['Variant_Classification'] = mapped_mutation_data['Variant_Classification'].replace( + {'Start_Codon_SNP': 'Translation_Start_Site', 'Start_Codon_Del': 'Translation_Start_Site', + 'Start_Codon_Ins': 'Translation_Start_Site', 'Stop_Codon_Del': 'Nonstop_Mutation', + 'Stop_Codon_Ins': 'Nonstop_Mutation'}) # some of the entrezID's have ?, but it must be an integer. check to see if we have an entrez ID using the hugo symbol and mapping using entrez_data. if there's any leftover, put in nothing questions = mapped_mutation_data[mapped_mutation_data['Entrez_Gene_Id'] == "?"].reset_index() # all rows with ? diff --git a/coderbuild/cptac/getCptacData.py b/coderbuild/cptac/getCptacData.py index 5100b399..abe3f1ce 100755 --- a/coderbuild/cptac/getCptacData.py +++ b/coderbuild/cptac/getCptacData.py @@ -30,16 +30,14 @@ def reclass_var(arr): 'Intron': ['Intron'], 'Missense_Mutation': ['Missense_Mutation'], 'Nonsense_Mutation': ['Nonsense_Mutation'], - 'Nonstop_Mutation': ['Nonstop_Mutation'], + # Stop_Codon_* / Start_Codon_* are named Nonstop_Mutation / Translation_Start_Site + # in the other datasets + 'Nonstop_Mutation': ['Nonstop_Mutation', 'Stop_Codon_Del', 'Stop_Codon_Ins'], 'RNA': ['RNA'], - 'Start_Codon_SNP': ['START_CODON_SNP', 'Start_Codon_SNP'], - 'Start_Codon_Del': ['Start_Codon_Del'], - 'Start_Codon_Ins': ['Start_Codon_Ins', 'START_CODON_INS'], - 'Stop_Codon_Del': ['Stop_Codon_Del'], - 'Stop_Codon_Ins': ['Stop_Codon_Ins'], 'Silent': ['Silent'], 'Splice_Site': ['Splice_Site'], - 'Translation_Start_Site': ['Translation_Start_Site'] + 'Translation_Start_Site': ['Translation_Start_Site', 'START_CODON_SNP', 'Start_Codon_SNP', 'Start_Codon_Del', + 'Start_Codon_Ins', 'START_CODON_INS'] } newarr = [] diff --git a/coderbuild/novartis/02-omics-novartis.py b/coderbuild/novartis/02-omics-novartis.py index b5e8688d..9db6debd 100644 --- a/coderbuild/novartis/02-omics-novartis.py +++ b/coderbuild/novartis/02-omics-novartis.py @@ -2,6 +2,7 @@ import numpy as np import os import math +import re import argparse import synapseclient @@ -219,6 +220,33 @@ def map_transcriptomics_novPDX(transcriptomics_data, improve_id_data, entrez_dat +def classify_protein_change(change): + """ + Variant class for a Novartis PDXE protein change. + + Notation in the source: "K1831N" substitution, "R24*" stop gained, + "G518GR" / "-423SR" residues inserted, "TS178T" / "PP43-" residues deleted, + and frameshifts as a bare position ("-659", "-1554?", "-349P?"). + """ + if re.fullmatch(r'-\d+([A-Z*]+)?\?|-\d+', change): + return 'Frameshift_Variant' + if re.fullmatch(r'[A-Z*]+\d+-', change): + return 'In_Frame_Del' + if re.fullmatch(r'-\d+[A-Z*]+', change): + return 'In_Frame_Ins' + if re.fullmatch(r'[A-Z]\d+\*', change): + return 'Nonsense_Mutation' + if re.fullmatch(r'\*\d+[A-Z]+', change): + return 'Nonstop_Mutation' + m = re.fullmatch(r'([A-Z*]+)\d+([A-Z*]+)', change) + if m: + ref, alt = m.groups() + if len(ref) == len(alt): + return 'Silent' if ref == alt else 'Missense_Mutation' + return 'In_Frame_Ins' if len(alt) > len(ref) else 'In_Frame_Del' + return 'Undetermined' + + def map_mutations_novPDX(mutation_data, improve_id_data, entrez_data): """ Maps transcriptomics data to improved sample id's and entrez gene data. Also does some data formatting. @@ -252,15 +280,15 @@ def map_mutations_novPDX(mutation_data, improve_id_data, entrez_data): # include only rows that are mutations (data had both cn and mutations) mutations_only_df = mutation_data[mutation_data['Category'].isin(["MutNovel","MutKnownFunctional","MutLikelyFunctional"])] - # turn details column into mutation column + # turn details column into mutation column (Details is ",") mutations_only_df['mutation'] = mutations_only_df['Details'].str.split(pat = ",", expand=True).iloc[:,0] - mutations_only_df[~mutations_only_df['mutation'].str.contains(r'\?')] # 1866 rows have a "?" in them. remove - # create variant classifications with information that we have - mutations_only_df['variant_classification'] = np.nan - mutations_only_df.loc[mutations_only_df['mutation'].str.contains("-"),['variant_classification']] = "Frame_Shift_Del" - mutations_only_df.loc[mutations_only_df['mutation'].str.contains(r'[A-Za-z]\d+[A-Za-z]$', regex=True, na=False),['variant_classification']] = "Missense_Mutation" - mutations_only_df['variant_classification'] = mutations_only_df['variant_classification'].fillna("Undetermined") + # Variant classes from the protein change. Frameshifts are written without + # residues ("-659", "-1554?", e.g. RNF43 G659fs, APC T1556fs) and don't say + # whether bases were inserted or deleted, so they stay Frameshift_Variant. + mutations_only_df['variant_classification'] = mutations_only_df['mutation'].astype(str).map(classify_protein_change) + # protein-level changes, marked "p." like HGVS (also keeps "-659" from reading as a number) + mutations_only_df['mutation'] = 'p.' + mutations_only_df['mutation'].astype(str) # missing entrex id's are not in genes.csv, so get rid of those rows mutations_only_df = mutations_only_df[mutations_only_df['Entrez'].notna()] @@ -276,7 +304,6 @@ def map_mutations_novPDX(mutation_data, improve_id_data, entrez_data): mutations_final['source'] = "CPDM" mutations_final['study'] = "novartis" mutations_final = mutations_final.astype({'entrez_id':'int', 'mutation':'str'}) - mutations_final['mutation'] = mutations_final['mutation'].astype('|S') return(mutations_final) diff --git a/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py b/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py index 9fd02171..63ffb87c 100644 --- a/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py +++ b/coderbuild/pancreatic/02a-getPancreaticDataFromSynapse.py @@ -59,9 +59,9 @@ def parseCNVFile(fpath, sampid, genes): 'SPLICE_SITE_REGION+INTRON':'Splice_Site', 'SPLICE_SITE_REGION+NON_CODING_EXON_VARIANT':'Splice_Site', 'SPLICE_SITE_REGION+SYNONYMOUS_CODING':'Silent', - 'START_GAINED+UTR_5_PRIME':'Start_Codon_Ins', - 'STOP_GAINED':'Stop_Codon_Ins', - 'STOP_GAINED+CODON_CHANGE_PLUS_CODON_INSERTION':'Stop_Codon_Ins', + 'START_GAINED+UTR_5_PRIME':"5'UTR", + 'STOP_GAINED':'Nonsense_Mutation', + 'STOP_GAINED+CODON_CHANGE_PLUS_CODON_INSERTION':'Nonsense_Mutation', 'SYNONYMOUS_CODING':'Silent', 'UPSTREAM':"5'Flank", 'UTR_3_PRIME':"3'UTR", @@ -77,6 +77,11 @@ def parseMutFile(fpath, sampid,genes): mutfile = mutfile.dropna(subset='SNPEFF_CDS_CHANGE') mutfile.columns = ['gene_symbol','SNPEFF_EFFECT','mutation'] fullfile = pd.merge(mutfile,pd.DataFrame({'SNPEFF_EFFECT':mutmap.keys(),'variant_classification':mutmap.values()})) + # Other datasets split frameshifts by indel type; HGVS dup/ins add bases, del removes them + frameshift = fullfile['variant_classification'] == 'Frameshift_Variant' + is_ins = fullfile['mutation'].astype(str).str.contains("dup|ins") & ~fullfile['mutation'].astype(str).str.contains("del") + fullfile.loc[frameshift & is_ins, 'variant_classification'] = 'Frame_Shift_Ins' + fullfile.loc[frameshift & ~is_ins, 'variant_classification'] = 'Frame_Shift_Del' fullfile = pd.merge(fullfile,genes) fullfile['improve_sample_id'] = sampid fullfile['source']='TiriacEtAl' diff --git a/coderbuild/sarcoma/01_createSarcomaOmicsFiles.py b/coderbuild/sarcoma/01_createSarcomaOmicsFiles.py index 2cfcaecb..9bde58c0 100644 --- a/coderbuild/sarcoma/01_createSarcomaOmicsFiles.py +++ b/coderbuild/sarcoma/01_createSarcomaOmicsFiles.py @@ -71,8 +71,12 @@ def download_and_format_genomic_mutation(synLoginObject, genesTable, samplesTabl mutation_merged.replace({'variant_classification': "synonymous"}, "Silent", inplace=True) mutation_merged.replace({'variant_classification': "Inframe_Del"}, "In_Frame_Del", inplace=True) mutation_merged.replace({'variant_classification': "5_prime_UTR"}, "5'UTR", inplace=True) - mutation_merged.replace({'variant_classification': "Frameshift"}, "Frameshift_Variant", inplace=True) - mutation_merged.replace({'variant_classification': "intergenic_variant"}, "Silent", inplace=True) + # Other datasets split frameshifts by indel type; HGVS dup/ins add bases, del removes them + frameshift = mutation_merged['variant_classification'] == "Frameshift" + is_ins = mutation_merged['Name'].astype(str).str.contains("dup|ins") & ~mutation_merged['Name'].astype(str).str.contains("del") + mutation_merged.loc[frameshift & is_ins, 'variant_classification'] = "Frame_Shift_Ins" + mutation_merged.loc[frameshift & ~is_ins, 'variant_classification'] = "Frame_Shift_Del" + mutation_merged.replace({'variant_classification': "intergenic_variant"}, "IGR", inplace=True) mutation_merged_select = mutation_merged[['entrez_id', 'Sample_ID_Tumor', 'Name', 'variant_classification']] #merge with improve_ids