From 3bec27071d8d9af2f61631ab673ddf793ed8a8e9 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 27 Oct 2026 16:27:29 -0400 Subject: [PATCH] Fix duplicate NCI60 and PRISM drug responses --- coderbuild/broad_sanger/03a-nci60Drugs.py | 159 +++++----- .../broad_sanger/04a-drugResponseData.R | 25 ++ coderbuild/broad_sanger/04b-nci60-updated.py | 92 ++++++ coderbuild/utils/map_improve_drug_ids.py | 25 ++ coderbuild/utils/pubchem_retrieval.py | 65 ++++- tests/test_build_drug_ids.py | 273 ++++++++++++++++++ 6 files changed, 564 insertions(+), 75 deletions(-) create mode 100644 tests/test_build_drug_ids.py diff --git a/coderbuild/broad_sanger/03a-nci60Drugs.py b/coderbuild/broad_sanger/03a-nci60Drugs.py index 568c397f..591851bc 100644 --- a/coderbuild/broad_sanger/03a-nci60Drugs.py +++ b/coderbuild/broad_sanger/03a-nci60Drugs.py @@ -9,33 +9,27 @@ import argparse import pubchem_retrieval as pr import random as rand -from urllib import request import download_utils -import io _BROWSER_UA = ( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) -def _fetch(url: str) -> bytes: - """Download url with a browser User-Agent (NCI wiki blocks Python's default).""" - req = request.Request(url, headers={"User-Agent": _BROWSER_UA}) - with request.urlopen(req) as resp: - return resp.read() - -def _read_csv_url(url: str, **kwargs) -> pl.DataFrame: - return pl.read_csv(io.BytesIO(_fetch(url)), **kwargs) - def _retrieve_url(url: str, dest: str) -> None: - """Download url to dest, resuming an interrupted transfer. + """Download url to dest with retries, resuming an interrupted transfer. - This also fetches the ~346MB DOSERESP.zip, and _fetch() reads the whole - body into memory in one call -- the same shape that truncated with - IncompleteRead in build v32. Small CSVs still go through _fetch(). + NCI's wiki truncates downloads now and then (DOSERESP.zip in build v32, + nsc_chemical_names.csv in the first 2.4 rebuild), so every file goes + through download_utils rather than a single urlopen().read(). A browser + User-Agent is sent because the wiki blocks Python's default. """ download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA}) +def _read_csv_url(url: str, dest: str, **kwargs) -> pl.DataFrame: + _retrieve_url(url, dest) + return pl.read_csv(dest, **kwargs) + ##drug files smi_strings='https://wiki.nci.nih.gov/download/attachments/155844992/nsc_smiles.csv?version=1&modificationDate=1710381820000&api=v2&download=true' #oct 2024 @@ -84,10 +78,10 @@ def main(): 'InChIKey':[],'formula':[],'weight':[],'pubchem_id':[]} print('Downloading NSC identifiers for nci60 data') - names = _read_csv_url(chemnames, ignore_errors=True) + names = _read_csv_url(chemnames, 'nsc_chemical_names.csv', ignore_errors=True) #castab = pd.read_csv(cas) - pubchems = _read_csv_url(pc_ids) - smiles = _read_csv_url(smi_strings) + pubchems = _read_csv_url(pc_ids, 'nsc_sid_cid.csv') + smiles = _read_csv_url(smi_strings, 'nsc_smiles.csv') print('Getting experimental data to filter drugs') if not os.path.exists('DOSERESP.csv'): @@ -100,6 +94,7 @@ def main(): raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).") dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True) pubchems = pubchems.filter(pl.col('NSC').is_in(dose_resp['NSC'])) + nsc_cid = pubchems # every NSC with dose data smiles = smiles.filter(pl.col("NSC").is_in(dose_resp['NSC']) ) ##first retreive pubchem data @@ -109,16 +104,11 @@ def main(): else: arr = set(pubchems['CID']) - ##first filter to see if there are structures/drugs in teh data already. i dont think this does much. - if os.path.exists(opts.output): - curdrugs = pl.read_csv(opts.output,separator='\t') - smiles_notnull = smiles.filter(pl.col('SMILES').is_not_null()) - upper=[a.upper() for a in smiles_notnull['SMILES']] - smiles_upper = pl.DataFrame({'NSC':smiles_notnull['NSC'],'upper':upper}) - ssmiles = smiles_upper.filter(~pl.col('upper').is_in(curdrugs['canSMILES'])) - pubchems = pubchems.filter(pl.col('NSC').is_in(ssmiles['NSC'])) - arr = set(pubchems['CID']) - + # A rerun after a failure finds this script's earlier output. Every CID is + # still requested: update_dataframe_and_write_tsv skips CIDs already in the + # output, and it also limits the output to the requested CIDs, so asking + # for fewer (as this used to on a rerun) deleted drugs it had already + # written -- about 560 NCI60 drugs in the second 2.4 rebuild. print("Querying pubchem from CIDs") pr.update_dataframe_and_write_tsv( unique_names=arr, @@ -149,49 +139,80 @@ def main(): ##add in improve ids, nsc name and structure for all. mdf = msmi.join(names,on='NSC',how='left').join(pubchems,on='NSC',how='left') - max_imp = max(int(a.split('_')[1]) for a in res['improve_drug_id']) - - smicount=len(set(mdf['SMILES'])) ## unique smiles in our missing data frame - newdf = pl.DataFrame( - { - "improve_drug_id": ["SMI_"+str(a) for a in range(max_imp+1,max_imp+1+smicount,1)], - 'canSMILES': [a for a in set(mdf['SMILES'])], - # 'isoSMILES': [a for a in set(mdf['SMILES'])], - 'InChIKey': [None for a in range(smicount)], - 'formula': [None for a in range(smicount)], - 'weight': [None for a in range(smicount)] - } - ) + res = add_missing_nscs(res, mdf) + res = add_nsc_names(res, nsc_cid) + res.write_csv(opts.output,separator='\t') - #create updated nsc ids and names - namedf = pl.DataFrame( - { - "nscid": ['nsc-'+str(a) for a in mdf['NSC']], - 'lower_name': [a if a is None else str(a).lower() for a in mdf['NAME']], - 'canSMILES': list(mdf['SMILES']), - 'pubchem_id': list(mdf['CID']) - } + +def _cid_str(column): + """A CID column as text without a float suffix ('359323.0' -> '359323').""" + return pl.col(column).cast(pl.Utf8).str.replace(r'\.0$', '') + + +def _drug_per_cid(res): + """Return res with text columns, and one drug row (id + structure) per CID.""" + str_cols = ['canSMILES', 'InChIKey', 'formula', 'weight'] + res = res.with_columns([pl.col(c).cast(pl.Utf8) for c in str_cols] + [_cid_str('pubchem_id')]) + # One structure per CID: the lowest SMI id if a CID somehow has several. + known = ( + res.with_columns(pl.col('improve_drug_id').str.extract(r'SMI_(\d+)', 1) + .cast(pl.Int64).alias('_n')) + .sort('_n') + .unique(subset=['pubchem_id'], keep='first', maintain_order=True) + .select(['pubchem_id', 'improve_drug_id', 'canSMILES', 'InChIKey', 'formula', 'weight']) ) - #merge and melt - merged = pl.concat([mdf,namedf],how='horizontal').select(['SMILES','pubchem_id','nscid','lower_name']) - melted = merged.melt(id_vars=['SMILES','pubchem_id'],value_vars=['nscid','lower_name']).select(['SMILES','pubchem_id','value']).unique() - melted.columns = ['canSMILES','pubchem_id','chem_name'] - - if newdf.shape[0] > 0: - res = res.with_columns([ - pl.col("InChIKey").cast(pl.Utf8), - pl.col("formula").cast(pl.Utf8), - pl.col("weight").cast(pl.Utf8) - ]) - newdf = newdf.with_columns([ - pl.col("InChIKey").cast(pl.Utf8), - pl.col("formula").cast(pl.Utf8), - pl.col("weight").cast(pl.Utf8) - ]) - - newdf = newdf.join(melted, on='canSMILES', how='inner').select(res.columns) - res = pl.concat([res, newdf], how='vertical') - res.write_csv(opts.output,separator='\t') + return res, known + + +def add_nsc_names(res, nsc_cid): + """ + Give every NSC an 'nsc-' name under its CID's drug id. + + 04b-nci60-updated.py joins dose responses to drugs on 'nsc-' only. + PubChem mostly lists NSCs as 'nsc123', and the drug cache no longer + carries 'nsc-' names between builds, so without this most NCI60 drugs + had no name 04b could match. NCI's NSC->CID table is used rather than + PubChem synonyms, which can list an NSC under a related compound. + """ + res, known = _drug_per_cid(res) + names = (nsc_cid.select([_cid_str('CID').alias('pubchem_id'), + pl.format('nsc-{}', _cid_str('NSC')).alias('chem_name')]) + .drop_nulls().unique()) + added = names.join(known, on='pubchem_id', how='inner').select(res.columns) + out = pl.concat([res, added], how='vertical').unique(maintain_order=True) + print(f'NSC names: {added.height} of {names.height} NSCs attached to a drug id') + return out + + +def add_missing_nscs(res, mdf): + """ + Attach NSC ids that PubChem does not list as synonyms to their drug. + + Every NSC in mdf maps (via NCI's NSC->CID table) to a CID already in res, + so its 'nsc-' name and NCI name are added under that CID's existing + improve_drug_id and structure. Minting a new id here instead gave one + chemical two ids; when both NSCs had dose data the build fit two curves + that the stable-id step later collapsed onto one id (issue #512). + """ + res, known = _drug_per_cid(res) + nsc_names = mdf.select([_cid_str('CID').alias('pubchem_id'), + pl.format('nsc-{}', pl.col('NSC')).alias('chem_name')]) + # NCI's own compound names are only added when no other drug already uses + # the name, so a PSet drug of that name cannot be matched to this CID. + taken = res.select(['chem_name', 'pubchem_id']).unique() + nci_names = (mdf.select([_cid_str('CID').alias('pubchem_id'), + pl.col('NAME').cast(pl.Utf8).str.to_lowercase().alias('chem_name')]) + .join(taken.filter(pl.col('pubchem_id').is_not_null()).rename({'pubchem_id': '_other'}), + on='chem_name', how='left') + .filter(pl.col('_other').is_null() | (pl.col('_other') == pl.col('pubchem_id'))) + .drop('_other')) + names = pl.concat([nsc_names, nci_names]).drop_nulls().unique() + added = names.join(known, on='pubchem_id', how='inner').select(res.columns) + unmatched = names.join(known, on='pubchem_id', how='anti') + if unmatched.height: + print(f'{unmatched.height} NSC names had no CID in the drug table and were skipped') + print(f'Attached {added.height} NSC names to existing drug ids') + return pl.concat([res, added], how='vertical').unique(maintain_order=True) if __name__=='__main__': main() diff --git a/coderbuild/broad_sanger/04a-drugResponseData.R b/coderbuild/broad_sanger/04a-drugResponseData.R index a5d943bb..54fd9571 100755 --- a/coderbuild/broad_sanger/04a-drugResponseData.R +++ b/coderbuild/broad_sanger/04a-drugResponseData.R @@ -15,6 +15,30 @@ source("pinned_psets.R") source("retry_utils.R") all.dsets<-PharmacoGx::availablePSets() + +#' one_drug_per_name +#' A drug name can sit under more than one improve_drug_id (e.g. a synonym +#' shared by two records of one chemical). Joining on it copied every +#' experiment to each id, which later surfaced as duplicate or copied drug +#' responses (issue #512). Keep the lowest SMI id for each name. +one_drug_per_name<-function(d.map,studyName){ + d.map<-d.map|> + mutate(smi_num=as.numeric(sub('SMI_','',improve_drug_id))) + multi<-d.map|> + group_by(chem_name)|> + summarize(n=n_distinct(improve_drug_id),.groups='drop')|> + subset(n>1) + if(nrow(multi)>0) + print(paste('WARNING:',nrow(multi),'drug names in',studyName, + 'match more than one drug id; keeping the lowest id for:', + paste(head(multi$chem_name,20),collapse=', '))) + d.map|> + group_by(chem_name)|> + dplyr::filter(smi_num==min(smi_num))|> + ungroup()|> + dplyr::select(-smi_num) +} + ##first define a generic dose response function @@ -52,6 +76,7 @@ getDoseRespData<-function(dset,studyName,improve_samples,drug.map){ ###reduced drug and sample maps new.d.map<-subset(drug.map,chem_name%in%mapping$treatmentid)|> mutate(chem_name=tolower(chem_name)) + new.d.map<-one_drug_per_name(new.d.map,studyName) new.s.map<-subset(improve_samples,other_names%in%mapping$sampleid) diff --git a/coderbuild/broad_sanger/04b-nci60-updated.py b/coderbuild/broad_sanger/04b-nci60-updated.py index 4c8022d1..0cff71fc 100644 --- a/coderbuild/broad_sanger/04b-nci60-updated.py +++ b/coderbuild/broad_sanger/04b-nci60-updated.py @@ -7,6 +7,7 @@ import argparse import os import re +import json from urllib import request import download_utils @@ -35,6 +36,96 @@ def _retrieve_url(url: str, dest: str) -> None: #may 2025 oneconc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/ONECONC.zip?version=19&modificationDate=1775183401634&api=v2' +#NCI's NSC -> PubChem CID table, the same release 03a-nci60Drugs.py uses +pc_ids = 'https://wiki.nci.nih.gov/download/attachments/155844992/nsc_sid_cid.csv?version=5&modificationDate=1735931099618&api=v2' + + +def _id_str(column): + """An id column as text without a float suffix ('359323.0' -> '359323').""" + return pl.col(column).cast(pl.Utf8).str.replace(r'\.0$', '') + + +def _load_nsc_cids(): + """Return NCI's NSC -> CID table as a DataFrame with str columns.""" + download_utils.retrieve_url(pc_ids, 'nsc_sid_cid.csv', headers={"User-Agent": _BROWSER_UA}) + tab = pl.read_csv('nsc_sid_cid.csv', ignore_errors=True) + return tab.select([_id_str('NSC'), _id_str('CID')]).drop_nulls().unique() + + +def _pubchem_inchikeys(cids): + """Return {CID: InChIKey} for the given CIDs from PubChem.""" + out = {} + for i in range(0, len(cids), 100): + batch = ','.join(cids[i:i + 100]) + url = f'https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/{batch}/property/InChIKey/JSON' + with request.urlopen(request.Request(url, headers={"User-Agent": _BROWSER_UA})) as resp: + for prop in json.load(resp)['PropertyTable']['Properties']: + out[str(prop['CID'])] = prop.get('InChIKey') + return out + + +def resolve_nsc_ids(drugmapping, drugs, load_nsc_cids, fetch_inchikeys=_pubchem_inchikeys): + """ + Keep one improve_drug_id per NSC. + + An 'nsc-' name can sit under more than one drug id when PubChem lists + the NSC as a synonym of a related compound. Joining on it copied that + NSC's dose response to every such drug. Ambiguous NSCs keep the id whose + PubChem CID matches NCI's own NSC->CID table (the lowest SMI id if several + share that CID). If NCI's CID is none of the + candidates (e.g. it is the non-stereo record of a salt), the candidate + with the same InChIKey skeleton (first block) is kept. Any NSC still + ambiguous stops the build. + """ + counts = drugmapping.group_by('NSC').agg(pl.col('improve_drug_id').n_unique().alias('n')) + ambiguous = counts.filter(pl.col('n') > 1)['NSC'].to_list() + if len(ambiguous) == 0: + return drugmapping + nsc_cid = load_nsc_cids().select([_id_str('NSC'), _id_str('CID')]).drop_nulls().unique() + id_cid = drugs.select([pl.col('improve_drug_id'), _id_str('pubchem_id')]).drop_nulls().unique() + amb = (drugmapping.filter(pl.col('NSC').is_in(ambiguous)) + .with_columns(_id_str('NSC')) + .join(id_cid, on='improve_drug_id', how='left') + .join(nsc_cid.rename({'CID': 'nci_cid'}), on='NSC', how='left')) + # Ids sharing NCI's CID are one chemical listed twice (the stable-id step + # later merges them by structure), so the lowest SMI id is kept. + keep = (amb.filter(pl.col('pubchem_id') == pl.col('nci_cid')) + .select(['NSC', 'improve_drug_id']).unique() + .with_columns(pl.col('improve_drug_id').str.extract(r'SMI_(\d+)', 1) + .cast(pl.Int64).alias('_n')) + .sort(['NSC', '_n']).unique(subset=['NSC'], keep='first').drop('_n')) + per = keep.group_by('NSC').agg(pl.col('improve_drug_id').n_unique().alias('n')) + unresolved = {str(a) for a in ambiguous} - set(per.filter(pl.col('n') == 1)['NSC']) + + if unresolved: + keep = keep.filter(~pl.col('NSC').is_in(list(unresolved))) + rest = amb.filter(pl.col('NSC').is_in(list(unresolved))) + nci_keys = fetch_inchikeys(sorted(set(rest['nci_cid'].drop_nulls().to_list()))) + id_keys = {} + if 'InChIKey' in drugs.columns: + for drug_id, key in drugs.select(['improve_drug_id', 'InChIKey']).drop_nulls().unique().rows(): + id_keys.setdefault(drug_id, key) + picked = [] + for nsc in sorted(unresolved): + cand = rest.filter(pl.col('NSC') == nsc) + nci_key = nci_keys.get(cand['nci_cid'][0]) if cand['nci_cid'][0] is not None else None + skeleton = nci_key.split('-')[0] if nci_key else None + matches = {d for d in cand['improve_drug_id'].to_list() + if skeleton and (id_keys.get(d) or '').split('-')[0] == skeleton} + if len(matches) == 1: + picked.append((nsc, matches.pop())) + if picked: + keep = pl.concat([keep, pl.DataFrame(picked, schema=['NSC', 'improve_drug_id'], orient='row')]) + unresolved -= {nsc for nsc, _ in picked} + if unresolved: + raise ValueError( + f"{len(unresolved)} NSC id(s) map to more than one drug and NCI's NSC->CID table " + f"does not pick one: {sorted(unresolved)[:20]}") + print(f'Resolved {len(ambiguous)} NSC ids that matched more than one drug id') + keep = keep.with_columns(pl.col('NSC').cast(drugmapping['NSC'].dtype)) + return pl.concat([drugmapping.filter(~pl.col('NSC').is_in(ambiguous)), keep]) + + def main(): parser = argparse.ArgumentParser() @@ -70,6 +161,7 @@ def main(): drugmapping = drugmapping.join(drugs,on='chem_name')[['NSC','improve_drug_id']] drugmapping = drugmapping.unique() + drugmapping = resolve_nsc_ids(drugmapping, drugs, _load_nsc_cids) ###update sample mapping on = samples[['other_names','improve_sample_id']].rename({'other_names': 'common_name'}) diff --git a/coderbuild/utils/map_improve_drug_ids.py b/coderbuild/utils/map_improve_drug_ids.py index 13db3c8d..8b411a8e 100644 --- a/coderbuild/utils/map_improve_drug_ids.py +++ b/coderbuild/utils/map_improve_drug_ids.py @@ -344,9 +344,19 @@ def rewrite_drugs_file(file_path, drug_id_mapping, datasets=None): recompress_if_needed(file_path_decompressed, was_gz, original_path=file_path if was_gz else None) print(f"No improve drug ID column found in {file_path_decompressed}") return + # Response rows are keyed by every column except the measured value. + # Old ids that unify to one stable id would leave two rows with the + # same key (two results for one sample/drug), so check for that here. + check_keys = '_experiments' in filename or '_combinations' in filename + key_cols = [c for c in fieldnames if c not in ('dose_response_value', 'drug_combination_value')] + # Only ids that absorb a remapped id can collide, so track just those. + merged_ids = {new for (ds, old), new in drug_id_mapping.items() if ds == dataset and old != new} + seen_keys = {} + collisions = [] writer = csv.DictWriter(fout, fieldnames=fieldnames, delimiter='\t') writer.writeheader() for row in reader: + original_ids = tuple(row.get(c, "").strip() for c in id_columns) for column in id_columns: original_id = row.get(column, "").strip() key = (dataset, original_id) @@ -356,8 +366,23 @@ def rewrite_drugs_file(file_path, drug_id_mapping, datasets=None): print(f"Replacing {column} '{original_id}' with stable_id '{new_id}' in {file_path_decompressed}") row[column] = new_id changes_made = True + if check_keys and any(row[c] in merged_ids for c in id_columns): + row_key = tuple(row.get(c, '') for c in key_cols) + if row_key in seen_keys and seen_keys[row_key] != original_ids: + collisions.append((row_key, seen_keys[row_key], original_ids)) + seen_keys.setdefault(row_key, original_ids) writer.writerow(row) + if collisions: + os.remove(tmp_path) + recompress_if_needed(file_path_decompressed, was_gz, original_path=file_path if was_gz else None) + examples = '; '.join(f"{'/'.join(a)} and {'/'.join(b)}" for _, a, b in collisions[:5]) + raise RuntimeError( + f"{filename}: unifying drug ids would give {len(collisions)} rows the same key, i.e. " + f"more than one result for a sample/drug/metric (old ids e.g. {examples}). The drug " + f"step assigned one chemical several ids; fix that upstream rather than publishing " + f"duplicate responses.") + if changes_made: os.replace(tmp_path, file_path_decompressed) print(f"Changes were made to {file_path_decompressed}. File has been updated.") diff --git a/coderbuild/utils/pubchem_retrieval.py b/coderbuild/utils/pubchem_retrieval.py index a3c95873..0daf4448 100644 --- a/coderbuild/utils/pubchem_retrieval.py +++ b/coderbuild/utils/pubchem_retrieval.py @@ -3,6 +3,7 @@ import csv from concurrent.futures import ThreadPoolExecutor, as_completed import os +import re import threading import time import signal @@ -32,17 +33,56 @@ def _ts(): # --------------------------------------------------------------------------- _pubchem_data_cache = {} +# NCI60 NSC names ('nsc-') are added by 03a-nci60Drugs.py from NCI's own +# NSC->CID table each build, so the cache must not carry them forward. +_NSC_NAME = re.compile(r'^nsc-\d+$', re.IGNORECASE) + + +def _cid_key(cid): + """ + CID as plain integer text. Concatenating files where any pubchem_id is + blank makes pandas read the column as float, and str(359323.0) would + neither match lookups by CID nor be a valid pubchem_id in the output. + """ + try: + return str(int(float(cid))) + except (TypeError, ValueError): + return str(cid) + + +def _cache_row(group): + """ + Pick the one row whose properties represent a CID in the cache. + + A CID can sit under several drug rows, e.g. NCI60 NSC rows that carry + NCI's SMILES string next to the PubChem-derived row. The PubChem row is + the one with an InChIKey; ties go to the lowest SMI id. + """ + has_key = group['InChIKey'].notna() & ~group['InChIKey'].astype(str).str.strip().isin(['', 'nan']) + pool = group[has_key] if has_key.any() else group + if 'improve_drug_id' in pool.columns: + num = pool['improve_drug_id'].astype(str).str.extract(r'SMI_(\d+)', expand=False) + pool = pool.assign(_n=pd.to_numeric(num, errors='coerce')).sort_values('_n', kind='stable') + return pool.iloc[0] + def load_pubchem_cache(file_paths): """Populate _pubchem_data_cache from previously built drug TSV files. Call this before the main drug-build loop to avoid re-querying PubChem for drugs whose properties are already known. ID assignment is unaffected. + + All files are combined before grouping so every CID gets exactly one + entry (one structure) regardless of file order, and every name for the + CID resolves to that entry. Previously each CID took the first row it + met and later files overwrote earlier ones, so a cached "PubChem" SMILES + could be NCI's string and names of one chemical could resolve to two + structures -- which gave one chemical two drug ids (issue #512). """ global _pubchem_data_cache if isinstance(file_paths, str): file_paths = [p.strip() for p in file_paths.split(',') if p.strip()] - loaded = 0 + frames = [] for path in file_paths: if not os.path.exists(path): print(f"[{_ts()}] Cache hint: file not found, skipping: {path}") @@ -59,22 +99,35 @@ def load_pubchem_cache(file_paths): if not needed.issubset(set(df.columns)): print(f"[{_ts()}] Cache file {path} missing expected columns, skipping.") continue + frames.append(df) + loaded = 0 + uncached = 0 + if frames: + df = pd.concat(frames, ignore_index=True) for cid, group in df.groupby('pubchem_id', dropna=True): - synonyms = [str(n) for n in group['chem_name'].dropna().tolist()] - row = group.iloc[0] + row = _cache_row(group) + synonyms = list(dict.fromkeys( + str(n) for n in group['chem_name'].dropna().tolist() if not _NSC_NAME.match(str(n)))) + if not synonyms: + # Only NSC names (e.g. CID 24202157, whose one PubChem synonym + # is 'nsc-630659'). A cache entry with no names writes no rows, + # so the drug would be dropped; leave it to be fetched live. + uncached += 1 + continue entry = { 'SMILES': str(row['canSMILES']), 'InChIKey': str(row['InChIKey']), 'MolecularFormula': str(row['formula']), 'MolecularWeight': str(row['weight']), - 'CID': str(cid), + 'CID': _cid_key(cid), 'synonyms': synonyms, } - _pubchem_data_cache[str(cid)] = entry + _pubchem_data_cache[_cid_key(cid)] = entry for syn in synonyms: _pubchem_data_cache[str(syn).lower()] = entry loaded += 1 - print(f"[{_ts()}] PubChem cache: {loaded} drugs loaded from {len(file_paths)} file(s).") + print(f"[{_ts()}] PubChem cache: {loaded} drugs loaded from {len(file_paths)} file(s); " + f"{uncached} with only NSC names left to fetch live.") # Auto-load from hint file written by build_all.py before the container starts. diff --git a/tests/test_build_drug_ids.py b/tests/test_build_drug_ids.py new file mode 100644 index 00000000..75e63e9e --- /dev/null +++ b/tests/test_build_drug_ids.py @@ -0,0 +1,273 @@ +""" +Tests for drug id assignment in the build (issue #512). + +One chemical must get one improve_drug_id, one experiment must map to one +drug, and the stable id step must refuse to merge ids into more than one +value for a sample/drug/time/metric. Fixtures are modelled on the NCI60 cases that +shipped in 2.4: pentamidine (CID 359323) screened as NSC 620107 and NSC +757400, and NSC 694279 listed under two related PubChem records. +""" +import csv +import gzip +import importlib.util +import os +import sys + +import pandas as pd +import polars as pl +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +UTILS = os.path.join(ROOT, 'coderbuild', 'utils') +BROAD = os.path.join(ROOT, 'coderbuild', 'broad_sanger') +sys.path.insert(0, UTILS) + +import map_improve_drug_ids # noqa: E402 +import pubchem_retrieval # noqa: E402 + + +def _load(name, filename): + spec = importlib.util.spec_from_file_location(name, os.path.join(BROAD, filename)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +nci60_drugs = _load('nci60_drugs', '03a-nci60Drugs.py') +nci60_exp = _load('nci60_exp', '04b-nci60-updated.py') + +NCI_SMILES = 'NC(=N)c1ccc(OCCCCCOc2ccc(cc2)C(=N)N)cc1' +PUBCHEM_SMILES = 'C1=CC(=CC=C1C(=N)N)OCCCCCOC2=CC=C(C=C2)C(=N)N.C(CS(=O)(=O)O)O' +DRUG_COLUMNS = ['improve_drug_id', 'chem_name', 'pubchem_id', 'canSMILES', 'InChIKey', 'formula', 'weight'] + + +def _write_tsv(path, rows, columns): + pd.DataFrame(rows, columns=columns).to_csv(path, sep='\t', index=False) + + +# --- pubchem_retrieval cache ------------------------------------------------- + +def test_cache_uses_pubchem_row_and_drops_nsc_names(tmp_path): + fill_in = ['SMI_9', 'nsc-620107', 359323, NCI_SMILES, '', '', ''] + pubchem = ['SMI_4', 'pentamidine isethionate', 359323, PUBCHEM_SMILES, 'XDRYMKDFEDOLFX-UHFFFAOYSA-N', + 'C21H32N4O10S2', '592.7'] + first, second = tmp_path / 'a.tsv', tmp_path / 'b.tsv' + _write_tsv(first, [fill_in], DRUG_COLUMNS) + _write_tsv(second, [pubchem], DRUG_COLUMNS) + + pubchem_retrieval._pubchem_data_cache.clear() + pubchem_retrieval.load_pubchem_cache([str(first), str(second)]) + entry = pubchem_retrieval._pubchem_data_cache['359323'] + + assert entry['SMILES'] == PUBCHEM_SMILES + assert 'nsc-620107' not in entry['synonyms'] + assert 'nsc-620107' not in pubchem_retrieval._pubchem_data_cache + assert pubchem_retrieval._pubchem_data_cache['pentamidine isethionate'] is entry + + +def test_cache_skips_drugs_named_only_by_nsc(tmp_path): + # CID 24202157's only PubChem synonym is 'nsc-630659'. Cached without it, + # the entry had no names, wrote no rows, and the drug was dropped (113 + # NCI60 drugs in the first 2.4 rebuild). It must be fetched live instead. + path = tmp_path / 'a.tsv' + _write_tsv(path, [['SMI_7', 'nsc-630659', 24202157, 'N.[Co+3]', 'KEY', 'F', '1']], DRUG_COLUMNS) + pubchem_retrieval._pubchem_data_cache.clear() + pubchem_retrieval.load_pubchem_cache([str(path)]) + assert '24202157' not in pubchem_retrieval._pubchem_data_cache + + +def test_rerun_keeps_drugs_already_written(tmp_path, monkeypatch): + # 03a reruns after a failed later step find their own earlier output. All + # CIDs are requested again; already-written ones must be kept, not fetched. + out = tmp_path / 'nci60_drugs.tsv' + rows = [['SMI_1', 'pentamidine', '359323', PUBCHEM_SMILES, 'K1', 'F', '1'], + ['SMI_2', 'nsc-630659', '24202157', 'N.[Co+3]', 'K2', 'F', '1']] + _write_tsv(out, rows, DRUG_COLUMNS) + monkeypatch.setattr(pubchem_retrieval, 'fetch_url', + lambda url: pytest.fail(f'already written, should not fetch {url}')) + pubchem_retrieval._pubchem_data_cache.clear() + pubchem_retrieval.update_dataframe_and_write_tsv( + unique_names={'359323', '24202157'}, output_filename=str(out), + ignore_chems=str(tmp_path / 'ignore.txt'), isname=False, + restrict_to_raw_names={'359323', '24202157'}) + kept = pd.read_csv(out, sep='\t', dtype=str) + assert set(kept['improve_drug_id']) == {'SMI_1', 'SMI_2'} + + +def test_cache_entry_does_not_depend_on_file_order(tmp_path): + rows = [['SMI_9', 'pentamidine', 359323, NCI_SMILES, '', '', ''], + ['SMI_4', 'pentamidine isethionate', 359323, PUBCHEM_SMILES, 'KEY', 'F', '1']] + a, b = tmp_path / 'a.tsv', tmp_path / 'b.tsv' + _write_tsv(a, rows[:1], DRUG_COLUMNS) + _write_tsv(b, rows[1:], DRUG_COLUMNS) + smiles = [] + for order in ([a, b], [b, a]): + pubchem_retrieval._pubchem_data_cache.clear() + pubchem_retrieval.load_pubchem_cache([str(p) for p in order]) + smiles.append(pubchem_retrieval._pubchem_data_cache['359323']['SMILES']) + assert smiles == [PUBCHEM_SMILES, PUBCHEM_SMILES] + + +# --- 03a-nci60Drugs: NSC names missing from PubChem --------------------------- + +def test_missing_nsc_reuses_existing_drug_id(): + res = pl.DataFrame({'improve_drug_id': ['SMI_4'], 'chem_name': ['pentamidine isethionate'], + 'pubchem_id': [359323], 'canSMILES': [PUBCHEM_SMILES], 'InChIKey': ['KEY'], + 'formula': ['F'], 'weight': ['1']}) + mdf = pl.DataFrame({'NSC': [620107, 757400], 'NAME': ['Pentamidine isethionate', 'Pentamidine'], + 'CID': [359323, 359323], 'SMILES': [NCI_SMILES, NCI_SMILES]}) + out = nci60_drugs.add_missing_nscs(res, mdf) + + assert set(out['improve_drug_id']) == {'SMI_4'} + assert {'nsc-620107', 'nsc-757400'} <= set(out['chem_name']) + assert set(out['canSMILES']) == {PUBCHEM_SMILES} + + +def test_missing_nsc_handles_float_cids_and_taken_names(): + # pubchem_id read as float (a blank somewhere in the file) must still match, + # and an NCI name already used by another compound must not be reused. + res = pl.DataFrame({'improve_drug_id': ['SMI_4', 'SMI_7'], 'chem_name': ['pentamidine', 'cisplatin'], + 'pubchem_id': [359323.0, 5702198.0], 'canSMILES': ['A', 'B'], + 'InChIKey': ['K1', 'K2'], 'formula': ['F', 'F'], 'weight': ['1', '1']}) + mdf = pl.DataFrame({'NSC': [757400], 'NAME': ['Cisplatin'], 'CID': [359323], 'SMILES': ['X']}) + out = nci60_drugs.add_missing_nscs(res, mdf) + added = out.filter(pl.col('chem_name') == 'nsc-757400') + assert added['improve_drug_id'].to_list() == ['SMI_4'] + assert out.filter(pl.col('chem_name') == 'cisplatin')['improve_drug_id'].to_list() == ['SMI_7'] + + +def test_every_nsc_gets_a_dash_name_from_the_nci_table(): + # PubChem lists the NSC as 'nsc620107' only; 04b needs 'nsc-620107'. + # NSC 99 has a CID not in the drug table and is skipped. + res = pl.DataFrame({'improve_drug_id': ['SMI_4', 'SMI_4'], 'chem_name': ['pentamidine', 'nsc620107'], + 'pubchem_id': [359323, 359323], 'canSMILES': ['A', 'A'], 'InChIKey': ['K', 'K'], + 'formula': ['F', 'F'], 'weight': ['1', '1']}) + nsc_cid = pl.DataFrame({'NSC': [620107, 757400, 99], 'CID': [359323.0, 359323.0, 1.0]}) + out = nci60_drugs.add_nsc_names(res, nsc_cid) + + dash = out.filter(pl.col('chem_name').str.starts_with('nsc-')) + assert sorted(dash['chem_name']) == ['nsc-620107', 'nsc-757400'] + assert set(dash['improve_drug_id']) == {'SMI_4'} + assert set(dash['canSMILES']) == {'A'} + assert nci60_drugs.add_nsc_names(out, nsc_cid).height == out.height # rerun adds nothing + + +def test_cache_keys_are_integer_cids_even_with_blank_ids(tmp_path): + rows = [['SMI_4', 'pentamidine', 359323, PUBCHEM_SMILES, 'KEY', 'F', '1'], + ['SMI_9', 'unknown drug', '', 'CCO', '', '', '']] + path = tmp_path / 'a.tsv' + _write_tsv(path, rows, DRUG_COLUMNS) + pubchem_retrieval._pubchem_data_cache.clear() + pubchem_retrieval.load_pubchem_cache([str(path)]) + assert pubchem_retrieval._pubchem_data_cache['359323']['CID'] == '359323' + assert '359323.0' not in pubchem_retrieval._pubchem_data_cache + + +# --- 04b-nci60-updated: one drug per NSC --------------------------------------- + +def _nsc_inputs(): + drugmapping = pl.DataFrame({'NSC': [694279, 694279, 1], 'improve_drug_id': ['SMI_1', 'SMI_2', 'SMI_3']}) + drugs = pl.DataFrame({'improve_drug_id': ['SMI_1', 'SMI_2', 'SMI_3'], + 'chem_name': ['nsc-694279', 'nsc-694279', 'nsc-1'], + 'pubchem_id': [6712523, 123664, 5]}) + return drugmapping, drugs + + +def test_ambiguous_nsc_resolved_with_nci_cid_table(): + drugmapping, drugs = _nsc_inputs() + nci = lambda: pl.DataFrame({'NSC': ['694279'], 'CID': ['123664']}) + out = nci60_exp.resolve_nsc_ids(drugmapping, drugs, nci) + assert sorted(out.rows()) == [(1, 'SMI_3'), (694279, 'SMI_2')] + + +def test_nsc_under_two_ids_with_the_same_cid_keeps_the_lowest(): + # One chemical listed twice in the drug table: both ids carry NCI's CID. + drugmapping = pl.DataFrame({'NSC': [179940, 179940], 'improve_drug_id': ['SMI_56129', 'SMI_6490']}) + drugs = pl.DataFrame({'improve_drug_id': ['SMI_56129', 'SMI_6490'], 'pubchem_id': [42640, 42640]}) + nci = lambda: pl.DataFrame({'NSC': ['179940'], 'CID': ['42640']}) + out = nci60_exp.resolve_nsc_ids(drugmapping, drugs, nci) + assert out.rows() == [(179940, 'SMI_6490')] + + +def test_ambiguous_nsc_resolved_with_float_cids(): + drugmapping, drugs = _nsc_inputs() + drugs = drugs.with_columns(pl.col('pubchem_id').cast(pl.Float64)) + nci = lambda: pl.DataFrame({'NSC': ['694279'], 'CID': ['123664.0']}) + out = nci60_exp.resolve_nsc_ids(drugmapping, drugs, nci) + assert sorted(out.rows()) == [(1, 'SMI_3'), (694279, 'SMI_2')] + + +def test_unresolvable_nsc_stops_the_build(): + drugmapping, drugs = _nsc_inputs() + nci = lambda: pl.DataFrame({'NSC': ['694279'], 'CID': ['999']}) + with pytest.raises(ValueError, match='more than one drug'): + nci60_exp.resolve_nsc_ids(drugmapping, drugs, nci, fetch_inchikeys=lambda cids: {}) + + +def test_nsc_falls_back_to_inchikey_skeleton(): + # NSC 141993: NCI lists minocycline HCl without stereo (CID 54706380); + # the drug table has the stereo salt record and the free base. + drugmapping = pl.DataFrame({'NSC': [141993, 141993], 'improve_drug_id': ['SMI_16296', 'SMI_57059']}) + drugs = pl.DataFrame({'improve_drug_id': ['SMI_16296', 'SMI_57059'], + 'pubchem_id': [54685925, 54675783], + 'InChIKey': ['WTJXVDPDEQKTCV-VQAITOIOSA-N', 'FFTVPQUHLQBXQZ-KVUCHLLUSA-N']}) + nci = lambda: pl.DataFrame({'NSC': ['141993'], 'CID': ['54706380']}) + keys = lambda cids: {'54706380': 'WTJXVDPDEQKTCV-UHFFFAOYSA-N'} + out = nci60_exp.resolve_nsc_ids(drugmapping, drugs, nci, fetch_inchikeys=keys) + assert out.rows() == [(141993, 'SMI_16296')] + + +def test_unambiguous_nscs_do_not_load_nci_table(): + drugmapping = pl.DataFrame({'NSC': [1], 'improve_drug_id': ['SMI_3']}) + def fail(): + raise AssertionError('NCI table should not be needed') + out = nci60_exp.resolve_nsc_ids(drugmapping, pl.DataFrame({'improve_drug_id': ['SMI_3'], 'pubchem_id': [5]}), fail) + assert out.equals(drugmapping) + + +# --- map_improve_drug_ids: refuse to publish two values for one key ------------ + +EXP_COLUMNS = ['source', 'improve_sample_id', 'improve_drug_id', 'study', 'time', 'time_unit', + 'dose_response_metric', 'dose_response_value'] + + +def _exp_file(tmp_path, rows, gz=False): + path = tmp_path / 'nci60_experiments.tsv' + _write_tsv(path, rows, EXP_COLUMNS) + if gz: + with open(path, 'rb') as f_in, gzip.open(str(path) + '.gz', 'wb') as f_out: + f_out.write(f_in.read()) + os.remove(path) + return str(path) + '.gz' + return str(path) + + +def test_rewrite_raises_when_two_ids_merge_onto_one_key(tmp_path): + path = _exp_file(tmp_path, [['NCI60_24', '46', 'SMI_9', 'NCI60', '48', 'hours', 'fit_auc', '0.8265'], + ['NCI60_24', '46', 'SMI_8', 'NCI60', '48', 'hours', 'fit_auc', '0.7799']]) + mapping = {('nci60', 'SMI_9'): 'SMI_13316', ('nci60', 'SMI_8'): 'SMI_13316'} + with pytest.raises(RuntimeError, match='same key'): + map_improve_drug_ids.rewrite_drugs_file(path, mapping) + # the file is left untouched + with open(path) as f: + assert [r['improve_drug_id'] for r in csv.DictReader(f, delimiter='\t')] == ['SMI_9', 'SMI_8'] + + +def test_rewrite_without_collision_still_maps_ids(tmp_path): + path = _exp_file(tmp_path, [['NCI60_24', '46', 'SMI_9', 'NCI60', '48', 'hours', 'fit_auc', '0.8'], + ['NCI60_24', '47', 'SMI_8', 'NCI60', '48', 'hours', 'fit_auc', '0.7']], gz=True) + mapping = {('nci60', 'SMI_9'): 'SMI_13316', ('nci60', 'SMI_8'): 'SMI_13316'} + map_improve_drug_ids.rewrite_drugs_file(path, mapping) + with gzip.open(path, 'rt') as f: + assert {r['improve_drug_id'] for r in csv.DictReader(f, delimiter='\t')} == {'SMI_13316'} + + +def test_rewrite_raises_on_colliding_combination_rows(tmp_path): + columns = ['improve_drug_1', 'improve_drug_2', 'improve_sample_id', 'time_unit', 'time', + 'drug_combination_metric', 'drug_combination_value', 'source', 'study'] + path = tmp_path / 'mpnst_combinations.tsv' + _write_tsv(path, [['SMI_9', 'SMI_1', '3971', 'days', '25', 'lmm', '-0.1', 'NF', 'PDX'], + ['SMI_8', 'SMI_1', '3971', 'days', '25', 'lmm', '-0.2', 'NF', 'PDX']], columns) + mapping = {('mpnst', 'SMI_9'): 'SMI_5', ('mpnst', 'SMI_8'): 'SMI_5'} + with pytest.raises(RuntimeError, match='same key'): + map_improve_drug_ids.rewrite_drugs_file(str(path), mapping)