Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 90 additions & 69 deletions coderbuild/broad_sanger/03a-nci60Drugs.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,27 @@
import argparse
import pubchem_retrieval as pr
import random as rand
from urllib import request
import download_utils
import io

_BROWSER_UA = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)

def _fetch(url: str) -> bytes:
"""Download url with a browser User-Agent (NCI wiki blocks Python's default)."""
req = request.Request(url, headers={"User-Agent": _BROWSER_UA})
with request.urlopen(req) as resp:
return resp.read()

def _read_csv_url(url: str, **kwargs) -> pl.DataFrame:
return pl.read_csv(io.BytesIO(_fetch(url)), **kwargs)

def _retrieve_url(url: str, dest: str) -> None:
"""Download url to dest, resuming an interrupted transfer.
"""Download url to dest with retries, resuming an interrupted transfer.

This also fetches the ~346MB DOSERESP.zip, and _fetch() reads the whole
body into memory in one call -- the same shape that truncated with
IncompleteRead in build v32. Small CSVs still go through _fetch().
NCI's wiki truncates downloads now and then (DOSERESP.zip in build v32,
nsc_chemical_names.csv in the first 2.4 rebuild), so every file goes
through download_utils rather than a single urlopen().read(). A browser
User-Agent is sent because the wiki blocks Python's default.
"""
download_utils.retrieve_url(url, dest, headers={"User-Agent": _BROWSER_UA})

def _read_csv_url(url: str, dest: str, **kwargs) -> pl.DataFrame:
_retrieve_url(url, dest)
return pl.read_csv(dest, **kwargs)

##drug files
smi_strings='https://wiki.nci.nih.gov/download/attachments/155844992/nsc_smiles.csv?version=1&modificationDate=1710381820000&api=v2&download=true'
#oct 2024
Expand Down Expand Up @@ -84,10 +78,10 @@ def main():
'InChIKey':[],'formula':[],'weight':[],'pubchem_id':[]}

print('Downloading NSC identifiers for nci60 data')
names = _read_csv_url(chemnames, ignore_errors=True)
names = _read_csv_url(chemnames, 'nsc_chemical_names.csv', ignore_errors=True)
#castab = pd.read_csv(cas)
pubchems = _read_csv_url(pc_ids)
smiles = _read_csv_url(smi_strings)
pubchems = _read_csv_url(pc_ids, 'nsc_sid_cid.csv')
smiles = _read_csv_url(smi_strings, 'nsc_smiles.csv')

print('Getting experimental data to filter drugs')
if not os.path.exists('DOSERESP.csv'):
Expand All @@ -100,6 +94,7 @@ def main():
raise RuntimeError(f"unzip of doseresp.zip failed (exit status {_rc}).")
dose_resp = pl.read_csv("DOSERESP.csv",quote_char='"',infer_schema_length=10000000,ignore_errors=True)
pubchems = pubchems.filter(pl.col('NSC').is_in(dose_resp['NSC']))
nsc_cid = pubchems # every NSC with dose data
smiles = smiles.filter(pl.col("NSC").is_in(dose_resp['NSC'])
)
##first retreive pubchem data
Expand All @@ -109,16 +104,11 @@ def main():
else:
arr = set(pubchems['CID'])

##first filter to see if there are structures/drugs in teh data already. i dont think this does much.
if os.path.exists(opts.output):
curdrugs = pl.read_csv(opts.output,separator='\t')
smiles_notnull = smiles.filter(pl.col('SMILES').is_not_null())
upper=[a.upper() for a in smiles_notnull['SMILES']]
smiles_upper = pl.DataFrame({'NSC':smiles_notnull['NSC'],'upper':upper})
ssmiles = smiles_upper.filter(~pl.col('upper').is_in(curdrugs['canSMILES']))
pubchems = pubchems.filter(pl.col('NSC').is_in(ssmiles['NSC']))
arr = set(pubchems['CID'])

# A rerun after a failure finds this script's earlier output. Every CID is
# still requested: update_dataframe_and_write_tsv skips CIDs already in the
# output, and it also limits the output to the requested CIDs, so asking
# for fewer (as this used to on a rerun) deleted drugs it had already
# written -- about 560 NCI60 drugs in the second 2.4 rebuild.
print("Querying pubchem from CIDs")
pr.update_dataframe_and_write_tsv(
unique_names=arr,
Expand Down Expand Up @@ -149,49 +139,80 @@ def main():
##add in improve ids, nsc name and structure for all.
mdf = msmi.join(names,on='NSC',how='left').join(pubchems,on='NSC',how='left')

max_imp = max(int(a.split('_')[1]) for a in res['improve_drug_id'])

smicount=len(set(mdf['SMILES'])) ## unique smiles in our missing data frame
newdf = pl.DataFrame(
{
"improve_drug_id": ["SMI_"+str(a) for a in range(max_imp+1,max_imp+1+smicount,1)],
'canSMILES': [a for a in set(mdf['SMILES'])],
# 'isoSMILES': [a for a in set(mdf['SMILES'])],
'InChIKey': [None for a in range(smicount)],
'formula': [None for a in range(smicount)],
'weight': [None for a in range(smicount)]
}
)
res = add_missing_nscs(res, mdf)
res = add_nsc_names(res, nsc_cid)
res.write_csv(opts.output,separator='\t')

#create updated nsc ids and names
namedf = pl.DataFrame(
{
"nscid": ['nsc-'+str(a) for a in mdf['NSC']],
'lower_name': [a if a is None else str(a).lower() for a in mdf['NAME']],
'canSMILES': list(mdf['SMILES']),
'pubchem_id': list(mdf['CID'])
}

def _cid_str(column):
"""A CID column as text without a float suffix ('359323.0' -> '359323')."""
return pl.col(column).cast(pl.Utf8).str.replace(r'\.0$', '')


def _drug_per_cid(res):
"""Return res with text columns, and one drug row (id + structure) per CID."""
str_cols = ['canSMILES', 'InChIKey', 'formula', 'weight']
res = res.with_columns([pl.col(c).cast(pl.Utf8) for c in str_cols] + [_cid_str('pubchem_id')])
# One structure per CID: the lowest SMI id if a CID somehow has several.
known = (
res.with_columns(pl.col('improve_drug_id').str.extract(r'SMI_(\d+)', 1)
.cast(pl.Int64).alias('_n'))
.sort('_n')
.unique(subset=['pubchem_id'], keep='first', maintain_order=True)
.select(['pubchem_id', 'improve_drug_id', 'canSMILES', 'InChIKey', 'formula', 'weight'])
)
#merge and melt
merged = pl.concat([mdf,namedf],how='horizontal').select(['SMILES','pubchem_id','nscid','lower_name'])
melted = merged.melt(id_vars=['SMILES','pubchem_id'],value_vars=['nscid','lower_name']).select(['SMILES','pubchem_id','value']).unique()
melted.columns = ['canSMILES','pubchem_id','chem_name']

if newdf.shape[0] > 0:
res = res.with_columns([
pl.col("InChIKey").cast(pl.Utf8),
pl.col("formula").cast(pl.Utf8),
pl.col("weight").cast(pl.Utf8)
])
newdf = newdf.with_columns([
pl.col("InChIKey").cast(pl.Utf8),
pl.col("formula").cast(pl.Utf8),
pl.col("weight").cast(pl.Utf8)
])

newdf = newdf.join(melted, on='canSMILES', how='inner').select(res.columns)
res = pl.concat([res, newdf], how='vertical')
res.write_csv(opts.output,separator='\t')
return res, known


def add_nsc_names(res, nsc_cid):
"""
Give every NSC an 'nsc-<id>' name under its CID's drug id.

04b-nci60-updated.py joins dose responses to drugs on 'nsc-<id>' only.
PubChem mostly lists NSCs as 'nsc123', and the drug cache no longer
carries 'nsc-' names between builds, so without this most NCI60 drugs
had no name 04b could match. NCI's NSC->CID table is used rather than
PubChem synonyms, which can list an NSC under a related compound.
"""
res, known = _drug_per_cid(res)
names = (nsc_cid.select([_cid_str('CID').alias('pubchem_id'),
pl.format('nsc-{}', _cid_str('NSC')).alias('chem_name')])
.drop_nulls().unique())
added = names.join(known, on='pubchem_id', how='inner').select(res.columns)
out = pl.concat([res, added], how='vertical').unique(maintain_order=True)
print(f'NSC names: {added.height} of {names.height} NSCs attached to a drug id')
return out


def add_missing_nscs(res, mdf):
"""
Attach NSC ids that PubChem does not list as synonyms to their drug.

Every NSC in mdf maps (via NCI's NSC->CID table) to a CID already in res,
so its 'nsc-<id>' name and NCI name are added under that CID's existing
improve_drug_id and structure. Minting a new id here instead gave one
chemical two ids; when both NSCs had dose data the build fit two curves
that the stable-id step later collapsed onto one id (issue #512).
"""
res, known = _drug_per_cid(res)
nsc_names = mdf.select([_cid_str('CID').alias('pubchem_id'),
pl.format('nsc-{}', pl.col('NSC')).alias('chem_name')])
# NCI's own compound names are only added when no other drug already uses
# the name, so a PSet drug of that name cannot be matched to this CID.
taken = res.select(['chem_name', 'pubchem_id']).unique()
nci_names = (mdf.select([_cid_str('CID').alias('pubchem_id'),
pl.col('NAME').cast(pl.Utf8).str.to_lowercase().alias('chem_name')])
.join(taken.filter(pl.col('pubchem_id').is_not_null()).rename({'pubchem_id': '_other'}),
on='chem_name', how='left')
.filter(pl.col('_other').is_null() | (pl.col('_other') == pl.col('pubchem_id')))
.drop('_other'))
names = pl.concat([nsc_names, nci_names]).drop_nulls().unique()
added = names.join(known, on='pubchem_id', how='inner').select(res.columns)
unmatched = names.join(known, on='pubchem_id', how='anti')
if unmatched.height:
print(f'{unmatched.height} NSC names had no CID in the drug table and were skipped')
print(f'Attached {added.height} NSC names to existing drug ids')
return pl.concat([res, added], how='vertical').unique(maintain_order=True)

if __name__=='__main__':
main()
25 changes: 25 additions & 0 deletions coderbuild/broad_sanger/04a-drugResponseData.R
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,30 @@ source("pinned_psets.R")
source("retry_utils.R")

all.dsets<-PharmacoGx::availablePSets()

#' one_drug_per_name
#' A drug name can sit under more than one improve_drug_id (e.g. a synonym
#' shared by two records of one chemical). Joining on it copied every
#' experiment to each id, which later surfaced as duplicate or copied drug
#' responses (issue #512). Keep the lowest SMI id for each name.
one_drug_per_name<-function(d.map,studyName){
d.map<-d.map|>
mutate(smi_num=as.numeric(sub('SMI_','',improve_drug_id)))
multi<-d.map|>
group_by(chem_name)|>
summarize(n=n_distinct(improve_drug_id),.groups='drop')|>
subset(n>1)
if(nrow(multi)>0)
print(paste('WARNING:',nrow(multi),'drug names in',studyName,
'match more than one drug id; keeping the lowest id for:',
paste(head(multi$chem_name,20),collapse=', ')))
d.map|>
group_by(chem_name)|>
dplyr::filter(smi_num==min(smi_num))|>
ungroup()|>
dplyr::select(-smi_num)
}

##first define a generic dose response function


Expand Down Expand Up @@ -52,6 +76,7 @@ getDoseRespData<-function(dset,studyName,improve_samples,drug.map){
###reduced drug and sample maps
new.d.map<-subset(drug.map,chem_name%in%mapping$treatmentid)|>
mutate(chem_name=tolower(chem_name))
new.d.map<-one_drug_per_name(new.d.map,studyName)
new.s.map<-subset(improve_samples,other_names%in%mapping$sampleid)


Expand Down
92 changes: 92 additions & 0 deletions coderbuild/broad_sanger/04b-nci60-updated.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import argparse
import os
import re
import json
from urllib import request
import download_utils

Expand Down Expand Up @@ -35,6 +36,96 @@ def _retrieve_url(url: str, dest: str) -> None:
#may 2025
oneconc_data = 'https://wiki.nci.nih.gov/download/attachments/147193864/ONECONC.zip?version=19&modificationDate=1775183401634&api=v2'

#NCI's NSC -> PubChem CID table, the same release 03a-nci60Drugs.py uses
pc_ids = 'https://wiki.nci.nih.gov/download/attachments/155844992/nsc_sid_cid.csv?version=5&modificationDate=1735931099618&api=v2'


def _id_str(column):
"""An id column as text without a float suffix ('359323.0' -> '359323')."""
return pl.col(column).cast(pl.Utf8).str.replace(r'\.0$', '')


def _load_nsc_cids():
"""Return NCI's NSC -> CID table as a DataFrame with str columns."""
download_utils.retrieve_url(pc_ids, 'nsc_sid_cid.csv', headers={"User-Agent": _BROWSER_UA})
tab = pl.read_csv('nsc_sid_cid.csv', ignore_errors=True)
return tab.select([_id_str('NSC'), _id_str('CID')]).drop_nulls().unique()


def _pubchem_inchikeys(cids):
"""Return {CID: InChIKey} for the given CIDs from PubChem."""
out = {}
for i in range(0, len(cids), 100):
batch = ','.join(cids[i:i + 100])
url = f'https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/{batch}/property/InChIKey/JSON'
with request.urlopen(request.Request(url, headers={"User-Agent": _BROWSER_UA})) as resp:
for prop in json.load(resp)['PropertyTable']['Properties']:
out[str(prop['CID'])] = prop.get('InChIKey')
return out


def resolve_nsc_ids(drugmapping, drugs, load_nsc_cids, fetch_inchikeys=_pubchem_inchikeys):
"""
Keep one improve_drug_id per NSC.

An 'nsc-<id>' name can sit under more than one drug id when PubChem lists
the NSC as a synonym of a related compound. Joining on it copied that
NSC's dose response to every such drug. Ambiguous NSCs keep the id whose
PubChem CID matches NCI's own NSC->CID table (the lowest SMI id if several
share that CID). If NCI's CID is none of the
candidates (e.g. it is the non-stereo record of a salt), the candidate
with the same InChIKey skeleton (first block) is kept. Any NSC still
ambiguous stops the build.
"""
counts = drugmapping.group_by('NSC').agg(pl.col('improve_drug_id').n_unique().alias('n'))
ambiguous = counts.filter(pl.col('n') > 1)['NSC'].to_list()
if len(ambiguous) == 0:
return drugmapping
nsc_cid = load_nsc_cids().select([_id_str('NSC'), _id_str('CID')]).drop_nulls().unique()
id_cid = drugs.select([pl.col('improve_drug_id'), _id_str('pubchem_id')]).drop_nulls().unique()
amb = (drugmapping.filter(pl.col('NSC').is_in(ambiguous))
.with_columns(_id_str('NSC'))
.join(id_cid, on='improve_drug_id', how='left')
.join(nsc_cid.rename({'CID': 'nci_cid'}), on='NSC', how='left'))
# Ids sharing NCI's CID are one chemical listed twice (the stable-id step
# later merges them by structure), so the lowest SMI id is kept.
keep = (amb.filter(pl.col('pubchem_id') == pl.col('nci_cid'))
.select(['NSC', 'improve_drug_id']).unique()
.with_columns(pl.col('improve_drug_id').str.extract(r'SMI_(\d+)', 1)
.cast(pl.Int64).alias('_n'))
.sort(['NSC', '_n']).unique(subset=['NSC'], keep='first').drop('_n'))
per = keep.group_by('NSC').agg(pl.col('improve_drug_id').n_unique().alias('n'))
unresolved = {str(a) for a in ambiguous} - set(per.filter(pl.col('n') == 1)['NSC'])

if unresolved:
keep = keep.filter(~pl.col('NSC').is_in(list(unresolved)))
rest = amb.filter(pl.col('NSC').is_in(list(unresolved)))
nci_keys = fetch_inchikeys(sorted(set(rest['nci_cid'].drop_nulls().to_list())))
id_keys = {}
if 'InChIKey' in drugs.columns:
for drug_id, key in drugs.select(['improve_drug_id', 'InChIKey']).drop_nulls().unique().rows():
id_keys.setdefault(drug_id, key)
picked = []
for nsc in sorted(unresolved):
cand = rest.filter(pl.col('NSC') == nsc)
nci_key = nci_keys.get(cand['nci_cid'][0]) if cand['nci_cid'][0] is not None else None
skeleton = nci_key.split('-')[0] if nci_key else None
matches = {d for d in cand['improve_drug_id'].to_list()
if skeleton and (id_keys.get(d) or '').split('-')[0] == skeleton}
if len(matches) == 1:
picked.append((nsc, matches.pop()))
if picked:
keep = pl.concat([keep, pl.DataFrame(picked, schema=['NSC', 'improve_drug_id'], orient='row')])
unresolved -= {nsc for nsc, _ in picked}
if unresolved:
raise ValueError(
f"{len(unresolved)} NSC id(s) map to more than one drug and NCI's NSC->CID table "
f"does not pick one: {sorted(unresolved)[:20]}")
print(f'Resolved {len(ambiguous)} NSC ids that matched more than one drug id')
keep = keep.with_columns(pl.col('NSC').cast(drugmapping['NSC'].dtype))
return pl.concat([drugmapping.filter(~pl.col('NSC').is_in(ambiguous)), keep])


def main():

parser = argparse.ArgumentParser()
Expand Down Expand Up @@ -70,6 +161,7 @@ def main():

drugmapping = drugmapping.join(drugs,on='chem_name')[['NSC','improve_drug_id']]
drugmapping = drugmapping.unique()
drugmapping = resolve_nsc_ids(drugmapping, drugs, _load_nsc_cids)

###update sample mapping
on = samples[['other_names','improve_sample_id']].rename({'other_names': 'common_name'})
Expand Down
Loading