Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 112 additions & 31 deletions coderbuild/utils/build_drug_desc.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,39 @@


import argparse
import os
import tempfile

# ---------------------------------------------------------------------------
# Keep multiprocessing's scratch files OFF the bind mount.
#
# This MUST run before multiprocessing creates any worker, so it sits above the
# imports that pull it in.
#
# build_all.py bind-mounts the host's local/ directory at /tmp inside every
# container. Python 3.14 changed the default multiprocessing start method on
# Linux from "fork" to "forkserver", and forkserver opens a Unix domain socket
# under tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS
# file sharing does not support chmod() on a socket there, so worker startup
# died with:
#
# OSError: [Errno 22] Invalid argument: '/tmp/pymp-8ai8jp5x/sock-2c162e8922a7'
#
# Mordred computes descriptors in parallel, so this killed the drugs step --
# and it did so at the very END of build_drugs.sh, after nci60, the PSets and
# the join had all completed, wasting the whole multi-hour drug build on every
# attempt. It is deterministic, so retries do not help.
#
# Pointing TMPDIR at container-local storage fixes it at the source. Verified:
# with /tmp as a bind mount a Pool raises EINVAL; with this set it succeeds.
# Moving the socket is preferred over forcing start_method="fork", which would
# reintroduce the fork-with-threads hazard that motivated the upstream change.
_MP_TMPDIR = os.environ.get("CODERDATA_MP_TMPDIR", "/opt/mp_tmp")
os.makedirs(_MP_TMPDIR, exist_ok=True)
os.environ["TMPDIR"] = _MP_TMPDIR
tempfile.tempdir = _MP_TMPDIR
# ---------------------------------------------------------------------------

from rdkit import Chem
from rdkit.Chem import AllChem
from rdkit.Chem import rdFingerprintGenerator
Expand All @@ -14,6 +47,7 @@
import numpy as np
from mordred import Calculator, descriptors
import multiprocessing
import gzip

# Remove all of the Deprecation warnings. There is a github issue to update the code and 50k lines of warnings in the build log so I'm hiding them for now.
from rdkit import RDLogger
Expand Down Expand Up @@ -73,50 +107,97 @@ def smiles_to_mordred(smiles,nproc=2):

return longtab

## Number of SMILES whose descriptors are held in memory at once.
##
## Mordred returns a WIDE frame of ~1,600 object-dtype descriptors per molecule,
## which pd.melt() then explodes into rows: at 56,890 SMILES that is roughly 92
## MILLION rows, before the merge, the concat and two more full-frame copies in
## the cleaning step. Build v23 was OOM-killed doing exactly that
## ("build_drugs.sh: line 21: Killed") on a 23.4GiB Docker VM.
##
## v19 completed the same step at 46,319 SMILES, so this was never comfortable
## -- a 23% growth in the drug table was enough to cross the limit. Chunking
## makes peak memory independent of how many drugs the release contains, so it
## does not silently creep back up to the ceiling next time.
SMILES_CHUNK = int(os.environ.get("DRUG_DESC_SMILES_CHUNK", "2000"))


def _chunks(seq, n):
for i in range(0, len(seq), n):
yield seq[i:i + n]


def _clean_block(block):
"""Coerce bad values and drop malformed ids for one block of rows."""
# Mordred puts the exception object into a cell when a descriptor cannot be
# computed for a molecule, so str() of the error lands in the table (e.g. the
# "module 'numpy' has no attribute 'float'" AttributeError from np.float being
# removed in NumPy >= 1.24, or messages containing "invalid"/"missing"/"min...").
# Every Mordred descriptor is numeric, so coerce any non-numeric value to
# "NaN" -- this robustly catches ANY such error string, unlike the previous
# fixed substring blocklist which missed the numpy AttributeError. Morgan
# fingerprints are legitimate bit strings and are left untouched.
block['descriptor_value'] = block['descriptor_value'].astype(str)
is_fingerprint = block['structural_descriptor'] == 'morgan fingerprint'
numeric = pd.to_numeric(block['descriptor_value'], errors='coerce')

# Render every Mordred value through float, so the text written does not
# depend on where the chunk boundaries fall. Mordred returns ints for some
# descriptors; whether a given column also picks up a NaN decides if pandas
# holds it as int64 ("0") or float64 ("0.0"). That makes the SAME descriptor
# serialise differently depending on how many molecules share a chunk, which
# would make the output non-deterministic. Fingerprints are bit strings and
# are left untouched.
block.loc[~is_fingerprint, 'descriptor_value'] = (
numeric[~is_fingerprint]
.map(lambda v: "NaN" if pd.isna(v) else repr(float(v)))
)

# Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens.
block['improve_drug_id'] = block['improve_drug_id'].astype(str).str.strip()
return block[block['improve_drug_id'].str.match(r'^SMI_\d+$')]


def main():
parser = argparse.ArgumentParser('Build drug descriptor table')
parser.add_argument('--drugtable',dest='drugtable')
parser.add_argument('--desctable',dest='outtable')

args = parser.parse_args()
args = parser.parse_args()

cores = multiprocessing.cpu_count()
ncors = cores-1
# print("Running with "+str(ncors)+' out of '+str(cores)+' processors')
# print('Adding drug table for '+args.drugtable)
tab = pd.read_csv(args.drugtable,sep='\t')

cansmiles = [a for a in set(tab.canSMILES) if str(a)!='nan']
# isosmiles = list(set(tab.isoSMILES))
morgs = smiles_to_fingerprint(cansmiles)

ids = pd.DataFrame(tab[['improve_drug_id','canSMILES']]).drop_duplicates()
# print("IDS columns:", ids.columns.tolist())
# print("MORGS columns:", morgs.columns.tolist())
id_morg = ids.rename({"canSMILES":'smile'},axis=1).merge(morgs)[['improve_drug_id','structural_descriptor','descriptor_value']]

mords = smiles_to_mordred(cansmiles,nproc=ncors)

id_mord = ids.rename({'canSMILES':'smile'},axis=1).merge(mords)[['improve_drug_id','structural_descriptor','descriptor_value']]

full = pd.concat([id_morg,id_mord],axis=0)

# Convert any values that contain the following strings to NA. I think this covers all of the cases, but add here if more are found.
strings_to_replace = ["min", "max", "invalid", "multiple", "missing"]
pattern = '|'.join(strings_to_replace)
full['descriptor_value'] = full['descriptor_value'].astype(str)
full.loc[full['descriptor_value'].str.contains(pattern, case=False, na=False), 'descriptor_value'] = "NaN"

# Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens.
full['improve_drug_id'] = full['improve_drug_id'].astype(str).str.strip()
mask = full['improve_drug_id'].str.match(r'^SMI_\d+$')
n_dropped = (~mask).sum()
# if n_dropped:
# print(f"Dropping {n_dropped} malformed improve_drug_id rows.")
full = full[mask].copy()


full.to_csv(args.outtable,sep='\t',index=False,compression='gzip')
ids = ids.rename({"canSMILES":'smile'},axis=1)
del tab

cols = ['improve_drug_id','structural_descriptor','descriptor_value']
total = len(cansmiles)
print(f'Building descriptors for {total} SMILES in chunks of {SMILES_CHUNK}',
flush=True)

# Stream each block straight to the gzip output. Morgan fingerprints are
# emitted first, then Mordred descriptors, preserving the original row order.
with gzip.open(args.outtable, 'wt', newline='') as out:
header = True
for label, fn in (('morgan', smiles_to_fingerprint),
('mordred', lambda c: smiles_to_mordred(c, nproc=ncors))):
for i, chunk in enumerate(_chunks(cansmiles, SMILES_CHUNK), start=1):
desc = fn(chunk)
if desc is None or desc.empty:
continue
block = ids.merge(desc)[cols]
block = _clean_block(block)
block.to_csv(out, sep='\t', index=False, header=header)
header = False
done = min(i * SMILES_CHUNK, total)
print(f' [{label}] {done}/{total} SMILES -> {len(block)} rows',
flush=True)
del desc, block

if __name__=='__main__':
main()
8 changes: 4 additions & 4 deletions coderbuild/utils/calc_pdx_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def mrecist(time, volume, min_time=10, return_detail=False):

exdf["volume_change"] = tumor_volume_change(volume)
exdf["average_response"] = avg_response(exdf["volume_change"])
nxdf = {'metric':'mRESCIST','value':None}
nxdf = {'metric':'mrecist','value':None}

df = pd.DataFrame({"time": time, "volume": volume})
df = df[df['time'] >= min_time]
Expand All @@ -131,7 +131,7 @@ def mrecist(time, volume, min_time=10, return_detail=False):
best_average_response = exdf["best_average_response"][0]

mrecist = None
nxdf = {'metric':'mRESCIST','value':mrecist}
nxdf = {'metric':'mrecist','value':mrecist}
# exdf["mRECIST"] = mrecist

if best_response is not None and best_average_response is not None:
Expand All @@ -148,7 +148,7 @@ def mrecist(time, volume, min_time=10, return_detail=False):
mrecist = "CR"

# exdf["mRECIST"] = mrecist
nxdf = {'metric':'mRESCIST','value':mrecist}
nxdf = {'metric':'mrecist','value':mrecist}
if not return_detail:
return nxdf

Expand Down Expand Up @@ -211,7 +211,7 @@ def TGI(contr_volume, treat_volume,time):

# Simulated batch response class object
rtx = {
"metric": "TGI",
"metric": "tgi",
"value": tgi,
'time': np.max(time)
}
Expand Down
105 changes: 80 additions & 25 deletions coderbuild/utils/fit_curve.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
from sklearn.metrics import r2_score
from scipy.optimize import curve_fit
import multiprocessing
from datetime import datetime

def _log(msg):
print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] {msg}', flush=True)

#import uno_data as ud

Expand Down Expand Up @@ -225,26 +229,69 @@ def process_single_drug(name_group_tuple):
metrics = compute_fit_metrics(xdata, ydata, popt, pcov)
return name, metrics

def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None):
cols = ['source', 'improve_sample_id', 'Drug', 'study','time','time_unit']
groups = df.groupby(cols)
count = count or (4484081 - start)
groups = islice(groups, start, start+count)
def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None,
workers=None, chunk_size=50000):
cols = ['source', 'improve_sample_id', 'Drug', 'study', 'time', 'time_unit']
cores = multiprocessing.cpu_count()
poolsize = round(cores-1)
print('we have '+str(cores)+' cores and '+str(poolsize)+' threads')
with multiprocessing.Pool(processes=poolsize) as pool:
results = pool.map(process_single_drug, groups)

with open(f'{fname}.{start}', 'w') as f:
header = None
for result in results:
name, metrics = result
if header is None:
header = cols + metrics.index.tolist()
print(sep.join(header), file=f)
print(sep.join(str(n) for n in name), end=sep, file=f)
print(sep.join(f'{x:.4g}' for x in metrics), file=f)
poolsize = workers if workers is not None else max(1, round(cores * 3 / 4))

# Stream lazily — never materialise more than one chunk at a time so
# fork() workers only copy chunk-sized data, not the entire groupby result.
groups_iter = islice(df.groupby(cols), start, start + count if count else None)

output_file = f'{fname}.{start}'
if os.path.exists(output_file):
os.remove(output_file)
header_written = False
groups_done = 0
chunks_skipped = 0
chunk_idx = 0

_log(f'fitting {fname}: {cores} cores, {poolsize} workers, chunk_size={chunk_size}')

while True:
chunk = list(islice(groups_iter, chunk_size))
if not chunk:
break
chunk_idx += 1
chunk_workers = poolsize

for attempt in range(1, 4):
try:
_log(f' chunk {chunk_idx} '
f'(groups {groups_done + 1}-{groups_done + len(chunk)}, '
f'attempt {attempt}/3, {chunk_workers} workers)')
with multiprocessing.Pool(processes=chunk_workers) as pool:
results = pool.map(process_single_drug, chunk)

with open(output_file, 'a') as f:
for result in results:
name, metrics = result
if not header_written:
header = cols + metrics.index.tolist()
print(sep.join(header), file=f)
header_written = True
print(sep.join(str(n) for n in name), end=sep, file=f)
print(sep.join(f'{x:.4g}' for x in metrics), file=f)

groups_done += len(chunk)
_log(f' chunk {chunk_idx} complete ({groups_done} groups so far)')
break

except Exception as e:
_log(f' chunk {chunk_idx} failed '
f'(attempt {attempt}/3): {type(e).__name__}: {e}')
if attempt < 3:
chunk_workers = max(1, chunk_workers // 2)
_log(f' retrying with {chunk_workers} workers')
else:
_log(f' WARNING: chunk {chunk_idx} skipped after 3 failed attempts')
groups_done += len(chunk)
chunks_skipped += 1

_log(f'fitting complete: {fname} — {groups_done} groups, '
f'{chunks_skipped} chunk(s) skipped, output: {output_file}')
return chunks_skipped


def main():
Expand All @@ -261,9 +308,11 @@ def main():
parser.add_argument('--output', help='prefix of output file')
parser.add_argument('--beataml', action='store_true', help='Include this if for BeatAML')
parser.add_argument('--debug',action='store_true',default=False)

parser.add_argument('--workers', type=int, default=None, help='Number of parallel worker processes (default: 3/4 of available CPUs)')
parser.add_argument('--chunk_size', type=int, default=50000, help='Max drug-groups per pool.map call (default: 50000; reduce for very large datasets)')

args = parser.parse_args()
print(args.input)
_log(f'loading {args.input}')
df_all = pd.read_table(args.input)
if args.debug:
df_all = df_all.iloc[0:1000000]
Expand All @@ -274,12 +323,18 @@ def main():
df_all.DOSE = -1.0 * np.log10(df_all.DOSE/1000000.0)
##need data to be between 0 and 1, not 0 and 100
df_all.GROWTH=df_all.GROWTH/100.00
print(df_all.head)
_log(f'loaded {len(df_all)} rows — starting curve fitting')
fname = args.output or 'combined_single_response_agg'
process_df_part(df_all, fname, beataml=args.beataml)#, start=args.start, count=args.count)

# if args.beataml == False:
skipped = process_df_part(df_all, fname, beataml=args.beataml, workers=args.workers,
chunk_size=args.chunk_size)
# Free the large input DataFrame before the schema-formatting step
del df_all
import gc; gc.collect()
_log(f'done: {fname}')
format_coderd_schema(fname+'.0')
if skipped:
_log(f'ERROR: {skipped} chunk(s) permanently skipped — output is incomplete')
sys.exit(1)

if __name__ == '__main__':
main()
Loading