diff --git a/coderbuild/utils/build_drug_desc.py b/coderbuild/utils/build_drug_desc.py index bbeeb81e..3a2c63ba 100644 --- a/coderbuild/utils/build_drug_desc.py +++ b/coderbuild/utils/build_drug_desc.py @@ -6,6 +6,39 @@ import argparse +import os +import tempfile + +# --------------------------------------------------------------------------- +# Keep multiprocessing's scratch files OFF the bind mount. +# +# This MUST run before multiprocessing creates any worker, so it sits above the +# imports that pull it in. +# +# build_all.py bind-mounts the host's local/ directory at /tmp inside every +# container. Python 3.14 changed the default multiprocessing start method on +# Linux from "fork" to "forkserver", and forkserver opens a Unix domain socket +# under tempfile.gettempdir() -- i.e. /tmp, i.e. the bind mount. Docker's macOS +# file sharing does not support chmod() on a socket there, so worker startup +# died with: +# +# OSError: [Errno 22] Invalid argument: '/tmp/pymp-8ai8jp5x/sock-2c162e8922a7' +# +# Mordred computes descriptors in parallel, so this killed the drugs step -- +# and it did so at the very END of build_drugs.sh, after nci60, the PSets and +# the join had all completed, wasting the whole multi-hour drug build on every +# attempt. It is deterministic, so retries do not help. +# +# Pointing TMPDIR at container-local storage fixes it at the source. Verified: +# with /tmp as a bind mount a Pool raises EINVAL; with this set it succeeds. +# Moving the socket is preferred over forcing start_method="fork", which would +# reintroduce the fork-with-threads hazard that motivated the upstream change. +_MP_TMPDIR = os.environ.get("CODERDATA_MP_TMPDIR", "/opt/mp_tmp") +os.makedirs(_MP_TMPDIR, exist_ok=True) +os.environ["TMPDIR"] = _MP_TMPDIR +tempfile.tempdir = _MP_TMPDIR +# --------------------------------------------------------------------------- + from rdkit import Chem from rdkit.Chem import AllChem from rdkit.Chem import rdFingerprintGenerator @@ -14,6 +47,7 @@ import numpy as np from mordred import Calculator, descriptors import multiprocessing +import gzip # Remove all of the Deprecation warnings. There is a github issue to update the code and 50k lines of warnings in the build log so I'm hiding them for now. from rdkit import RDLogger @@ -73,50 +107,97 @@ def smiles_to_mordred(smiles,nproc=2): return longtab +## Number of SMILES whose descriptors are held in memory at once. +## +## Mordred returns a WIDE frame of ~1,600 object-dtype descriptors per molecule, +## which pd.melt() then explodes into rows: at 56,890 SMILES that is roughly 92 +## MILLION rows, before the merge, the concat and two more full-frame copies in +## the cleaning step. Build v23 was OOM-killed doing exactly that +## ("build_drugs.sh: line 21: Killed") on a 23.4GiB Docker VM. +## +## v19 completed the same step at 46,319 SMILES, so this was never comfortable +## -- a 23% growth in the drug table was enough to cross the limit. Chunking +## makes peak memory independent of how many drugs the release contains, so it +## does not silently creep back up to the ceiling next time. +SMILES_CHUNK = int(os.environ.get("DRUG_DESC_SMILES_CHUNK", "2000")) + + +def _chunks(seq, n): + for i in range(0, len(seq), n): + yield seq[i:i + n] + + +def _clean_block(block): + """Coerce bad values and drop malformed ids for one block of rows.""" + # Mordred puts the exception object into a cell when a descriptor cannot be + # computed for a molecule, so str() of the error lands in the table (e.g. the + # "module 'numpy' has no attribute 'float'" AttributeError from np.float being + # removed in NumPy >= 1.24, or messages containing "invalid"/"missing"/"min..."). + # Every Mordred descriptor is numeric, so coerce any non-numeric value to + # "NaN" -- this robustly catches ANY such error string, unlike the previous + # fixed substring blocklist which missed the numpy AttributeError. Morgan + # fingerprints are legitimate bit strings and are left untouched. + block['descriptor_value'] = block['descriptor_value'].astype(str) + is_fingerprint = block['structural_descriptor'] == 'morgan fingerprint' + numeric = pd.to_numeric(block['descriptor_value'], errors='coerce') + + # Render every Mordred value through float, so the text written does not + # depend on where the chunk boundaries fall. Mordred returns ints for some + # descriptors; whether a given column also picks up a NaN decides if pandas + # holds it as int64 ("0") or float64 ("0.0"). That makes the SAME descriptor + # serialise differently depending on how many molecules share a chunk, which + # would make the output non-deterministic. Fingerprints are bit strings and + # are left untouched. + block.loc[~is_fingerprint, 'descriptor_value'] = ( + numeric[~is_fingerprint] + .map(lambda v: "NaN" if pd.isna(v) else repr(float(v))) + ) + + # Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens. + block['improve_drug_id'] = block['improve_drug_id'].astype(str).str.strip() + return block[block['improve_drug_id'].str.match(r'^SMI_\d+$')] + + def main(): parser = argparse.ArgumentParser('Build drug descriptor table') parser.add_argument('--drugtable',dest='drugtable') parser.add_argument('--desctable',dest='outtable') - args = parser.parse_args() + args = parser.parse_args() cores = multiprocessing.cpu_count() ncors = cores-1 - # print("Running with "+str(ncors)+' out of '+str(cores)+' processors') - # print('Adding drug table for '+args.drugtable) tab = pd.read_csv(args.drugtable,sep='\t') cansmiles = [a for a in set(tab.canSMILES) if str(a)!='nan'] - # isosmiles = list(set(tab.isoSMILES)) - morgs = smiles_to_fingerprint(cansmiles) ids = pd.DataFrame(tab[['improve_drug_id','canSMILES']]).drop_duplicates() - # print("IDS columns:", ids.columns.tolist()) - # print("MORGS columns:", morgs.columns.tolist()) - id_morg = ids.rename({"canSMILES":'smile'},axis=1).merge(morgs)[['improve_drug_id','structural_descriptor','descriptor_value']] - - mords = smiles_to_mordred(cansmiles,nproc=ncors) - - id_mord = ids.rename({'canSMILES':'smile'},axis=1).merge(mords)[['improve_drug_id','structural_descriptor','descriptor_value']] - - full = pd.concat([id_morg,id_mord],axis=0) - - # Convert any values that contain the following strings to NA. I think this covers all of the cases, but add here if more are found. - strings_to_replace = ["min", "max", "invalid", "multiple", "missing"] - pattern = '|'.join(strings_to_replace) - full['descriptor_value'] = full['descriptor_value'].astype(str) - full.loc[full['descriptor_value'].str.contains(pattern, case=False, na=False), 'descriptor_value'] = "NaN" - - # Remove Data that is incorrectly written by mordred or rdkit. - Very rare bug, but it happens. - full['improve_drug_id'] = full['improve_drug_id'].astype(str).str.strip() - mask = full['improve_drug_id'].str.match(r'^SMI_\d+$') - n_dropped = (~mask).sum() - # if n_dropped: - # print(f"Dropping {n_dropped} malformed improve_drug_id rows.") - full = full[mask].copy() - - - full.to_csv(args.outtable,sep='\t',index=False,compression='gzip') + ids = ids.rename({"canSMILES":'smile'},axis=1) + del tab + + cols = ['improve_drug_id','structural_descriptor','descriptor_value'] + total = len(cansmiles) + print(f'Building descriptors for {total} SMILES in chunks of {SMILES_CHUNK}', + flush=True) + + # Stream each block straight to the gzip output. Morgan fingerprints are + # emitted first, then Mordred descriptors, preserving the original row order. + with gzip.open(args.outtable, 'wt', newline='') as out: + header = True + for label, fn in (('morgan', smiles_to_fingerprint), + ('mordred', lambda c: smiles_to_mordred(c, nproc=ncors))): + for i, chunk in enumerate(_chunks(cansmiles, SMILES_CHUNK), start=1): + desc = fn(chunk) + if desc is None or desc.empty: + continue + block = ids.merge(desc)[cols] + block = _clean_block(block) + block.to_csv(out, sep='\t', index=False, header=header) + header = False + done = min(i * SMILES_CHUNK, total) + print(f' [{label}] {done}/{total} SMILES -> {len(block)} rows', + flush=True) + del desc, block if __name__=='__main__': main() diff --git a/coderbuild/utils/calc_pdx_metrics.py b/coderbuild/utils/calc_pdx_metrics.py index 347e1f78..ec7eb54d 100755 --- a/coderbuild/utils/calc_pdx_metrics.py +++ b/coderbuild/utils/calc_pdx_metrics.py @@ -111,7 +111,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): exdf["volume_change"] = tumor_volume_change(volume) exdf["average_response"] = avg_response(exdf["volume_change"]) - nxdf = {'metric':'mRESCIST','value':None} + nxdf = {'metric':'mrecist','value':None} df = pd.DataFrame({"time": time, "volume": volume}) df = df[df['time'] >= min_time] @@ -131,7 +131,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): best_average_response = exdf["best_average_response"][0] mrecist = None - nxdf = {'metric':'mRESCIST','value':mrecist} + nxdf = {'metric':'mrecist','value':mrecist} # exdf["mRECIST"] = mrecist if best_response is not None and best_average_response is not None: @@ -148,7 +148,7 @@ def mrecist(time, volume, min_time=10, return_detail=False): mrecist = "CR" # exdf["mRECIST"] = mrecist - nxdf = {'metric':'mRESCIST','value':mrecist} + nxdf = {'metric':'mrecist','value':mrecist} if not return_detail: return nxdf @@ -211,7 +211,7 @@ def TGI(contr_volume, treat_volume,time): # Simulated batch response class object rtx = { - "metric": "TGI", + "metric": "tgi", "value": tgi, 'time': np.max(time) } diff --git a/coderbuild/utils/fit_curve.py b/coderbuild/utils/fit_curve.py index c712cc3d..f050e268 100755 --- a/coderbuild/utils/fit_curve.py +++ b/coderbuild/utils/fit_curve.py @@ -17,6 +17,10 @@ from sklearn.metrics import r2_score from scipy.optimize import curve_fit import multiprocessing +from datetime import datetime + +def _log(msg): + print(f'[{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}] {msg}', flush=True) #import uno_data as ud @@ -225,26 +229,69 @@ def process_single_drug(name_group_tuple): metrics = compute_fit_metrics(xdata, ydata, popt, pcov) return name, metrics -def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None): - cols = ['source', 'improve_sample_id', 'Drug', 'study','time','time_unit'] - groups = df.groupby(cols) - count = count or (4484081 - start) - groups = islice(groups, start, start+count) +def process_df_part(df, fname, beataml=False, sep='\t', start=0, count=None, + workers=None, chunk_size=50000): + cols = ['source', 'improve_sample_id', 'Drug', 'study', 'time', 'time_unit'] cores = multiprocessing.cpu_count() - poolsize = round(cores-1) - print('we have '+str(cores)+' cores and '+str(poolsize)+' threads') - with multiprocessing.Pool(processes=poolsize) as pool: - results = pool.map(process_single_drug, groups) - - with open(f'{fname}.{start}', 'w') as f: - header = None - for result in results: - name, metrics = result - if header is None: - header = cols + metrics.index.tolist() - print(sep.join(header), file=f) - print(sep.join(str(n) for n in name), end=sep, file=f) - print(sep.join(f'{x:.4g}' for x in metrics), file=f) + poolsize = workers if workers is not None else max(1, round(cores * 3 / 4)) + + # Stream lazily — never materialise more than one chunk at a time so + # fork() workers only copy chunk-sized data, not the entire groupby result. + groups_iter = islice(df.groupby(cols), start, start + count if count else None) + + output_file = f'{fname}.{start}' + if os.path.exists(output_file): + os.remove(output_file) + header_written = False + groups_done = 0 + chunks_skipped = 0 + chunk_idx = 0 + + _log(f'fitting {fname}: {cores} cores, {poolsize} workers, chunk_size={chunk_size}') + + while True: + chunk = list(islice(groups_iter, chunk_size)) + if not chunk: + break + chunk_idx += 1 + chunk_workers = poolsize + + for attempt in range(1, 4): + try: + _log(f' chunk {chunk_idx} ' + f'(groups {groups_done + 1}-{groups_done + len(chunk)}, ' + f'attempt {attempt}/3, {chunk_workers} workers)') + with multiprocessing.Pool(processes=chunk_workers) as pool: + results = pool.map(process_single_drug, chunk) + + with open(output_file, 'a') as f: + for result in results: + name, metrics = result + if not header_written: + header = cols + metrics.index.tolist() + print(sep.join(header), file=f) + header_written = True + print(sep.join(str(n) for n in name), end=sep, file=f) + print(sep.join(f'{x:.4g}' for x in metrics), file=f) + + groups_done += len(chunk) + _log(f' chunk {chunk_idx} complete ({groups_done} groups so far)') + break + + except Exception as e: + _log(f' chunk {chunk_idx} failed ' + f'(attempt {attempt}/3): {type(e).__name__}: {e}') + if attempt < 3: + chunk_workers = max(1, chunk_workers // 2) + _log(f' retrying with {chunk_workers} workers') + else: + _log(f' WARNING: chunk {chunk_idx} skipped after 3 failed attempts') + groups_done += len(chunk) + chunks_skipped += 1 + + _log(f'fitting complete: {fname} — {groups_done} groups, ' + f'{chunks_skipped} chunk(s) skipped, output: {output_file}') + return chunks_skipped def main(): @@ -261,9 +308,11 @@ def main(): parser.add_argument('--output', help='prefix of output file') parser.add_argument('--beataml', action='store_true', help='Include this if for BeatAML') parser.add_argument('--debug',action='store_true',default=False) - + parser.add_argument('--workers', type=int, default=None, help='Number of parallel worker processes (default: 3/4 of available CPUs)') + parser.add_argument('--chunk_size', type=int, default=50000, help='Max drug-groups per pool.map call (default: 50000; reduce for very large datasets)') + args = parser.parse_args() - print(args.input) + _log(f'loading {args.input}') df_all = pd.read_table(args.input) if args.debug: df_all = df_all.iloc[0:1000000] @@ -274,12 +323,18 @@ def main(): df_all.DOSE = -1.0 * np.log10(df_all.DOSE/1000000.0) ##need data to be between 0 and 1, not 0 and 100 df_all.GROWTH=df_all.GROWTH/100.00 - print(df_all.head) + _log(f'loaded {len(df_all)} rows — starting curve fitting') fname = args.output or 'combined_single_response_agg' - process_df_part(df_all, fname, beataml=args.beataml)#, start=args.start, count=args.count) - -# if args.beataml == False: + skipped = process_df_part(df_all, fname, beataml=args.beataml, workers=args.workers, + chunk_size=args.chunk_size) + # Free the large input DataFrame before the schema-formatting step + del df_all + import gc; gc.collect() + _log(f'done: {fname}') format_coderd_schema(fname+'.0') + if skipped: + _log(f'ERROR: {skipped} chunk(s) permanently skipped — output is incomplete') + sys.exit(1) if __name__ == '__main__': main() diff --git a/coderbuild/utils/pubchem_retrieval.py b/coderbuild/utils/pubchem_retrieval.py index a72c5c68..a3c95873 100644 --- a/coderbuild/utils/pubchem_retrieval.py +++ b/coderbuild/utils/pubchem_retrieval.py @@ -1,11 +1,16 @@ import pandas as pd import requests +import csv from concurrent.futures import ThreadPoolExecutor, as_completed import os import threading import time import signal import sys +from datetime import datetime + +def _ts(): + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Global variables request_counter = 0 @@ -17,67 +22,131 @@ existing_structures = dict() existing_pubchemids = set() +# --------------------------------------------------------------------------- +# Local PubChem data cache — populated from previous-run drug files so that +# retrieve_drug_info can return cached properties+synonyms without HTTP requests. +# Keys: pubchem_id (str) AND chem_name (str.lower()). +# Values: {'SMILES': ..., 'InChIKey': ..., 'MolecularFormula': ..., +# 'MolecularWeight': ..., 'CID': ..., 'synonyms': [...]} +# improve_drug_id is NOT stored here; ID assignment is always fresh. +# --------------------------------------------------------------------------- +_pubchem_data_cache = {} + -def fetch_url(url, retries=3, backoff_factor=1): +def load_pubchem_cache(file_paths): + """Populate _pubchem_data_cache from previously built drug TSV files. + + Call this before the main drug-build loop to avoid re-querying PubChem for + drugs whose properties are already known. ID assignment is unaffected. + """ + global _pubchem_data_cache + if isinstance(file_paths, str): + file_paths = [p.strip() for p in file_paths.split(',') if p.strip()] + loaded = 0 + for path in file_paths: + if not os.path.exists(path): + print(f"[{_ts()}] Cache hint: file not found, skipping: {path}") + continue + try: + df = pd.read_csv(path, sep='\t', quoting=3) + except Exception: + try: + df = pd.read_csv(path, sep='\t') + except Exception as e: + print(f"[{_ts()}] Could not read cache file {path}: {e}") + continue + needed = {'pubchem_id', 'chem_name', 'canSMILES', 'InChIKey', 'formula', 'weight'} + if not needed.issubset(set(df.columns)): + print(f"[{_ts()}] Cache file {path} missing expected columns, skipping.") + continue + for cid, group in df.groupby('pubchem_id', dropna=True): + synonyms = [str(n) for n in group['chem_name'].dropna().tolist()] + row = group.iloc[0] + entry = { + 'SMILES': str(row['canSMILES']), + 'InChIKey': str(row['InChIKey']), + 'MolecularFormula': str(row['formula']), + 'MolecularWeight': str(row['weight']), + 'CID': str(cid), + 'synonyms': synonyms, + } + _pubchem_data_cache[str(cid)] = entry + for syn in synonyms: + _pubchem_data_cache[str(syn).lower()] = entry + loaded += 1 + print(f"[{_ts()}] PubChem cache: {loaded} drugs loaded from {len(file_paths)} file(s).") + + +# Auto-load from hint file written by build_all.py before the container starts. +_CACHE_HINT_FILE = "/tmp/prev_drug_files.txt" +if os.path.exists(_CACHE_HINT_FILE): + try: + _hint_paths = [l.strip() for l in open(_CACHE_HINT_FILE) if l.strip()] + if _hint_paths: + load_pubchem_cache(_hint_paths) + except Exception as _cache_err: + print(f"[{_ts()}] Warning: could not load PubChem cache from {_CACHE_HINT_FILE}: {_cache_err}") + + +def fetch_url(url, retries=4, backoff_factor=1): """ Fetches a URL with retry mechanism and backoff. - - Parameters: - - url (str): The URL to fetch. - - retries (int): Number of retry attempts. - - backoff_factor (float): Factor to calculate backoff time. - - Returns: - - dict: JSON response if successful. - - Raises: - - Exception: If all retry attempts fail. + + 503 responses use a 30s base backoff (honouring Retry-After if present) + because PubChem rate-throttles aggressively and a 1s retry just gets + another 503. """ global last_request_time, lock, request_counter with lock: current_time = time.time() - # Reset counter if more than 1 second has passed if current_time - last_request_time >= 1: request_counter = 0 last_request_time = current_time - - # Wait if the limit is reached - while request_counter >= 4: - time.sleep(0.2) # Sleep a bit to check again + while request_counter >= 3: # stay at ≤3 req/s; PubChem limit is 5 + time.sleep(0.1) current_time = time.time() if current_time - last_request_time >= 1: request_counter = 0 last_request_time = current_time - request_counter += 1 - - for attempt in range(retries + 1): # Total attempts = retries + 1 + + for attempt in range(retries + 1): + status_code = None + retry_after_header = None try: - response = requests.get(url, timeout=10) - if response.status_code == 200: - return response.json() - if response.status_code == 404: # permanent, no existing CID/name + response = requests.get(url, timeout=30) + status_code = response.status_code + if status_code == 200: + return response.json() + if status_code == 404: raise FileNotFoundError("404") - raise Exception(f"Failed to fetch {url}, Status Code: {response.status_code}") + retry_after_header = response.headers.get('Retry-After', '') + raise Exception(f"Failed to fetch {url}, Status Code: {status_code}") + except FileNotFoundError: + raise except Exception as exc: - if attempt < retries: - wait = backoff_factor * (2 ** attempt) - print(f"Attempt {attempt + 1} for URL {url} failed with error: {exc}. Retrying in {wait} seconds...") - time.sleep(wait) - else: - print(f"All {retries + 1} attempts failed for URL {url}.") + if attempt >= retries: + print(f"[{_ts()}] All {retries + 1} attempts failed for URL {url}.") raise + if status_code == 503: + wait = (int(retry_after_header) + if retry_after_header and retry_after_header.isdigit() + else min(15 * (attempt + 1), 60)) + else: + wait = min(15 * (attempt + 1), 60) + print(f"[{_ts()}] Attempt {attempt + 1} for URL {url} failed with error: {exc}. Retrying in {wait} seconds...") + time.sleep(wait) def retrieve_drug_info(compound, ignore_chems, isname=True): """ - Retrieves information for a given compound from PubChem. + Retrieves information for a given compound from PubChem (or local cache). Parameters: - compound (str or int): Name or CID of the compound. - ignore_chems (str): File path to log ignored compounds. - isname (bool): True if the compound is a name, False if it's a CID. - + Returns: - list: List of dictionaries containing drug information, or None if unsuccessful. """ @@ -85,77 +154,85 @@ def retrieve_drug_info(compound, ignore_chems, isname=True): if pd.isna(compound): return None - if isname: - urls = { - "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", - "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/synonyms/JSON" + # --- Check local cache before making any HTTP requests --- + cache_key = str(compound).lower() if isname else str(compound) + cached = _pubchem_data_cache.get(cache_key) + if cached is not None: + properties = { + 'SMILES': cached['SMILES'], + 'InChIKey': cached['InChIKey'], + 'MolecularFormula': cached['MolecularFormula'], + 'MolecularWeight': cached['MolecularWeight'], + 'CID': cached['CID'], } + synonyms_list = cached['synonyms'] else: - urls = { - "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", - "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/synonyms/JSON" - } - - with ThreadPoolExecutor(max_workers=4) as executor: - future_to_url = {executor.submit(fetch_url, url): key for key, url in urls.items()} - results = {} + # --- Normal PubChem fetch via HTTP --- + if isname: + urls = { + "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", + "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{compound}/synonyms/JSON" + } + else: + urls = { + "properties": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/property/SMILES,InChIKey,MolecularFormula,MolecularWeight/JSON", + "synonyms": f"https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/CID/{compound}/synonyms/JSON" + } + + with ThreadPoolExecutor(max_workers=4) as executor: + future_to_url = {executor.submit(fetch_url, url): key for key, url in urls.items()} + results = {} + for future in as_completed(future_to_url): + key = future_to_url[future] + try: + results[key] = future.result() + except FileNotFoundError: + print(f"[{_ts()}] {compound} not found in PubChem. Adding to ignore list.") + with open(ignore_chems, "a") as f: + f.write(f"{compound}\n") + return None + except Exception as exc: + print(f"[{_ts()}] {compound} generated a transient exception: {exc}") + return None + + if not all(key in results for key in ["properties", "synonyms"]): + return None - for future in as_completed(future_to_url): - key = future_to_url[future] - try: - data = future.result() - results[key] = data - - except FileNotFoundError: # only 404s are added - print(f"{compound} not found in PubChem. Adding to ignore list.") - with open(ignore_chems, "a") as f: - f.write(f"{compound}\n") - return None - except Exception as exc: # transient error, don't blacklist - print(f"{compound} generated a transient exception: {exc}") - return None - - if all(key in results for key in ["properties", "synonyms"]): properties = results["properties"]['PropertyTable']['Properties'][0] synonyms_list = results["synonyms"]['InformationList']['Information'][0]['Synonym'] - # Check if this compound or any of its synonyms already has an assigned improve_drug_id - new_syns = set() - if isname: - sl = synonyms_list + [compound] - else: - sl = synonyms_list - for synonym in sl: - synonym_lower = str(synonym).lower() - if synonym_lower not in existing_synonyms: - new_syns.add(synonym_lower) - if len(new_syns) == 0: # Ensure there are new synonyms before proceeding - return None - for synonym in new_syns: - synonym_lower = str(synonym).lower() - existing_synonyms.add(synonym_lower) - - # Check for structure - if properties['SMILES'] in existing_structures.keys(): - print(f'Found structure for {compound}') - SMI_assignment = existing_structures[properties['SMILES']] - else: - if improve_drug_id == 0: - improve_drug_id = 1 - SMI_assignment = f"SMI_{improve_drug_id}" - existing_structures[properties['SMILES']] = SMI_assignment - improve_drug_id += 1 - - #print(new_syns) - data_for_tsv = [{ - 'improve_drug_id': SMI_assignment, - 'name': str(synonym).lower(), - **properties - } for synonym in new_syns] - - return data_for_tsv - else: + # --- Shared: synonym dedup + SMI ID assignment (identical for both paths) --- + new_syns = set() + sl = synonyms_list + ([compound] if isname else []) + for synonym in sl: + synonym_lower = str(synonym).lower() + if synonym_lower not in existing_synonyms: + new_syns.add(synonym_lower) + if len(new_syns) == 0: return None + for synonym in new_syns: + existing_synonyms.add(str(synonym).lower()) + + if properties['SMILES'] in existing_structures: + print(f'[{_ts()}] Found structure for {compound}') + SMI_assignment = existing_structures[properties['SMILES']] + else: + if improve_drug_id == 0: + improve_drug_id = 1 + SMI_assignment = f"SMI_{improve_drug_id}" + existing_structures[properties['SMILES']] = SMI_assignment + improve_drug_id += 1 + + # Sanitise every value: these rows are later written to TSV with plain + # f.write() and no quoting, so an embedded tab/newline/quote would corrupt + # the record. See _sanitise_field(). + data_for_tsv = [{ + 'improve_drug_id': SMI_assignment, + 'name': _sanitise_field(str(synonym).lower()), + **{k: _sanitise_field(v) for k, v in properties.items()} + } for synonym in new_syns] + + return data_for_tsv def fetch_data_for_batch(batch, ignore_chems, isname): @@ -166,18 +243,91 @@ def fetch_data_for_batch(batch, ignore_chems, isname): - batch (list): List of compound names or CIDs. - ignore_chems (str): File path to log ignored compounds. - isname (bool): True if compounds are names, False if they're CIDs. - + Returns: - list: Combined list of drug information for the batch. """ all_data = [] - for compound_name in batch: - data = retrieve_drug_info(compound_name, ignore_chems, isname) + for compound in batch: + data = retrieve_drug_info(compound, ignore_chems, isname) if data: all_data.extend(data) return all_data +DRUG_TSV_COLUMNS = 7 # improve_drug_id, chem_name, pubchem_id, canSMILES, + # InChIKey, formula, weight + + +def _sanitise_field(value): + """Make a value safe to write into a raw (unescaped) TSV cell. + + Several writers in this pipeline emit TSV with plain f.write() and no + quoting, so any tab, carriage return, newline or double quote inside a + value silently corrupts the row: the record gains extra columns and every + later reader fails with + + pandas.errors.ParserError: Expected 7 fields in line N, saw 12 + + That is what broke the 2026-09-02 build after ~10 hours -- a single + malformed synonym row for SMI_20392 in nci60_drugs.tsv, whose weight cell + ended up as `367.4"` followed by five stray tabs. + + Drug synonyms carry no meaning in these characters, so replacing them is + lossless for our purposes and makes a structurally broken row impossible. + """ + if value is None: + return "" + s = str(value) + for ch in ("\t", "\r", "\n"): + s = s.replace(ch, " ") + s = s.replace('"', "") + return s.strip() + + +def _read_drug_tsv(path, **kwargs): + """Read a drug TSV, tolerating rows padded with trailing EMPTY columns. + + Historic files (written before _sanitise_field existed) can contain rows + with extra trailing tabs. Those extra cells hold no data, so trimming them + loses nothing -- unlike dropping the row, which would silently remove a + drug from the release. + + A row with extra NON-empty cells is a different matter: that is real data + in an unexpected shape, so this raises with the offending line number and + content rather than guessing. + """ + try: + return pd.read_csv(path, sep="\t", **kwargs) + except pd.errors.ParserError: + pass # fall through to the repairing path below + + print(f"[{_ts()}] {path}: malformed row(s) detected; inspecting.") + cleaned, repaired, bad = [], 0, [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for lineno, raw in enumerate(fh, start=1): + parts = raw.rstrip("\n").split("\t") + if len(parts) > DRUG_TSV_COLUMNS: + extra = parts[DRUG_TSV_COLUMNS:] + if any(x.strip() for x in extra): + bad.append((lineno, raw[:200])) + continue + parts = parts[:DRUG_TSV_COLUMNS] + repaired += 1 + cleaned.append("\t".join(parts)) + + if bad: + detail = "\n".join(f" line {n}: {t!r}" for n, t in bad[:5]) + raise ValueError( + f"{path} contains {len(bad)} row(s) with unexpected NON-EMPTY extra " + f"columns. Refusing to guess at their meaning, because silently " + f"dropping them would remove drugs from the release.\n{detail}") + + print(f"[{_ts()}] {path}: trimmed trailing empty columns from {repaired} row(s).") + from io import StringIO + return pd.read_csv(StringIO("\n".join(cleaned) + "\n"), sep="\t", **kwargs) + + def read_existing_data(output_filename): """ Reads existing data from the output file to prevent duplication. @@ -190,13 +340,13 @@ def read_existing_data(output_filename): """ global improve_drug_id, existing_synonyms, existing_structures, existing_pubchemids try: - df = pd.read_csv(output_filename, sep='\t', quoting=3) + df = _read_drug_tsv(output_filename, quoting=3) existing_synonyms = set([str(a).lower() for a in set(df.chem_name)]) existing_pubchemids = set([str(a) for a in df['pubchem_id']]) max_id = df['improve_drug_id'].str.extract(r'SMI_(\d+)').astype(float).max() improve_drug_id = int(max_id[0]) + 1 if pd.notna(max_id[0]) else 1 existing_structures = {row['canSMILES']: row['improve_drug_id'] for _, row in df.iterrows()} - print(f'Read in {len(existing_synonyms)} drug names and {len(existing_pubchemids)} pubchem IDs') + print(f'[{_ts()}] Read in {len(existing_synonyms)} drug names and {len(existing_pubchemids)} pubchem IDs') except FileNotFoundError: return {} @@ -206,7 +356,7 @@ def timeout_handler(signum, frame): Handles timeouts by setting the global `should_continue` flag to False. """ global should_continue - print("Time limit reached, exiting gracefully...") + print(f"[{_ts()}] Time limit reached, exiting gracefully...") should_continue = False @@ -223,7 +373,7 @@ def _load_prev_drugs_union(prevDrugFilepath: str) -> pd.DataFrame: dfs = [] for p in paths: if not os.path.exists(p): - print(f"Warning: previous drug file '{p}' not found; skipping.") + print(f"[{_ts()}] Warning: previous drug file '{p}' not found; skipping.") continue try: if p.lower().endswith(".tsv"): @@ -232,7 +382,7 @@ def _load_prev_drugs_union(prevDrugFilepath: str) -> pd.DataFrame: df = pd.read_csv(p) dfs.append(df) except Exception as e: - print(f"Warning: failed to read previous drug file '{p}': {e}; skipping.") + print(f"[{_ts()}] Warning: failed to read previous drug file '{p}': {e}; skipping.") if not dfs: return pd.DataFrame(columns=["improve_drug_id", "chem_name", "pubchem_id", "canSMILES", "InChIKey", "formula", "weight"]) @@ -304,15 +454,15 @@ def update_dataframe_and_write_tsv(unique_names, else: restrict_set = raw_names # default filtering - print(f"Starting with {len(raw_names)} provided {'names' if isname else 'IDs'}; restricting output to {len(restrict_set)} of them.") + print(f"[{_ts()}] Starting with {len(raw_names)} provided {'names' if isname else 'IDs'}; restricting output to {len(restrict_set)} of them.") # --- 1) read existing output to bootstrap state --- - print(f"Reading existing data from {output_filename}") + print(f"[{_ts()}] Reading existing data from {output_filename}") # capture existing output file (if any) to include in base existing_output_df = pd.DataFrame() if os.path.exists(output_filename): try: - existing_output_df = pd.read_csv(output_filename, sep="\t", quoting=3) + existing_output_df = _read_drug_tsv(output_filename, quoting=3) except Exception: existing_output_df = pd.read_csv(output_filename, sep="\t") # read_existing_data populates globals (synonyms, pubchemids, and sets improve_drug_id based on output) @@ -327,7 +477,7 @@ def update_dataframe_and_write_tsv(unique_names, desired_start = max(existing_output_max, prev_union_max) + 1 if improve_drug_id < desired_start: improve_drug_id = desired_start - print(f"SMI numbering will start from {improve_drug_id} (max prior was {desired_start - 1})") + print(f"[{_ts()}] SMI numbering will start from {improve_drug_id} (max prior was {desired_start - 1})") # build seen names/IDs (to avoid re-query) seen_names = set(existing_synonyms) @@ -341,10 +491,10 @@ def update_dataframe_and_write_tsv(unique_names, # --- 3) determine new candidates to query --- if isname: candidates = raw_names - seen_names - print(f"{len(raw_names)} raw names provided; {len(seen_names)} already seen; {len(candidates)} new to fetch.") + print(f"[{_ts()}] {len(raw_names)} raw names provided; {len(seen_names)} already seen; {len(candidates)} new to fetch.") else: candidates = raw_names - seen_pubchemids - print(f"{len(raw_names)} raw IDs provided; {len(seen_pubchemids)} already seen; {len(candidates)} new to fetch.") + print(f"[{_ts()}] {len(raw_names)} raw IDs provided; {len(seen_pubchemids)} already seen; {len(candidates)} new to fetch.") # apply ignore_chems filtering ignore_chem_set = set() @@ -353,10 +503,13 @@ def update_dataframe_and_write_tsv(unique_names, for line in file: ignore_chem_set.add(line.strip()) candidates = set(candidates) - ignore_chem_set - print(f"{len(candidates)} candidates remain after removing ignored.") + print(f"[{_ts()}] {len(candidates)} candidates remain after removing ignored.") # --- 4) make a temp union file with previous union + existing output --- - temp_file = output_filename.replace(".tsv", "_temp.tsv") + if output_filename.endswith(".tsv"): + temp_file = output_filename[:-4] + "_temp.tsv" + else: + temp_file = output_filename + "_temp" base_dfs = [] if not prev_union_df.empty: base_dfs.append(prev_union_df) @@ -380,34 +533,76 @@ def update_dataframe_and_write_tsv(unique_names, # create empty temp file so fetch logic can append headers open(temp_file, "a").close() - # --- 5) fetch new ones in batches and append to temp_file --- - candidates_list = list(candidates) - for i in range(0, len(candidates_list), batch_size): - if not should_continue: + # --- 5) fetch new ones in batches and append to temp_file (3 sweeps) --- + remaining_candidates = set(candidates) + + for sweep in range(3): + if not should_continue or not remaining_candidates: break - batch = candidates_list[i : i + batch_size] - data = fetch_data_for_batch(batch, ignore_chems, isname) - if data: - file_exists = os.path.isfile(temp_file) - mode = "a" if file_exists else "w" - with open(temp_file, mode) as f: - if os.path.getsize(temp_file) == 0: - f.write("improve_drug_id\tchem_name\tpubchem_id\tcanSMILES\tInChIKey\tformula\tweight\n") - for entry in data: - f.write( - f"{entry['improve_drug_id']}\t{entry['name']}\t{entry.get('CID', '')}\t" - f"{entry['SMILES']}\t{entry['InChIKey']}\t" - f"{entry['MolecularFormula']}\t{entry['MolecularWeight']}\n" - ) - with open(ignore_chems, "a") as ig_f: - for entry in data: - if isname: - ig_f.write(f"{entry['name']}\n") - else: - ig_f.write(f"{entry.get('CID', '')}\n") + + if sweep > 0: + print(f"[{_ts()}] Sweep {sweep + 1}/3: waiting 60 seconds before retry pass...") + time.sleep(60) + + # refresh ignore set + ignore_chem_set = set() + if os.path.exists(ignore_chems): + with open(ignore_chems, "r") as f: + for line in f: + ignore_chem_set.add(line.strip()) + + # remove already-fetched candidates by reading what landed in temp_file + if os.path.exists(temp_file) and os.path.getsize(temp_file) > 0: + try: + # QUOTE_NONE: temp_file is written with raw f.write() + tmp_df = pd.read_csv(temp_file, sep="\t", quoting=3) + if not isname and "pubchem_id" in tmp_df.columns: + fetched = {str(x) for x in tmp_df["pubchem_id"].dropna()} + remaining_candidates -= fetched + elif isname and "chem_name" in tmp_df.columns: + fetched = {str(x).lower() for x in tmp_df["chem_name"].dropna()} + remaining_candidates -= fetched + except Exception: + pass + + remaining_candidates -= ignore_chem_set + + print(f"[{_ts()}] Sweep {sweep + 1}/3: {len(remaining_candidates)} candidates to fetch.") + + candidates_list = list(remaining_candidates) + for i in range(0, len(candidates_list), batch_size): + if not should_continue: + break + batch = candidates_list[i : i + batch_size] + data = fetch_data_for_batch(batch, ignore_chems, isname) + if data: + file_exists = os.path.isfile(temp_file) + mode = "a" if file_exists else "w" + with open(temp_file, mode) as f: + if os.path.getsize(temp_file) == 0: + f.write("improve_drug_id\tchem_name\tpubchem_id\tcanSMILES\tInChIKey\tformula\tweight\n") + for entry in data: + # Every cell goes through _sanitise_field: this write + # is unescaped, so a stray tab/newline/quote here is + # what produced the malformed nci60_drugs.tsv row that + # failed the 2026-09-02 build. + f.write("\t".join(_sanitise_field(x) for x in ( + entry['improve_drug_id'], entry['name'], + entry.get('CID', ''), entry['SMILES'], + entry['InChIKey'], entry['MolecularFormula'], + entry['MolecularWeight'])) + "\n") + with open(ignore_chems, "a") as ig_f: + for entry in data: + if isname: + ig_f.write(f"{entry['name']}\n") + else: + ig_f.write(f"{entry.get('CID', '')}\n") # --- 6) load combined temp results --- - combined = pd.read_csv(temp_file, sep="\t") + # temp_file is written with raw f.write() (no quoting), so it must be read + # with QUOTE_NONE. Reading it with pandas' default QUOTE_MINIMAL would + # mis-parse any synonym containing a double quote. + combined = _read_drug_tsv(temp_file, quoting=3) # Determine previous max (before new fetches) to identify newly assigned SMI IDs previous_max = desired_start - 1 # desired_start was max(existing_output_max, prev_union_max) + 1 @@ -437,18 +632,40 @@ def update_dataframe_and_write_tsv(unique_names, if keep_ids: final_df = combined[combined["improve_drug_id"].isin(keep_ids)].copy() else: - print("Warning: no relevant drugs were retained/fetched for the restriction set.") + print(f"[{_ts()}] Warning: no relevant drugs were retained/fetched for the restriction set.") final_df = pd.DataFrame(columns=combined.columns) # --- 10) write final filtered output --- final_df.drop_duplicates(inplace=True) - final_df.to_csv(output_filename, sep="\t", index=False) + # QUOTE_NONE/quotechar=None is REQUIRED here, not cosmetic. + # + # These files are read back with quoting=3 (QUOTE_NONE) -- by + # read_existing_data(), by _load_prev_drugs_union(), and by the next build + # via --prev_drugs. Writing with pandas' default QUOTE_MINIMAL wraps any + # field containing a double quote and doubles its internal quotes, but the + # QUOTE_NONE reader does not strip the wrapper or collapse the doubling. + # So every read/write cycle DOUBLES the quote count in that field. + # + # Real drug synonyms contain quotes (e.g. cimetidine's + # n-cyano-n'-methyl-n''-[2-...]guanidine + # ), so this compounded silently across releases: v2.3 shipped a chem_name + # of ~2^16 characters, and by v2.4 the same field had reached ~2^24 (16 MB), + # inflating ctrpv2/nci60/prism_drugs.tsv to ~300 MB each and finally + # breaking map_improve_drug_ids.py with + # _csv.Error: field larger than field limit (131072) + # + # Writing QUOTE_NONE makes the writer symmetric with the readers, so a + # value survives any number of round trips unchanged. Verified stable over + # repeated cycles. Drug names never contain tabs, so QUOTE_NONE is safe for + # this TSV. + final_df.to_csv(output_filename, sep="\t", index=False, + quoting=csv.QUOTE_NONE, quotechar=None) if os.path.exists(temp_file): try: os.remove(temp_file) except OSError as e: - print(f"Warning: failed to delete temp file {temp_file}: {e}") + print(f"[{_ts()}] Warning: failed to delete temp file {temp_file}: {e}") return final_df