diff --git a/scripts/align_drug_descriptors.py b/scripts/align_drug_descriptors.py index 0e190af9..6dac8265 100644 --- a/scripts/align_drug_descriptors.py +++ b/scripts/align_drug_descriptors.py @@ -1,9 +1,45 @@ #!/usr/bin/env python3 +"""Harmonize drug_descriptor values across datasets. + +For each (improve_drug_id, structural_descriptor) seen across the per-dataset +`*_drug_descriptors.tsv[.gz]` files, the FIRST value encountered (files processed +in sorted order) is treated as canonical, and any later file whose value differs +is rewritten to match. + +Memory-bounded implementation +----------------------------- +The reference map is kept in an on-disk SQLite database, not a Python dict, so the +step scales to very large descriptor files (e.g. nci60 ~89M rows / 2.7 GB). The +previous in-memory dict held every (drug_id, descriptor) -> value pair at once and +was OOM-killed by the container on large builds. + +Fingerprint descriptors are skipped from harmonization: a fingerprint is a +deterministic function of chemical structure, so the same drug always yields the +same value -- there is nothing to harmonize, and these rows dominate both memory +and file size. They are passed through unchanged. +""" import os +import sys import gzip import shutil import csv import argparse +import sqlite3 +import tempfile + +# Descriptor values (e.g. Morgan fingerprints) can be long strings. +csv.field_size_limit(min(sys.maxsize, 2**31 - 1)) + +# Descriptor names (case-insensitive substring match) excluded from harmonization. +SKIP_DESCRIPTOR_SUBSTRINGS = ("fingerprint",) + +_INSERT_BATCH = 100_000 + + +def _skip_descriptor(name): + n = name.lower() + return any(s in n for s in SKIP_DESCRIPTOR_SUBSTRINGS) + # Helper scripts def decompress_gz_if_needed(path): @@ -15,6 +51,7 @@ def decompress_gz_if_needed(path): return out, True return path, False + def recompress_if_needed(decompressed, was_gz, original): """If was_gz, recompress decompressed back to original and remove decompressed.""" if was_gz: @@ -22,6 +59,7 @@ def recompress_if_needed(decompressed, was_gz, original): shutil.copyfileobj(f_in, f_out) os.remove(decompressed) + def find_descriptor_files(directory): """Find all *_drug_descriptors.tsv and .tsv.gz files in directory.""" files = [] @@ -32,23 +70,39 @@ def find_descriptor_files(directory): # Actual work -def build_reference_map(files): - """Build map from (drug_id, descriptor) with first found value.""" - ref = {} +def build_reference_db(files, conn): + """Populate SQLite `ref` with the first-seen value per (drug_id, descriptor). + + INSERT OR IGNORE keeps the first value encountered, matching the original + 'first found wins' semantics. Fingerprint descriptors are not stored. + """ + cur = conn.cursor() + cur.execute( + "CREATE TABLE ref (drug_id TEXT, descriptor TEXT, value TEXT, " + "PRIMARY KEY (drug_id, descriptor)) WITHOUT ROWID" + ) for fp in files: path, gz = decompress_gz_if_needed(fp) with open(path, newline='', encoding='utf-8') as f: reader = csv.DictReader(f, delimiter='\t') + batch = [] for row in reader: - key = (row['improve_drug_id'], row['structural_descriptor']) - val = row['descriptor_value'] - if key not in ref: - ref[key] = val + desc = row['structural_descriptor'] + if _skip_descriptor(desc): + continue + batch.append((row['improve_drug_id'], desc, row['descriptor_value'])) + if len(batch) >= _INSERT_BATCH: + cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch) + batch.clear() + if batch: + cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch) + conn.commit() recompress_if_needed(path, gz, fp) - return ref -def rewrite_files(files, ref): - """Go back and rewrite any mismatches in-place.""" + +def rewrite_files(files, conn): + """Rewrite any mismatched descriptor_value in-place to the canonical value.""" + cur = conn.cursor() for fp in files: path, gz = decompress_gz_if_needed(fp) tmp = path + '.tmp' @@ -62,13 +116,16 @@ def rewrite_files(files, ref): writer.writeheader() for row in reader: - key = (row['improve_drug_id'], row['structural_descriptor']) - correct = ref.get(key) - if correct is not None and row['descriptor_value'] != correct: - # print(f"Fixing {key} in {os.path.basename(fp)}: " - # f"{row['descriptor_value']} to {correct}") - row['descriptor_value'] = correct - changed = True + desc = row['structural_descriptor'] + if not _skip_descriptor(desc): + cur.execute( + "SELECT value FROM ref WHERE drug_id=? AND descriptor=?", + (row['improve_drug_id'], desc), + ) + res = cur.fetchone() + if res is not None and row['descriptor_value'] != res[0]: + row['descriptor_value'] = res[0] + changed = True writer.writerow(row) if changed: @@ -78,21 +135,31 @@ def rewrite_files(files, ref): recompress_if_needed(path, gz, fp) + def main(): parser = argparse.ArgumentParser( description="Harmonize drug_descriptor values across multiple files." ) parser.add_argument('--local_dir', default='.', help='Folder containing *_drug_descriptors.tsv[.gz]') - parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input + parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input args = parser.parse_args() files = find_descriptor_files(args.local_dir) if not files: print("No drug_descriptor files found in", args.local_dir) return - ref = build_reference_map(files) - rewrite_files(files, ref) + with tempfile.TemporaryDirectory() as tmpdir: + conn = sqlite3.connect(os.path.join(tmpdir, "descriptors.sqlite")) + # Temp DB: durability is irrelevant, so favour speed. + conn.execute("PRAGMA journal_mode=OFF") + conn.execute("PRAGMA synchronous=OFF") + try: + build_reference_db(files, conn) + rewrite_files(files, conn) + finally: + conn.close() print("Done.") + if __name__ == '__main__': main() diff --git a/scripts/check_schema.py b/scripts/check_schema.py index f547e54d..a0daba46 100644 --- a/scripts/check_schema.py +++ b/scripts/check_schema.py @@ -1,89 +1,128 @@ -import subprocess import argparse -import concurrent.futures +import logging +import os import sys +from concurrent.futures import ProcessPoolExecutor + import yaml -import os -def run_validations_for_dataset(dataset_name, validations): - """ - Runs validations for a dataset and returns whether all validations passed. +# Path to the LinkML schema, relative to the repo root (the working directory +# used both locally and inside the `upload` Docker image). +SCHEMA_PATH = os.path.join('schema', 'coderdata.yaml') + +# One Validator per worker process. Constructing a Validator compiles the +# schema (and, lazily, the per-target-class JSON Schema) exactly once and +# caches it, so reusing a single instance across many files avoids the +# per-file process startup + schema recompilation that dominated runtime when +# this script shelled out to `linkml-validate` once per file. +# +# The plugin configuration below reproduces the exact defaults of the +# `linkml-validate` CLI (JsonschemaValidationPlugin with closed=True, and +# strict=False), so validation is identical to the previous implementation -- +# same engine, same closed-world JSON Schema, same results. +_VALIDATOR = None + - Parameters: - dataset_name (str): The name of the dataset. - validations (list): List of validations, each a dict with 'target_class' and 'file'. +def _init_worker(): + """Initializer run once per worker process: build the shared Validator.""" + global _VALIDATOR + # LinkML emits chatty INFO logs while importing the schema; silence them so + # the validation output stays readable. + logging.disable(logging.INFO) + from linkml.validator import Validator + from linkml.validator.plugins import JsonschemaValidationPlugin - Returns: - tuple: (dataset_name, bool) where bool is True if all validations passed, False otherwise. + _VALIDATOR = Validator( + SCHEMA_PATH, + validation_plugins=[JsonschemaValidationPlugin(closed=True)], + ) + + +def _validate_one(task): + """Validate a single file against its target class using the shared Validator. + + Returns (dataset_name, target_class, file_path, [error_messages]). """ - validation_failed = False - for validation in validations: - target_class = validation['target_class'] - file_path = validation['file'] - print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...") - - # Run the validation command - try: - result = subprocess.run( - ['linkml-validate', '--schema', 'schema/coderdata.yaml', '--target-class', target_class, file_path], - stdout=sys.stdout, stderr=sys.stderr, text=True - ) - - # Print output and error from command - if result.stdout: - print(result.stdout) - if result.stderr: - print(f"Error in validating {target_class} in {file_path}:", result.stderr) - # Check the exit status - if result.returncode != 0: - print(f"Validation failed for {target_class} in file {file_path}.") - validation_failed = True - else: - print(f"Validation succeeded for {target_class} in file {file_path}.") - except Exception as e: - print(f"An error occurred while validating {target_class} in {file_path}: {e}") - validation_failed = True + dataset_name, target_class, file_path = task + from linkml.validator.loaders import default_loader_for_file + + # A declared file that was never produced is reported as a clean failure + # rather than crashing the whole validation run with a FileNotFoundError. + if not os.path.exists(file_path): + return ( + dataset_name, target_class, file_path, + [f"[ERROR] declared file not found (was it produced by the build?): {file_path}"], + ) + + loader = default_loader_for_file(file_path) + errors = [] + for result in _VALIDATOR.iter_results_from_source(loader, target_class): + severity = getattr(result.severity, 'value', str(result.severity)) + message = ( + f"[{severity}] [{loader.source}/{result.instance_index}] " + f"{result.message}" + ) + # linkml-validate exits non-zero only on ERROR (or worse); mirror that + # so a WARNING/INFO does not fail the build. + if str(severity).upper() in ('ERROR', 'FATAL', 'CRITICAL'): + errors.append(message) + return (dataset_name, target_class, file_path, errors) - if validation_failed: - print(f"One or more validations failed for dataset {dataset_name}.") - return (dataset_name, False) - else: - print(f"All validations succeeded for dataset {dataset_name}.") - return (dataset_name, True) def main(): - parser = argparse.ArgumentParser(description="Run schema validations for specified datasets.") - parser.add_argument('-d', '--datasets', nargs='*', help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")', default=None) + parser = argparse.ArgumentParser( + description="Run schema validations for specified datasets." + ) + parser.add_argument( + '-d', '--datasets', nargs='*', + help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")', + default=None, + ) args = parser.parse_args() - # Read the config file config_path = os.path.join('schema', 'expected_files.yaml') with open(config_path, 'r') as f: config = yaml.safe_load(f) - available_datasets = config['datasets'].keys() + available_datasets = list(config['datasets'].keys()) datasets_to_validate = args.datasets if args.datasets else available_datasets - datasets_to_validate = [dataset for dataset in datasets_to_validate if dataset in available_datasets] + datasets_to_validate = [d for d in datasets_to_validate if d in available_datasets] print(f"Datasets to validate: {datasets_to_validate}") - all_passed = True - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = { - executor.submit( - run_validations_for_dataset, - dataset, - config['datasets'][dataset] - ): dataset for dataset in datasets_to_validate - } - - for future in concurrent.futures.as_completed(futures): - dataset_name, result = future.result() - if not result: - all_passed = False - print(f"Validation failed for dataset {dataset_name}") + # Flatten to per-file tasks so all files validate in parallel across + # processes (the previous version only parallelized across datasets). + tasks = [] + for dataset in datasets_to_validate: + for validation in config['datasets'][dataset]: + tasks.append((dataset, validation['target_class'], validation['file'])) + + # Track pass/fail per dataset so the summary matches the old output. + dataset_failed = {d: False for d in datasets_to_validate} + + # Cap workers: each worker can load a multi-GB decompressed omics file into + # memory, so bound concurrency to avoid exhausting RAM on large builds. + max_workers = min(len(tasks), os.cpu_count() or 1, 8) or 1 + with ProcessPoolExecutor(max_workers=max_workers, initializer=_init_worker) as executor: + for dataset_name, target_class, file_path, errors in executor.map(_validate_one, tasks): + print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...") + if errors: + for message in errors: + print(message) + print(f"Validation failed for {target_class} in file {file_path}.") + dataset_failed[dataset_name] = True else: - print(f"Validation passed for dataset {dataset_name}") + print(f"Validation succeeded for {target_class} in file {file_path}.") + + all_passed = True + for dataset in datasets_to_validate: + if dataset_failed[dataset]: + all_passed = False + print(f"One or more validations failed for dataset {dataset}.") + print(f"Validation failed for dataset {dataset}") + else: + print(f"All validations succeeded for dataset {dataset}.") + print(f"Validation passed for dataset {dataset}") if all_passed: print("All schema validations passed successfully.") @@ -92,5 +131,6 @@ def main(): print("Some schema validations failed.") sys.exit(1) + if __name__ == '__main__': main() diff --git a/scripts/map_improve_drug_ids.py b/scripts/map_improve_drug_ids.py index 5c12b1ca..417fe2ac 100644 --- a/scripts/map_improve_drug_ids.py +++ b/scripts/map_improve_drug_ids.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 import json import os +import sys import csv import argparse from datetime import datetime import gzip import shutil +# Drug synonym fields can legitimately be long, and a quoting bug in +# pubchem_retrieval.py (fixed 2026-08-29) historically inflated some chem_name +# values well past Python's default 131072-byte csv field limit. Matches the +# limit already set in align_drug_descriptors.py so the two stay consistent. +csv.field_size_limit(min(sys.maxsize, 2**31 - 1)) + ##### Load, make structure, save functions for improve_drug_mapping.json def load_mapping(mapping_file='improve_drug_mapping.json'): """ diff --git a/scripts/prepare_data_for_improve.py b/scripts/prepare_data_for_improve.py index 573b5c92..00cb199b 100644 --- a/scripts/prepare_data_for_improve.py +++ b/scripts/prepare_data_for_improve.py @@ -194,21 +194,21 @@ def process_datasets(args): # Logic for PDX datasets that don't have `auc` but mRECIST (note # the typo currently in the `drugresponse_metric` column). - elif experiments_raw['dose_response_metric'].isin(['mRESCIST']).any(): + elif experiments_raw['dose_response_metric'].isin(['mrecist']).any(): experiment = data_sets[data_set].format( data_type='experiments', shape='wide', metrics=[ - 'mRESCIST', + 'mrecist', ], ) # conversion logic from mRECIST -> auc - experiment.loc[experiment['mRESCIST'] == 'CR', 'mRESCIST'] = "0.1" - experiment.loc[experiment['mRESCIST'] == 'PR', 'mRESCIST'] = "0.2" - experiment.loc[experiment['mRESCIST'] == 'SD', 'mRESCIST'] = "0.5" - experiment.loc[experiment['mRESCIST'] == 'PD', 'mRESCIST'] = "1.0" + experiment.loc[experiment['mrecist'] == 'CR', 'mrecist'] = "0.1" + experiment.loc[experiment['mrecist'] == 'PR', 'mrecist'] = "0.2" + experiment.loc[experiment['mrecist'] == 'SD', 'mrecist'] = "0.5" + experiment.loc[experiment['mrecist'] == 'PD', 'mrecist'] = "1.0" - experiment.rename(columns={'mRESCIST': 'auc'}, inplace=True) + experiment.rename(columns={'mrecist': 'auc'}, inplace=True) experiments.append(experiment) # The remaining datasets should have `auc` as