Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 87 additions & 20 deletions scripts/align_drug_descriptors.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,45 @@
#!/usr/bin/env python3
"""Harmonize drug_descriptor values across datasets.

For each (improve_drug_id, structural_descriptor) seen across the per-dataset
`*_drug_descriptors.tsv[.gz]` files, the FIRST value encountered (files processed
in sorted order) is treated as canonical, and any later file whose value differs
is rewritten to match.

Memory-bounded implementation
-----------------------------
The reference map is kept in an on-disk SQLite database, not a Python dict, so the
step scales to very large descriptor files (e.g. nci60 ~89M rows / 2.7 GB). The
previous in-memory dict held every (drug_id, descriptor) -> value pair at once and
was OOM-killed by the container on large builds.

Fingerprint descriptors are skipped from harmonization: a fingerprint is a
deterministic function of chemical structure, so the same drug always yields the
same value -- there is nothing to harmonize, and these rows dominate both memory
and file size. They are passed through unchanged.
"""
import os
import sys
import gzip
import shutil
import csv
import argparse
import sqlite3
import tempfile

# Descriptor values (e.g. Morgan fingerprints) can be long strings.
csv.field_size_limit(min(sys.maxsize, 2**31 - 1))

# Descriptor names (case-insensitive substring match) excluded from harmonization.
SKIP_DESCRIPTOR_SUBSTRINGS = ("fingerprint",)

_INSERT_BATCH = 100_000


def _skip_descriptor(name):
n = name.lower()
return any(s in n for s in SKIP_DESCRIPTOR_SUBSTRINGS)


# Helper scripts
def decompress_gz_if_needed(path):
Expand All @@ -15,13 +51,15 @@ def decompress_gz_if_needed(path):
return out, True
return path, False


def recompress_if_needed(decompressed, was_gz, original):
"""If was_gz, recompress decompressed back to original and remove decompressed."""
if was_gz:
with open(decompressed, 'rb') as f_in, gzip.open(original, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(decompressed)


def find_descriptor_files(directory):
"""Find all *_drug_descriptors.tsv and .tsv.gz files in directory."""
files = []
Expand All @@ -32,23 +70,39 @@ def find_descriptor_files(directory):


# Actual work
def build_reference_map(files):
"""Build map from (drug_id, descriptor) with first found value."""
ref = {}
def build_reference_db(files, conn):
"""Populate SQLite `ref` with the first-seen value per (drug_id, descriptor).

INSERT OR IGNORE keeps the first value encountered, matching the original
'first found wins' semantics. Fingerprint descriptors are not stored.
"""
cur = conn.cursor()
cur.execute(
"CREATE TABLE ref (drug_id TEXT, descriptor TEXT, value TEXT, "
"PRIMARY KEY (drug_id, descriptor)) WITHOUT ROWID"
)
for fp in files:
path, gz = decompress_gz_if_needed(fp)
with open(path, newline='', encoding='utf-8') as f:
reader = csv.DictReader(f, delimiter='\t')
batch = []
for row in reader:
key = (row['improve_drug_id'], row['structural_descriptor'])
val = row['descriptor_value']
if key not in ref:
ref[key] = val
desc = row['structural_descriptor']
if _skip_descriptor(desc):
continue
batch.append((row['improve_drug_id'], desc, row['descriptor_value']))
if len(batch) >= _INSERT_BATCH:
cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch)
batch.clear()
if batch:
cur.executemany("INSERT OR IGNORE INTO ref VALUES (?,?,?)", batch)
conn.commit()
recompress_if_needed(path, gz, fp)
return ref

def rewrite_files(files, ref):
"""Go back and rewrite any mismatches in-place."""

def rewrite_files(files, conn):
"""Rewrite any mismatched descriptor_value in-place to the canonical value."""
cur = conn.cursor()
for fp in files:
path, gz = decompress_gz_if_needed(fp)
tmp = path + '.tmp'
Expand All @@ -62,13 +116,16 @@ def rewrite_files(files, ref):
writer.writeheader()

for row in reader:
key = (row['improve_drug_id'], row['structural_descriptor'])
correct = ref.get(key)
if correct is not None and row['descriptor_value'] != correct:
# print(f"Fixing {key} in {os.path.basename(fp)}: "
# f"{row['descriptor_value']} to {correct}")
row['descriptor_value'] = correct
changed = True
desc = row['structural_descriptor']
if not _skip_descriptor(desc):
cur.execute(
"SELECT value FROM ref WHERE drug_id=? AND descriptor=?",
(row['improve_drug_id'], desc),
)
res = cur.fetchone()
if res is not None and row['descriptor_value'] != res[0]:
row['descriptor_value'] = res[0]
changed = True
writer.writerow(row)

if changed:
Expand All @@ -78,21 +135,31 @@ def rewrite_files(files, ref):

recompress_if_needed(path, gz, fp)


def main():
parser = argparse.ArgumentParser(
description="Harmonize drug_descriptor values across multiple files."
)
parser.add_argument('--local_dir', default='.', help='Folder containing *_drug_descriptors.tsv[.gz]')
parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input
parser.add_argument('--version', help=argparse.SUPPRESS) # ignore the version input
args = parser.parse_args()
files = find_descriptor_files(args.local_dir)
if not files:
print("No drug_descriptor files found in", args.local_dir)
return

ref = build_reference_map(files)
rewrite_files(files, ref)
with tempfile.TemporaryDirectory() as tmpdir:
conn = sqlite3.connect(os.path.join(tmpdir, "descriptors.sqlite"))
# Temp DB: durability is irrelevant, so favour speed.
conn.execute("PRAGMA journal_mode=OFF")
conn.execute("PRAGMA synchronous=OFF")
try:
build_reference_db(files, conn)
rewrite_files(files, conn)
finally:
conn.close()
print("Done.")


if __name__ == '__main__':
main()
170 changes: 105 additions & 65 deletions scripts/check_schema.py
Original file line number Diff line number Diff line change
@@ -1,89 +1,128 @@
import subprocess
import argparse
import concurrent.futures
import logging
import os
import sys
from concurrent.futures import ProcessPoolExecutor

import yaml
import os

def run_validations_for_dataset(dataset_name, validations):
"""
Runs validations for a dataset and returns whether all validations passed.
# Path to the LinkML schema, relative to the repo root (the working directory
# used both locally and inside the `upload` Docker image).
SCHEMA_PATH = os.path.join('schema', 'coderdata.yaml')

# One Validator per worker process. Constructing a Validator compiles the
# schema (and, lazily, the per-target-class JSON Schema) exactly once and
# caches it, so reusing a single instance across many files avoids the
# per-file process startup + schema recompilation that dominated runtime when
# this script shelled out to `linkml-validate` once per file.
#
# The plugin configuration below reproduces the exact defaults of the
# `linkml-validate` CLI (JsonschemaValidationPlugin with closed=True, and
# strict=False), so validation is identical to the previous implementation --
# same engine, same closed-world JSON Schema, same results.
_VALIDATOR = None


Parameters:
dataset_name (str): The name of the dataset.
validations (list): List of validations, each a dict with 'target_class' and 'file'.
def _init_worker():
"""Initializer run once per worker process: build the shared Validator."""
global _VALIDATOR
# LinkML emits chatty INFO logs while importing the schema; silence them so
# the validation output stays readable.
logging.disable(logging.INFO)
from linkml.validator import Validator
from linkml.validator.plugins import JsonschemaValidationPlugin

Returns:
tuple: (dataset_name, bool) where bool is True if all validations passed, False otherwise.
_VALIDATOR = Validator(
SCHEMA_PATH,
validation_plugins=[JsonschemaValidationPlugin(closed=True)],
)


def _validate_one(task):
"""Validate a single file against its target class using the shared Validator.

Returns (dataset_name, target_class, file_path, [error_messages]).
"""
validation_failed = False
for validation in validations:
target_class = validation['target_class']
file_path = validation['file']
print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...")

# Run the validation command
try:
result = subprocess.run(
['linkml-validate', '--schema', 'schema/coderdata.yaml', '--target-class', target_class, file_path],
stdout=sys.stdout, stderr=sys.stderr, text=True
)

# Print output and error from command
if result.stdout:
print(result.stdout)
if result.stderr:
print(f"Error in validating {target_class} in {file_path}:", result.stderr)
# Check the exit status
if result.returncode != 0:
print(f"Validation failed for {target_class} in file {file_path}.")
validation_failed = True
else:
print(f"Validation succeeded for {target_class} in file {file_path}.")
except Exception as e:
print(f"An error occurred while validating {target_class} in {file_path}: {e}")
validation_failed = True
dataset_name, target_class, file_path = task
from linkml.validator.loaders import default_loader_for_file

# A declared file that was never produced is reported as a clean failure
# rather than crashing the whole validation run with a FileNotFoundError.
if not os.path.exists(file_path):
return (
dataset_name, target_class, file_path,
[f"[ERROR] declared file not found (was it produced by the build?): {file_path}"],
)

loader = default_loader_for_file(file_path)
errors = []
for result in _VALIDATOR.iter_results_from_source(loader, target_class):
severity = getattr(result.severity, 'value', str(result.severity))
message = (
f"[{severity}] [{loader.source}/{result.instance_index}] "
f"{result.message}"
)
# linkml-validate exits non-zero only on ERROR (or worse); mirror that
# so a WARNING/INFO does not fail the build.
if str(severity).upper() in ('ERROR', 'FATAL', 'CRITICAL'):
errors.append(message)
return (dataset_name, target_class, file_path, errors)

if validation_failed:
print(f"One or more validations failed for dataset {dataset_name}.")
return (dataset_name, False)
else:
print(f"All validations succeeded for dataset {dataset_name}.")
return (dataset_name, True)

def main():
parser = argparse.ArgumentParser(description="Run schema validations for specified datasets.")
parser.add_argument('-d', '--datasets', nargs='*', help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")', default=None)
parser = argparse.ArgumentParser(
description="Run schema validations for specified datasets."
)
parser.add_argument(
'-d', '--datasets', nargs='*',
help='List of datasets to validate (e.g., "beataml cptac ccle hcmi")',
default=None,
)
args = parser.parse_args()

# Read the config file
config_path = os.path.join('schema', 'expected_files.yaml')
with open(config_path, 'r') as f:
config = yaml.safe_load(f)

available_datasets = config['datasets'].keys()
available_datasets = list(config['datasets'].keys())
datasets_to_validate = args.datasets if args.datasets else available_datasets
datasets_to_validate = [dataset for dataset in datasets_to_validate if dataset in available_datasets]
datasets_to_validate = [d for d in datasets_to_validate if d in available_datasets]

print(f"Datasets to validate: {datasets_to_validate}")

all_passed = True
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(
run_validations_for_dataset,
dataset,
config['datasets'][dataset]
): dataset for dataset in datasets_to_validate
}

for future in concurrent.futures.as_completed(futures):
dataset_name, result = future.result()
if not result:
all_passed = False
print(f"Validation failed for dataset {dataset_name}")
# Flatten to per-file tasks so all files validate in parallel across
# processes (the previous version only parallelized across datasets).
tasks = []
for dataset in datasets_to_validate:
for validation in config['datasets'][dataset]:
tasks.append((dataset, validation['target_class'], validation['file']))

# Track pass/fail per dataset so the summary matches the old output.
dataset_failed = {d: False for d in datasets_to_validate}

# Cap workers: each worker can load a multi-GB decompressed omics file into
# memory, so bound concurrency to avoid exhausting RAM on large builds.
max_workers = min(len(tasks), os.cpu_count() or 1, 8) or 1
with ProcessPoolExecutor(max_workers=max_workers, initializer=_init_worker) as executor:
for dataset_name, target_class, file_path, errors in executor.map(_validate_one, tasks):
print(f"Validating {target_class} in file {file_path} for dataset {dataset_name}...")
if errors:
for message in errors:
print(message)
print(f"Validation failed for {target_class} in file {file_path}.")
dataset_failed[dataset_name] = True
else:
print(f"Validation passed for dataset {dataset_name}")
print(f"Validation succeeded for {target_class} in file {file_path}.")

all_passed = True
for dataset in datasets_to_validate:
if dataset_failed[dataset]:
all_passed = False
print(f"One or more validations failed for dataset {dataset}.")
print(f"Validation failed for dataset {dataset}")
else:
print(f"All validations succeeded for dataset {dataset}.")
print(f"Validation passed for dataset {dataset}")

if all_passed:
print("All schema validations passed successfully.")
Expand All @@ -92,5 +131,6 @@ def main():
print("Some schema validations failed.")
sys.exit(1)


if __name__ == '__main__':
main()
7 changes: 7 additions & 0 deletions scripts/map_improve_drug_ids.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
#!/usr/bin/env python3
import json
import os
import sys
import csv
import argparse
from datetime import datetime
import gzip
import shutil

# Drug synonym fields can legitimately be long, and a quoting bug in
# pubchem_retrieval.py (fixed 2026-08-29) historically inflated some chem_name
# values well past Python's default 131072-byte csv field limit. Matches the
# limit already set in align_drug_descriptors.py so the two stay consistent.
csv.field_size_limit(min(sys.maxsize, 2**31 - 1))

##### Load, make structure, save functions for improve_drug_mapping.json
def load_mapping(mapping_file='improve_drug_mapping.json'):
"""
Expand Down
Loading