From ce17d59a83760cf06334e3042eac7c3297bc1e4b Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 27 Oct 2026 15:52:27 -0400 Subject: [PATCH 1/3] Add validation suite --- coderbuild/README.md | 43 ++- coderbuild/build_all.py | 42 ++- coderbuild/utils/validate_build.py | 511 +++++++++++++++++++++++++++++ tests/conftest.py | 12 + tests/test_build_data.py | 74 +++++ tests/test_validate_build.py | 286 ++++++++++++++++ 6 files changed, 961 insertions(+), 7 deletions(-) create mode 100644 coderbuild/utils/validate_build.py create mode 100644 tests/conftest.py create mode 100644 tests/test_build_data.py create mode 100644 tests/test_validate_build.py diff --git a/coderbuild/README.md b/coderbuild/README.md index dc6c2eef..379b5374 100755 --- a/coderbuild/README.md +++ b/coderbuild/README.md @@ -13,7 +13,7 @@ are added. This script initializes all docker containers, builds all datasets, validates them, and uploads them to figshare. It requires the following authorization tokens to be set in the local environment depending on the use case: -`SYNAPSE_AUTH_TOKEN`: Required for beataml and mpnst datasets. Join the [CoderData team](https://www.synapse.org/#!Team:3503472) on Synapse and generate an access token. +`SYNAPSE_AUTH_TOKEN`: Required for the beataml, mpnst, bladder, pancreatic, liver, sarcoma, novartis, colorectal and broad_sanger (DepMap files) datasets. Join the [CoderData team](https://www.synapse.org/#!Team:3503472) on Synapse and generate an access token. `FIGSHARE_TOKEN`: This token is required to upload to Figshare. `GITHUB_TOKEN`: This token is required to upload to GitHub. @@ -24,11 +24,17 @@ It requires the following authorization tokens to be set in the local environmen - `--omics`: Processes and builds the omics data files. - `--drugs`: Processes and builds the drug data files. - `--exp`: Processes and builds the experiment data files. -- `--all`: Executes all available processes above (docker, samples, omics, drugs, exp). This does not run the validate or figshare commands. -- `--validate`: Validates the generated datasets using the schema check scripts. This is automatically included if data upload occurs. +- `--misc`: Runs the final post-build step (e.g. splitting broad_sanger into its datasets). +- `--prev_drugs`: Comma-separated list of previously built drug files. They seed the PubChem lookup so only new drugs are queried. +- `--all`: Executes all available processes above (docker, samples, omics, drugs, exp, misc). This does not run the validate or figshare commands. +- `--validate`: Validates the generated datasets with the build data checks (see below) and the schema check scripts. This is automatically included if data upload occurs. +- `--previous-summary`: Optional. `validation_summary.json(.gz)` from the previous release, compared against by the build data checks. +- `--skip-build-checks`: Skips the build data checks during `--validate` (not recommended). - `--figshare`: Uploads the datasets to Figshare. FIGSHARE_TOKEN must be set in local environment. - `--high_mem`: Utilizes high memory mode for concurrent data processing. This has been successfully tested using 32 or more vCPUs. -- `--dataset`: Specifies the datasets to process (default='broad_sanger,hcmi,beataml,mpnst,cptac'). +- `--dataset`: Specifies the datasets to process (default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,cptac,hcmi'). +- `--depmap-ready`: Confirms that the DepMap files on Synapse are up to date for the current DepMap release. Required for any build that includes broad_sanger. +- `--continue`: Resumes a partial build, skipping steps whose output already exists in `local/`. Progress is logged to `local/build_progress.log`. - `--version`: Specifies the version number for the Figshare upload title (e.g., "0.1.29"). This must be a higher version than previously published versions. - `--github-username`: GitHub username matching the GITHUB_TOKEN. Required to push the new Tag to the GitHub Repository. - `--github-email`: GitHub email matching the GITHUB_TOKEN. Required to push the new Tag to the GitHub Repository. @@ -37,7 +43,7 @@ It requires the following authorization tokens to be set in the local environmen - Build all datasets and upload to Figshare and GitHub. Required tokens for the following command: `SYNAPSE_AUTH_TOKEN`, `FIGSHARE_TOKEN`, `GITHUB_TOKEN`. ```bash -python coderbuild/build_all.py --all --high_mem --validate --figshare --version 0.1.41 --github-username jjacobson95 --github-email jeremy.jacobson3402@gmail.com +python coderbuild/build_all.py --all --high_mem --validate --depmap-ready --figshare --version 0.1.41 --github-username jjacobson95 --github-email jeremy.jacobson3402@gmail.com ``` - Build only the experiment files. @@ -51,7 +57,7 @@ This script builds a single dataset for **debugging purposes only**. It can help It requires the following authorization tokens to be set in the local environment depending on the dataset: -`SYNAPSE_AUTH_TOKEN`: Required for beataml and mpnst datasets. Follow the directions above to use gain access. +`SYNAPSE_AUTH_TOKEN`: Required for most datasets (see the list above). Follow the directions above to gain access. Available arguments: - `--dataset`: Required. Name of the dataset to build. At a minimum, this will build the docker images. @@ -59,6 +65,8 @@ Available arguments: - `--build`: Optional. Build the desired Dataset. - `--validate`: Optional. Run the schema checker on the built files. - `--continue`: Optional. Continues from where the build left off by skipping existing files in "local" directory. +- `--depmap-ready`: Required when building broad_sanger. Confirms the DepMap files on Synapse are up to date. + Example usage: Build the broad_sanger dataset: @@ -78,6 +86,29 @@ Build the broad_sanger dataset but skip previously built files in "local" direct python coderbuild/build_dataset.py --dataset broad_sanger --continue ``` +## Build data checks (validate_build.py) + +These checks look for problems the schema check cannot catch: duplicate results, ids that do not match between files, sample ids shared between datasets, copied responses, and large changes from the previous release. `build_all.py --validate` runs them automatically. They can also be run on their own against any folder of CoderData files, either a local build (`local/all_files_dir`) or a release downloaded from Figshare. Files may be plain or `.gz`. They need `pandas` and `numpy`; the pytest form also needs `pytest`. + +As a command: +```bash +python coderbuild/utils/validate_build.py --dir local/all_files_dir +``` +- `--dir`: Required. Folder of files to check. +- `--datasets`: Optional. Check only these datasets (default: all found in `--dir`). +- `--previous-summary`: Optional. Summary of an earlier build to compare counts with. Each release publishes one as `validation_summary.json.gz`. +- `--write-summary`: Optional. Save this build's counts, to use as the next baseline. + +Results are printed to the terminal. The command exits with code 1 if there are errors; warnings do not change the exit code. + +As tests: +```bash +pytest tests/test_build_data.py --build-dir local/all_files_dir +``` +Add `--build-datasets beataml,hcmi` to check only some datasets, or `--previous-summary ` to compare with a previous build. Each dataset is one test. Errors fail the test, and warnings show as pytest warnings. Without `--build-dir` these tests are skipped. + +In `build_all.py` the checks run after the id mapping and before the schema check, and write `validation_summary.json` into `local/all_files_dir` so it is published with the release. If they find errors, the build stops after the schema check. + diff --git a/coderbuild/build_all.py b/coderbuild/build_all.py index 37290fd1..209eb8fa 100644 --- a/coderbuild/build_all.py +++ b/coderbuild/build_all.py @@ -71,6 +71,11 @@ def main(): parser.add_argument('--high_mem',dest='high_mem',default=False,action='store_true',help = "If you have 32 or more CPUs, this option is recommended. It will run many code portions in parallel. If you don't have enough memory, this will cause a run failure.") parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,cptac,hcmi',help='Datasets to process. Defaults to all available.') parser.add_argument('--version', type=str, required=False, help='Version number for the Figshare upload title (e.g., "0.1.29"). This is required for Figshare upload. This must be a higher version than previously published versions.') + parser.add_argument('--previous-summary', dest='previous_summary', default=None, + help='validation_summary.json(.gz) from an earlier build or release; --validate then ' + 'warns on large changes in samples, drugs, pairs, omics coverage and model types.') + parser.add_argument('--skip-build-checks', dest='skip_build_checks', default=False, action='store_true', + help='Skip validate_build.py data checks during --validate (not recommended).') parser.add_argument('--github-username', type=str, required=False, help='GitHub username for the repository.') parser.add_argument('--github-email', type=str, required=False, help='GitHub email for the repository.') parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', @@ -1019,10 +1024,45 @@ def get_latest_commit_hash(owner, repo, branch='main'): drug_mapping_command_2 = ['python3', 'coderbuild/utils/align_drug_descriptors.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command_2, 'all_files_dir', 'Align_Drug_Descriptors', args.version) + # Data checks the schema cannot express (duplicate results, orphan or + # overlapping ids, copied responses, large changes from the previous + # build). The summary is written into all_files_dir so it is published + # with the release and can be the next build's --previous-summary. + # Errors stop the build after the schema check has also reported. + build_check_failed = False + if not args.skip_build_checks: + summary_path = os.path.join(all_files_dir, 'validation_summary.json') + check_cmd = [sys.executable, 'coderbuild/utils/validate_build.py', '--dir', all_files_dir, + '--write-summary', summary_path] + if args.previous_summary: + check_cmd += ['--previous-summary', args.previous_summary] + _log('Running build data checks') + # The summary is written once every check has run, so its absence + # means the checker never finished. + if os.path.exists(summary_path): + os.remove(summary_path) + sys.stdout.flush() + # stdin can be closed when the build runs under nohup ... &, and a + # new Python then dies at startup (init_sys_streams: Bad file descriptor). + rc = subprocess.run(check_cmd, stdin=subprocess.DEVNULL).returncode + build_check_failed = rc != 0 + if rc == 0: + _log('Build data checks passed') + elif os.path.exists(summary_path): + _log(f'Build data checks found errors (exit code {rc}); see the results above') + else: + # The 2.4 rebuild stopped here with no output. + _log(f'Build data checks did not finish (exit code {rc}). ' + f'Rerun by hand: {" ".join(check_cmd)}') + # Run schema checker - This will always run if uploading data. schema_check_command = ['python3', 'coderbuild/utils/check_schema.py', '--datasets'] + datasets run_docker_upload_cmd(schema_check_command, 'all_files_dir', 'validate', args.version) - + + if build_check_failed: + raise RuntimeError('Build data checks failed; see the log above. ' + 'Fix them, or rerun with --skip-build-checks to override.') + _log("Validation complete. Proceeding with file compression/decompression adjustments") # Compress or decompress files based on specific conditions after checking diff --git a/coderbuild/utils/validate_build.py b/coderbuild/utils/validate_build.py new file mode 100644 index 00000000..ed253186 --- /dev/null +++ b/coderbuild/utils/validate_build.py @@ -0,0 +1,511 @@ +#!/usr/bin/env python3 +""" +Data checks on a CoderData build that the LinkML schema cannot express. + +The schema checks that each row is well formed. These checks look across +rows, files and releases, at the problems that shipped in 2.4 without failing +the schema: two results for one sample/drug (NCI60, PRISM), one experiment +reported under two drugs, drug responses attached to the wrong specimens +(BeatAML) and whole sample classes dropped (HCMI). + +Errors are definite integrity violations and make the script exit 1. +Warnings need a person to look; they never fail the build. + +Usage: + python coderbuild/utils/validate_build.py --dir local/all_files_dir \\ + [--previous-summary prev_summary.json] [--write-summary this_summary.json] + +Results are printed to the terminal. + +--previous-summary compares this build with an earlier one (a summary written +by --write-summary; each release publishes one as validation_summary.json.gz) and warns on large +changes, drops of more than half and datasets that are no longer built. +Files may be plain or .gz. +""" +import argparse +import glob +import gzip +import hashlib +import json +import os +import re +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd + +CELL_LINE_DATASETS = {'ccle', 'ctrpv2', 'fimm', 'gcsi', 'gdscv1', 'gdscv2', 'nci60', 'prism'} +OMICS = ['transcriptomics', 'proteomics', 'phosphoproteomics', 'mutations', 'copy_number'] +CHUNK_ROWS = 2_000_000 + +EXPERIMENT_KEY = ['source', 'improve_sample_id', 'improve_drug_id', 'study', + 'time', 'time_unit', 'dose_response_metric'] +COMBINATION_KEY = ['source', 'improve_sample_id', 'improve_drug_1', 'improve_drug_2', + 'study', 'time', 'time_unit', 'drug_combination_metric'] +# Metrics on a 0..~1 scale; values far outside are worth a look. +BOUNDED_METRICS = {'auc': (0, 1.5), 'fit_auc': (0, 1.5), 'aac': (-0.5, 1.5)} +# Relative change versus the previous build that triggers a warning. +CHANGE_THRESHOLD = 0.10 +# A drop this large versus the previous build is reported on its own: it is far +# more likely a lost download or broken join than a real change in the data. +DROP_THRESHOLD = 0.5 +# Values that stand for "no value" in sample metadata. +PLACEHOLDERS = {'', 'na', 'nan', 'none', 'not reported'} + + +# --------------------------------------------------------------------------- io + +def _find(directory, dataset, kind): + for ext in ('.csv', '.tsv', '.csv.gz', '.tsv.gz'): + path = os.path.join(directory, f'{dataset}_{kind}{ext}') + if os.path.exists(path): + return path + return None + + +def _sep(path): + return '\t' if '.tsv' in path else ',' + + +def _columns(path): + return list(pd.read_csv(path, sep=_sep(path), nrows=0).columns) + + +def _chunks(path, columns): + present = [c for c in _columns(path) if c in columns] + return pd.read_csv(path, sep=_sep(path), usecols=present, dtype=str, + keep_default_na=False, chunksize=CHUNK_ROWS) + + +def _read(path, columns): + return pd.concat(_chunks(path, columns), ignore_index=True) + + +def _unique(path, column, keep_blank=False): + values = set() + if column not in _columns(path): + return values + for chunk in _chunks(path, [column]): + values.update(chunk[column].unique()) + if not keep_blank: + values.discard('') + return values + + +def discover_datasets(directory): + names = set() + for path in glob.glob(os.path.join(directory, '*_samples.csv*')): + names.add(os.path.basename(path).split('_samples.csv')[0]) + return sorted(names) + + +# ----------------------------------------------------------------------- report + +class Report: + def __init__(self): + self.errors = [] + self.warnings = [] + + def error(self, check, dataset, message): + self.errors.append((check, dataset, message)) + + def warn(self, check, dataset, message): + self.warnings.append((check, dataset, message)) + + def markdown(self): + lines = ['# CoderData build validation', '', + f'{len(self.errors)} error(s), {len(self.warnings)} warning(s).', ''] + for title, items in (('Errors', self.errors), ('Warnings', self.warnings)): + lines.append(f'## {title}') + lines.append('') + if not items: + lines.append('None.') + for check, dataset, message in items: + lines.append(f'- **{check}** ({dataset}): {message}') + lines.append('') + return '\n'.join(lines) + + +def _examples(values, n=5): + values = list(values) + shown = ', '.join(str(v) for v in values[:n]) + return shown + (f', ... ({len(values)} total)' if len(values) > n else '') + + +# ----------------------------------------------------------------------- checks + +def duplicate_keys(path, key, report, check, dataset): + """Rows sharing every key column (two values for one sample/drug/metric).""" + cols = _columns(path) + missing = [c for c in key if c not in cols] + if missing: + report.error(check, dataset, f'{os.path.basename(path)} lacks key column(s) {missing}') + return + hashes = [pd.util.hash_pandas_object(c[key], index=False).to_numpy() for c in _chunks(path, key)] + if not hashes: + return + values, counts = np.unique(np.concatenate(hashes), return_counts=True) + suspect = values[counts > 1] + if not len(suspect): + return + rows = pd.concat([c[key][np.isin(pd.util.hash_pandas_object(c[key], index=False).to_numpy(), suspect)] + for c in _chunks(path, key)], ignore_index=True) + dup = rows[rows.duplicated(key, keep=False)] + if len(dup): + pairs = dup.drop_duplicates().apply(lambda r: f"{r['improve_sample_id']}/" + f"{r.get('improve_drug_id', r.get('improve_drug_1'))}", axis=1) + report.error(check, dataset, + f'{len(dup)} rows in {os.path.basename(path)} share a key with another row ' + f'(more than one value for a sample/drug/time/metric); sample/drug e.g. ' + f'{_examples(pairs.unique())}') + + +def copied_responses(path, report, dataset, metric='fit_auc', min_rows=3): + """Different drug ids with identical responses in every sample.""" + cols = ['improve_sample_id', 'improve_drug_id', 'time', 'dose_response_metric', 'dose_response_value'] + parts = [c[c['dose_response_metric'] == metric] for c in _chunks(path, cols)] + df = pd.concat(parts, ignore_index=True) if parts else pd.DataFrame() + if df.empty: + return + key = df['improve_sample_id'] + '|' + df['time'] + '|' + df['dose_response_value'] + sig = key.groupby(df['improve_drug_id']).agg( + lambda s: hashlib.md5('\n'.join(sorted(s)).encode()).hexdigest() if len(s) >= min_rows else '') + sig = sig[sig != ''] + shared = sig[sig.duplicated(keep=False)] + if len(shared): + groups = [sorted(g.index) for _, g in shared.groupby(shared)] + report.warn('copied-responses', dataset, + f'{len(groups)} group(s) of drug ids report identical {metric} for every sample ' + f'(one experiment attached to several drugs?): {_examples(groups)}') + + +def value_ranges(path, report, dataset): + """Bounded metrics far outside their range, and metrics that are mostly blank.""" + cols = ['dose_response_metric', 'dose_response_value'] + total, blank, out = defaultdict(int), defaultdict(int), defaultdict(int) + for chunk in _chunks(path, cols): + values = pd.to_numeric(chunk['dose_response_value'], errors='coerce') + for metric, idx in chunk.groupby('dose_response_metric').groups.items(): + v = values.loc[idx] + total[metric] += len(v) + blank[metric] += int(v.isna().sum() + np.isinf(v).sum()) + if metric in BOUNDED_METRICS: + lo, hi = BOUNDED_METRICS[metric] + out[metric] += int(((v < lo) | (v > hi)).sum()) + for metric in total: + if blank[metric] == total[metric]: + continue # categorical metric (e.g. mrecist: PD/SD/PR/CR) + if blank[metric] / total[metric] > 0.5: + report.warn('value-ranges', dataset, + f'{metric}: {blank[metric] / total[metric]:.0%} of values are blank or infinite') + if out.get(metric): + lo, hi = BOUNDED_METRICS[metric] + report.warn('value-ranges', dataset, + f'{metric}: {out[metric]} of {total[metric]} values outside [{lo}, {hi}]') + + +def _id_values(report, dataset, kind, where, values): + """Report blank ids and ids written as decimals; return the ids without blanks.""" + if '' in values: + report.error('missing-id', dataset, f'{where} has rows with a blank {kind} id') + decimal = sorted(v for v in values if re.fullmatch(r'\d+\.0', v)) + if decimal: + report.error('decimal-id', dataset, + f'{len(decimal)} {kind} ids in {where} are written as decimals (e.g. "{decimal[0]}")') + return values - {''} + + +def _orphans(report, dataset, kind, where, missing, known): + if not missing: + return + decimal = {m for m in missing if re.fullmatch(r'\d+\.0', m) and m[:-2] in known} + hint = (f'; {len(decimal)} of them are known ids written as decimals (e.g. "{sorted(decimal)[0]}")' + if decimal else '') + report.error(f'orphan-{kind}-id', dataset, + f'{len(missing)} {kind} ids in {where} are not in the {kind}s file: ' + f'{_examples(sorted(missing))}{hint}') + + +def sample_checks(dataset, path, report): + """Per-sample metadata problems; returns the dataset's sample ids.""" + wanted = ['improve_sample_id', 'other_id', 'other_id_source', 'model_type', + 'cancer_type', 'common_name', 'other_names'] + s = _read(path, wanted).reindex(columns=wanted, fill_value='') + ids = _id_values(report, dataset, 'sample', 'samples', set(s['improve_sample_id'])) + mt = s.groupby('improve_sample_id')['model_type'].nunique() + if (mt > 1).any(): + report.error('sample-model-type', dataset, + f'{int((mt > 1).sum())} sample ids carry more than one model_type: ' + f'{_examples(mt[mt > 1].index)}') + # One sample, conflicting descriptions. other_names are aliases and may + # differ; 'NA' is a placeholder, not a name. Cell lines combine DepMap and + # Sanger records whose cancer_type vocabularies differ, so only their names + # are compared. + cols = ('common_name',) if dataset in CELL_LINE_DATASETS else ('common_name', 'cancer_type') + for col in cols: + named = s[~s[col].str.strip().str.lower().isin(PLACEHOLDERS)] + n = named.groupby('improve_sample_id')[col].nunique() + if (n > 1).any(): + report.warn('sample-metadata-conflict', dataset, + f'{int((n > 1).sum())} sample ids have more than one {col}: {_examples(n[n > 1].index)}') + return ids, s + + +def copied_samples(path, value_cols, report, dataset, what, min_rows=3): + """ + Different sample ids whose rows in a file are identical. + + Each sample gets an order-independent fingerprint: two independent 64-bit + row hashes summed with wrap-around (numpy uint64; pandas sums can spill + into float64 and collide), plus the row count. The file is streamed, and + samples whose fingerprints match are then compared row by row, so only + exact copies are reported. + """ + cols = ['improve_sample_id'] + value_cols + if not all(c in _columns(path) for c in cols): + return + keys = ('0123456789123456', 'fedcba9876543210') + acc = {} # sample -> [sum1, sum2, rows] + for chunk in _chunks(path, cols): + sid = chunk['improve_sample_id'].str.replace(r'\.0$', '', regex=True) + codes, uniq = pd.factorize(sid) + n = np.bincount(codes, minlength=len(uniq)) + sums = [] + for key in keys: + h = pd.util.hash_pandas_object(chunk[value_cols], index=False, hash_key=key).to_numpy() + s = np.zeros(len(uniq), dtype=np.uint64) + np.add.at(s, codes, h) + sums.append(s) + for k, sample in enumerate(uniq): + a = acc.setdefault(sample, np.zeros(3, dtype=np.uint64)) + a += np.array((sums[0][k], sums[1][k], n[k]), dtype=np.uint64) # array add wraps silently + if not acc: + return + sig = pd.DataFrame.from_dict(acc, orient='index', columns=['h1', 'h2', 'n']) + sig = sig[sig['n'] >= min_rows] + suspect = sig[sig.duplicated(['h1', 'h2', 'n'], keep=False)] + if suspect.empty: + return + # Confirm exactly: re-read only the suspect samples and compare their rows. + wanted = set(suspect.index) + rows = pd.concat([c.assign(improve_sample_id=c['improve_sample_id'].str.replace(r'\.0$', '', regex=True)) + .loc[lambda x: x['improve_sample_id'].isin(wanted)] + for c in _chunks(path, cols)], ignore_index=True) + content = {s: tuple(map(tuple, g[value_cols].sort_values(value_cols).to_numpy())) + for s, g in rows.groupby('improve_sample_id')} + groups = [] + for _, g in suspect.groupby(['h1', 'h2', 'n']): + by_content = defaultdict(list) + for s in g.index: + by_content[content[s]].append(s) + groups += [sorted(v) for v in by_content.values() if len(v) > 1] + if groups: + report.warn('copied-samples', dataset, + f'{len(groups)} group(s) of sample ids have identical {what}: {_examples(sorted(groups))}') + + +def drug_checks(dataset, path, report): + """One structure per drug id and one drug id per CID; returns the drug ids. + + One CID under two drug ids is one chemical listed twice; its responses + are split between the ids and merging them later gives duplicate results + (issue #512), so it is an error. + """ + d = _read(path, ['improve_drug_id', 'canSMILES', 'pubchem_id']) + ids = _id_values(report, dataset, 'drug', 'drugs', set(d['improve_drug_id'])) + smiles = d[d['canSMILES'] != ''].groupby('improve_drug_id')['canSMILES'].nunique() + if (smiles > 1).any(): + report.error('drug-structure', dataset, + f'{int((smiles > 1).sum())} drug ids have more than one canSMILES: ' + f'{_examples(smiles[smiles > 1].index)}') + if 'pubchem_id' in d.columns: + cid = d[d['pubchem_id'] != ''].assign(pubchem_id=lambda x: x['pubchem_id'].str.replace(r'\.0$', '', regex=True)) + per_cid = cid.groupby('pubchem_id')['improve_drug_id'].agg(lambda x: sorted(set(x))) + split = per_cid[per_cid.apply(len) > 1] + if len(split): + report.error('drug-split', dataset, + f'{len(split)} PubChem CID(s) sit under more than one drug id: ' + f'{_examples(f"{k}: {v}" for k, v in split.items())}') + return ids + + +def mapping_checks(directory, report): + for name, items_key, key_fn in ( + ('improve_sample_mapping.json', 'samples', + lambda e: [tuple(q.get(k, '').strip() for k in ('other_id', 'other_id_source', 'model_type', 'other_names')) + for q in e.get('quadruples', [])]), + ('improve_drug_mapping.json', 'drugs', lambda e: [e.get('canSMILES')])): + path = next((p for p in (os.path.join(directory, name), os.path.join(directory, name + '.gz')) + if os.path.exists(p)), None) + if not path: + report.warn('mapping-file', 'all', f'{name} not found') + continue + opener = gzip.open if path.endswith('.gz') else open + with opener(path, 'rt') as f: + entries = json.load(f).get(items_key, []) + ids = [e['stable_id'] for e in entries] + if len(ids) != len(set(ids)): + report.error('mapping-file', 'all', f'{name}: stable ids listed more than once') + owner = defaultdict(set) + for e in entries: + for k in key_fn(e): + owner[k].add(e['stable_id']) + multi = {k: v for k, v in owner.items() if len(v) > 1} + if multi: + report.error('mapping-file', 'all', + f'{name}: {len(multi)} identities map to more than one stable id, e.g. ' + f'{_examples(f"{k}: {sorted(v)}" for k, v in multi.items())}') + + +# ------------------------------------------------------------------ main runner + +def validate(directory, datasets=None, previous=None): + report = Report() + requested = datasets + datasets = datasets or discover_datasets(directory) + summary = {'datasets': {}} + sample_owner = defaultdict(set) + + for ds in datasets: + info = {} + spath = _find(directory, ds, 'samples') + if not spath: + report.error('missing-file', ds, 'no samples file') + continue + sids, samples = sample_checks(ds, spath, report) + info['samples'] = len(sids) + info['model_types'] = samples.drop_duplicates('improve_sample_id')['model_type'].value_counts().to_dict() + for sid in sids: + sample_owner[sid].add(ds) + + dpath = _find(directory, ds, 'drugs') + dids = drug_checks(ds, dpath, report) if dpath else set() + info['drugs'] = len(dids) + + info['omics'] = {} + for kind in OMICS: + path = _find(directory, ds, kind) + if not path: + continue + used = _id_values(report, ds, 'sample', kind, _unique(path, 'improve_sample_id', keep_blank=True)) + if kind == 'transcriptomics': + copied_samples(path, ['entrez_id', 'transcriptomics'], report, ds, 'transcriptomics') + info['omics'][kind] = len(used) + _orphans(report, ds, 'sample', kind, used - sids, sids) + + epath = _find(directory, ds, 'experiments') + if epath: + duplicate_keys(epath, EXPERIMENT_KEY, report, 'duplicate-result', ds) + copied_responses(epath, report, ds) + copied_samples(epath, ['improve_drug_id', 'time', 'dose_response_metric', 'dose_response_value'], + report, ds, 'drug responses') + value_ranges(epath, report, ds) + pairs, esids, edids = set(), set(), set() + for chunk in _chunks(epath, ['improve_sample_id', 'improve_drug_id']): + pairs.update(pd.util.hash_pandas_object(chunk[['improve_sample_id', 'improve_drug_id']], + index=False).to_numpy().tolist()) + esids.update(chunk['improve_sample_id'].unique()) + edids.update(chunk['improve_drug_id'].unique()) + esids = _id_values(report, ds, 'sample', 'experiments', esids) + edids = _id_values(report, ds, 'drug', 'experiments', edids) + info['pairs'] = len(pairs) + info['samples_with_response'] = len(esids) + info['drugs_with_response'] = len(edids) + _orphans(report, ds, 'sample', 'experiments', esids - sids, sids) + _orphans(report, ds, 'drug', 'experiments', edids - dids, dids) + no_omics = [k for k, n in info['omics'].items() if esids and n == 0] + if no_omics: + report.warn('omics-coverage', ds, f'no samples with drug response have {no_omics}') + + cpath = _find(directory, ds, 'combinations') + if cpath: + duplicate_keys(cpath, COMBINATION_KEY, report, 'duplicate-result', ds) + c = _read(cpath, ['improve_sample_id', 'improve_drug_1', 'improve_drug_2']) + csids = _id_values(report, ds, 'sample', 'combinations', set(c['improve_sample_id'])) + cdids = _id_values(report, ds, 'drug', 'combinations', + set(c['improve_drug_1']) | set(c['improve_drug_2'])) + _orphans(report, ds, 'sample', 'combinations', csids - sids, sids) + _orphans(report, ds, 'drug', 'combinations', cdids - dids, dids) + info['combination_rows'] = len(c) + + xpath = _find(directory, ds, 'drug_descriptors') + if xpath and dids: + xdids = _id_values(report, ds, 'drug', 'drug_descriptors', + _unique(xpath, 'improve_drug_id', keep_blank=True)) + _orphans(report, ds, 'drug', 'drug_descriptors', xdids - dids, dids) + summary['datasets'][ds] = info + + # Cell lines share ids by design; every other dataset owns its ids. + leaked = {sid: sorted(owners) for sid, owners in sample_owner.items() + if len(owners) > 1 and not owners <= CELL_LINE_DATASETS} + if leaked: + report.error('sample-id-overlap', 'all', + f'{len(leaked)} sample ids are used by more than one dataset: ' + f'{_examples(f"{k}: {v}" for k, v in leaked.items())}') + + mapping_checks(directory, report) + if previous: + compare(summary, previous, report, requested) + return report, summary + + +def compare(summary, previous, report, datasets=None): + """Warn on large changes versus a previous build's summary.""" + prev = previous.get('datasets', {}) + # Datasets left out on purpose (--datasets) are not missing. + expected = set(prev) if datasets is None else set(prev) & set(datasets) + for ds in sorted(expected - set(summary['datasets'])): + report.warn('dataset-missing', ds, 'in the previous build but not in this one') + for ds, cur in summary['datasets'].items(): + old = prev.get(ds) + if old is None: + report.warn('release-change', ds, 'new dataset (not in the previous build)') + continue + for field in ('samples', 'drugs', 'pairs', 'samples_with_response', 'drugs_with_response'): + a, b = old.get(field), cur.get(field) + if a and b is not None and (a - b) / a > DROP_THRESHOLD: + report.warn('release-drop', ds, f'{field}: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + elif a and b is not None and abs(b - a) / a > CHANGE_THRESHOLD: + report.warn('release-change', ds, f'{field}: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + for kind in set(old.get('omics', {})) | set(cur.get('omics', {})): + a, b = old.get('omics', {}).get(kind), cur.get('omics', {}).get(kind) + if a is None or b is None: + report.warn('release-change', ds, f'{kind}: {"added" if a is None else "removed"}') + elif a and (a - b) / a > DROP_THRESHOLD: + report.warn('release-drop', ds, f'{kind} samples: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + elif a and abs(b - a) / a > CHANGE_THRESHOLD: + report.warn('release-change', ds, f'{kind} samples: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + old_mt, new_mt = set(old.get('model_types', {})), set(cur.get('model_types', {})) + if old_mt != new_mt: + report.warn('release-change', ds, + f'model types changed: removed {sorted(old_mt - new_mt) or "none"}, ' + f'added {sorted(new_mt - old_mt) or "none"}') + + +def load_summary(path): + """Read a summary written by --write-summary (plain or .gz, as published).""" + opener = gzip.open if path.endswith('.gz') else open + with opener(path, 'rt') as f: + return json.load(f) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--dir', required=True, help='folder of build output files') + ap.add_argument('--datasets', nargs='*', default=None, help='datasets to check (default: all found)') + ap.add_argument('--previous-summary', default=None, help='summary JSON of the build to compare with') + ap.add_argument('--write-summary', default=None, help='write this build\'s summary JSON here') + args = ap.parse_args(argv) + + previous = load_summary(args.previous_summary) if args.previous_summary else None + report, summary = validate(args.dir, args.datasets, previous) + print(report.markdown()) + if args.write_summary: + with open(args.write_summary, 'w') as f: + json.dump(summary, f, indent=1, sort_keys=True) + return 1 if report.errors else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..f8aec3fc --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,12 @@ +import os + + +def pytest_addoption(parser): + group = parser.getgroup('coderdata build data') + group.addoption('--build-dir', default=os.environ.get('CODERDATA_BUILD_DIR'), + help='folder of CoderData build or release files to check with ' + 'tests/test_build_data.py (or set CODERDATA_BUILD_DIR)') + group.addoption('--previous-summary', default=os.environ.get('CODERDATA_PREVIOUS_SUMMARY'), + help='validation_summary.json of an earlier build, to report large changes') + group.addoption('--build-datasets', default=os.environ.get('CODERDATA_BUILD_DATASETS'), + help='comma-separated datasets to check (default: all found in --build-dir)') diff --git a/tests/test_build_data.py b/tests/test_build_data.py new file mode 100644 index 00000000..7d4e921f --- /dev/null +++ b/tests/test_build_data.py @@ -0,0 +1,74 @@ +""" +Data checks on a real CoderData build or release, run as tests. + +These are the checks `coderbuild/build_all.py --validate` runs, exposed to +pytest so any folder of CoderData files can be checked on its own: a local +build (local/all_files_dir) or a downloaded release. + + pytest tests/test_build_data.py --build-dir local/all_files_dir + pytest tests/test_build_data.py --build-dir data/ --build-datasets beataml,hcmi + CODERDATA_BUILD_DIR=local/all_files_dir pytest tests/test_build_data.py + +Each dataset is one test, plus one for cross-dataset checks. Errors fail the +test; warnings are reported as pytest warnings. Without --build-dir the tests +are skipped. The checks themselves are unit tested in test_validate_build.py. +""" +import os +import sys +import warnings + +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, 'coderbuild', 'utils')) + +import validate_build as vb # noqa: E402 + + +class BuildDataWarning(UserWarning): + """A validation warning: worth a look, not a failure.""" + + +def _options(config): + build_dir = config.getoption('--build-dir') + datasets = config.getoption('--build-datasets') + return build_dir, (datasets.split(',') if datasets else None) + + +def pytest_generate_tests(metafunc): + if 'dataset' not in metafunc.fixturenames: + return + build_dir, datasets = _options(metafunc.config) + if not build_dir: + metafunc.parametrize('dataset', [pytest.param(None, marks=pytest.mark.skip( + reason='no build to check: pass --build-dir or set CODERDATA_BUILD_DIR'))]) + return + metafunc.parametrize('dataset', datasets or vb.discover_datasets(build_dir)) + + +@pytest.fixture(scope='session') +def build_report(pytestconfig): + build_dir, datasets = _options(pytestconfig) + if not build_dir: + pytest.skip('no build to check: pass --build-dir or set CODERDATA_BUILD_DIR') + if not os.path.isdir(build_dir): + pytest.fail(f'--build-dir {build_dir} is not a folder') + summary = pytestconfig.getoption('--previous-summary') + previous = vb.load_summary(summary) if summary else None + report, _ = vb.validate(build_dir, datasets, previous) + return report + + +def _check(report, scope): + for check, _, message in [w for w in report.warnings if w[1] == scope]: + warnings.warn(f'{check}: {message}', BuildDataWarning) + errors = [f'{check}: {message}' for check, ds, message in report.errors if ds == scope] + assert not errors, '\n'.join(errors) + + +def test_dataset(build_report, dataset): + _check(build_report, dataset) + + +def test_cross_dataset(build_report): + _check(build_report, 'all') diff --git a/tests/test_validate_build.py b/tests/test_validate_build.py new file mode 100644 index 00000000..2c4421d4 --- /dev/null +++ b/tests/test_validate_build.py @@ -0,0 +1,286 @@ +""" +Tests for coderbuild/utils/validate_build.py. + +Each test writes a tiny build with one planted problem modelled on what +shipped in CoderData 2.4, and checks that the validator reports it (and that +a clean build passes). +""" +import gzip +import json +import os +import sys + +import pandas as pd +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, 'coderbuild', 'utils')) + +import validate_build as vb # noqa: E402 + +EXP_COLS = ['source', 'improve_sample_id', 'improve_drug_id', 'study', 'time', 'time_unit', + 'dose_response_metric', 'dose_response_value'] +DRUG_COLS = ['improve_drug_id', 'chem_name', 'pubchem_id', 'canSMILES', 'InChIKey', 'formula', 'weight'] +SAMPLE_COLS = ['other_id', 'improve_sample_id', 'other_names', 'common_name', 'cancer_type', + 'model_type', 'other_id_source', 'species'] + + +def _tsv(path, rows, cols, sep='\t'): + pd.DataFrame(rows, columns=cols).to_csv(path, sep=sep, index=False) + + +def _samples(d, ds, ids, model='patient derived organoid', cancer='Colon cancer'): + _tsv(d / f'{ds}_samples.csv', [[f'X{i}', i, f'n{i}', f'c{i}', cancer, model, 'src', 'Homo sapiens'] + for i in ids], SAMPLE_COLS, sep=',') + + +def _drugs(d, ds, rows): + _tsv(d / f'{ds}_drugs.tsv', rows, DRUG_COLS) + + +def _exp(d, ds, rows): + _tsv(d / f'{ds}_experiments.tsv', rows, EXP_COLS) + + +def _mappings(d, samples=None, drugs=None): + samples = samples or [{'stable_id': '1', 'quadruples': [ + {'other_id': 'X1', 'other_id_source': 'src', 'model_type': 'patient derived organoid', 'other_names': 'n1'}]}] + drugs = drugs or [{'stable_id': 'SMI_1', 'canSMILES': 'CCO'}] + with gzip.open(d / 'improve_sample_mapping.json.gz', 'wt') as f: + json.dump({'metadata': {'builds': []}, 'samples': samples}, f) + with gzip.open(d / 'improve_drug_mapping.json.gz', 'wt') as f: + json.dump({'metadata': {'builds': []}, 'drugs': drugs}, f) + + +def _clean_build(d): + _samples(d, 'liver', [1, 2, 3]) + _drugs(d, 'liver', [['SMI_1', 'ethanol', '702', 'CCO', 'K', 'F', '1'], + ['SMI_2', 'taxol', '36314', 'CCC', 'K2', 'F', '1']]) + rows = [] + for s, a, b in ((1, '0.5', '0.9'), (2, '0.6', '0.2'), (3, '0.7', '0.4')): + rows.append(['syn', s, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', a]) + rows.append(['syn', s, 'SMI_2', 'L', '72', 'hrs', 'fit_auc', b]) + _exp(d, 'liver', rows) + _tsv(d / 'liver_transcriptomics.csv', [[1, 7105, 1.0], [2, 7105, 2.0]], + ['improve_sample_id', 'entrez_id', 'transcriptomics'], sep=',') + _mappings(d) + + +def _checks(report, kind='errors'): + return {c for c, _, _ in getattr(report, kind)} + + +def test_clean_build_passes(tmp_path): + _clean_build(tmp_path) + report, summary = vb.validate(str(tmp_path)) + assert report.errors == [] + assert summary['datasets']['liver']['pairs'] == 6 + assert vb.main(['--dir', str(tmp_path)]) == 0 + + +def test_duplicate_results_fail(tmp_path): + # issue #512: two curves for one sample/drug + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.82'], + ['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.78']]) + report, _ = vb.validate(str(tmp_path)) + assert 'duplicate-result' in _checks(report) + assert vb.main(['--dir', str(tmp_path)]) == 1 + + +def test_decimal_sample_ids_are_orphans(tmp_path): + # novartis 2.4: experiments wrote sample ids as '3524.0' + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', '1.0', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5']]) + report, _ = vb.validate(str(tmp_path)) + msg = [m for c, _, m in report.errors if c == 'orphan-sample-id'][0] + assert 'written as decimals' in msg + + +def test_orphan_drug_ids_fail(tmp_path): + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_9', 'L', '72', 'hrs', 'fit_auc', '0.5']]) + report, _ = vb.validate(str(tmp_path)) + assert 'orphan-drug-id' in _checks(report) + + +def test_copied_responses_warn(tmp_path): + _clean_build(tmp_path) + rows = [['syn', s, d, 'L', '72', 'hrs', 'fit_auc', v] + for d in ('SMI_1', 'SMI_2') for s, v in ((1, '0.5'), (2, '0.6'), (3, '0.7'))] + _exp(tmp_path, 'liver', rows) + report, _ = vb.validate(str(tmp_path)) + assert 'copied-responses' in _checks(report, 'warnings') + assert report.errors == [] + + +def test_sample_ids_shared_by_two_datasets_fail(tmp_path): + _clean_build(tmp_path) + _samples(tmp_path, 'bladder', [3, 4]) + report, _ = vb.validate(str(tmp_path)) + assert 'sample-id-overlap' in _checks(report) + + +def test_cell_lines_may_share_ids(tmp_path): + for ds in ('ccle', 'prism'): + _samples(tmp_path, ds, [1, 2], model='cell line', cancer='Lung cancer') + _mappings(tmp_path) + report, _ = vb.validate(str(tmp_path)) + assert 'sample-id-overlap' not in _checks(report) + + +def test_conflicting_model_types_fail(tmp_path): + _clean_build(tmp_path) + rows = [['X1', 1, 'n1', 'c1', 'Colon cancer', 'tumor', 'src', 'Homo sapiens'], + ['X1b', 1, 'n1', 'c1', 'Colon cancer', 'patient derived organoid', 'src2', 'Homo sapiens'], + ['X2', 2, 'n2', 'c2', 'Colon cancer', 'tumor', 'src', 'Homo sapiens'], + ['X3', 3, 'n3', 'c3', 'Colon cancer', 'tumor', 'src', 'Homo sapiens']] + _tsv(tmp_path / 'liver_samples.csv', rows, SAMPLE_COLS, sep=',') + report, _ = vb.validate(str(tmp_path)) + assert 'sample-model-type' in _checks(report) + + +def test_drug_with_two_structures_or_split_cid_fails(tmp_path): + _clean_build(tmp_path) + _drugs(tmp_path, 'liver', [['SMI_1', 'ethanol', '702', 'CCO', 'K', 'F', '1'], + ['SMI_1', 'ethyl alcohol', '702', 'OCC', 'K', 'F', '1'], + ['SMI_2', 'taxol', '36314', 'CCC', 'K2', 'F', '1'], + ['SMI_3', 'paclitaxel', '36314.0', 'CCCC', 'K2', 'F', '1']]) + report, _ = vb.validate(str(tmp_path)) + assert 'drug-structure' in _checks(report) + assert 'drug-split' in _checks(report) # one chemical under two ids (issue #512) + + +def test_blank_ids_fail(tmp_path): + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5'], + ['syn', '', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.6'], + ['syn', 2, '', 'L', '72', 'hrs', 'fit_auc', '0.7']]) + report, _ = vb.validate(str(tmp_path)) + blank = [m for c, _, m in report.errors if c == 'missing-id'] + assert any('blank sample id' in m for m in blank) and any('blank drug id' in m for m in blank) + assert 'orphan-sample-id' not in _checks(report) # blanks are not reported twice + + +def test_decimal_ids_in_the_samples_file_fail(tmp_path): + # ids written as decimals everywhere still match each other, so only the + # format check catches them + _clean_build(tmp_path) + _tsv(tmp_path / 'liver_samples.csv', [[f'X{i}', f'{i}.0', f'n{i}', f'c{i}', 'Colon cancer', + 'patient derived organoid', 'src', 'Homo sapiens'] for i in (1, 2, 3)], + SAMPLE_COLS, sep=',') + rows = [['syn', f'{s}.0', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', v] for s, v in ((1, '0.5'), (2, '0.6'), (3, '0.7'))] + _exp(tmp_path, 'liver', rows) + report, _ = vb.validate(str(tmp_path)) + assert 'decimal-id' in _checks(report) + + +def test_mapping_identity_with_two_ids_fails(tmp_path): + _clean_build(tmp_path) + _mappings(tmp_path, drugs=[{'stable_id': 'SMI_1', 'canSMILES': 'CCO'}, + {'stable_id': 'SMI_2', 'canSMILES': 'CCO'}]) + report, _ = vb.validate(str(tmp_path)) + assert 'mapping-file' in _checks(report) + + +def test_categorical_metrics_are_not_range_checked(tmp_path): + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'mrecist', 'PD'], + ['syn', 2, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '9.0']]) + report, _ = vb.validate(str(tmp_path)) + assert report.errors == [] + assert any('outside' in m for c, _, m in report.warnings if c == 'value-ranges') + + +def test_release_changes_and_large_drops_warn(tmp_path): + # hcmi 2.2 -> 2.4 lost its tumor and cell-line samples + _clean_build(tmp_path) + previous = {'datasets': {'liver': {'samples': 10, 'drugs': 2, 'pairs': 6, + 'model_types': {'tumor': 5, 'patient derived organoid': 5}, + 'omics': {'transcriptomics': 2, 'mutations': 2}}, + 'hcmi': {'samples': 5}}} + report, _ = vb.validate(str(tmp_path), previous=previous) + messages = [m for c, _, m in report.warnings if c == 'release-change'] + drops = [m for c, _, m in report.warnings if c == 'release-drop'] + assert any(m.startswith('samples: 10 -> 3') for m in drops) # -70%: flagged as a drop + assert report.errors == [] + assert any('removed' in m and 'tumor' in m for m in messages) + assert any('mutations' in m for m in messages) + assert ('dataset-missing', 'hcmi') in {(c, ds) for c, ds, _ in report.warnings} + + +def test_moderate_drop_only_warns(tmp_path): + # nci60 2.4 rebuild lost 98% of its drugs; a 30% change is only a warning + _clean_build(tmp_path) + _, summary = vb.validate(str(tmp_path)) + liver = summary['datasets']['liver'] + previous = {'datasets': {'liver': dict(liver, samples=round(liver['samples'] / 0.7))}} + report, _ = vb.validate(str(tmp_path), previous=previous) + assert 'release-drop' not in _checks(report, 'warnings') + assert any(m.startswith('samples:') for c, _, m in report.warnings if c == 'release-change') + + +def test_datasets_left_out_on_purpose_are_not_missing(tmp_path): + _clean_build(tmp_path) + _, summary = vb.validate(str(tmp_path)) + previous = {'datasets': dict(summary['datasets'], hcmi={'samples': 5})} + report, _ = vb.validate(str(tmp_path), ['liver'], previous) + assert 'dataset-missing' not in _checks(report, 'warnings') + + +def test_summary_round_trip(tmp_path): + _clean_build(tmp_path) + out = tmp_path / 'summary.json' + assert vb.main(['--dir', str(tmp_path), '--write-summary', str(out)]) == 0 + assert vb.main(['--dir', str(tmp_path), '--previous-summary', str(out)]) == 0 + assert json.loads(out.read_text())['datasets']['liver']['pairs'] == 6 + # releases publish the summary gzipped + gz = tmp_path / 'validation_summary.json.gz' + with gzip.open(gz, 'wt') as f: + f.write(out.read_text()) + assert vb.load_summary(str(gz)) == json.loads(out.read_text()) + assert vb.main(['--dir', str(tmp_path), '--previous-summary', str(gz)]) == 0 + + +def test_copied_samples_warn(tmp_path): + # novartis: model NIBRX-2043 was attached to sample ids 3828 and 3829 + _clean_build(tmp_path) + rows = [['syn', s, d, 'L', '72', 'hrs', 'fit_auc', v] + for s in (1, 2) for d, v in (('SMI_1', '0.5'), ('SMI_2', '0.6'), ('SMI_1', '0.7'))] + rows = [r for i, r in enumerate(rows)] + rows[2][6] = 'auc' + rows[5][6] = 'auc' + rows.append(['syn', 3, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.1']) + _exp(tmp_path, 'liver', rows) + report, _ = vb.validate(str(tmp_path)) + msg = [m for c, _, m in report.warnings if c == 'copied-samples'] + assert msg and "['1', '2']" in msg[0] + + +def test_na_names_and_cell_line_cancer_wording_do_not_warn(tmp_path): + rows = [['CVCL_1', 1, 'RH-30', 'RH-30', 'Alveolar Rhabdomyosarcoma', 'cell line', 'Cellosaurus', 'Homo sapiens'], + ['ACH-1', 1, 'RH-30', 'NA', 'Rhabdomyosarcoma', 'cell line', 'DepMap', 'Homo sapiens']] + _tsv(tmp_path / 'ccle_samples.csv', rows, SAMPLE_COLS, sep=',') + _mappings(tmp_path) + report, _ = vb.validate(str(tmp_path)) + assert 'sample-metadata-conflict' not in _checks(report, 'warnings') + rows[1][3] = 'SJRH-30' + _tsv(tmp_path / 'ccle_samples.csv', rows, SAMPLE_COLS, sep=',') + report, _ = vb.validate(str(tmp_path)) + assert 'sample-metadata-conflict' in _checks(report, 'warnings') + + +def test_same_size_samples_with_different_data_are_not_copies(tmp_path): + # PRISM 2.4 false positive: hundreds of samples with the same number of + # rows collided when per-sample hash sums overflowed in pandas. + import numpy as np + rng = np.random.default_rng(0) + ids = list(range(1, 301)) + _samples(tmp_path, 'liver', ids) + _drugs(tmp_path, 'liver', [[f'SMI_{d}', f'd{d}', str(d), f'C{d}', 'K', 'F', '1'] for d in range(1, 41)]) + rows = [['syn', s, f'SMI_{d}', 'L', '72', 'hrs', 'fit_auc', f'{rng.random():.4f}'] + for s in ids for d in range(1, 41)] + _exp(tmp_path, 'liver', rows) + _mappings(tmp_path) + report, _ = vb.validate(str(tmp_path)) + assert 'copied-samples' not in _checks(report, 'warnings') From 21973104c305dfd409def7a0cea08d484c748f1c Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 28 Oct 2026 18:52:50 -0400 Subject: [PATCH 2/3] Add a comment to each validation test --- tests/test_build_data.py | 4 +++ tests/test_validate_build.py | 55 ++++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/tests/test_build_data.py b/tests/test_build_data.py index 7d4e921f..14e2bc7c 100644 --- a/tests/test_build_data.py +++ b/tests/test_build_data.py @@ -67,8 +67,12 @@ def _check(report, scope): def test_dataset(build_report, dataset): + # Fails if this dataset has any errors. Its warnings are shown in the pytest + # output without failing. _check(build_report, dataset) def test_cross_dataset(build_report): + # Same as above, for checks that compare datasets (shared sample IDs, + # mapping files). _check(build_report, 'all') diff --git a/tests/test_validate_build.py b/tests/test_validate_build.py index 2c4421d4..8e5a062b 100644 --- a/tests/test_validate_build.py +++ b/tests/test_validate_build.py @@ -71,6 +71,7 @@ def _checks(report, kind='errors'): def test_clean_build_passes(tmp_path): + # A build with no problems should pass with no errors. _clean_build(tmp_path) report, summary = vb.validate(str(tmp_path)) assert report.errors == [] @@ -79,7 +80,8 @@ def test_clean_build_passes(tmp_path): def test_duplicate_results_fail(tmp_path): - # issue #512: two curves for one sample/drug + # Two results for the same sample, drug, time and metric should fail. + # This is what broke the wide format in issue #512. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.82'], ['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.78']]) @@ -89,7 +91,9 @@ def test_duplicate_results_fail(tmp_path): def test_decimal_sample_ids_are_orphans(tmp_path): - # novartis 2.4: experiments wrote sample ids as '3524.0' + # Sample IDs written as decimals ("1.0") don't match the samples file, so + # they should fail, and the message should say why. Novartis shipped + # IDs like "3524.0" in 2.4. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', '1.0', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5']]) report, _ = vb.validate(str(tmp_path)) @@ -98,6 +102,7 @@ def test_decimal_sample_ids_are_orphans(tmp_path): def test_orphan_drug_ids_fail(tmp_path): + # A response for a drug that isn't in the drugs file should fail. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', 1, 'SMI_9', 'L', '72', 'hrs', 'fit_auc', '0.5']]) report, _ = vb.validate(str(tmp_path)) @@ -105,6 +110,8 @@ def test_orphan_drug_ids_fail(tmp_path): def test_copied_responses_warn(tmp_path): + # Two drugs with exactly the same responses in every sample are probably + # one drug copied under two IDs. This is a warning, not an error. _clean_build(tmp_path) rows = [['syn', s, d, 'L', '72', 'hrs', 'fit_auc', v] for d in ('SMI_1', 'SMI_2') for s, v in ((1, '0.5'), (2, '0.6'), (3, '0.7'))] @@ -115,6 +122,8 @@ def test_copied_responses_warn(tmp_path): def test_sample_ids_shared_by_two_datasets_fail(tmp_path): + # Two datasets using the same sample ID should fail (cell lines are the + # exception, see the next test). _clean_build(tmp_path) _samples(tmp_path, 'bladder', [3, 4]) report, _ = vb.validate(str(tmp_path)) @@ -122,6 +131,8 @@ def test_sample_ids_shared_by_two_datasets_fail(tmp_path): def test_cell_lines_may_share_ids(tmp_path): + # Cell line datasets (such as CCLE and PRISM) share sample IDs on purpose, + # so this should not fail. for ds in ('ccle', 'prism'): _samples(tmp_path, ds, [1, 2], model='cell line', cancer='Lung cancer') _mappings(tmp_path) @@ -130,6 +141,8 @@ def test_cell_lines_may_share_ids(tmp_path): def test_conflicting_model_types_fail(tmp_path): + # One sample ID listed with two model types (e.g. tumor and organoid) + # should fail. _clean_build(tmp_path) rows = [['X1', 1, 'n1', 'c1', 'Colon cancer', 'tumor', 'src', 'Homo sapiens'], ['X1b', 1, 'n1', 'c1', 'Colon cancer', 'patient derived organoid', 'src2', 'Homo sapiens'], @@ -141,6 +154,9 @@ def test_conflicting_model_types_fail(tmp_path): def test_drug_with_two_structures_or_split_cid_fails(tmp_path): + # Two things should fail here: one drug ID with two structures, and one + # PubChem compound under two drug IDs (issue #512). "36314.0" and + # "36314" count as the same compound. _clean_build(tmp_path) _drugs(tmp_path, 'liver', [['SMI_1', 'ethanol', '702', 'CCO', 'K', 'F', '1'], ['SMI_1', 'ethyl alcohol', '702', 'OCC', 'K', 'F', '1'], @@ -152,6 +168,8 @@ def test_drug_with_two_structures_or_split_cid_fails(tmp_path): def test_blank_ids_fail(tmp_path): + # Blank sample or drug IDs should fail. Each blank is reported once, not + # again as an ID missing from the samples or drugs file. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5'], ['syn', '', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.6'], @@ -163,8 +181,8 @@ def test_blank_ids_fail(tmp_path): def test_decimal_ids_in_the_samples_file_fail(tmp_path): - # ids written as decimals everywhere still match each other, so only the - # format check catches them + # IDs written as decimals in every file still match each other, so only + # the decimal check catches them. They should still fail. _clean_build(tmp_path) _tsv(tmp_path / 'liver_samples.csv', [[f'X{i}', f'{i}.0', f'n{i}', f'c{i}', 'Colon cancer', 'patient derived organoid', 'src', 'Homo sapiens'] for i in (1, 2, 3)], @@ -176,6 +194,8 @@ def test_decimal_ids_in_the_samples_file_fail(tmp_path): def test_mapping_identity_with_two_ids_fails(tmp_path): + # The stable ID mapping file should never give one drug structure two + # IDs. _clean_build(tmp_path) _mappings(tmp_path, drugs=[{'stable_id': 'SMI_1', 'canSMILES': 'CCO'}, {'stable_id': 'SMI_2', 'canSMILES': 'CCO'}]) @@ -184,6 +204,8 @@ def test_mapping_identity_with_two_ids_fails(tmp_path): def test_categorical_metrics_are_not_range_checked(tmp_path): + # Text results like mRECIST ("PD") are not range checked. A number + # outside the expected range is only a warning. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'mrecist', 'PD'], ['syn', 2, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '9.0']]) @@ -193,7 +215,10 @@ def test_categorical_metrics_are_not_range_checked(tmp_path): def test_release_changes_and_large_drops_warn(tmp_path): - # hcmi 2.2 -> 2.4 lost its tumor and cell-line samples + # Compared with the previous build, these should all warn (not fail): + # samples dropping by more than half, a model type disappearing, an + # omics type disappearing, and a dataset no longer being built. HCMI + # lost its tumor and cell line samples between 2.2 and 2.4. _clean_build(tmp_path) previous = {'datasets': {'liver': {'samples': 10, 'drugs': 2, 'pairs': 6, 'model_types': {'tumor': 5, 'patient derived organoid': 5}, @@ -210,7 +235,8 @@ def test_release_changes_and_large_drops_warn(tmp_path): def test_moderate_drop_only_warns(tmp_path): - # nci60 2.4 rebuild lost 98% of its drugs; a 30% change is only a warning + # A 30% drop is reported as a change, but not as a large drop (that + # needs more than 50%). _clean_build(tmp_path) _, summary = vb.validate(str(tmp_path)) liver = summary['datasets']['liver'] @@ -221,6 +247,8 @@ def test_moderate_drop_only_warns(tmp_path): def test_datasets_left_out_on_purpose_are_not_missing(tmp_path): + # If only some datasets were built (--datasets), the others should not + # be reported as missing. _clean_build(tmp_path) _, summary = vb.validate(str(tmp_path)) previous = {'datasets': dict(summary['datasets'], hcmi={'samples': 5})} @@ -229,6 +257,9 @@ def test_datasets_left_out_on_purpose_are_not_missing(tmp_path): def test_summary_round_trip(tmp_path): + # The summary written by one build can be read back by the next as + # --previous-summary, whether plain or gzipped (releases publish it + # gzipped). _clean_build(tmp_path) out = tmp_path / 'summary.json' assert vb.main(['--dir', str(tmp_path), '--write-summary', str(out)]) == 0 @@ -243,7 +274,9 @@ def test_summary_round_trip(tmp_path): def test_copied_samples_warn(tmp_path): - # novartis: model NIBRX-2043 was attached to sample ids 3828 and 3829 + # Two sample IDs with exactly the same responses are probably one sample + # listed twice. This is a warning. Novartis model NIBRX-2043 had this + # under sample IDs 3828 and 3829. _clean_build(tmp_path) rows = [['syn', s, d, 'L', '72', 'hrs', 'fit_auc', v] for s in (1, 2) for d, v in (('SMI_1', '0.5'), ('SMI_2', '0.6'), ('SMI_1', '0.7'))] @@ -258,6 +291,9 @@ def test_copied_samples_warn(tmp_path): def test_na_names_and_cell_line_cancer_wording_do_not_warn(tmp_path): + # A cell line named "NA" in one source, or with slightly different + # cancer type wording, should not warn. Two real, different names for + # the same sample ID should. rows = [['CVCL_1', 1, 'RH-30', 'RH-30', 'Alveolar Rhabdomyosarcoma', 'cell line', 'Cellosaurus', 'Homo sapiens'], ['ACH-1', 1, 'RH-30', 'NA', 'Rhabdomyosarcoma', 'cell line', 'DepMap', 'Homo sapiens']] _tsv(tmp_path / 'ccle_samples.csv', rows, SAMPLE_COLS, sep=',') @@ -271,8 +307,9 @@ def test_na_names_and_cell_line_cancer_wording_do_not_warn(tmp_path): def test_same_size_samples_with_different_data_are_not_copies(tmp_path): - # PRISM 2.4 false positive: hundreds of samples with the same number of - # rows collided when per-sample hash sums overflowed in pandas. + # Many samples with the same number of results but different values + # should not be flagged as copies. An earlier version of this check + # flagged hundreds of PRISM samples by mistake. import numpy as np rng = np.random.default_rng(0) ids = list(range(1, 301)) From 5294e0838a6364fdc61070168e4c1b00e8481ac5 Mon Sep 17 00:00:00 2001 From: Jeremy Jacobson <85139244+jjacobson95@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:30:39 -0500 Subject: [PATCH 3/3] updated comments in test_validate_build.py --- tests/test_validate_build.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_validate_build.py b/tests/test_validate_build.py index 8e5a062b..6eb98c0a 100644 --- a/tests/test_validate_build.py +++ b/tests/test_validate_build.py @@ -91,9 +91,8 @@ def test_duplicate_results_fail(tmp_path): def test_decimal_sample_ids_are_orphans(tmp_path): - # Sample IDs written as decimals ("1.0") don't match the samples file, so - # they should fail, and the message should say why. Novartis shipped - # IDs like "3524.0" in 2.4. + # Sample IDs written as decimals ("1.0", "3524.0") don't match the samples file, + # so they should fail, and the message should say why. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', '1.0', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5']]) report, _ = vb.validate(str(tmp_path)) @@ -102,7 +101,7 @@ def test_decimal_sample_ids_are_orphans(tmp_path): def test_orphan_drug_ids_fail(tmp_path): - # A response for a drug that isn't in the drugs file should fail. + # An experimental response for a drug that isn't in the drugs file should fail. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', 1, 'SMI_9', 'L', '72', 'hrs', 'fit_auc', '0.5']]) report, _ = vb.validate(str(tmp_path)) @@ -122,8 +121,8 @@ def test_copied_responses_warn(tmp_path): def test_sample_ids_shared_by_two_datasets_fail(tmp_path): - # Two datasets using the same sample ID should fail (cell lines are the - # exception, see the next test). + # Two datasets using the same sample ID should fail. Cell lines (broad/sanger) + # are allowed to have overlap, see the next test. _clean_build(tmp_path) _samples(tmp_path, 'bladder', [3, 4]) report, _ = vb.validate(str(tmp_path)) @@ -142,7 +141,7 @@ def test_cell_lines_may_share_ids(tmp_path): def test_conflicting_model_types_fail(tmp_path): # One sample ID listed with two model types (e.g. tumor and organoid) - # should fail. + # should fail. These should have different sample IDs. _clean_build(tmp_path) rows = [['X1', 1, 'n1', 'c1', 'Colon cancer', 'tumor', 'src', 'Homo sapiens'], ['X1b', 1, 'n1', 'c1', 'Colon cancer', 'patient derived organoid', 'src2', 'Homo sapiens'], @@ -168,7 +167,7 @@ def test_drug_with_two_structures_or_split_cid_fails(tmp_path): def test_blank_ids_fail(tmp_path): - # Blank sample or drug IDs should fail. Each blank is reported once, not + # Empty sample or drug IDs should fail. Each empty is reported once, not # again as an ID missing from the samples or drugs file. _clean_build(tmp_path) _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5'], @@ -182,7 +181,7 @@ def test_blank_ids_fail(tmp_path): def test_decimal_ids_in_the_samples_file_fail(tmp_path): # IDs written as decimals in every file still match each other, so only - # the decimal check catches them. They should still fail. + # the decimal check catches them. They should still fail. (Should be int) _clean_build(tmp_path) _tsv(tmp_path / 'liver_samples.csv', [[f'X{i}', f'{i}.0', f'n{i}', f'c{i}', 'Colon cancer', 'patient derived organoid', 'src', 'Homo sapiens'] for i in (1, 2, 3)], @@ -215,6 +214,7 @@ def test_categorical_metrics_are_not_range_checked(tmp_path): def test_release_changes_and_large_drops_warn(tmp_path): + # This is a cross build comparison --- # Compared with the previous build, these should all warn (not fail): # samples dropping by more than half, a model type disappearing, an # omics type disappearing, and a dataset no longer being built. HCMI