diff --git a/coderbuild/README.md b/coderbuild/README.md index dc6c2eef..379b5374 100755 --- a/coderbuild/README.md +++ b/coderbuild/README.md @@ -13,7 +13,7 @@ are added. This script initializes all docker containers, builds all datasets, validates them, and uploads them to figshare. It requires the following authorization tokens to be set in the local environment depending on the use case: -`SYNAPSE_AUTH_TOKEN`: Required for beataml and mpnst datasets. Join the [CoderData team](https://www.synapse.org/#!Team:3503472) on Synapse and generate an access token. +`SYNAPSE_AUTH_TOKEN`: Required for the beataml, mpnst, bladder, pancreatic, liver, sarcoma, novartis, colorectal and broad_sanger (DepMap files) datasets. Join the [CoderData team](https://www.synapse.org/#!Team:3503472) on Synapse and generate an access token. `FIGSHARE_TOKEN`: This token is required to upload to Figshare. `GITHUB_TOKEN`: This token is required to upload to GitHub. @@ -24,11 +24,17 @@ It requires the following authorization tokens to be set in the local environmen - `--omics`: Processes and builds the omics data files. - `--drugs`: Processes and builds the drug data files. - `--exp`: Processes and builds the experiment data files. -- `--all`: Executes all available processes above (docker, samples, omics, drugs, exp). This does not run the validate or figshare commands. -- `--validate`: Validates the generated datasets using the schema check scripts. This is automatically included if data upload occurs. +- `--misc`: Runs the final post-build step (e.g. splitting broad_sanger into its datasets). +- `--prev_drugs`: Comma-separated list of previously built drug files. They seed the PubChem lookup so only new drugs are queried. +- `--all`: Executes all available processes above (docker, samples, omics, drugs, exp, misc). This does not run the validate or figshare commands. +- `--validate`: Validates the generated datasets with the build data checks (see below) and the schema check scripts. This is automatically included if data upload occurs. +- `--previous-summary`: Optional. `validation_summary.json(.gz)` from the previous release, compared against by the build data checks. +- `--skip-build-checks`: Skips the build data checks during `--validate` (not recommended). - `--figshare`: Uploads the datasets to Figshare. FIGSHARE_TOKEN must be set in local environment. - `--high_mem`: Utilizes high memory mode for concurrent data processing. This has been successfully tested using 32 or more vCPUs. -- `--dataset`: Specifies the datasets to process (default='broad_sanger,hcmi,beataml,mpnst,cptac'). +- `--dataset`: Specifies the datasets to process (default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,cptac,hcmi'). +- `--depmap-ready`: Confirms that the DepMap files on Synapse are up to date for the current DepMap release. Required for any build that includes broad_sanger. +- `--continue`: Resumes a partial build, skipping steps whose output already exists in `local/`. Progress is logged to `local/build_progress.log`. - `--version`: Specifies the version number for the Figshare upload title (e.g., "0.1.29"). This must be a higher version than previously published versions. - `--github-username`: GitHub username matching the GITHUB_TOKEN. Required to push the new Tag to the GitHub Repository. - `--github-email`: GitHub email matching the GITHUB_TOKEN. Required to push the new Tag to the GitHub Repository. @@ -37,7 +43,7 @@ It requires the following authorization tokens to be set in the local environmen - Build all datasets and upload to Figshare and GitHub. Required tokens for the following command: `SYNAPSE_AUTH_TOKEN`, `FIGSHARE_TOKEN`, `GITHUB_TOKEN`. ```bash -python coderbuild/build_all.py --all --high_mem --validate --figshare --version 0.1.41 --github-username jjacobson95 --github-email jeremy.jacobson3402@gmail.com +python coderbuild/build_all.py --all --high_mem --validate --depmap-ready --figshare --version 0.1.41 --github-username jjacobson95 --github-email jeremy.jacobson3402@gmail.com ``` - Build only the experiment files. @@ -51,7 +57,7 @@ This script builds a single dataset for **debugging purposes only**. It can help It requires the following authorization tokens to be set in the local environment depending on the dataset: -`SYNAPSE_AUTH_TOKEN`: Required for beataml and mpnst datasets. Follow the directions above to use gain access. +`SYNAPSE_AUTH_TOKEN`: Required for most datasets (see the list above). Follow the directions above to gain access. Available arguments: - `--dataset`: Required. Name of the dataset to build. At a minimum, this will build the docker images. @@ -59,6 +65,8 @@ Available arguments: - `--build`: Optional. Build the desired Dataset. - `--validate`: Optional. Run the schema checker on the built files. - `--continue`: Optional. Continues from where the build left off by skipping existing files in "local" directory. +- `--depmap-ready`: Required when building broad_sanger. Confirms the DepMap files on Synapse are up to date. + Example usage: Build the broad_sanger dataset: @@ -78,6 +86,29 @@ Build the broad_sanger dataset but skip previously built files in "local" direct python coderbuild/build_dataset.py --dataset broad_sanger --continue ``` +## Build data checks (validate_build.py) + +These checks look for problems the schema check cannot catch: duplicate results, ids that do not match between files, sample ids shared between datasets, copied responses, and large changes from the previous release. `build_all.py --validate` runs them automatically. They can also be run on their own against any folder of CoderData files, either a local build (`local/all_files_dir`) or a release downloaded from Figshare. Files may be plain or `.gz`. They need `pandas` and `numpy`; the pytest form also needs `pytest`. + +As a command: +```bash +python coderbuild/utils/validate_build.py --dir local/all_files_dir +``` +- `--dir`: Required. Folder of files to check. +- `--datasets`: Optional. Check only these datasets (default: all found in `--dir`). +- `--previous-summary`: Optional. Summary of an earlier build to compare counts with. Each release publishes one as `validation_summary.json.gz`. +- `--write-summary`: Optional. Save this build's counts, to use as the next baseline. + +Results are printed to the terminal. The command exits with code 1 if there are errors; warnings do not change the exit code. + +As tests: +```bash +pytest tests/test_build_data.py --build-dir local/all_files_dir +``` +Add `--build-datasets beataml,hcmi` to check only some datasets, or `--previous-summary ` to compare with a previous build. Each dataset is one test. Errors fail the test, and warnings show as pytest warnings. Without `--build-dir` these tests are skipped. + +In `build_all.py` the checks run after the id mapping and before the schema check, and write `validation_summary.json` into `local/all_files_dir` so it is published with the release. If they find errors, the build stops after the schema check. + diff --git a/coderbuild/build_all.py b/coderbuild/build_all.py index 37290fd1..209eb8fa 100644 --- a/coderbuild/build_all.py +++ b/coderbuild/build_all.py @@ -71,6 +71,11 @@ def main(): parser.add_argument('--high_mem',dest='high_mem',default=False,action='store_true',help = "If you have 32 or more CPUs, this option is recommended. It will run many code portions in parallel. If you don't have enough memory, this will cause a run failure.") parser.add_argument('--dataset',dest='datasets',default='broad_sanger,beataml,pancreatic,bladder,sarcoma,liver,novartis,colorectal,mpnst,cptac,hcmi',help='Datasets to process. Defaults to all available.') parser.add_argument('--version', type=str, required=False, help='Version number for the Figshare upload title (e.g., "0.1.29"). This is required for Figshare upload. This must be a higher version than previously published versions.') + parser.add_argument('--previous-summary', dest='previous_summary', default=None, + help='validation_summary.json(.gz) from an earlier build or release; --validate then ' + 'warns on large changes in samples, drugs, pairs, omics coverage and model types.') + parser.add_argument('--skip-build-checks', dest='skip_build_checks', default=False, action='store_true', + help='Skip validate_build.py data checks during --validate (not recommended).') parser.add_argument('--github-username', type=str, required=False, help='GitHub username for the repository.') parser.add_argument('--github-email', type=str, required=False, help='GitHub email for the repository.') parser.add_argument('--depmap-ready', dest='depmap_ready', default=False, action='store_true', @@ -1019,10 +1024,45 @@ def get_latest_commit_hash(owner, repo, branch='main'): drug_mapping_command_2 = ['python3', 'coderbuild/utils/align_drug_descriptors.py', '--local_dir', "/tmp"] + version_args run_docker_upload_cmd(drug_mapping_command_2, 'all_files_dir', 'Align_Drug_Descriptors', args.version) + # Data checks the schema cannot express (duplicate results, orphan or + # overlapping ids, copied responses, large changes from the previous + # build). The summary is written into all_files_dir so it is published + # with the release and can be the next build's --previous-summary. + # Errors stop the build after the schema check has also reported. + build_check_failed = False + if not args.skip_build_checks: + summary_path = os.path.join(all_files_dir, 'validation_summary.json') + check_cmd = [sys.executable, 'coderbuild/utils/validate_build.py', '--dir', all_files_dir, + '--write-summary', summary_path] + if args.previous_summary: + check_cmd += ['--previous-summary', args.previous_summary] + _log('Running build data checks') + # The summary is written once every check has run, so its absence + # means the checker never finished. + if os.path.exists(summary_path): + os.remove(summary_path) + sys.stdout.flush() + # stdin can be closed when the build runs under nohup ... &, and a + # new Python then dies at startup (init_sys_streams: Bad file descriptor). + rc = subprocess.run(check_cmd, stdin=subprocess.DEVNULL).returncode + build_check_failed = rc != 0 + if rc == 0: + _log('Build data checks passed') + elif os.path.exists(summary_path): + _log(f'Build data checks found errors (exit code {rc}); see the results above') + else: + # The 2.4 rebuild stopped here with no output. + _log(f'Build data checks did not finish (exit code {rc}). ' + f'Rerun by hand: {" ".join(check_cmd)}') + # Run schema checker - This will always run if uploading data. schema_check_command = ['python3', 'coderbuild/utils/check_schema.py', '--datasets'] + datasets run_docker_upload_cmd(schema_check_command, 'all_files_dir', 'validate', args.version) - + + if build_check_failed: + raise RuntimeError('Build data checks failed; see the log above. ' + 'Fix them, or rerun with --skip-build-checks to override.') + _log("Validation complete. Proceeding with file compression/decompression adjustments") # Compress or decompress files based on specific conditions after checking diff --git a/coderbuild/utils/validate_build.py b/coderbuild/utils/validate_build.py new file mode 100644 index 00000000..ed253186 --- /dev/null +++ b/coderbuild/utils/validate_build.py @@ -0,0 +1,511 @@ +#!/usr/bin/env python3 +""" +Data checks on a CoderData build that the LinkML schema cannot express. + +The schema checks that each row is well formed. These checks look across +rows, files and releases, at the problems that shipped in 2.4 without failing +the schema: two results for one sample/drug (NCI60, PRISM), one experiment +reported under two drugs, drug responses attached to the wrong specimens +(BeatAML) and whole sample classes dropped (HCMI). + +Errors are definite integrity violations and make the script exit 1. +Warnings need a person to look; they never fail the build. + +Usage: + python coderbuild/utils/validate_build.py --dir local/all_files_dir \\ + [--previous-summary prev_summary.json] [--write-summary this_summary.json] + +Results are printed to the terminal. + +--previous-summary compares this build with an earlier one (a summary written +by --write-summary; each release publishes one as validation_summary.json.gz) and warns on large +changes, drops of more than half and datasets that are no longer built. +Files may be plain or .gz. +""" +import argparse +import glob +import gzip +import hashlib +import json +import os +import re +import sys +from collections import defaultdict + +import numpy as np +import pandas as pd + +CELL_LINE_DATASETS = {'ccle', 'ctrpv2', 'fimm', 'gcsi', 'gdscv1', 'gdscv2', 'nci60', 'prism'} +OMICS = ['transcriptomics', 'proteomics', 'phosphoproteomics', 'mutations', 'copy_number'] +CHUNK_ROWS = 2_000_000 + +EXPERIMENT_KEY = ['source', 'improve_sample_id', 'improve_drug_id', 'study', + 'time', 'time_unit', 'dose_response_metric'] +COMBINATION_KEY = ['source', 'improve_sample_id', 'improve_drug_1', 'improve_drug_2', + 'study', 'time', 'time_unit', 'drug_combination_metric'] +# Metrics on a 0..~1 scale; values far outside are worth a look. +BOUNDED_METRICS = {'auc': (0, 1.5), 'fit_auc': (0, 1.5), 'aac': (-0.5, 1.5)} +# Relative change versus the previous build that triggers a warning. +CHANGE_THRESHOLD = 0.10 +# A drop this large versus the previous build is reported on its own: it is far +# more likely a lost download or broken join than a real change in the data. +DROP_THRESHOLD = 0.5 +# Values that stand for "no value" in sample metadata. +PLACEHOLDERS = {'', 'na', 'nan', 'none', 'not reported'} + + +# --------------------------------------------------------------------------- io + +def _find(directory, dataset, kind): + for ext in ('.csv', '.tsv', '.csv.gz', '.tsv.gz'): + path = os.path.join(directory, f'{dataset}_{kind}{ext}') + if os.path.exists(path): + return path + return None + + +def _sep(path): + return '\t' if '.tsv' in path else ',' + + +def _columns(path): + return list(pd.read_csv(path, sep=_sep(path), nrows=0).columns) + + +def _chunks(path, columns): + present = [c for c in _columns(path) if c in columns] + return pd.read_csv(path, sep=_sep(path), usecols=present, dtype=str, + keep_default_na=False, chunksize=CHUNK_ROWS) + + +def _read(path, columns): + return pd.concat(_chunks(path, columns), ignore_index=True) + + +def _unique(path, column, keep_blank=False): + values = set() + if column not in _columns(path): + return values + for chunk in _chunks(path, [column]): + values.update(chunk[column].unique()) + if not keep_blank: + values.discard('') + return values + + +def discover_datasets(directory): + names = set() + for path in glob.glob(os.path.join(directory, '*_samples.csv*')): + names.add(os.path.basename(path).split('_samples.csv')[0]) + return sorted(names) + + +# ----------------------------------------------------------------------- report + +class Report: + def __init__(self): + self.errors = [] + self.warnings = [] + + def error(self, check, dataset, message): + self.errors.append((check, dataset, message)) + + def warn(self, check, dataset, message): + self.warnings.append((check, dataset, message)) + + def markdown(self): + lines = ['# CoderData build validation', '', + f'{len(self.errors)} error(s), {len(self.warnings)} warning(s).', ''] + for title, items in (('Errors', self.errors), ('Warnings', self.warnings)): + lines.append(f'## {title}') + lines.append('') + if not items: + lines.append('None.') + for check, dataset, message in items: + lines.append(f'- **{check}** ({dataset}): {message}') + lines.append('') + return '\n'.join(lines) + + +def _examples(values, n=5): + values = list(values) + shown = ', '.join(str(v) for v in values[:n]) + return shown + (f', ... ({len(values)} total)' if len(values) > n else '') + + +# ----------------------------------------------------------------------- checks + +def duplicate_keys(path, key, report, check, dataset): + """Rows sharing every key column (two values for one sample/drug/metric).""" + cols = _columns(path) + missing = [c for c in key if c not in cols] + if missing: + report.error(check, dataset, f'{os.path.basename(path)} lacks key column(s) {missing}') + return + hashes = [pd.util.hash_pandas_object(c[key], index=False).to_numpy() for c in _chunks(path, key)] + if not hashes: + return + values, counts = np.unique(np.concatenate(hashes), return_counts=True) + suspect = values[counts > 1] + if not len(suspect): + return + rows = pd.concat([c[key][np.isin(pd.util.hash_pandas_object(c[key], index=False).to_numpy(), suspect)] + for c in _chunks(path, key)], ignore_index=True) + dup = rows[rows.duplicated(key, keep=False)] + if len(dup): + pairs = dup.drop_duplicates().apply(lambda r: f"{r['improve_sample_id']}/" + f"{r.get('improve_drug_id', r.get('improve_drug_1'))}", axis=1) + report.error(check, dataset, + f'{len(dup)} rows in {os.path.basename(path)} share a key with another row ' + f'(more than one value for a sample/drug/time/metric); sample/drug e.g. ' + f'{_examples(pairs.unique())}') + + +def copied_responses(path, report, dataset, metric='fit_auc', min_rows=3): + """Different drug ids with identical responses in every sample.""" + cols = ['improve_sample_id', 'improve_drug_id', 'time', 'dose_response_metric', 'dose_response_value'] + parts = [c[c['dose_response_metric'] == metric] for c in _chunks(path, cols)] + df = pd.concat(parts, ignore_index=True) if parts else pd.DataFrame() + if df.empty: + return + key = df['improve_sample_id'] + '|' + df['time'] + '|' + df['dose_response_value'] + sig = key.groupby(df['improve_drug_id']).agg( + lambda s: hashlib.md5('\n'.join(sorted(s)).encode()).hexdigest() if len(s) >= min_rows else '') + sig = sig[sig != ''] + shared = sig[sig.duplicated(keep=False)] + if len(shared): + groups = [sorted(g.index) for _, g in shared.groupby(shared)] + report.warn('copied-responses', dataset, + f'{len(groups)} group(s) of drug ids report identical {metric} for every sample ' + f'(one experiment attached to several drugs?): {_examples(groups)}') + + +def value_ranges(path, report, dataset): + """Bounded metrics far outside their range, and metrics that are mostly blank.""" + cols = ['dose_response_metric', 'dose_response_value'] + total, blank, out = defaultdict(int), defaultdict(int), defaultdict(int) + for chunk in _chunks(path, cols): + values = pd.to_numeric(chunk['dose_response_value'], errors='coerce') + for metric, idx in chunk.groupby('dose_response_metric').groups.items(): + v = values.loc[idx] + total[metric] += len(v) + blank[metric] += int(v.isna().sum() + np.isinf(v).sum()) + if metric in BOUNDED_METRICS: + lo, hi = BOUNDED_METRICS[metric] + out[metric] += int(((v < lo) | (v > hi)).sum()) + for metric in total: + if blank[metric] == total[metric]: + continue # categorical metric (e.g. mrecist: PD/SD/PR/CR) + if blank[metric] / total[metric] > 0.5: + report.warn('value-ranges', dataset, + f'{metric}: {blank[metric] / total[metric]:.0%} of values are blank or infinite') + if out.get(metric): + lo, hi = BOUNDED_METRICS[metric] + report.warn('value-ranges', dataset, + f'{metric}: {out[metric]} of {total[metric]} values outside [{lo}, {hi}]') + + +def _id_values(report, dataset, kind, where, values): + """Report blank ids and ids written as decimals; return the ids without blanks.""" + if '' in values: + report.error('missing-id', dataset, f'{where} has rows with a blank {kind} id') + decimal = sorted(v for v in values if re.fullmatch(r'\d+\.0', v)) + if decimal: + report.error('decimal-id', dataset, + f'{len(decimal)} {kind} ids in {where} are written as decimals (e.g. "{decimal[0]}")') + return values - {''} + + +def _orphans(report, dataset, kind, where, missing, known): + if not missing: + return + decimal = {m for m in missing if re.fullmatch(r'\d+\.0', m) and m[:-2] in known} + hint = (f'; {len(decimal)} of them are known ids written as decimals (e.g. "{sorted(decimal)[0]}")' + if decimal else '') + report.error(f'orphan-{kind}-id', dataset, + f'{len(missing)} {kind} ids in {where} are not in the {kind}s file: ' + f'{_examples(sorted(missing))}{hint}') + + +def sample_checks(dataset, path, report): + """Per-sample metadata problems; returns the dataset's sample ids.""" + wanted = ['improve_sample_id', 'other_id', 'other_id_source', 'model_type', + 'cancer_type', 'common_name', 'other_names'] + s = _read(path, wanted).reindex(columns=wanted, fill_value='') + ids = _id_values(report, dataset, 'sample', 'samples', set(s['improve_sample_id'])) + mt = s.groupby('improve_sample_id')['model_type'].nunique() + if (mt > 1).any(): + report.error('sample-model-type', dataset, + f'{int((mt > 1).sum())} sample ids carry more than one model_type: ' + f'{_examples(mt[mt > 1].index)}') + # One sample, conflicting descriptions. other_names are aliases and may + # differ; 'NA' is a placeholder, not a name. Cell lines combine DepMap and + # Sanger records whose cancer_type vocabularies differ, so only their names + # are compared. + cols = ('common_name',) if dataset in CELL_LINE_DATASETS else ('common_name', 'cancer_type') + for col in cols: + named = s[~s[col].str.strip().str.lower().isin(PLACEHOLDERS)] + n = named.groupby('improve_sample_id')[col].nunique() + if (n > 1).any(): + report.warn('sample-metadata-conflict', dataset, + f'{int((n > 1).sum())} sample ids have more than one {col}: {_examples(n[n > 1].index)}') + return ids, s + + +def copied_samples(path, value_cols, report, dataset, what, min_rows=3): + """ + Different sample ids whose rows in a file are identical. + + Each sample gets an order-independent fingerprint: two independent 64-bit + row hashes summed with wrap-around (numpy uint64; pandas sums can spill + into float64 and collide), plus the row count. The file is streamed, and + samples whose fingerprints match are then compared row by row, so only + exact copies are reported. + """ + cols = ['improve_sample_id'] + value_cols + if not all(c in _columns(path) for c in cols): + return + keys = ('0123456789123456', 'fedcba9876543210') + acc = {} # sample -> [sum1, sum2, rows] + for chunk in _chunks(path, cols): + sid = chunk['improve_sample_id'].str.replace(r'\.0$', '', regex=True) + codes, uniq = pd.factorize(sid) + n = np.bincount(codes, minlength=len(uniq)) + sums = [] + for key in keys: + h = pd.util.hash_pandas_object(chunk[value_cols], index=False, hash_key=key).to_numpy() + s = np.zeros(len(uniq), dtype=np.uint64) + np.add.at(s, codes, h) + sums.append(s) + for k, sample in enumerate(uniq): + a = acc.setdefault(sample, np.zeros(3, dtype=np.uint64)) + a += np.array((sums[0][k], sums[1][k], n[k]), dtype=np.uint64) # array add wraps silently + if not acc: + return + sig = pd.DataFrame.from_dict(acc, orient='index', columns=['h1', 'h2', 'n']) + sig = sig[sig['n'] >= min_rows] + suspect = sig[sig.duplicated(['h1', 'h2', 'n'], keep=False)] + if suspect.empty: + return + # Confirm exactly: re-read only the suspect samples and compare their rows. + wanted = set(suspect.index) + rows = pd.concat([c.assign(improve_sample_id=c['improve_sample_id'].str.replace(r'\.0$', '', regex=True)) + .loc[lambda x: x['improve_sample_id'].isin(wanted)] + for c in _chunks(path, cols)], ignore_index=True) + content = {s: tuple(map(tuple, g[value_cols].sort_values(value_cols).to_numpy())) + for s, g in rows.groupby('improve_sample_id')} + groups = [] + for _, g in suspect.groupby(['h1', 'h2', 'n']): + by_content = defaultdict(list) + for s in g.index: + by_content[content[s]].append(s) + groups += [sorted(v) for v in by_content.values() if len(v) > 1] + if groups: + report.warn('copied-samples', dataset, + f'{len(groups)} group(s) of sample ids have identical {what}: {_examples(sorted(groups))}') + + +def drug_checks(dataset, path, report): + """One structure per drug id and one drug id per CID; returns the drug ids. + + One CID under two drug ids is one chemical listed twice; its responses + are split between the ids and merging them later gives duplicate results + (issue #512), so it is an error. + """ + d = _read(path, ['improve_drug_id', 'canSMILES', 'pubchem_id']) + ids = _id_values(report, dataset, 'drug', 'drugs', set(d['improve_drug_id'])) + smiles = d[d['canSMILES'] != ''].groupby('improve_drug_id')['canSMILES'].nunique() + if (smiles > 1).any(): + report.error('drug-structure', dataset, + f'{int((smiles > 1).sum())} drug ids have more than one canSMILES: ' + f'{_examples(smiles[smiles > 1].index)}') + if 'pubchem_id' in d.columns: + cid = d[d['pubchem_id'] != ''].assign(pubchem_id=lambda x: x['pubchem_id'].str.replace(r'\.0$', '', regex=True)) + per_cid = cid.groupby('pubchem_id')['improve_drug_id'].agg(lambda x: sorted(set(x))) + split = per_cid[per_cid.apply(len) > 1] + if len(split): + report.error('drug-split', dataset, + f'{len(split)} PubChem CID(s) sit under more than one drug id: ' + f'{_examples(f"{k}: {v}" for k, v in split.items())}') + return ids + + +def mapping_checks(directory, report): + for name, items_key, key_fn in ( + ('improve_sample_mapping.json', 'samples', + lambda e: [tuple(q.get(k, '').strip() for k in ('other_id', 'other_id_source', 'model_type', 'other_names')) + for q in e.get('quadruples', [])]), + ('improve_drug_mapping.json', 'drugs', lambda e: [e.get('canSMILES')])): + path = next((p for p in (os.path.join(directory, name), os.path.join(directory, name + '.gz')) + if os.path.exists(p)), None) + if not path: + report.warn('mapping-file', 'all', f'{name} not found') + continue + opener = gzip.open if path.endswith('.gz') else open + with opener(path, 'rt') as f: + entries = json.load(f).get(items_key, []) + ids = [e['stable_id'] for e in entries] + if len(ids) != len(set(ids)): + report.error('mapping-file', 'all', f'{name}: stable ids listed more than once') + owner = defaultdict(set) + for e in entries: + for k in key_fn(e): + owner[k].add(e['stable_id']) + multi = {k: v for k, v in owner.items() if len(v) > 1} + if multi: + report.error('mapping-file', 'all', + f'{name}: {len(multi)} identities map to more than one stable id, e.g. ' + f'{_examples(f"{k}: {sorted(v)}" for k, v in multi.items())}') + + +# ------------------------------------------------------------------ main runner + +def validate(directory, datasets=None, previous=None): + report = Report() + requested = datasets + datasets = datasets or discover_datasets(directory) + summary = {'datasets': {}} + sample_owner = defaultdict(set) + + for ds in datasets: + info = {} + spath = _find(directory, ds, 'samples') + if not spath: + report.error('missing-file', ds, 'no samples file') + continue + sids, samples = sample_checks(ds, spath, report) + info['samples'] = len(sids) + info['model_types'] = samples.drop_duplicates('improve_sample_id')['model_type'].value_counts().to_dict() + for sid in sids: + sample_owner[sid].add(ds) + + dpath = _find(directory, ds, 'drugs') + dids = drug_checks(ds, dpath, report) if dpath else set() + info['drugs'] = len(dids) + + info['omics'] = {} + for kind in OMICS: + path = _find(directory, ds, kind) + if not path: + continue + used = _id_values(report, ds, 'sample', kind, _unique(path, 'improve_sample_id', keep_blank=True)) + if kind == 'transcriptomics': + copied_samples(path, ['entrez_id', 'transcriptomics'], report, ds, 'transcriptomics') + info['omics'][kind] = len(used) + _orphans(report, ds, 'sample', kind, used - sids, sids) + + epath = _find(directory, ds, 'experiments') + if epath: + duplicate_keys(epath, EXPERIMENT_KEY, report, 'duplicate-result', ds) + copied_responses(epath, report, ds) + copied_samples(epath, ['improve_drug_id', 'time', 'dose_response_metric', 'dose_response_value'], + report, ds, 'drug responses') + value_ranges(epath, report, ds) + pairs, esids, edids = set(), set(), set() + for chunk in _chunks(epath, ['improve_sample_id', 'improve_drug_id']): + pairs.update(pd.util.hash_pandas_object(chunk[['improve_sample_id', 'improve_drug_id']], + index=False).to_numpy().tolist()) + esids.update(chunk['improve_sample_id'].unique()) + edids.update(chunk['improve_drug_id'].unique()) + esids = _id_values(report, ds, 'sample', 'experiments', esids) + edids = _id_values(report, ds, 'drug', 'experiments', edids) + info['pairs'] = len(pairs) + info['samples_with_response'] = len(esids) + info['drugs_with_response'] = len(edids) + _orphans(report, ds, 'sample', 'experiments', esids - sids, sids) + _orphans(report, ds, 'drug', 'experiments', edids - dids, dids) + no_omics = [k for k, n in info['omics'].items() if esids and n == 0] + if no_omics: + report.warn('omics-coverage', ds, f'no samples with drug response have {no_omics}') + + cpath = _find(directory, ds, 'combinations') + if cpath: + duplicate_keys(cpath, COMBINATION_KEY, report, 'duplicate-result', ds) + c = _read(cpath, ['improve_sample_id', 'improve_drug_1', 'improve_drug_2']) + csids = _id_values(report, ds, 'sample', 'combinations', set(c['improve_sample_id'])) + cdids = _id_values(report, ds, 'drug', 'combinations', + set(c['improve_drug_1']) | set(c['improve_drug_2'])) + _orphans(report, ds, 'sample', 'combinations', csids - sids, sids) + _orphans(report, ds, 'drug', 'combinations', cdids - dids, dids) + info['combination_rows'] = len(c) + + xpath = _find(directory, ds, 'drug_descriptors') + if xpath and dids: + xdids = _id_values(report, ds, 'drug', 'drug_descriptors', + _unique(xpath, 'improve_drug_id', keep_blank=True)) + _orphans(report, ds, 'drug', 'drug_descriptors', xdids - dids, dids) + summary['datasets'][ds] = info + + # Cell lines share ids by design; every other dataset owns its ids. + leaked = {sid: sorted(owners) for sid, owners in sample_owner.items() + if len(owners) > 1 and not owners <= CELL_LINE_DATASETS} + if leaked: + report.error('sample-id-overlap', 'all', + f'{len(leaked)} sample ids are used by more than one dataset: ' + f'{_examples(f"{k}: {v}" for k, v in leaked.items())}') + + mapping_checks(directory, report) + if previous: + compare(summary, previous, report, requested) + return report, summary + + +def compare(summary, previous, report, datasets=None): + """Warn on large changes versus a previous build's summary.""" + prev = previous.get('datasets', {}) + # Datasets left out on purpose (--datasets) are not missing. + expected = set(prev) if datasets is None else set(prev) & set(datasets) + for ds in sorted(expected - set(summary['datasets'])): + report.warn('dataset-missing', ds, 'in the previous build but not in this one') + for ds, cur in summary['datasets'].items(): + old = prev.get(ds) + if old is None: + report.warn('release-change', ds, 'new dataset (not in the previous build)') + continue + for field in ('samples', 'drugs', 'pairs', 'samples_with_response', 'drugs_with_response'): + a, b = old.get(field), cur.get(field) + if a and b is not None and (a - b) / a > DROP_THRESHOLD: + report.warn('release-drop', ds, f'{field}: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + elif a and b is not None and abs(b - a) / a > CHANGE_THRESHOLD: + report.warn('release-change', ds, f'{field}: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + for kind in set(old.get('omics', {})) | set(cur.get('omics', {})): + a, b = old.get('omics', {}).get(kind), cur.get('omics', {}).get(kind) + if a is None or b is None: + report.warn('release-change', ds, f'{kind}: {"added" if a is None else "removed"}') + elif a and (a - b) / a > DROP_THRESHOLD: + report.warn('release-drop', ds, f'{kind} samples: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + elif a and abs(b - a) / a > CHANGE_THRESHOLD: + report.warn('release-change', ds, f'{kind} samples: {a:,} -> {b:,} ({(b - a) / a:+.0%})') + old_mt, new_mt = set(old.get('model_types', {})), set(cur.get('model_types', {})) + if old_mt != new_mt: + report.warn('release-change', ds, + f'model types changed: removed {sorted(old_mt - new_mt) or "none"}, ' + f'added {sorted(new_mt - old_mt) or "none"}') + + +def load_summary(path): + """Read a summary written by --write-summary (plain or .gz, as published).""" + opener = gzip.open if path.endswith('.gz') else open + with opener(path, 'rt') as f: + return json.load(f) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--dir', required=True, help='folder of build output files') + ap.add_argument('--datasets', nargs='*', default=None, help='datasets to check (default: all found)') + ap.add_argument('--previous-summary', default=None, help='summary JSON of the build to compare with') + ap.add_argument('--write-summary', default=None, help='write this build\'s summary JSON here') + args = ap.parse_args(argv) + + previous = load_summary(args.previous_summary) if args.previous_summary else None + report, summary = validate(args.dir, args.datasets, previous) + print(report.markdown()) + if args.write_summary: + with open(args.write_summary, 'w') as f: + json.dump(summary, f, indent=1, sort_keys=True) + return 1 if report.errors else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..f8aec3fc --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,12 @@ +import os + + +def pytest_addoption(parser): + group = parser.getgroup('coderdata build data') + group.addoption('--build-dir', default=os.environ.get('CODERDATA_BUILD_DIR'), + help='folder of CoderData build or release files to check with ' + 'tests/test_build_data.py (or set CODERDATA_BUILD_DIR)') + group.addoption('--previous-summary', default=os.environ.get('CODERDATA_PREVIOUS_SUMMARY'), + help='validation_summary.json of an earlier build, to report large changes') + group.addoption('--build-datasets', default=os.environ.get('CODERDATA_BUILD_DATASETS'), + help='comma-separated datasets to check (default: all found in --build-dir)') diff --git a/tests/test_build_data.py b/tests/test_build_data.py new file mode 100644 index 00000000..14e2bc7c --- /dev/null +++ b/tests/test_build_data.py @@ -0,0 +1,78 @@ +""" +Data checks on a real CoderData build or release, run as tests. + +These are the checks `coderbuild/build_all.py --validate` runs, exposed to +pytest so any folder of CoderData files can be checked on its own: a local +build (local/all_files_dir) or a downloaded release. + + pytest tests/test_build_data.py --build-dir local/all_files_dir + pytest tests/test_build_data.py --build-dir data/ --build-datasets beataml,hcmi + CODERDATA_BUILD_DIR=local/all_files_dir pytest tests/test_build_data.py + +Each dataset is one test, plus one for cross-dataset checks. Errors fail the +test; warnings are reported as pytest warnings. Without --build-dir the tests +are skipped. The checks themselves are unit tested in test_validate_build.py. +""" +import os +import sys +import warnings + +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, 'coderbuild', 'utils')) + +import validate_build as vb # noqa: E402 + + +class BuildDataWarning(UserWarning): + """A validation warning: worth a look, not a failure.""" + + +def _options(config): + build_dir = config.getoption('--build-dir') + datasets = config.getoption('--build-datasets') + return build_dir, (datasets.split(',') if datasets else None) + + +def pytest_generate_tests(metafunc): + if 'dataset' not in metafunc.fixturenames: + return + build_dir, datasets = _options(metafunc.config) + if not build_dir: + metafunc.parametrize('dataset', [pytest.param(None, marks=pytest.mark.skip( + reason='no build to check: pass --build-dir or set CODERDATA_BUILD_DIR'))]) + return + metafunc.parametrize('dataset', datasets or vb.discover_datasets(build_dir)) + + +@pytest.fixture(scope='session') +def build_report(pytestconfig): + build_dir, datasets = _options(pytestconfig) + if not build_dir: + pytest.skip('no build to check: pass --build-dir or set CODERDATA_BUILD_DIR') + if not os.path.isdir(build_dir): + pytest.fail(f'--build-dir {build_dir} is not a folder') + summary = pytestconfig.getoption('--previous-summary') + previous = vb.load_summary(summary) if summary else None + report, _ = vb.validate(build_dir, datasets, previous) + return report + + +def _check(report, scope): + for check, _, message in [w for w in report.warnings if w[1] == scope]: + warnings.warn(f'{check}: {message}', BuildDataWarning) + errors = [f'{check}: {message}' for check, ds, message in report.errors if ds == scope] + assert not errors, '\n'.join(errors) + + +def test_dataset(build_report, dataset): + # Fails if this dataset has any errors. Its warnings are shown in the pytest + # output without failing. + _check(build_report, dataset) + + +def test_cross_dataset(build_report): + # Same as above, for checks that compare datasets (shared sample IDs, + # mapping files). + _check(build_report, 'all') diff --git a/tests/test_validate_build.py b/tests/test_validate_build.py new file mode 100644 index 00000000..6eb98c0a --- /dev/null +++ b/tests/test_validate_build.py @@ -0,0 +1,323 @@ +""" +Tests for coderbuild/utils/validate_build.py. + +Each test writes a tiny build with one planted problem modelled on what +shipped in CoderData 2.4, and checks that the validator reports it (and that +a clean build passes). +""" +import gzip +import json +import os +import sys + +import pandas as pd +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, 'coderbuild', 'utils')) + +import validate_build as vb # noqa: E402 + +EXP_COLS = ['source', 'improve_sample_id', 'improve_drug_id', 'study', 'time', 'time_unit', + 'dose_response_metric', 'dose_response_value'] +DRUG_COLS = ['improve_drug_id', 'chem_name', 'pubchem_id', 'canSMILES', 'InChIKey', 'formula', 'weight'] +SAMPLE_COLS = ['other_id', 'improve_sample_id', 'other_names', 'common_name', 'cancer_type', + 'model_type', 'other_id_source', 'species'] + + +def _tsv(path, rows, cols, sep='\t'): + pd.DataFrame(rows, columns=cols).to_csv(path, sep=sep, index=False) + + +def _samples(d, ds, ids, model='patient derived organoid', cancer='Colon cancer'): + _tsv(d / f'{ds}_samples.csv', [[f'X{i}', i, f'n{i}', f'c{i}', cancer, model, 'src', 'Homo sapiens'] + for i in ids], SAMPLE_COLS, sep=',') + + +def _drugs(d, ds, rows): + _tsv(d / f'{ds}_drugs.tsv', rows, DRUG_COLS) + + +def _exp(d, ds, rows): + _tsv(d / f'{ds}_experiments.tsv', rows, EXP_COLS) + + +def _mappings(d, samples=None, drugs=None): + samples = samples or [{'stable_id': '1', 'quadruples': [ + {'other_id': 'X1', 'other_id_source': 'src', 'model_type': 'patient derived organoid', 'other_names': 'n1'}]}] + drugs = drugs or [{'stable_id': 'SMI_1', 'canSMILES': 'CCO'}] + with gzip.open(d / 'improve_sample_mapping.json.gz', 'wt') as f: + json.dump({'metadata': {'builds': []}, 'samples': samples}, f) + with gzip.open(d / 'improve_drug_mapping.json.gz', 'wt') as f: + json.dump({'metadata': {'builds': []}, 'drugs': drugs}, f) + + +def _clean_build(d): + _samples(d, 'liver', [1, 2, 3]) + _drugs(d, 'liver', [['SMI_1', 'ethanol', '702', 'CCO', 'K', 'F', '1'], + ['SMI_2', 'taxol', '36314', 'CCC', 'K2', 'F', '1']]) + rows = [] + for s, a, b in ((1, '0.5', '0.9'), (2, '0.6', '0.2'), (3, '0.7', '0.4')): + rows.append(['syn', s, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', a]) + rows.append(['syn', s, 'SMI_2', 'L', '72', 'hrs', 'fit_auc', b]) + _exp(d, 'liver', rows) + _tsv(d / 'liver_transcriptomics.csv', [[1, 7105, 1.0], [2, 7105, 2.0]], + ['improve_sample_id', 'entrez_id', 'transcriptomics'], sep=',') + _mappings(d) + + +def _checks(report, kind='errors'): + return {c for c, _, _ in getattr(report, kind)} + + +def test_clean_build_passes(tmp_path): + # A build with no problems should pass with no errors. + _clean_build(tmp_path) + report, summary = vb.validate(str(tmp_path)) + assert report.errors == [] + assert summary['datasets']['liver']['pairs'] == 6 + assert vb.main(['--dir', str(tmp_path)]) == 0 + + +def test_duplicate_results_fail(tmp_path): + # Two results for the same sample, drug, time and metric should fail. + # This is what broke the wide format in issue #512. + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.82'], + ['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.78']]) + report, _ = vb.validate(str(tmp_path)) + assert 'duplicate-result' in _checks(report) + assert vb.main(['--dir', str(tmp_path)]) == 1 + + +def test_decimal_sample_ids_are_orphans(tmp_path): + # Sample IDs written as decimals ("1.0", "3524.0") don't match the samples file, + # so they should fail, and the message should say why. + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', '1.0', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5']]) + report, _ = vb.validate(str(tmp_path)) + msg = [m for c, _, m in report.errors if c == 'orphan-sample-id'][0] + assert 'written as decimals' in msg + + +def test_orphan_drug_ids_fail(tmp_path): + # An experimental response for a drug that isn't in the drugs file should fail. + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_9', 'L', '72', 'hrs', 'fit_auc', '0.5']]) + report, _ = vb.validate(str(tmp_path)) + assert 'orphan-drug-id' in _checks(report) + + +def test_copied_responses_warn(tmp_path): + # Two drugs with exactly the same responses in every sample are probably + # one drug copied under two IDs. This is a warning, not an error. + _clean_build(tmp_path) + rows = [['syn', s, d, 'L', '72', 'hrs', 'fit_auc', v] + for d in ('SMI_1', 'SMI_2') for s, v in ((1, '0.5'), (2, '0.6'), (3, '0.7'))] + _exp(tmp_path, 'liver', rows) + report, _ = vb.validate(str(tmp_path)) + assert 'copied-responses' in _checks(report, 'warnings') + assert report.errors == [] + + +def test_sample_ids_shared_by_two_datasets_fail(tmp_path): + # Two datasets using the same sample ID should fail. Cell lines (broad/sanger) + # are allowed to have overlap, see the next test. + _clean_build(tmp_path) + _samples(tmp_path, 'bladder', [3, 4]) + report, _ = vb.validate(str(tmp_path)) + assert 'sample-id-overlap' in _checks(report) + + +def test_cell_lines_may_share_ids(tmp_path): + # Cell line datasets (such as CCLE and PRISM) share sample IDs on purpose, + # so this should not fail. + for ds in ('ccle', 'prism'): + _samples(tmp_path, ds, [1, 2], model='cell line', cancer='Lung cancer') + _mappings(tmp_path) + report, _ = vb.validate(str(tmp_path)) + assert 'sample-id-overlap' not in _checks(report) + + +def test_conflicting_model_types_fail(tmp_path): + # One sample ID listed with two model types (e.g. tumor and organoid) + # should fail. These should have different sample IDs. + _clean_build(tmp_path) + rows = [['X1', 1, 'n1', 'c1', 'Colon cancer', 'tumor', 'src', 'Homo sapiens'], + ['X1b', 1, 'n1', 'c1', 'Colon cancer', 'patient derived organoid', 'src2', 'Homo sapiens'], + ['X2', 2, 'n2', 'c2', 'Colon cancer', 'tumor', 'src', 'Homo sapiens'], + ['X3', 3, 'n3', 'c3', 'Colon cancer', 'tumor', 'src', 'Homo sapiens']] + _tsv(tmp_path / 'liver_samples.csv', rows, SAMPLE_COLS, sep=',') + report, _ = vb.validate(str(tmp_path)) + assert 'sample-model-type' in _checks(report) + + +def test_drug_with_two_structures_or_split_cid_fails(tmp_path): + # Two things should fail here: one drug ID with two structures, and one + # PubChem compound under two drug IDs (issue #512). "36314.0" and + # "36314" count as the same compound. + _clean_build(tmp_path) + _drugs(tmp_path, 'liver', [['SMI_1', 'ethanol', '702', 'CCO', 'K', 'F', '1'], + ['SMI_1', 'ethyl alcohol', '702', 'OCC', 'K', 'F', '1'], + ['SMI_2', 'taxol', '36314', 'CCC', 'K2', 'F', '1'], + ['SMI_3', 'paclitaxel', '36314.0', 'CCCC', 'K2', 'F', '1']]) + report, _ = vb.validate(str(tmp_path)) + assert 'drug-structure' in _checks(report) + assert 'drug-split' in _checks(report) # one chemical under two ids (issue #512) + + +def test_blank_ids_fail(tmp_path): + # Empty sample or drug IDs should fail. Each empty is reported once, not + # again as an ID missing from the samples or drugs file. + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.5'], + ['syn', '', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.6'], + ['syn', 2, '', 'L', '72', 'hrs', 'fit_auc', '0.7']]) + report, _ = vb.validate(str(tmp_path)) + blank = [m for c, _, m in report.errors if c == 'missing-id'] + assert any('blank sample id' in m for m in blank) and any('blank drug id' in m for m in blank) + assert 'orphan-sample-id' not in _checks(report) # blanks are not reported twice + + +def test_decimal_ids_in_the_samples_file_fail(tmp_path): + # IDs written as decimals in every file still match each other, so only + # the decimal check catches them. They should still fail. (Should be int) + _clean_build(tmp_path) + _tsv(tmp_path / 'liver_samples.csv', [[f'X{i}', f'{i}.0', f'n{i}', f'c{i}', 'Colon cancer', + 'patient derived organoid', 'src', 'Homo sapiens'] for i in (1, 2, 3)], + SAMPLE_COLS, sep=',') + rows = [['syn', f'{s}.0', 'SMI_1', 'L', '72', 'hrs', 'fit_auc', v] for s, v in ((1, '0.5'), (2, '0.6'), (3, '0.7'))] + _exp(tmp_path, 'liver', rows) + report, _ = vb.validate(str(tmp_path)) + assert 'decimal-id' in _checks(report) + + +def test_mapping_identity_with_two_ids_fails(tmp_path): + # The stable ID mapping file should never give one drug structure two + # IDs. + _clean_build(tmp_path) + _mappings(tmp_path, drugs=[{'stable_id': 'SMI_1', 'canSMILES': 'CCO'}, + {'stable_id': 'SMI_2', 'canSMILES': 'CCO'}]) + report, _ = vb.validate(str(tmp_path)) + assert 'mapping-file' in _checks(report) + + +def test_categorical_metrics_are_not_range_checked(tmp_path): + # Text results like mRECIST ("PD") are not range checked. A number + # outside the expected range is only a warning. + _clean_build(tmp_path) + _exp(tmp_path, 'liver', [['syn', 1, 'SMI_1', 'L', '72', 'hrs', 'mrecist', 'PD'], + ['syn', 2, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '9.0']]) + report, _ = vb.validate(str(tmp_path)) + assert report.errors == [] + assert any('outside' in m for c, _, m in report.warnings if c == 'value-ranges') + + +def test_release_changes_and_large_drops_warn(tmp_path): + # This is a cross build comparison --- + # Compared with the previous build, these should all warn (not fail): + # samples dropping by more than half, a model type disappearing, an + # omics type disappearing, and a dataset no longer being built. HCMI + # lost its tumor and cell line samples between 2.2 and 2.4. + _clean_build(tmp_path) + previous = {'datasets': {'liver': {'samples': 10, 'drugs': 2, 'pairs': 6, + 'model_types': {'tumor': 5, 'patient derived organoid': 5}, + 'omics': {'transcriptomics': 2, 'mutations': 2}}, + 'hcmi': {'samples': 5}}} + report, _ = vb.validate(str(tmp_path), previous=previous) + messages = [m for c, _, m in report.warnings if c == 'release-change'] + drops = [m for c, _, m in report.warnings if c == 'release-drop'] + assert any(m.startswith('samples: 10 -> 3') for m in drops) # -70%: flagged as a drop + assert report.errors == [] + assert any('removed' in m and 'tumor' in m for m in messages) + assert any('mutations' in m for m in messages) + assert ('dataset-missing', 'hcmi') in {(c, ds) for c, ds, _ in report.warnings} + + +def test_moderate_drop_only_warns(tmp_path): + # A 30% drop is reported as a change, but not as a large drop (that + # needs more than 50%). + _clean_build(tmp_path) + _, summary = vb.validate(str(tmp_path)) + liver = summary['datasets']['liver'] + previous = {'datasets': {'liver': dict(liver, samples=round(liver['samples'] / 0.7))}} + report, _ = vb.validate(str(tmp_path), previous=previous) + assert 'release-drop' not in _checks(report, 'warnings') + assert any(m.startswith('samples:') for c, _, m in report.warnings if c == 'release-change') + + +def test_datasets_left_out_on_purpose_are_not_missing(tmp_path): + # If only some datasets were built (--datasets), the others should not + # be reported as missing. + _clean_build(tmp_path) + _, summary = vb.validate(str(tmp_path)) + previous = {'datasets': dict(summary['datasets'], hcmi={'samples': 5})} + report, _ = vb.validate(str(tmp_path), ['liver'], previous) + assert 'dataset-missing' not in _checks(report, 'warnings') + + +def test_summary_round_trip(tmp_path): + # The summary written by one build can be read back by the next as + # --previous-summary, whether plain or gzipped (releases publish it + # gzipped). + _clean_build(tmp_path) + out = tmp_path / 'summary.json' + assert vb.main(['--dir', str(tmp_path), '--write-summary', str(out)]) == 0 + assert vb.main(['--dir', str(tmp_path), '--previous-summary', str(out)]) == 0 + assert json.loads(out.read_text())['datasets']['liver']['pairs'] == 6 + # releases publish the summary gzipped + gz = tmp_path / 'validation_summary.json.gz' + with gzip.open(gz, 'wt') as f: + f.write(out.read_text()) + assert vb.load_summary(str(gz)) == json.loads(out.read_text()) + assert vb.main(['--dir', str(tmp_path), '--previous-summary', str(gz)]) == 0 + + +def test_copied_samples_warn(tmp_path): + # Two sample IDs with exactly the same responses are probably one sample + # listed twice. This is a warning. Novartis model NIBRX-2043 had this + # under sample IDs 3828 and 3829. + _clean_build(tmp_path) + rows = [['syn', s, d, 'L', '72', 'hrs', 'fit_auc', v] + for s in (1, 2) for d, v in (('SMI_1', '0.5'), ('SMI_2', '0.6'), ('SMI_1', '0.7'))] + rows = [r for i, r in enumerate(rows)] + rows[2][6] = 'auc' + rows[5][6] = 'auc' + rows.append(['syn', 3, 'SMI_1', 'L', '72', 'hrs', 'fit_auc', '0.1']) + _exp(tmp_path, 'liver', rows) + report, _ = vb.validate(str(tmp_path)) + msg = [m for c, _, m in report.warnings if c == 'copied-samples'] + assert msg and "['1', '2']" in msg[0] + + +def test_na_names_and_cell_line_cancer_wording_do_not_warn(tmp_path): + # A cell line named "NA" in one source, or with slightly different + # cancer type wording, should not warn. Two real, different names for + # the same sample ID should. + rows = [['CVCL_1', 1, 'RH-30', 'RH-30', 'Alveolar Rhabdomyosarcoma', 'cell line', 'Cellosaurus', 'Homo sapiens'], + ['ACH-1', 1, 'RH-30', 'NA', 'Rhabdomyosarcoma', 'cell line', 'DepMap', 'Homo sapiens']] + _tsv(tmp_path / 'ccle_samples.csv', rows, SAMPLE_COLS, sep=',') + _mappings(tmp_path) + report, _ = vb.validate(str(tmp_path)) + assert 'sample-metadata-conflict' not in _checks(report, 'warnings') + rows[1][3] = 'SJRH-30' + _tsv(tmp_path / 'ccle_samples.csv', rows, SAMPLE_COLS, sep=',') + report, _ = vb.validate(str(tmp_path)) + assert 'sample-metadata-conflict' in _checks(report, 'warnings') + + +def test_same_size_samples_with_different_data_are_not_copies(tmp_path): + # Many samples with the same number of results but different values + # should not be flagged as copies. An earlier version of this check + # flagged hundreds of PRISM samples by mistake. + import numpy as np + rng = np.random.default_rng(0) + ids = list(range(1, 301)) + _samples(tmp_path, 'liver', ids) + _drugs(tmp_path, 'liver', [[f'SMI_{d}', f'd{d}', str(d), f'C{d}', 'K', 'F', '1'] for d in range(1, 41)]) + rows = [['syn', s, f'SMI_{d}', 'L', '72', 'hrs', 'fit_auc', f'{rng.random():.4f}'] + for s in ids for d in range(1, 41)] + _exp(tmp_path, 'liver', rows) + _mappings(tmp_path) + report, _ = vb.validate(str(tmp_path)) + assert 'copied-samples' not in _checks(report, 'warnings')