From efefbe7f1bee89e0860ca5555f26aa129d765702 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 13 Oct 2026 11:26:22 -0600 Subject: [PATCH] Refresh datasets page from the republished 2.4 data (Figshare v2) Regenerate every number on the Datasets page from the build now served as version 2 of Figshare article 33990850. gen_dataset_stats.py now also writes dataset_summary_statistics.csv and dataset_curve_metrics_wide.csv, which were previously produced by notebooks/dataset_documentation.ipynb against hardcoded local paths and had not been recomputed for 2.4. It also fixes two counting bugs: - Lines with quoted fields (e.g. samples other_names "b1_4, b1_5, b1_6") were split on raw delimiters, shifting columns. This undercounted MPNST samples (47 vs 133) and Novartis (348 vs 386), overcounted the cell-line datasets by one, and garbled model_type values. - Sample ids written as floats ("3524.0", novartis experiments) are normalized so they match the integer ids in the omics files. Also add a delta_auc entry to the dose-response metric glossary and update the refresh instructions in docs/README.md. --- docs/README.md | 2 +- docs/source/_static/dataset_counts.csv | 24 +-- .../_static/dataset_curve_metrics_wide.csv | 34 ++--- .../_static/dataset_summary_statistics.csv | 20 +-- docs/source/datasets_included.rst | 21 +-- scripts/gen_dataset_stats.py | 144 +++++++++++++++--- 6 files changed, 177 insertions(+), 68 deletions(-) diff --git a/docs/README.md b/docs/README.md index 4be16c0e..4cdf8c5e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -68,7 +68,7 @@ To update pages navigate to `docs/source/`, all .rst and .md files live here. He **Note:** - When adding pages, reStructuredText files are the most sphinx friendly and allow for use of helpful directives. However, Markdown files can be used and will render properly when the myst-parser extension is used. - New pages must be added to the `.. toctree::` in the `index.rst` to be recognized by Sphinx. -- The counts on the Datasets page (`datasets_included.rst`) are generated directly from a local CoderData build using `scripts/gen_dataset_stats.py`, which writes a machine-readable summary to `docs/source/_static/dataset_counts.csv`. To refresh the numbers for a new release, run `python scripts/gen_dataset_stats.py --build-dir --output docs/source/_static/dataset_counts.csv` and update the tables in `datasets_included.rst` accordingly. +- The counts on the Datasets page (`datasets_included.rst`) are generated directly from a local CoderData build using `scripts/gen_dataset_stats.py`. To refresh the numbers for a new release, run `python scripts/gen_dataset_stats.py --build-dir ` from the repository root. This rewrites `dataset_counts.csv`, `dataset_summary_statistics.csv` and `dataset_curve_metrics_wide.csv` in `docs/source/_static/`. The last two are rendered on the page directly; copy the sample and drug counts from `dataset_counts.csv` into the Datasets and Modalities table by hand. - Currently the API reference does not include documentation for `plot_2D_respones_metric`, `format`, `split_train_test_validate`, and `split_train_other`. It seems that the most updated `coderdata` directory currently does not include docstrings for these functions. ### Tutorial diff --git a/docs/source/_static/dataset_counts.csv b/docs/source/_static/dataset_counts.csv index 0905b013..634e2f47 100644 --- a/docs/source/_static/dataset_counts.csv +++ b/docs/source/_static/dataset_counts.csv @@ -1,19 +1,19 @@ dataset,model_type,samples,drugs,genes,experiments,transcriptomics,proteomics,phosphoproteomics,mutations,copy_number,drug_descriptors,combinations,dose_response_metrics -beataml,ex vivo,1022,164,19221,324980,13573491,1437831,0,7233,0,264696,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +beataml,ex vivo,1022,164,19221,322110,13573491,1437831,0,7233,0,264696,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 bladder,patient derived organoid;tumor;patient derived xenograft;xenograft derived organoid,134,50,19403,33000,815975,0,0,3568,1135661,80700,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -ccle,cell line,503,24,19380,115430,9114448,2180178,0,224974,6788934,38736,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +ccle,cell line,502,24,19380,115430,9114448,2180178,0,224974,6788934,38736,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 colorectal,patient derived organoid;tumor,61,10,20839,1400,156993,0,0,44634,689155,16140,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 cptac,tumor,1139,0,19278,0,21170373,9486498,0,252398,19261183,0,0, -ctrpv2,cell line,848,460,19381,3101330,15739656,3011948,0,389773,10906812,742440,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -fimm,cell line,53,52,19353,26630,919104,250970,0,12743,760509,83928,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -gcsi,cell line,571,43,19380,132290,10148440,2247800,0,276904,7197012,69402,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -gdscv1,cell line,985,291,19158,2454890,18501510,4694495,0,3385316,37266972,469674,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -gdscv2,cell line,807,170,19158,1146910,15192913,3879361,0,2852337,30512750,274380,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -hcmi,patient derived organoid,1087,0,19218,0,12540435,0,0,100602,6044025,0,0, +ctrpv2,cell line,847,460,19381,3101330,15739656,3011948,0,389773,10906812,742440,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +fimm,cell line,52,52,19353,26630,919104,250970,0,12743,760509,83928,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +gcsi,cell line,570,43,19380,132290,10148440,2247800,0,276904,7197012,69402,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +gdscv1,cell line,984,291,19158,2454890,18501510,4694495,0,3385316,37266972,469674,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +gdscv2,cell line,806,169,19158,1139640,15192913,3879361,0,2852337,30512750,272766,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +hcmi,patient derived organoid,1087,0,19220,0,12540435,0,0,104261,6044025,0,0, liver,patient derived organoid,62,76,27163,44530,1063454,452662,0,12810,1686901,122664,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -mpnst,3D-MEDS;patient derived xenograft;tumor,47,39,19731,27985,1612748,52638,0,84720,541907,62946,75,aac;abc;auc;delta_auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2;lmm;mrecist;tgi -nci60,cell line,84,23267,19368,12364040,1340360,387298,0,54400,1094391,37551324,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -novartis,patient derived xenograft,348,25,19042,6950,6215451,0,0,106692,6219933,40350,0,abc;lmm;mrecist;tgi +mpnst,3D-MEDS;patient derived xenograft;tumor,133,39,19731,27985,1612748,52638,0,84720,541907,62946,75,aac;abc;auc;delta_auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2;lmm;mrecist;tgi +nci60,cell line,83,55143,19368,29189940,1340360,387298,0,54400,1094391,89025012,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +novartis,patient derived xenograft,386,25,19042,6950,6215451,0,0,106692,6219933,40350,0,abc;lmm;mrecist;tgi pancreatic,patient derived organoid,49,25,19070,1900,1031014,0,0,78589,955688,40350,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 -prism,cell line,479,1418,19381,6385340,9076152,2144440,0,249890,6992973,2288652,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 +prism,cell line,478,1416,19381,6381390,9076152,2144440,0,249890,6992973,2287038,0,aac;auc;dss;fit_auc;fit_ec50;fit_ec50se;fit_einf;fit_hs;fit_ic50;fit_r2 sarcoma,tumor;patient derived organoid,36,34,17895,275,573198,0,0,92,0,54876,0,published_auc diff --git a/docs/source/_static/dataset_curve_metrics_wide.csv b/docs/source/_static/dataset_curve_metrics_wide.csv index 24650ef6..9d2eadee 100644 --- a/docs/source/_static/dataset_curve_metrics_wide.csv +++ b/docs/source/_static/dataset_curve_metrics_wide.csv @@ -1,17 +1,17 @@ -dataset,num_drugs,aac,abc,auc,dss,fit_auc,fit_ec50,fit_ec50se,fit_einf,fit_hs,fit_ic50,fit_r2,lmm,mRESCIST,published_auc,TGI -beataml,164,X,,X,X,X,X,X,X,X,X,X,,,, -bladder,50,X,,X,X,X,X,X,X,X,X,X,,,, -ccle,24,X,,X,X,X,X,X,X,X,X,X,,,, -colorectal,10,X,,X,X,X,X,X,X,X,X,X,,,, -ctrpv2,459,X,,X,X,X,X,X,X,X,X,X,,,, -fimm,52,X,,X,X,X,X,X,X,X,X,X,,,, -gcsi,44,X,,X,X,X,X,X,X,X,X,X,,,, -gdscv1,294,X,,X,X,X,X,X,X,X,X,X,,,, -gdscv2,171,X,,X,X,X,X,X,X,X,X,X,,,, -liver,76,X,,X,X,X,X,X,X,X,X,X,,,, -mpnst,30,X,X,X,X,X,X,X,X,X,X,X,X,X,,X -nci60,55157,X,,X,X,X,X,X,X,X,X,X,,,, -novartis,25,,X,,,,,,,,,,X,X,,X -pancreatic,25,X,,X,X,X,X,X,X,X,X,X,,,, -prism,1419,X,,X,X,X,X,X,X,X,X,X,,,, -sarcoma,34,,,,,,,,,,,,,,X, +dataset,num_drugs,aac,abc,auc,delta_auc,dss,fit_auc,fit_ec50,fit_ec50se,fit_einf,fit_hs,fit_ic50,fit_r2,lmm,mrecist,published_auc,tgi +beataml,139,X,,X,,X,X,X,X,X,X,X,X,,,, +bladder,50,X,,X,,X,X,X,X,X,X,X,X,,,, +ccle,24,X,,X,,X,X,X,X,X,X,X,X,,,, +colorectal,10,X,,X,,X,X,X,X,X,X,X,X,,,, +ctrpv2,460,X,,X,,X,X,X,X,X,X,X,X,,,, +fimm,52,X,,X,,X,X,X,X,X,X,X,X,,,, +gcsi,43,X,,X,,X,X,X,X,X,X,X,X,,,, +gdscv1,291,X,,X,,X,X,X,X,X,X,X,X,,,, +gdscv2,169,X,,X,,X,X,X,X,X,X,X,X,,,, +liver,73,X,,X,,X,X,X,X,X,X,X,X,,,, +mpnst,39,X,X,X,X,X,X,X,X,X,X,X,X,X,X,,X +nci60,55143,X,,X,,X,X,X,X,X,X,X,X,,,, +novartis,23,,X,,,,,,,,,,,X,X,,X +pancreatic,5,X,,X,,X,X,X,X,X,X,X,X,,,, +prism,1416,X,,X,,X,X,X,X,X,X,X,X,,,, +sarcoma,32,,,,,,,,,,,,,,,X, diff --git a/docs/source/_static/dataset_summary_statistics.csv b/docs/source/_static/dataset_summary_statistics.csv index 867b7f37..ba0f07eb 100644 --- a/docs/source/_static/dataset_summary_statistics.csv +++ b/docs/source/_static/dataset_summary_statistics.csv @@ -1,19 +1,19 @@ dataset,sample_drug_pairs,sample_drug_transcript_pairs,sample_drug_transcriptomics_mutation_pairs,sample_drug_transcriptomics_copynumber_pairs,sample_drug_mutation_copynumber_pairs -beataml,31926,4137,3958,, +beataml,32211,4176,3995,, bladder,3300,840,640,640,3100 -ccle,11543,10887,10792,10887,11118 +ccle,11543,10933,10885,8375,8398 colorectal,140,60,60,60,140 cptac,,,,, -ctrpv2,309401,300507,295742,299698,300616 -fimm,2663,2457,2457,2457,2611 -gcsi,13398,12506,12338,12506,13112 -gdscv1,247753,245220,241999,241240,242570 -gdscv2,115440,114373,112829,112523,113133 +ctrpv2,310133,300871,297687,216046,216830 +fimm,2663,2457,2457,2098,2098 +gcsi,13229,12368,12259,9186,9248 +gdscv1,245489,242980,239787,239034,240351 +gdscv2,113964,112911,111387,111085,111687 hcmi,,,,, liver,4453,4453,4453,4453,4453 -mpnst,272,193,184,191,184 -nci60,2960756,2329149,2329132,2329149,2784474 +mpnst,2736,2657,181,181,181 +nci60,2909092,2318378,2318362,2063572,2063572 novartis,1766,1734,1734,1723,1723 pancreatic,190,190,185,185,185 -prism,638983,632078,630672,632078,636226 +prism,637674,632199,632199,497743,499088 sarcoma,275,234,187,, diff --git a/docs/source/datasets_included.rst b/docs/source/datasets_included.rst index d9994859..4621529a 100644 --- a/docs/source/datasets_included.rst +++ b/docs/source/datasets_included.rst @@ -18,21 +18,21 @@ Dataset Overview "BeatAML", "[1]_, [2]_", "ex vivo", "1022", "164", "X", "X", "X", "X", "X", "" "Bladder", "[3]_", "patient derived organoid; patient derived xenograft; tumor; xenograft derived organoid", "134", "50", "X", "X", "X", "", "X", "X" - "CCLE", "[4]_", "cell line", "503", "24", "X", "X", "X", "X", "X", "X" + "CCLE", "[4]_", "cell line", "502", "24", "X", "X", "X", "X", "X", "X" "Colorectal", "[18]_", "patient derived organoid; tumor", "61", "10", "X", "X", "X", "", "X", "X" "CPTAC", "[5]_", "tumor", "1139", "", "", "", "X", "X", "X", "X" - "CTRPv2", "[6]_, [7]_, [8]_", "cell line", "848", "460", "X", "X", "X", "X", "X", "X" - "FIMM", "[9]_, [10]_", "cell line", "53", "52", "X", "X", "X", "X", "X", "X" - "GDSC v1", "[23]_, [24]_, [25]_", "cell line", "985", "291", "X", "X", "X", "X", "X", "X" - "GDSC v2", "[23]_, [24]_, [25]_", "cell line", "807", "170", "X", "X", "X", "X", "X", "X" - "gCSI", "[21]_, [22]_", "cell line", "571", "43", "X", "X", "X", "X", "X", "X" + "CTRPv2", "[6]_, [7]_, [8]_", "cell line", "847", "460", "X", "X", "X", "X", "X", "X" + "FIMM", "[9]_, [10]_", "cell line", "52", "52", "X", "X", "X", "X", "X", "X" + "GDSC v1", "[23]_, [24]_, [25]_", "cell line", "984", "291", "X", "X", "X", "X", "X", "X" + "GDSC v2", "[23]_, [24]_, [25]_", "cell line", "806", "169", "X", "X", "X", "X", "X", "X" + "gCSI", "[21]_, [22]_", "cell line", "570", "43", "X", "X", "X", "X", "X", "X" "HCMI", "[11]_", "patient derived organoid", "1087", "", "", "", "X", "", "X", "X" "Liver", "[19]_", "patient derived organoid", "62", "76", "X", "X", "X", "X", "X", "X" - "MPNST", "[12]_", "3D-MEDS; patient derived xenograft; tumor", "47", "39", "X", "X", "X", "X", "X", "X" - "NCI60", "[13]_", "cell line", "84", "23267", "X", "X", "X", "X", "X", "X" - "Novartis", "[20]_", "patient derived xenograft", "348", "25", "X", "X", "X", "", "X", "X" + "MPNST", "[12]_", "3D-MEDS; patient derived xenograft; tumor", "133", "39", "X", "X", "X", "X", "X", "X" + "NCI60", "[13]_", "cell line", "83", "55143", "X", "X", "X", "X", "X", "X" + "Novartis", "[20]_", "patient derived xenograft", "386", "25", "X", "X", "X", "", "X", "X" "Pancreatic", "[14]_", "patient derived organoid", "49", "25", "X", "X", "X", "", "X", "X" - "PRISM", "[15]_, [16]_", "cell line", "479", "1418", "X", "X", "X", "X", "X", "X" + "PRISM", "[15]_, [16]_", "cell line", "478", "1416", "X", "X", "X", "X", "X", "X" "Sarcoma", "[17]_", "tumor; patient derived organoid", "36", "34", "X", "X", "X", "", "X", "" @@ -78,6 +78,7 @@ Types of dose-response metrics collected include: - AAC - Area above the response curve; the complement value of AUC. - ABC - Area between curves, the difference between the AUC of the control and the treated cells. (PDX) - AUC - Area under the fitted hill slope curve across all doses present. Lower AUC signifies lower levels of growth. +- delta_auc - Difference in AUC between a single drug and its combination (single-drug AUC minus combination AUC); used for MPNST treated-microtissue combination experiments. - DSS - A multiparametric dose response value, that takes into account control and treated cells; the Drug Sensitivity Score is described in Yadav et al., 2014. - fit_auc - Area under the fitted hill slope curve across the common interval of −log10[M], where the molar concentration ranges from 10⁻⁴ to 10⁻¹⁰. - fit_ec50 - The fitted curve prediction of the −log10M concentration at which 50% of the maximal effect is observed. diff --git a/scripts/gen_dataset_stats.py b/scripts/gen_dataset_stats.py index 346ea843..51e5aa62 100644 --- a/scripts/gen_dataset_stats.py +++ b/scripts/gen_dataset_stats.py @@ -1,11 +1,18 @@ #!/usr/bin/env python3 -"""Generate a comprehensive per-dataset counts table from a local CoderData build. +"""Generate the per-dataset tables on the docs Datasets page from a local build. Reads a build output directory (the ``all_files_dir`` produced by -``coderbuild/build_all.py``) and writes a single wide CSV summarizing every -dataset: unique samples, unique drugs, unique genes, per-modality row counts, -and the set of dose-response metrics present. The output is written to the -docs static directory so the documentation site can render current numbers. +``coderbuild/build_all.py``) and writes three CSVs to the docs static +directory so the documentation site can render current numbers: + +* ``dataset_counts.csv``: unique samples, unique drugs, unique genes, + per-modality row counts, and the dose-response metrics present. The + Datasets and Modalities table in ``datasets_included.rst`` is copied + from this file. +* ``dataset_summary_statistics.csv``: unique sample-drug pairs, overall and + restricted to samples with each omics combination. +* ``dataset_curve_metrics_wide.csv``: drugs per dataset and which + dose-response metrics each dataset provides. This script reads the build files directly (no network, no installed ``coderdata`` package required), so it can be run against any local build. @@ -85,8 +92,10 @@ def _count_rows(path): def _iter_column(path, column): """Yield the value of one column per data row using fast line splitting. - Assumes the target column contains no embedded delimiter (true for the - id/metric columns we tally here). Skips the header. + Lines containing a quote character are parsed with the csv module, since + a quoted field earlier in the row (e.g. a samples ``other_names`` value + with embedded commas) would otherwise shift the column index. Skips the + header. """ delim, idx = _header_index(path, column) if idx is None: @@ -94,7 +103,11 @@ def _iter_column(path, column): with _open_any(path) as f: f.readline() # skip header for line in f: - parts = line.rstrip("\n").rstrip("\r").split(delim) + line = line.rstrip("\n").rstrip("\r") + if '"' in line: + parts = next(csv.reader([line], delimiter=delim)) + else: + parts = line.split(delim) if idx < len(parts): v = parts[idx] if v and v != "NA": @@ -173,6 +186,100 @@ def build_rows(build_dir, datasets): return rows +def _sample_key(v): + """Normalize an improve_sample_id so '3524.0' and '3524' match. + + Some experiments files (e.g. novartis) write sample ids as floats while + the omics files write integers. + """ + return v[:-2] if v.endswith(".0") else v + + +def _iter_pairs(path): + """Yield (improve_sample_id, improve_drug_id) per experiments row.""" + delim, s_idx = _header_index(path, "improve_sample_id") + _, d_idx = _header_index(path, "improve_drug_id") + with _open_any(path) as f: + f.readline() # skip header + for line in f: + parts = line.rstrip("\n").rstrip("\r").split(delim) + yield _sample_key(parts[s_idx]), parts[d_idx] + + +def build_summary_rows(build_dir, datasets): + """Unique sample-drug pairs, overall and restricted to samples with omics. + + Mirrors the definitions in notebooks/dataset_documentation.ipynb: a cell + is blank when the experiments file or any required omics file is absent. + """ + combos = [ + ("sample_drug_transcript_pairs", ["transcriptomics"]), + ("sample_drug_transcriptomics_mutation_pairs", ["transcriptomics", "mutations"]), + ("sample_drug_transcriptomics_copynumber_pairs", ["transcriptomics", "copy_number"]), + ("sample_drug_mutation_copynumber_pairs", ["mutations", "copy_number"]), + ] + rows = [] + for d in datasets: + rec = OrderedDict(dataset=d, sample_drug_pairs="") + for name, _ in combos: + rec[name] = "" + exp_path = _find_file(build_dir, d, "experiments") + if exp_path: + pairs = set(_iter_pairs(exp_path)) + rec["sample_drug_pairs"] = len(pairs) + omics_samples = {} + for modality in ("transcriptomics", "mutations", "copy_number"): + p = _find_file(build_dir, d, modality) + if p: + omics_samples[modality] = { + _sample_key(v) for v in _iter_column(p, "improve_sample_id")} + for name, needed in combos: + if all(m in omics_samples for m in needed): + keep = set.intersection(*(omics_samples[m] for m in needed)) + rec[name] = sum(1 for s, _ in pairs if s in keep) + rows.append(rec) + return rows + + +def build_curve_metric_rows(build_dir, datasets): + """Per-dataset drug count and X markers for each dose_response_metric.""" + per_dataset = OrderedDict() + for d in datasets: + p = _find_file(build_dir, d, "experiments") + if not p: + continue + delim, d_idx = _header_index(p, "improve_drug_id") + _, m_idx = _header_index(p, "dose_response_metric") + drugs, metrics = set(), set() + with _open_any(p) as f: + f.readline() # skip header + for line in f: + parts = line.rstrip("\n").rstrip("\r").split(delim) + drugs.add(parts[d_idx]) + if parts[m_idx] and parts[m_idx] != "NA": + metrics.add(parts[m_idx]) + per_dataset[d] = (len(drugs), metrics) + all_metrics = sorted(set().union(*(m for _, m in per_dataset.values())), + key=str.lower) + rows = [] + for d, (n_drugs, metrics) in per_dataset.items(): + rec = OrderedDict(dataset=d, num_drugs=n_drugs) + for m in all_metrics: + rec[m] = "X" if m in metrics else "" + rows.append(rec) + return rows + + +def _write_csv(path, rows): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + fieldnames = list(rows[0].keys()) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + print(f"Wrote {path} with {len(rows)} datasets and {len(fieldnames)} columns.") + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -180,6 +287,14 @@ def main(): help="build output directory (default: local/all_files_dir)") ap.add_argument("--output", default="docs/source/_static/dataset_counts.csv", help="output CSV path (default: docs/source/_static/dataset_counts.csv)") + ap.add_argument("--summary-output", + default="docs/source/_static/dataset_summary_statistics.csv", + help="sample-drug pair summary CSV path " + "(default: docs/source/_static/dataset_summary_statistics.csv)") + ap.add_argument("--curve-output", + default="docs/source/_static/dataset_curve_metrics_wide.csv", + help="curve metric presence CSV path " + "(default: docs/source/_static/dataset_curve_metrics_wide.csv)") ap.add_argument("--datasets", nargs="*", default=None, help="restrict to these dataset names (default: all discovered)") args = ap.parse_args() @@ -191,16 +306,9 @@ def main(): if not datasets: raise SystemExit(f"no *_samples.csv found in {args.build_dir}") - rows = build_rows(args.build_dir, datasets) - - os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) - fieldnames = list(rows[0].keys()) - with open(args.output, "w", newline="", encoding="utf-8") as f: - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(rows) - - print(f"Wrote {args.output} with {len(rows)} datasets and {len(fieldnames)} columns.") + _write_csv(args.output, build_rows(args.build_dir, datasets)) + _write_csv(args.summary_output, build_summary_rows(args.build_dir, datasets)) + _write_csv(args.curve_output, build_curve_metric_rows(args.build_dir, datasets)) print("Datasets:", ", ".join(datasets))