From a3e06f11443f926fd9adad432fc91b4da43c654c Mon Sep 17 00:00:00 2001 From: MVChem Date: Thu, 15 Oct 2026 16:35:05 +0800 Subject: [PATCH] Add vertebrae consensus backend --- README.md | 9 + config.yaml | 3 + docs/VERTEBRAE_CONSENSUS.md | 192 ++++++++++++ main.py | 33 +- tests/test_vertebrae_consensus.py | 264 ++++++++++++++++ tools/verify_vertebrae_consensus.py | 95 ++++++ .../SOURCE_PROVENANCE.json | 69 ++++ utils/_vertebrae_consensus/__init__.py | 1 + .../_vertebrae_consensus/body_root_guided.py | 166 ++++++++++ .../body_root_guided_consistency.py | 69 ++++ .../cervicothoracic_causal.py | 103 ++++++ .../component_identity_transfer.py | 41 +++ .../consensus_boundary.py | 72 +++++ .../global_body_identity_graph.py | 106 +++++++ .../global_body_offset_gate.py | 60 ++++ .../native_anchor_pipeline.py | 57 ++++ .../postprocessing_causal.py | 112 +++++++ utils/vertebrae_consensus.py | 294 ++++++++++++++++++ 18 files changed, 1741 insertions(+), 5 deletions(-) create mode 100644 docs/VERTEBRAE_CONSENSUS.md create mode 100644 tests/test_vertebrae_consensus.py create mode 100644 tools/verify_vertebrae_consensus.py create mode 100644 utils/_vertebrae_consensus/SOURCE_PROVENANCE.json create mode 100644 utils/_vertebrae_consensus/__init__.py create mode 100644 utils/_vertebrae_consensus/body_root_guided.py create mode 100644 utils/_vertebrae_consensus/body_root_guided_consistency.py create mode 100644 utils/_vertebrae_consensus/cervicothoracic_causal.py create mode 100644 utils/_vertebrae_consensus/component_identity_transfer.py create mode 100644 utils/_vertebrae_consensus/consensus_boundary.py create mode 100644 utils/_vertebrae_consensus/global_body_identity_graph.py create mode 100644 utils/_vertebrae_consensus/global_body_offset_gate.py create mode 100644 utils/_vertebrae_consensus/native_anchor_pipeline.py create mode 100644 utils/_vertebrae_consensus/postprocessing_causal.py create mode 100644 utils/vertebrae_consensus.py diff --git a/README.md b/README.md index 1bdc0e6..a4ce90e 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,15 @@ target_organs: (example) Before running any commands, please ensure that `config.yaml` is properly configured. But don't worry! **Most of the configurations do not need to be changed at all.** +# Multi-model Vertebrae Consensus (optional) + +`shapekit_consensus` integrates Hongkang Chu's warmup postprocessing chain: +original Hao refinement, SuPreM identity anchors, dual-chain checks, v3 identity +alignment, and agreement-based boundary edits. Supply raw predictions from all +three models, or replay the final stage from prepared predictions. It runs on +CPU, writes each completed case atomically, and leaves the default engine +unchanged. See [inputs, commands, limitations, and regression checks](docs/VERTEBRAE_CONSENSUS.md). + # Native-Grid Two-Stage Vertebrae Engine (ShapeKit-Hao) An optional `shapekit_hao` backend integrates v2 component cleanup with diff --git a/config.yaml b/config.yaml index 2c7453f..6649c52 100644 --- a/config.yaml +++ b/config.yaml @@ -92,6 +92,9 @@ affine_reference_file_name: liver.nii.gz if_save_combined_label: True # vertebrae engine: +# shapekit_consensus - opt-in multi-model warmup integration; requires aligned +# TotalSegmentator v2, SuPreM, v3 predictions and mm CT. +# See docs/VERTEBRAE_CONSENSUS.md. Existing default is unchanged. # shapekit_hao - opt-in native-grid two-stage vertebrae refinement; # requires matching calibrated CT and target_organs [vertebrae]. # Fails on grid mismatch/missing CT; no legacy fallback. diff --git a/docs/VERTEBRAE_CONSENSUS.md b/docs/VERTEBRAE_CONSENSUS.md new file mode 100644 index 0000000..13ab3e5 --- /dev/null +++ b/docs/VERTEBRAE_CONSENSUS.md @@ -0,0 +1,192 @@ +# Multi-model vertebrae consensus backend + +`shapekit_consensus` integrates Hongkang Chu's submitted warmup CPU +postprocessing into ShapeKit's case selection, worker pool, logging, label +mapping, and output layout. It is optional; the existing default is unchanged. + +The method uses predictions from **TotalSegmentator v2, SuPreM, and +TotalSegmentator v3**. It does not train or run these networks. Predictions are +not reference annotations. No ground-truth mask, case-specific rule, or score +is an algorithm input. + +## Method + +Raw mode runs the following chain with the submitted numerical parameters: + +1. Original upstream Hao component cleanup and anatomical correction on SuPreM. +2. The native-grid caudal and cervicothoracic anchor refinements. +3. Original Hao refinement on TotalSegmentator v2, followed by the dual ordered + body-chain identity check using the corrected SuPreM anchors. +4. Global identity proposals on **raw v3**, then the coherent-offset gate. + V3 is not passed through Hao in this path. +5. Two-source agreement for boundary additions/deletions within 2 mm of the + incumbent foreground. Additions must agree with the nearest incumbent + identity; retained foreground IDs and complete C1/C2 masks are protected. + The submitted variant has `ct_guard=False`. CT is still required by the + upstream stages and by the physical-grid contract. + +`utils/_vertebrae_consensus/` contains the numerical functions extracted from +the submission, with cohort-specific CLIs removed and imports made package +relative. Function bodies are unchanged. `SOURCE_PROVENANCE.json` records +original source hashes and AST fingerprints. Hao is reused from the existing +`utils/vertebrae_hao_engine.py`, not copied or modified. + +## Installation + +Use the existing small CPU environment; PyTorch and model weights are not needed: + +```console +python -m pip install -r requirements-vertebrae-hao.txt +python -m pip install pytest +python -m pytest tests -q +``` + +Run ShapeKit commands from the repository root: its existing entrypoint reads +`config.yaml` from the working directory. + +## Raw prediction input + +All three roots have the same case IDs and mask naming: + +```text +v2/CASE/segmentations/vertebrae_L5.nii.gz +v2/CASE/segmentations/vertebrae_L4.nii.gz +... through vertebrae_C1.nii.gz +suprem/CASE/segmentations/vertebrae_L5.nii.gz +v3/CASE/segmentations/vertebrae_L5.nii.gz +ct/CASE/ct.nii.gz +``` + +Use **original raw predictions** for each model in this mode. The adapter reads +named binary masks and does not infer label conventions from combined maps. +Missing mask files are reported and treated as empty; insufficient anatomical +cores cause the numerical stages to abstain. Additional vertebra levels such +as L6 or T13 are rejected, not dropped. Each mask must be binary, nonoverlapping, +and on the same grid as the CT. No resampling or orientation conversion occurs. + +Raw mode supports the original **axis-aligned LAS/RAS** grids. Other scanner +orientations fail explicitly; they have not been adapted or validated. +All grids must be orthogonal, finite, and have matching shape, affine, and +spacing. The CT must declare millimeter units and contain calibrated HU after +NIfTI scaling. Legacy prediction headers with unknown units are accepted only +on the matching explicitly-mm CT grid; their original flags are retained and +reported. Meter/micron prediction coordinates are rejected. + +```console +python main.py --vertebrae_engine shapekit_consensus --vertebrae_only \ + --input_folder /data/v2 --suprem_root /data/suprem \ + --totalseg_v3_root /data/v3 --ct_root /data/ct \ + --output_folder /data/consensus --cpu_count 1 \ + --log_folder /data/consensus_logs +``` + +Repeat `--case CASE` to select cases, or use the existing `--csv` filter. +Without either filter, all primary-root case directories are selected. +Use `--consensus_save_stages` to save intermediate combined volumes for +regression checks. They use **internal IDs 1=L5 through 24=C1**, unlike the +ShapeKit combined output. + +## Prepared boundary-stage replay + +If the upstream chain has already run, set `--consensus_input_stage prepared`. +The roles change explicitly: + +| Argument | Prepared input | +| --- | --- | +| `--input_folder` | V2 + Hao + dual-chain incumbent | +| `--suprem_root` | Corrected SuPreM anchors | +| `--totalseg_v3_root` | Global-gated v3 prediction | + +Each root still supplies named binary masks in `CASE/segmentations/`. +Prepared mode runs **only** the final boundary agreement stage. Reports state +the input stage; a prepared run must not be described as a full replay. + +## Outputs and batch behavior + +- All 24 binary masks are written, including empty masks. Unrelated organ + masks from the primary input are copied unchanged. +- `combined_labels.nii.gz` follows `class_map`, normally 26=L5 through 49=C1. + Existing ascending-ID overlap priority is retained for combined multi-organ + output. Per-vertebra masks remain the authoritative binary exports. +- qform, sform, codes, affine, spacing, orientation, and header units are + retained from the primary reference mask. +- `vertebrae_consensus_report.json` records completion, input hashes, source + hashes, missing masks, label conversion, unit provenance, stage decisions, + and runtime. `dsc` remains null: this adapter does not read ground truth. +- New output directories must be disjoint from **all** input roots, including + both auxiliary models and CT. Existing case outputs are never overwritten. +- A case is written under `.incomplete` and renamed only after every output + and its completion report has been saved. Failed writes remain available + for diagnosis; a failed case does not appear as a completed case directory. +- The existing worker pool continues processing other selected cases and + exits nonzero if any case fails. Do not use `--continue_prediction`; its + legacy nonempty-folder check does not establish completion of this backend. + +This backend defaults to **one worker**; use `--cpu_count` to opt into concurrency. +Full-resolution 3D graph stages can require substantial +RAM; concurrency should follow measured peak RSS on the target CT resolution. +The adapter writes one binary mask at a time rather than constructing 24 +full-volume mask arrays simultaneously. This is a memory-management change in +the writer, not a change to the numerical method. + +## Regression and evidence limits + +Run the full chain on both supplied cases and compare **all 48 binary masks** +and metadata with the submitted standalone results. Convert ShapeKit combined +IDs using the report's `label_map` before comparing internal combined maps. +Also compare the saved `suprem_anchors`, `incumbent`, and `aligned_v3` stages +against the submitted frozen upstream maps. Unit tests alone cannot establish +real-case equivalence. + +```console +python tools/verify_vertebrae_consensus.py \ + --actual /data/consensus --reference /data/submitted_predictions \ + --case CASE1 --case CASE2 \ + --prepared-inputs /data/submitted_boundary_inputs \ + --report /data/consensus_equivalence.json +``` + +The optional `--prepared-inputs` folder has `SuPreM_anchors/CASE/`, +`incumbent/CASE/`, and `aligned_v3/CASE/` directories, each with a standalone +`combined_labels.nii.gz` in 0..24 encoding. It requires `--consensus_save_stages` +on the full run. The checker compares all masks, combined-map conversion, +units, qform/sform, affine, and spacing, and returns a nonzero exit on a mismatch. + +The previous public-reference two-case development mean was 90.106%; the +instructor's separate evaluation reported 91.5%. These are different protocols. +On three additional public cases, the candidate scored 88.640% versus 89.313% +for the stronger v2 + Hao baseline. The development improvement did **not** +generalize to that cohort. Packaging and output equivalence do not establish +clinical accuracy or large-cohort generalization. + +Credit: SuPreM authors; TotalSegmentator / Jakob Wasserthal and collaborators; +Hao Yu / BodyMaps ShapeKit for the unchanged Hao engine; Hongkang Chu for the +submitted anchor, identity-consistency, and boundary-consensus refinements. +The causal-root idea was informed by the public ShapeKit-Pro discussion; no +ShapeKit-Pro source was copied into these numerical modules. + +Keep CT volumes, predictions, reference masks, checkpoints, and private reports +outside the repository. Local validation artifacts are not PR source files. + +## Local integration validation — 15 October 2026 + +- All **84 tests** passed, including the existing suite, malformed-input checks, + partial-write handling, failed-case isolation, and verifier corruption tests. +- Both supplied warmup cases ran through the complete **raw** CPU chain using + ShapeKit's entrypoint and two concurrent case workers. +- All **48 exported binary masks** matched the submitted standalone predictions + voxel-for-voxel. Both combined maps matched after class-ID conversion, and all + **six frozen intermediate maps** matched. Shape, affine, spacing, units, qform, + and sform checks passed. +- Numerical function fingerprints matched the submitted source, and the run + reports' source hashes matched the current integration files. +- On this host, case 06 took 4.68 minutes and case 31 took 31.87 minutes. The + concurrent batch took 31 minutes 55 seconds. GNU time reported a maximum + process RSS of about **14.5 GiB**; this is not aggregate concurrent memory or + a large-cohort throughput benchmark. BLAS/OMP threads were set to one per + worker. The backend still defaults to one case worker. + +These are integration and output-equivalence checks. They do not rerun neural +inference, recompute the instructor's DSC, or establish accuracy on new cases. +Local CTs, predictions, logs, and regression reports are kept outside the Git +working tree. diff --git a/main.py b/main.py index 52d92bb..33a1718 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ from utils.vertebrae_pro import postprocessing_vertebrae_pro from utils.vertebrae_hao import process_case as process_hao_case from utils.vertebrae_ct_refinement import process_case as process_ct_refinement_case +from utils.vertebrae_consensus import process_case as process_consensus_case import logging import yaml import traceback @@ -228,7 +229,7 @@ def main(input_path, input_folder_name, output_path=None): input_path: the folder path """ - if vertebrae_engine in ('shapekit_hao', 'shapekit_ct_refinement'): + if vertebrae_engine in ('shapekit_hao', 'shapekit_ct_refinement', 'shapekit_consensus'): # Isolate the evaluated algorithm from legacy organ cleanup and IO. # The strict adapter validates each mask/CT grid and writes zero masks. if target_organs != {'vertebrae'}: @@ -236,6 +237,14 @@ def main(input_path, input_folder_name, output_path=None): ct_path = os.path.join(input_path, ct_file_name) if not os.path.exists(ct_path) and ct_root is not None: ct_path = os.path.join(ct_root, input_folder_name, ct_file_name) + if vertebrae_engine == 'shapekit_consensus': + return process_consensus_case( + input_path, os.path.join(output_path, input_folder_name), + ct_path, class_map, subfolder_name=subfolder_name, + suprem_path=os.path.join(args.suprem_root, input_folder_name), + v3_path=os.path.join(args.totalseg_v3_root, input_folder_name), + input_stage=args.consensus_input_stage, + save_stages=args.consensus_save_stages) processor = process_ct_refinement_case if vertebrae_engine == 'shapekit_ct_refinement' else process_hao_case return processor( input_path, os.path.join(output_path, input_folder_name), @@ -345,14 +354,21 @@ def run_in_parallel(sub_folders, input_folder, output_folder, max_workers=4, tqd parser.add_argument('--output_folder', type=str, help='Output files folder location, /path/to/save/results') parser.add_argument('--log_folder', type=str, default='./logs/task_001', help='Logging folder location') parser.add_argument('--csv', type=str, default=None, help='Guidence csv file telling ShapeKit specific ones for processing') -parser.add_argument('--cpu_count', type=int, default=cpu_count(), help='Number of CPU cores to use for parallel processing (default: system max)') +parser.add_argument('--cpu_count', type=int, default=None, + help='Case workers (default: 1 for consensus, system max for existing engines)') parser.add_argument('--continue_prediction', action="store_true", help='If continue from last processing record') parser.add_argument('--tqdm_ncols', type=int, default=100, help='Width of tqdm progress bar in characters') -parser.add_argument('--vertebrae_engine', choices=['shapekit', 'shapekit_songlin', 'shapekit_pro', 'shapekit_hao', 'shapekit_ct_refinement'], +parser.add_argument('--vertebrae_engine', choices=['shapekit', 'shapekit_songlin', 'shapekit_pro', 'shapekit_hao', 'shapekit_ct_refinement', 'shapekit_consensus'], help='Override config.yaml vertebrae engine for this run') parser.add_argument('--vertebrae_only', action='store_true', help='Process vertebrae only') parser.add_argument('--ct_root', help='Override external case CT root for this run') parser.add_argument('--case', action='append', help='Process only this case folder (repeatable)') +parser.add_argument('--suprem_root', help='Consensus: raw SuPreM cases, or corrected anchors in prepared mode') +parser.add_argument('--totalseg_v3_root', help='Consensus: raw TotalSegmentator v3 cases, or aligned v3 in prepared mode') +parser.add_argument('--consensus_input_stage', choices=['raw', 'prepared'], default='raw', + help='Consensus: full CPU chain from raw predictions, or final boundary replay') +parser.add_argument('--consensus_save_stages', action='store_true', + help='Consensus: save intermediate combined maps with internal IDs 1..24') args = parser.parse_args() if args.vertebrae_engine is not None: vertebrae_engine = args.vertebrae_engine @@ -360,6 +376,8 @@ def run_in_parallel(sub_folders, input_folder, output_folder, max_workers=4, tqd target_organs = {'vertebrae'} if args.ct_root is not None: ct_root = args.ct_root +if args.cpu_count is None: + args.cpu_count = 1 if vertebrae_engine == 'shapekit_consensus' else cpu_count() @@ -397,14 +415,19 @@ def run_in_parallel(sub_folders, input_folder, output_folder, max_workers=4, tqd parser.error('--input_folder and --output_folder are required') if args.cpu_count < 1: parser.error('--cpu_count must be at least 1') - if vertebrae_engine in ('shapekit_hao', 'shapekit_ct_refinement'): + if vertebrae_engine in ('shapekit_hao', 'shapekit_ct_refinement', 'shapekit_consensus'): if target_organs != {'vertebrae'}: parser.error(f'{vertebrae_engine} requires --vertebrae_only') if args.continue_prediction: parser.error(f'{vertebrae_engine} uses new case directories; do not use --continue_prediction') from pathlib import Path hao_output = Path(output_folder).resolve() - for source in (input_folder, ct_root): + if vertebrae_engine == 'shapekit_consensus': + if not args.suprem_root or not args.totalseg_v3_root: + parser.error('shapekit_consensus requires --suprem_root and --totalseg_v3_root') + for source in (input_folder, ct_root, + args.suprem_root if vertebrae_engine == 'shapekit_consensus' else None, + args.totalseg_v3_root if vertebrae_engine == 'shapekit_consensus' else None): if source is not None: hao_source = Path(source).resolve() if (hao_source == hao_output or hao_source in hao_output.parents diff --git a/tests/test_vertebrae_consensus.py b/tests/test_vertebrae_consensus.py new file mode 100644 index 0000000..d4ad38a --- /dev/null +++ b/tests/test_vertebrae_consensus.py @@ -0,0 +1,264 @@ +"""Contracts for the multi-model integration, with real numerical stages.""" +import ast +import hashlib +import json +from pathlib import Path +import subprocess +import sys + +import nibabel as nib +import numpy as np +import pytest + +from utils import vertebrae_consensus as adapter + + +def save(path, data, units='unknown', affine=None): + path.parent.mkdir(parents=True, exist_ok=True) + image = nib.Nifti1Image(data, np.eye(4) if affine is None else affine) + image.header.set_xyzt_units(units) + image.header['descrip'] = b'consensus contract fixture' + image.set_qform(image.affine, 1) + image.set_sform(image.affine, 2) + nib.save(image, path) + + +def masks(root, labels): + for i, name in enumerate(adapter.NAMES, 1): + # Present empty masks are distinct from missing masks in reports. + save(root/'segmentations'/(name+'.nii.gz'), (labels==i).astype(np.uint8)) + + +@pytest.fixture +def case(tmp_path): + primary = np.zeros((20, 20, 24), np.uint8) + primary[4:10, 4:10, 4:10] = 1 + primary[2, 2, 2] = 23 + primary[2, 2, 3] = 24 + auxiliary = primary.copy() + auxiliary[10, 6, 6] = 1 + auxiliary[4, 6, 6] = 0 + # Unanimity alone cannot expand beyond 2 mm or change the nearest ID. + auxiliary[16, 6, 6] = 1 + auxiliary[10, 7, 6] = 2 + paths = {r:tmp_path/r/'case' for r in ('primary','suprem','v3')} + for role, root in paths.items(): + masks(root, primary if role=='primary' else auxiliary) + ct = tmp_path/'ct/case/ct.nii.gz' + save(ct, np.full(primary.shape, 300, np.float32), units='mm') + mapping = {i+26:n for i,n in enumerate(adapter.NAMES)} + return dict(paths=paths,ct=ct,mapping=mapping,primary=primary,auxiliary=auxiliary, + output=tmp_path/'output/case') + + +def run(case, **kwargs): + return adapter.process_case(case['paths']['primary'],case['output'],case['ct'], + case['mapping'],suprem_path=case['paths']['suprem'], + v3_path=case['paths']['v3'],input_stage='prepared',**kwargs) + + +def test_numerical_function_bodies_match_submitted_sources(): + folder=Path(adapter.__file__).with_name('_vertebrae_consensus') + provenance=json.loads((folder/'SOURCE_PROVENANCE.json').read_text()) + for module, row in provenance['modules'].items(): + tree=ast.parse((folder/(module+'.py')).read_text()) + found={n.name:hashlib.sha256(ast.dump(n,include_attributes=False).encode()).hexdigest() + for n in tree.body if isinstance(n,(ast.FunctionDef,ast.ClassDef))} + assert found==row['function_ast_sha256'] + + +def test_loader_preserves_values_and_c_order_at_numerical_boundary(case): + reference,labels,missing,hashes=adapter.load_masks(case['paths']['suprem']) + np.testing.assert_array_equal(labels,case['auxiliary']) + assert labels.flags.c_contiguous + assert not missing + assert len(hashes)==24 + + +def test_real_boundary_geometry_protected_masks_and_source_immutability(case): + files=[p for root in case['paths'].values() for p in root.rglob('*.nii.gz')]+[case['ct']] + hashes={p:adapter.digest(p) for p in files} + report=run(case,save_stages=True) + expected=case['primary'].copy();expected[10,6,6]=1;expected[4,6,6]=0 + assert report['reference_labels_used'] is False + assert report['dsc'] is None + assert report['input_stage']=='prepared' + assert report['stage_reports'][0]['changed_voxels']==2 + combined=nib.load(case['output']/'combined_labels.nii.gz') + want=np.where(expected>0,expected+25,0) + np.testing.assert_array_equal(np.asarray(combined.dataobj),want) + assert combined.header.get_xyzt_units()[0]=='unknown' + assert int(combined.header['qform_code'])==1 + assert int(combined.header['sform_code'])==2 + for i,name in enumerate(adapter.NAMES,1): + image=nib.load(case['output']/'segmentations'/(name+'.nii.gz')) + np.testing.assert_array_equal(np.asarray(image.dataobj),expected==i) + assert image.header['descrip']==combined.header['descrip'] + assert len(list((case['output']/'segmentations').glob('*.nii.gz')))==24 + assert (case['output']/'stages/boundary_agreement.nii.gz').is_file() + assert hashes=={p:adapter.digest(p) for p in files} + assert json.loads((case['output']/'vertebrae_consensus_report.json').read_text())==report + + +def test_copy_other_organ_and_nonstandard_combined_ids(case): + other=np.zeros_like(case['primary']);other[0,0,0]=1 + path=case['paths']['primary']/'segmentations/liver.nii.gz';save(path,other) + case['mapping']={i+301:n for i,n in enumerate(adapter.NAMES)} + case['mapping'][5]='liver' + run(case) + assert path.read_bytes()==(case['output']/'segmentations/liver.nii.gz').read_bytes() + result=nib.load(case['output']/'combined_labels.nii.gz') + assert result.get_data_dtype()==np.dtype(np.uint16) + assert result.dataobj[6,6,6]==301 + assert result.dataobj[0,0,0]==5 + + +def test_raw_pipeline_runs_all_stages_without_cached_case_data(case): + empty=np.zeros_like(case['primary']) + final, stages=adapter.run_pipeline(empty,empty,empty,np.zeros_like(empty,dtype=np.float32), + nib.load(case['ct']),input_stage='raw') + np.testing.assert_array_equal(final,empty) + assert [s['stage'] for s in stages]==['suprem_hao','suprem_anchors','primary_hao', + 'incumbent','v3_proposal','aligned_v3','boundary_agreement'] + assert all(s.get('changed_voxels',0)==0 for s in stages) + + +@pytest.mark.parametrize('role',['primary','suprem','v3']) +def test_nonbinary_or_overlapping_masks_rejected(case,role): + bad=np.zeros_like(case['primary']);bad[0,0,0]=2 + save(case['paths'][role]/'segmentations/vertebrae_L5.nii.gz',bad) + with pytest.raises(ValueError,match='binary'):run(case) + assert not case['output'].exists() + + +def test_overlapping_classes_rejected(case): + save(case['paths']['suprem']/'segmentations/vertebrae_L4.nii.gz', + (case['auxiliary']==1).astype(np.uint8)) + with pytest.raises(ValueError,match='Overlapping'):run(case) + + +def test_missing_auxiliary_is_not_silently_dropped(case): + case['paths']['suprem']=case['paths']['suprem']/'missing' + with pytest.raises(ValueError,match='No named vertebra'):run(case) + assert not case['output'].exists() + + +def test_missing_single_mask_is_reported_and_empty_output_is_written(case): + (case['paths']['primary']/'segmentations/vertebrae_C7.nii.gz').unlink() + report=run(case) + assert report['missing_input_masks']['primary']==['vertebrae_C7'] + assert not np.any(nib.load(case['output']/'segmentations/vertebrae_C7.nii.gz').dataobj) + + +@pytest.mark.parametrize('units',['unknown','meter','micron']) +def test_ct_must_declare_mm(case,units): + save(case['ct'],np.zeros_like(case['primary'],dtype=np.float32),units=units) + with pytest.raises(ValueError,match='millimeter'):run(case) + + +def test_scaled_ct_is_loaded_as_hu(case,monkeypatch): + image=nib.Nifti1Image(np.ones_like(case['primary'],dtype=np.int16),np.eye(4)) + image.header.set_xyzt_units('mm');image.header.set_slope_inter(2,-1024);nib.save(image,case['ct']) + real=adapter.run_pipeline + def check(primary,suprem,v3,ct,*args,**kwargs): + assert ct.dtype==np.float32 + assert np.all(ct==-1022) + return real(primary,suprem,v3,ct,*args,**kwargs) + monkeypatch.setattr(adapter,'run_pipeline',check) + run(case) + + +def test_ct_nan_rejected(case): + data=np.zeros_like(case['primary'],dtype=np.float32);data[0,0,0]=np.nan + save(case['ct'],data,units='mm') + with pytest.raises(ValueError,match='Finite CT'):run(case) + + +def test_auxiliary_grid_mismatch_rejected(case): + affine=np.eye(4);affine[0,3]=2 + masks_root=case['paths']['v3']/'segmentations' + for p in masks_root.glob('*.nii.gz'): + data=np.asarray(nib.load(p).dataobj);save(p,data,affine=affine) + with pytest.raises(ValueError,match='Native-grid'):run(case) + + +def test_shear_rejected(case): + affine=np.eye(4);affine[0,1]=.1 + save(case['ct'],np.zeros_like(case['primary'],dtype=np.float32),units='mm',affine=affine) + with pytest.raises(ValueError,match='Sheared'):run(case) + + +def test_extra_vertebra_level_rejected(case): + save(case['paths']['v3']/'segmentations/vertebrae_L6.nii.gz',np.zeros_like(case['primary'])) + with pytest.raises(ValueError,match='24-level'):run(case) + + +def test_output_overlap_and_overwrite_rejected(case): + case['output']=case['paths']['suprem']/'new-output' + with pytest.raises(ValueError,match='disjoint'):run(case) + case['output']=case['paths']['v3'] + with pytest.raises(FileExistsError):run(case) + + +def test_partial_write_is_not_published(case,monkeypatch): + def fail(*args,**kwargs):raise OSError('injected write failure') + monkeypatch.setattr(adapter,'save_volume',fail) + with pytest.raises(OSError,match='injected'):run(case) + assert not case['output'].exists() + assert len(list((case['output'].parent/'.incomplete').iterdir()))==1 + + +def test_cli_isolates_failed_case_with_nonzero_batch_exit(case,tmp_path): + repo=Path(__file__).resolve().parents[1] + for root in case['paths'].values(): + (root.parent/'bad'/'segmentations').mkdir(parents=True) + command=[sys.executable,'main.py','--vertebrae_engine','shapekit_consensus', + '--vertebrae_only','--consensus_input_stage','prepared', + '--input_folder',str(case['paths']['primary'].parent), + '--suprem_root',str(case['paths']['suprem'].parent), + '--totalseg_v3_root',str(case['paths']['v3'].parent), + '--ct_root',str(case['ct'].parents[1]),'--output_folder',str(case['output'].parent), + '--cpu_count','2','--log_folder',str(tmp_path/'logs')] + result=subprocess.run(command,cwd=repo,capture_output=True,text=True,timeout=90) + assert result.returncode==1,result.stdout+result.stderr + assert 'Failed cases: bad' in result.stdout + assert (case['output']/'vertebrae_consensus_report.json').is_file() + assert not (case['output'].parent/'bad').exists() + + +def make_reference(case): + expected=case['primary'].copy();expected[10,6,6]=1;expected[4,6,6]=0 + root=case['output'].parent.parent/'reference/case' + masks(root,expected) + save(root/'combined_labels.nii.gz',expected) + return root + + +def test_equivalence_checker_accepts_label_conversion_and_checks_every_mask(case): + from tools.verify_vertebrae_consensus import compare_case + run(case);reference=make_reference(case) + report=compare_case(case['output'].parent,reference.parent,'case') + assert report['passed'] + assert len(report['masks'])==24 + assert report['changed_combined_voxels']==0 + + +@pytest.mark.parametrize('corruption',['mask','units','label_map']) +def test_equivalence_checker_detects_corruption(case,corruption): + from tools.verify_vertebrae_consensus import compare_case + run(case);reference=make_reference(case) + target=case['output']/'segmentations/vertebrae_L5.nii.gz' + data=np.asarray(nib.load(target).dataobj).copy() + if corruption=='mask': + data[0,0,0]=1;save(target,data) + elif corruption=='units':save(target,data,units='mm') + else: + path=case['output']/'vertebrae_consensus_report.json' + report=json.loads(path.read_text());report['label_map']['2']=report['label_map']['1'] + path.write_text(json.dumps(report)) + with pytest.raises(ValueError,match='label map'): + compare_case(case['output'].parent,reference.parent,'case') + return + result=compare_case(case['output'].parent,reference.parent,'case') + assert not result['passed'] + assert not result['masks'][0]['passed'] diff --git a/tools/verify_vertebrae_consensus.py b/tools/verify_vertebrae_consensus.py new file mode 100644 index 0000000..903da6e --- /dev/null +++ b/tools/verify_vertebrae_consensus.py @@ -0,0 +1,95 @@ +"""Compare the integration with submitted predictions, not ground truth.""" +import argparse +import json +from pathlib import Path +import sys + +import nibabel as nib +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from utils.vertebrae_consensus import NAMES + + +def geometry(a, b): + result = dict(shape=a.shape==b.shape, + affine=bool(np.allclose(a.affine,b.affine,rtol=0,atol=1e-5)), + spacing=bool(np.allclose(a.header.get_zooms(),b.header.get_zooms(),rtol=0,atol=1e-5)), + units=a.header.get_xyzt_units()==b.header.get_xyzt_units()) + for key in ('qform','sform'): + x,xc=getattr(a,'get_'+key)(coded=True);y,yc=getattr(b,'get_'+key)(coded=True) + result[key]=bool(xc==yc and ((x is None and y is None) or + (x is not None and y is not None and np.allclose(x,y,rtol=0,atol=1e-5)))) + return result + + +def compare_case(actual, reference, case, prepared_inputs=None): + aroot,rroot=Path(actual)/case,Path(reference)/case + report=json.loads((aroot/'vertebrae_consensus_report.json').read_text()) + if report.get('integration_status')!='complete': + raise ValueError('Completed consensus report required') + ids=[report['label_map'][str(i)] for i in range(1,25)] + if (any(type(i) is not int or not 1<=i<=65535 for i in ids) + or len(set(ids))!=24): + raise ValueError('Invalid or ambiguous label map') + ai=nib.load(aroot/'combined_labels.nii.gz');ri=nib.load(rroot/'combined_labels.nii.gz') + grid=geometry(ai,ri) + if ai.shape!=ri.shape: + return dict(case=case,passed=False,combined_geometry=grid) + a,r=np.asanyarray(ai.dataobj),np.asanyarray(ri.dataobj) + if a.dtype.kind not in 'ui' or a.min()<0 or a.max()>65535: + raise ValueError('Actual combined IDs must be nonnegative uint16-range integers') + if r.dtype.kind not in 'ui' or r.min()<0 or r.max()>24: + raise ValueError('Standalone reference combined map must use IDs 0..24') + lookup=np.zeros(65536,np.uint8) + for i,value in enumerate(ids,1):lookup[value]=i + changed=int(np.count_nonzero(lookup[a]!=r)) + rows=[] + for i,name in enumerate(NAMES,1): + am=nib.load(aroot/'segmentations'/(name+'.nii.gz')) + rm=nib.load(rroot/'segmentations'/(name+'.nii.gz')) + g=geometry(am,rm);ad,rd=np.asanyarray(am.dataobj),np.asanyarray(rm.dataobj) + shape=ad.shape==rd.shape==a.shape + binary=bool(np.all((ad==0)|(ad==1)) and np.all((rd==0)|(rd==1))) + mismatch=int(np.count_nonzero(ad!=rd)) if shape else None + consistent=shape and bool(np.array_equal(ad>0,a==ids[i-1]) and np.array_equal(rd>0,r==i)) + rows.append(dict(name=name,passed=all(g.values()) and mismatch==0 and binary and consistent, + geometry=g,changed_voxels=mismatch,binary=binary,combined_consistent=consistent)) + del ad,rd + print(f'{case}/{name}: {"PASS" if rows[-1]["passed"] else "FAIL"}',flush=True) + del a,r + stage_rows=[] + if prepared_inputs is not None: + for stage,role in [('suprem_anchors','SuPreM_anchors'),('incumbent','incumbent'),('aligned_v3','aligned_v3')]: + x=nib.load(aroot/'stages'/(stage+'.nii.gz')) + y=nib.load(Path(prepared_inputs)/role/case/'combined_labels.nii.gz') + g=geometry(x,y) + n=int(np.count_nonzero(np.asanyarray(x.dataobj)!=np.asanyarray(y.dataobj))) if x.shape==y.shape else None + stage_rows.append(dict(stage=stage,passed=all(g.values()) and n==0,geometry=g,changed_voxels=n)) + return dict(case=case,passed=all(grid.values()) and changed==0 and all(x['passed'] for x in rows+stage_rows), + combined_geometry=grid,changed_combined_voxels=changed,masks=rows,intermediate_stages=stage_rows) + + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--actual',type=Path,required=True) + p.add_argument('--reference',type=Path,required=True) + p.add_argument('--case',action='append',required=True) + p.add_argument('--prepared-inputs',type=Path) + p.add_argument('--report',type=Path,required=True) + args=p.parse_args() + if args.report.exists():p.error('Report already exists; choose a new path') + if not args.case:p.error('At least one case is required') + rows=[] + for case in args.case: + try:rows.append(compare_case(args.actual,args.reference,case,args.prepared_inputs)) + except Exception as exc:rows.append(dict(case=case,passed=False,error=str(exc))) + result=dict(purpose='Output equivalence, not ground-truth accuracy',dsc=None,cases=rows, + passed=bool(rows) and all(r['passed'] for r in rows)) + args.report.parent.mkdir(parents=True,exist_ok=True) + args.report.write_text(json.dumps(result,indent=2)+'\n') + print('PASS' if result['passed'] else 'FAIL',flush=True) + raise SystemExit(0 if result['passed'] else 1) + + +if __name__=='__main__':main() diff --git a/utils/_vertebrae_consensus/SOURCE_PROVENANCE.json b/utils/_vertebrae_consensus/SOURCE_PROVENANCE.json new file mode 100644 index 0000000..6c86757 --- /dev/null +++ b/utils/_vertebrae_consensus/SOURCE_PROVENANCE.json @@ -0,0 +1,69 @@ +{ + "source": "Hongkang Chu, submitted warmup code (10 October 2026)", + "packaging_changes": "Removed cohort-specific CLIs, unused imports, and absolute paths; local imports are package relative. Numerical function ASTs are unchanged. Hao is reused from upstream utils/vertebrae_hao_engine.py.", + "modules": { + "postprocessing_causal": { + "original_source_sha256": "575407a582224d227d6aca3fd0d17025a2fb544a4875623dc6a4e91fcb35cfd5", + "function_ast_sha256": { + "body_anchors": "c73b06799617d0a14fd3afbbc5b114c200f07907b447da018987899e28148474", + "sweep_one": "66b87bb514cc7bf375f399a45fb423c2e2abc5c2c81378769949aac126a5fcf5", + "refine": "383a8c74216dd7c4b95d7a8f2116a9d012892e196e657131d0ad8adee475ba55" + } + }, + "cervicothoracic_causal": { + "original_source_sha256": "9bd8d45aba3b06ea08e99cc13461c3cd299ae0297b2bf76a7a107785a923f675", + "function_ast_sha256": { + "body_anchors": "8f777f115de7e1978a1a7827a421dfdcb8e1cc190929e9dcf29f566b60478738", + "sweep": "48f4bb81c11cec1a8fa0b076898595c2a290841e8c41fdb816ba689342c96500", + "refine": "20ad28b7f7dda80dd3b9124e4f19232fc0606c3dd906e5bc6f8564fc3937979b" + } + }, + "native_anchor_pipeline": { + "original_source_sha256": "3678bb2b0de2443bc9ce979c0c79f6d48b89f6913b62ea91fb542dc0969cc08a", + "function_ast_sha256": { + "crop_band": "de12b85e031bd40b43fe7e297ce47e72e98720b2689db7fe321623558f782728", + "refine": "ace3ddfab513f5a9b57c8ed2cadf47249353c5a9b0e4267178600c172fdb6899" + } + }, + "body_root_guided": { + "original_source_sha256": "3928b011f576391904b3582434d599d3986af656dff947fe173cb1ade16a4459", + "function_ast_sha256": { + "Region": "a9fc8219404ed96bddffa7bf6b9de51565343276b5820463f5673b3378d8c8d9", + "centers_and_seeds": "2d5a9fcb86d899f8385856b5ecf9f47299b946e61930d46ce57891a096fe6353", + "refine_region": "77eb4498b23a0bc1653baf7a3f60cd5ca0b14a0ab6c83f4bdd4ecc035de1fb8e", + "refine": "b5be93ecb7054533ba478d13bff1ad66754243e27c7c7f27adb1136d289bd428" + } + }, + "body_root_guided_consistency": { + "original_source_sha256": "c234844cdd3eb0e5796a9111d9696abfbb4cf608a909964f6c666cfcc6f17241", + "function_ast_sha256": { + "chain_audit": "2243d0fe2fe92a7f6a7e43032a9fe07ef414baae9340da3b0d137c4c3237cd91", + "refine": "d53f1fe5eeaf60e9ee8ec1ad56a77148ddb7d06433fcec88a1f3685687edf0db" + } + }, + "component_identity_transfer": { + "original_source_sha256": "eee91e5eda30652e944a4ea08a9df69266b290b62f0dcf28b4e911563c4852b4", + "function_ast_sha256": { + "source_cores": "403c14f06588bac89c3474b84c444ff755d3cc1ba4e6436d674bbd932e45637f" + } + }, + "global_body_identity_graph": { + "original_source_sha256": "a34ad0e6fc8223ee37b64a94a36143e42a5bc3f4d3fa952ce642da8fd4e033fe", + "function_ast_sha256": { + "refine": "1bb00a2792f35c29affdfb4a1255965f8ce695a977f8468547518022c39752d3" + } + }, + "global_body_offset_gate": { + "original_source_sha256": "c1dc2b6aa1c6f9f58be91fcb6227e084fa8653e1e27ed85e581134711c839aa7", + "function_ast_sha256": { + "filter_proposal": "d93764869b1689d63bc938ff49658356cf46544f3676c2eb372246b3246102df" + } + }, + "consensus_boundary": { + "original_source_sha256": "ddb66b7e6c6d53ca39453a1bba1a102b4879c6bb27a64d349716d154cd01d3c3", + "function_ast_sha256": { + "refine": "e82d876d91d9502d58de01715d6a739b82b3a989572c131b00d136197f93fdcf" + } + } + } +} diff --git a/utils/_vertebrae_consensus/__init__.py b/utils/_vertebrae_consensus/__init__.py new file mode 100644 index 0000000..2963f4d --- /dev/null +++ b/utils/_vertebrae_consensus/__init__.py @@ -0,0 +1 @@ +"""Numerical stages for the opt-in vertebrae consensus backend.""" diff --git a/utils/_vertebrae_consensus/body_root_guided.py b/utils/_vertebrae_consensus/body_root_guided.py new file mode 100644 index 0000000..d2c3ee9 --- /dev/null +++ b/utils/_vertebrae_consensus/body_root_guided.py @@ -0,0 +1,166 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +from dataclasses import asdict,dataclass + +import nibabel as nib + +import numpy as np + +from scipy import ndimage as ndi + +from skimage.graph import MCP_Geometric + +from .postprocessing_causal import NAMES + +@dataclass(frozen=True) +class Region: + name:str + first_anchor:int + last_anchor:int + first_edit:int + last_edit:int + body_core_radius_mm:float + minimum_core_mm3:float + posterior_offset_mm:float + atlas_marker:bool=False + +REGIONS=[Region('thoracolumbar',3,14,4,13,5.,300.,22.), + Region('upper_thoracic_and_cervical',14,23,15,22,3.,60.,12.,True)] + +def centers_and_seeds(base,affine,region): + spacing=nib.affines.voxel_sizes(affine);centers=[];cores={} + for label in range(region.first_anchor,region.last_anchor+1): + depth=ndi.distance_transform_edt(base==label,sampling=spacing) + cc,count=ndi.label(depth>=region.body_core_radius_mm,np.ones((3,3,3))) + sizes=np.bincount(cc.ravel());sizes[0]=0 + if not count or sizes.max()*np.prod(spacing)60):raise ValueError('body_cores_not_anatomically_ordered') + origin=centers.mean(0);_,_,vh=np.linalg.svd(centers-origin,full_matrices=False) + t=vh[0].copy() + if t[2]<0:t=-t + anterior=np.array([0.,1.,0.])-t[1]*t + if np.linalg.norm(anterior)<.2:raise ValueError('ambiguous_anterior_direction') + anterior/=np.linalg.norm(anterior);right=np.cross(anterior,t);right/=np.linalg.norm(right) + basis=np.column_stack([right,anterior,t]);local_centers=(centers-origin)@basis + if np.any(np.diff(local_centers[:,2])<5):raise ValueError('intrinsic_core_order_invalid') + return centers,cores,origin,basis,local_centers + +def refine_region(base,ct,affine,region,anchors): + spacing=nib.affines.voxel_sizes(affine);direction=affine[:3,:3]/spacing + if not np.allclose(direction.T@direction,np.eye(3),atol=1e-5):raise ValueError('Sheared grids unsupported') + upper=24 if region.atlas_marker else region.last_anchor + pos=np.where((base>=region.first_anchor)&(base<=upper)) + if not len(pos[0]):raise ValueError('empty_region') + padding=np.ceil(5/spacing).astype(int) + roi=tuple(slice(max(0,int(p.min())-int(q)),min(base.shape[i],int(p.max())+int(q)+1)) + for i,(p,q) in enumerate(zip(pos,padding))) + b=base[roi];hu=ct[roi] + a=affine.copy();a[:3,3]=nib.affines.apply_affine(affine,[s.start for s in roi]) + centers,cores,origin,basis,local_centers=centers_and_seeds(anchors[roi],a,region) + domain=(b>=region.first_anchor)&(b<=upper) + outside=ndi.distance_transform_edt(~domain,sampling=spacing) + shell=(outside>0)&(outside<=2.5) + contrast=float(np.median(hu[domain])-np.median(hu[shell])) if shell.any() else 0. + del outside + if contrast<80.:raise ValueError('insufficient_CT_bone_to_surroundings_contrast') + protected=np.zeros(b.shape,bool) + for label,core in cores.items():protected|=core&(b==label) + # Exclude thick posterior pieces from the identity seeds. Some baseline + # IDs incorrectly include a neighbor's process; only anterior body cores + # are trusted, even if those pieces join the largest eroded component. + markers={} + for label,core in cores.items(): + coords=np.array(np.where(core)).T + local_core=(nib.affines.apply_affine(a,coords)-origin)@basis + center=local_centers[label-region.first_anchor] + selected=local_core[:,1]>=center[1]-region.body_core_radius_mm + selected &= domain[tuple(coords.T)] + seeds=coords[selected] + if len(seeds)*np.prod(spacing)=100) + cc,count=ndi.label(atlas,np.ones((3,3,3)));sizes=np.bincount(cc.ravel());sizes[0]=0 + if count and sizes.max()*np.prod(spacing)>=300: + markers[24]=[tuple(int(v) for v in q) for q in np.array(np.where(cc==sizes.argmax())).T] + depth=ndi.distance_transform_edt(domain,sampling=spacing).astype(np.float32) + CT_penalty=1+np.maximum(0,150-hu)/150 + votes=[];margins=[] + for power in [1.,1.5,2.]: + print(region.name,'3D geodesic thickness power',power,flush=True) + costs=np.full(b.shape,np.inf,np.float32) + costs[domain]=CT_penalty[domain]/(depth[domain]+.5)**power + best=np.full(b.shape,np.inf,np.float64);second=best.copy();owner=np.zeros(b.shape,np.uint8) + for label,seeds in markers.items(): + solver=MCP_Geometric(costs,sampling=tuple(spacing)) + distance,_=solver.find_costs(seeds) + better=distance=.10)&(margins[1]>=.10)&(margins[2]>=.10) + target=votes[0] + editable=((b>=region.first_edit)&(b<=region.last_edit)&(~protected)&(hu>=100) + &(target>=region.first_anchor)&(target<=upper) + &(np.abs(target.astype(int)-b.astype(int))==1)&consensus&confidence) + result=b.copy();result[editable]=target[editable] + # The volume limits are evaluated on the actual native proposal, then + # rechecked after every rollback so neighboring rollbacks cannot bypass it. + capped=[] + while True: + reject=[] + for label in range(region.first_anchor,upper+1): + n=int(np.count_nonzero(b==label));m=int(np.count_nonzero(result==label)) + affected=int(np.count_nonzero((b!=result)&((b==label)|(result==label)))) + if affected/max(n,1)>.50 or abs(m-n)/max(n,1)>.25:reject.append(label) + if not reject:break + restore=np.isin(b,reject)|np.isin(result,reject) + if not np.any(result[restore]!=b[restore]):raise ValueError('unresolved_volume_gate') + result[restore]=b[restore];capped.extend(reject) + assert np.array_equal(result>0,b>0) + assert not np.any((result!=b)&protected) + assert np.array_equal(result==24,b==24) + if region.atlas_marker:assert np.all(result[b==23]==23) + assert np.all(result[b==24]==24) + out=base.copy();out[roi]=result + return out,{'region':asdict(region),'reference_labels_used':False, + 'body_centers_RAS_mm':centers.tolist(),'axis_basis_world_columns':basis.tolist(), + 'physical_cost_powers':[1.,1.5,2.],'required_each_scale_runner_up_margin':.10, + 'marker_voxels':{str(k):len(v) for k,v in markers.items()}, + 'native_ROI_shape':list(b.shape),'capped_labels':sorted(set(capped)), + 'median_CT_foreground_to_surrounding_shell_contrast_HU':contrast, + 'foreground_exact':True,'concordant_source_body_core_voxels_exact':True,'posterior_gate_used':False, + 'C1_mask_exact':True,'existing_C2_voxels_frozen':True if region.atlas_marker else None,'changed_voxels':int(np.count_nonzero(out!=base)), + 'status':'applied' if np.any(out!=base) else 'abstained','DSC':None} + +def refine(base,ct,affine,anchors): + if base.shape!=ct.shape or base.ndim!=3 or not np.isfinite(ct).all():raise ValueError('Matching finite 3D CT required') + if not np.issubdtype(base.dtype,np.integer) or base.min()<0 or base.max()>24:raise ValueError('Integer labels0..24 required') + if anchors.shape!=base.shape or not np.issubdtype(anchors.dtype,np.integer) or anchors.min()<0 or anchors.max()>24:raise ValueError('Matching integer SuPreM anchor labels0..24 required') + out=base.copy();reports=[] + for region in REGIONS: + try:out,r=refine_region(out,ct,affine,region,anchors) + except ValueError as e:r={'region':asdict(region),'status':'abstained','reason':str(e),'changed_voxels':0,'reference_labels_used':False,'DSC':None} + reports.append(r) + assert np.array_equal(out>0,base>0) + return out,{'method':'Independent SuPreM body anchors classify CT neural foreground with 3D geodesic consensus', + 'reference_labels_used':False,'foreground_exact':True,'regions':reports, + 'changed_voxels':int(np.count_nonzero(out!=base)), + 'status':'applied' if np.any(out!=base) else 'abstained','DSC':None, + 'credit':'Foreground: Jakob Wasserthal et al. / TotalSegmentator and Hao Yu / unchanged ShapeKit-Hao. Identity anchors: authentic SuPreM plus credited Hao and original fixed roots. Guided graph: original experimental research code.'} diff --git a/utils/_vertebrae_consensus/body_root_guided_consistency.py b/utils/_vertebrae_consensus/body_root_guided_consistency.py new file mode 100644 index 0000000..47a3505 --- /dev/null +++ b/utils/_vertebrae_consensus/body_root_guided_consistency.py @@ -0,0 +1,69 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +from dataclasses import asdict + +import nibabel as nib + +import numpy as np + +from .body_root_guided import REGIONS, centers_and_seeds, refine_region + +from .postprocessing_causal import NAMES + +def chain_audit(labels, affine, region): + positions = np.where((labels >= region.first_anchor) & (labels <= region.last_anchor)) + if not len(positions[0]): + raise ValueError('empty_anchor_band') + spacing = nib.affines.voxel_sizes(affine) + padding = np.ceil(5/spacing).astype(int) + roi = tuple(slice(max(0, int(p.min())-int(padding[i])), + min(labels.shape[i], int(p.max())+int(padding[i])+1)) + for i, p in enumerate(positions)) + cropped_affine = affine.copy() + cropped_affine[:3, 3] = nib.affines.apply_affine(affine, [s.start for s in roi]) + centers, _, _, _, _ = centers_and_seeds(labels[roi], cropped_affine, region) + return {'body_centers_RAS_mm': centers.tolist(), 'minimum_world_z_gap_mm': float(np.diff(centers[:, 2]).min())} + +def refine(base, ct, affine, anchors, regions=tuple(REGIONS)): + if base.ndim != 3 or base.shape != anchors.shape or base.shape != ct.shape or not np.isfinite(ct).all(): + raise ValueError('Matching finite 3D CT and prediction grids required') + for labels in (base, anchors): + if not np.issubdtype(labels.dtype, np.integer) or labels.min() < 0 or labels.max() > 24: + raise ValueError('Integer prediction labels0..24 required') + spacing = nib.affines.voxel_sizes(affine) + direction = affine[:3, :3]/spacing + if not np.allclose(direction.T@direction, np.eye(3), atol=1e-5): + raise ValueError('Sheared physical grids unsupported') + checks = [] + for region in regions: + for role, labels in [('neural_guide', base), ('independent_SuPreM_anchors', anchors)]: + row = {'region': asdict(region), 'role': role} + try: + row.update(chain_audit(labels, affine, region), status='valid') + except ValueError as error: + row.update(status='invalid', reason=str(error)) + checks.append(row) + report = {'method': 'Dual ordered body-chain gate followed by original guided 3D identity graph', + 'reference_labels_used': False, 'global_prerequisite': 'all configured bands ordered in both predictions', + 'chain_checks': checks, 'foreground_exact': True, 'DSC': None, + 'proposed_after_development_DSC_inspection': True, + 'new_model_used': True, 'pure_original_SuPreM_postprocessing': False, + 'credit': 'TotalSegmentator foreground plus unchanged Hao; original SuPreM anchors plus Hao and fixed roots; own graph and consistency gate.'} + if not checks or any(row['status'] != 'valid' for row in checks): + return base.copy(), dict(report, status='abstained', reason='dual_body_chain_prerequisite_failed', + changed_voxels=0, regions=[]) + actual = base.copy() + reports = [] + for region in regions: + try: + actual, row = refine_region(actual, ct, affine, region, anchors) + except ValueError as error: + row = {'region': asdict(region), 'status': 'abstained', 'reason': str(error), 'changed_voxels': 0} + reports.append(row) + assert np.array_equal(actual > 0, base > 0) + assert np.array_equal(actual == 24, base == 24) + assert np.all(actual[base == 23] == 23) + return actual, dict(report, status='applied' if np.any(actual != base) else 'abstained', + changed_voxels=int(np.count_nonzero(actual != base)), regions=reports) diff --git a/utils/_vertebrae_consensus/cervicothoracic_causal.py b/utils/_vertebrae_consensus/cervicothoracic_causal.py new file mode 100644 index 0000000..2698955 --- /dev/null +++ b/utils/_vertebrae_consensus/cervicothoracic_causal.py @@ -0,0 +1,103 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +from __future__ import annotations + +import nibabel as nib + +import numpy as np + +from scipy import ndimage as ndi + +from skimage.segmentation import watershed + +from .postprocessing_causal import NAMES + +def body_anchors(base, affine): + spacing = nib.affines.voxel_sizes(affine) + centers = [] + for label in range(16, 24): + depth = ndi.distance_transform_edt(base == label, sampling=spacing) + cc, count = ndi.label(depth >= 3., np.ones((3,3,3))) + sizes = np.bincount(cc.ravel()); sizes[0] = 0 + if count == 0 or sizes.max()*np.prod(spacing) < 60: + raise ValueError(f'insufficient_cervical_body_seed_{label}') + xyz = ndi.center_of_mass(cc == sizes.argmax()) + centers.append(nib.affines.apply_affine(affine, xyz)) + centers = np.array(centers) + gaps = np.diff(centers[:,2]) + if np.any(gaps<5) or np.any(gaps>30): + raise ValueError('cervical_anchors_not_ordered') + return centers + +def sweep(base, ct, affine, centers, width, root_depth): + spacing = nib.affines.voxel_sizes(affine) + direction = affine[:3,:3]/spacing + if not (np.allclose(np.abs(direction),np.eye(3),atol=1e-5) and direction[2,2]>0 and direction[1,1]>0): + raise ValueError('Native superior-positive axial LAS/RAS required') + x = affine[0,3]+affine[0,0]*np.arange(base.shape[0])[:,None] + y = affine[1,3]+affine[1,1]*np.arange(base.shape[1])[None,:] + z = affine[2,3]+affine[2,2]*np.arange(base.shape[2]) + boundaries = (centers[:-1,2]+centers[1:,2])/2 + previous = np.zeros(base.shape[:2],np.uint8) + out = base.copy() + root_counts = {str(i):0 for i in range(16,24)} + for k in range(base.shape[2]-1,-1,-1): + cx = np.interp(z[k],centers[:,2],centers[:,0]); cy=np.interp(z[k],centers[:,2],centers[:,1]) + expected = 16+int(np.clip(np.searchsorted(boundaries,z[k]),0,7)) + field = base[:,:,k] + domain = ((np.abs(x-cx)<=width)&(y=cy-65) + &(field>=16)&(field<=23)&(ct[:,:,k]>=100)) + own_bone = (field==expected)&(ct[:,:,k]>=100) + cc,_=ndi.label(own_bone,np.ones((3,3))) + body_region=(np.abs(x-cx)<=18)&(y>=cy-12)&(y<=cy+16) + ids=np.unique(cc[own_bone&body_region]);ids=ids[ids>0] + connected=np.isin(cc,ids) if len(ids) else np.zeros(field.shape,bool) + roots=domain&connected&(y>=cy-root_depth) + root_counts[str(expected)]+=int(roots.sum()) + # Permit only actual 26-neighbor continuity across the native slices. + # Dilation is a marker transfer, not foreground growth. + continued=ndi.grey_dilation(previous,size=(3,3)) + markers=np.where(domain,continued,0).astype(np.uint8) + markers[roots]=expected + if expected==23: + markers[domain&(field==23)]=23 + if markers.any(): + depth=ndi.distance_transform_edt(domain,sampling=spacing[:2]) + owner=watershed(-depth,markers=markers,mask=domain,connectivity=np.ones((3,3))).astype(np.uint8) + else:owner=np.zeros(field.shape,np.uint8) + editable=(field>=17)&(field<=22)&domain&(owner>=17)&(owner<=23) + out[:,:,k][editable]=owner[editable] + previous=owner + return out,root_counts + +def refine(base,ct,affine,centers): + settings=[(6.,26.),(8.,30.),(10.,34.)] + votes=[];root_rows=[] + for width,depth in settings: + print(' cervical root corridor',width,'mm; depth',depth,'mm',flush=True) + v,roots=sweep(base,ct,affine,centers,width,depth);votes.append(v);root_rows.append(roots) + proposed=votes[0] + change=(proposed==votes[1])&(proposed==votes[2])&(proposed.astype(int)-base.astype(int)==1) + out=base.copy();out[change]=proposed[change] + capped=[] + per_label=[] + for label in range(17,24): + n=int(np.count_nonzero(base==label));m=int(np.count_nonzero(out==label)) + affected=int(np.count_nonzero((out!=base)&((out==label)|(base==label)))) + row={'label':label,'before_voxels':n,'proposed_voxels':m,'changed_fraction':affected/max(n,1),'net_volume_change_fraction':(m-n)/max(n,1)} + per_label.append(row) + if affected/max(n,1)>.50 or abs(m-n)/max(n,1)>.25:capped.append(label) + revert=np.isin(base,capped)|np.isin(out,capped);out[revert]=base[revert] + assert np.array_equal(out>0,base>0) + assert np.all(out[base==23]==23) # C2's existing identity is frozen. + for label in list(range(1,17))+[24]:assert np.array_equal(out==label,base==label) + return out,{'method':'original cervical/upper-thoracic body-root-supported caudal ownership', + 'reference_labels_used':False,'settings_width_root_depth_mm':settings, + 'body_seed_radius_mm':3.,'minimum_body_seed_mm3':60., + 'capped_labels':capped,'per_label_proposed_changes':per_label,'root_voxels_by_scale':root_rows, + 'changed_voxels':int(np.count_nonzero(out!=base)),'foreground_exact':True, + 'C1_exact':True,'existing_C2_voxels_frozen':True, + 'status':'applied' if np.any(out!=base) else 'abstained','dsc':None, + 'limitation':'C1/atlas ownership unresolved; this anatomical exception is included in overall evaluation'} diff --git a/utils/_vertebrae_consensus/component_identity_transfer.py b/utils/_vertebrae_consensus/component_identity_transfer.py new file mode 100644 index 0000000..b344b7d --- /dev/null +++ b/utils/_vertebrae_consensus/component_identity_transfer.py @@ -0,0 +1,41 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +import nibabel as nib + +import numpy as np + +from scipy import ndimage as ndi + +from .postprocessing_causal import NAMES + +def source_cores(source, affine, labels): + spacing = nib.affines.voxel_sizes(affine) + rows = []; centers = [] + for label in labels: + positions = np.where(source == label) + if not len(positions[0]): raise ValueError(f'missing_source_body_{label}') + radius, minimum = (5., 300.) if label <= 14 else (3., 60.) + padding = np.ceil((radius+2)/spacing).astype(int) + starts = np.array([max(0, int(p.min())-int(q)) for p, q in zip(positions, padding)]) + roi = tuple(slice(int(starts[i]), min(source.shape[i], int(p.max())+int(padding[i])+1)) + for i, p in enumerate(positions)) + mask = source[roi] == label + depth = ndi.distance_transform_edt(mask, sampling=spacing) + tiers = [] + for offset in (-1., 0., 1.): + cc, count = ndi.label(depth >= radius+offset, np.ones((3, 3, 3))) + sizes = np.bincount(cc.ravel()); sizes[0] = 0 + required = minimum if offset == 0 else minimum/3 + if not count or sizes.max()*np.prod(spacing) < required: + raise ValueError(f'insufficient_scale_stable_body_{label}_radius_{radius+offset}') + coordinates = np.array(np.where(cc == sizes.argmax())).T+starts + tiers.append(coordinates) + if offset == 0: centers.append(nib.affines.apply_affine(affine, coordinates.mean(0))) + rows.append({'label': int(label), 'radius_mm': radius, 'minimum_core_mm3': minimum, 'tiers': tiers}) + centers = np.array(centers) + gaps = np.diff(centers[:, 2]) + if np.any(gaps < 5) or np.any(gaps > 60): + raise ValueError('source_body_chain_not_anatomically_ordered') + return rows, centers diff --git a/utils/_vertebrae_consensus/consensus_boundary.py b/utils/_vertebrae_consensus/consensus_boundary.py new file mode 100644 index 0000000..fc67fc6 --- /dev/null +++ b/utils/_vertebrae_consensus/consensus_boundary.py @@ -0,0 +1,72 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +import nibabel as nib + +import numpy as np + +from scipy import ndimage as ndi + +from .postprocessing_causal import NAMES + +def refine(base, source, auxiliary, ct, affine, ct_guard=True): + if base.ndim != 3 or any(x.shape != base.shape for x in (source, auxiliary, ct)): + raise ValueError('Matching3D arrays required') + for x in (base, source, auxiliary): + if not np.issubdtype(x.dtype, np.integer) or x.min() < 0 or x.max() > 24: + raise ValueError('Integer labels0..24 required') + spacing = nib.affines.voxel_sizes(affine) + direction = affine[:3, :3]/spacing + if not np.allclose(direction.T@direction, np.eye(3), atol=1e-5): + raise ValueError('Sheared grids unsupported') + if not np.isfinite(ct).all(): + raise ValueError('Finite CT intensities required') + report = {'method': 'Two-source boundary agreement with physical band and CT guard', + 'reference_labels_used': False, 'band_mm': 2.0, 'bone_threshold_HU': 100, + 'ct_guard': bool(ct_guard), 'incumbent_foreground_id_changes': 0, + 'C1_C2_masks_exact': True, 'DSC': None} + out = base.copy() + foreground = base > 0 + axes = [np.flatnonzero(np.any(foreground, axis=tuple(j for j in range(3) if j != i))) for i in range(3)] + if not axes[0].size: + return out, dict(report, status='abstained', reason='incumbent_has_no_foreground', changed_voxels=0) + # Crop by incumbent extent plus3mm; artificial crop boundaries are outside + # the2mm acceptance band. One extra zero layer models the scan exterior. + pad = np.ceil(3/spacing).astype(int)+1 + starts = np.maximum(0, [x[0] for x in axes]-pad) + stops = np.minimum(base.shape, np.array([x[-1]+1 for x in axes])+pad) + crop = tuple(slice(int(i), int(j)) for i, j in zip(starts, stops)) + b, s, v, h = [x[crop] for x in (base, source, auxiliary, ct)] + fg = b > 0 + distances, nearest = ndi.distance_transform_edt(~fg, sampling=spacing, return_indices=True) + nearest_label = b[tuple(nearest)] + add = (b == 0) & (s == v) & (s > 0) & (s <= 22) & (s == nearest_label) & (distances <= 2.0+1e-7) + del distances, nearest, nearest_label + inside = ndi.distance_transform_edt(np.pad(fg, 1), sampling=spacing)[1:-1, 1:-1, 1:-1] + remove = fg & (b <= 22) & (s == 0) & (v == 0) & (inside <= 2.0+1e-7) + del inside + agreement_add, agreement_remove = int(add.sum()), int(remove.sum()) + if ct_guard: + add &= h >= 100 + remove &= h < 100 + # Retain any class for which the accepted deletions would erase its mask. + erased = [] + for label in range(1, 23): + if np.any(b == label) and not np.any((b == label) & ~remove): + remove[b == label] = False + erased.append(label) + target = out[crop] + target[add] = s[add] + target[remove] = 0 + assert not np.any((base > 0) & (out > 0) & (base != out)) + for label in (23, 24): + assert np.array_equal(base == label, out == label) + rows = [{'id': label, 'name': name, 'added_voxels': int(np.count_nonzero(add & (s == label))), + 'removed_voxels': int(np.count_nonzero(remove & (b == label)))} for label, name in enumerate(NAMES, 1)] + changed = int(add.sum()+remove.sum()) + return out, dict(report, status='applied' if changed else 'abstained', changed_voxels=changed, + foreground_added_voxels=int(add.sum()), foreground_removed_voxels=int(remove.sum()), + agreement_band_add_candidates=agreement_add, agreement_band_remove_candidates=agreement_remove, + empty_class_deletion_abstentions=erased, per_class_edits=rows, + native_crop_start=starts.tolist(), native_crop_end_exclusive=stops.tolist()) diff --git a/utils/_vertebrae_consensus/global_body_identity_graph.py b/utils/_vertebrae_consensus/global_body_identity_graph.py new file mode 100644 index 0000000..c5a5136 --- /dev/null +++ b/utils/_vertebrae_consensus/global_body_identity_graph.py @@ -0,0 +1,106 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +import nibabel as nib + +import numpy as np + +from scipy import ndimage as ndi + +from skimage.graph import MCP_Geometric + +from .component_identity_transfer import source_cores + +from .postprocessing_causal import NAMES + +def refine(base, anchors, ct, affine, labels=tuple(range(1, 23))): + if base.ndim != 3 or base.shape != anchors.shape or base.shape != ct.shape or not np.isfinite(ct).all(): + raise ValueError('Matching finite3D inputs required') + for image in (base, anchors): + if not np.issubdtype(image.dtype, np.integer) or image.min() < 0 or image.max() > 24: + raise ValueError('Integer labels0..24 required') + spacing = nib.affines.voxel_sizes(affine) + direction = affine[:3, :3]/spacing + if not np.allclose(direction.T@direction, np.eye(3), atol=1e-5): + raise ValueError('Sheared grids unsupported') + report = {'method': 'Scale-consistent independent body-seeded global physical geodesics', + 'reference_labels_used': False, 'foreground_exact': True, 'original_C1_C2_masks_exact': True, + 'source_scale_offsets_mm': [-1., 0., 1.], 'source_chain_gap_range_mm': [5., 60.], + 'minimum_CT_seed_HU': 100., 'minimum_CT_foreground_contrast_HU': 80., + 'physical_cost_powers': [1., 1.5, 2.], 'each_scale_runner_up_margin': .10, + 'minimum_target_CT_HU': None, 'marrow_identity_can_change': True, + 'whole_body_identity_changes_can_exceed_adjacent_or_original_volume_gates': True, + 'proposed_after_development_error_review': True, 'DSC': None, + 'credit': 'Neural foreground: TotalSegmentator; independent anchors: SuPreM plus credited Hao and original roots; global graph: original experimental code'} + domain = (base > 0) & (base < 23) + pos = np.where(domain) + if not len(pos[0]): + return base.copy(), dict(report, status='abstained', reason='empty_domain', changed_voxels=0) + pad = np.ceil(5/spacing).astype(int) + starts = np.array([max(0, int(q.min())-int(v)) for q,v in zip(pos,pad)]) + roi = tuple(slice(int(starts[i]), min(base.shape[i], int(q.max())+int(pad[i])+1)) for i,q in enumerate(pos)) + d = domain[roi]; hu = ct[roi]; del domain, pos + outside = ndi.distance_transform_edt(~d, sampling=spacing) + shell = (outside > 0) & (outside <= 2.5) + contrast = float(np.median(hu[d])-np.median(hu[shell])) if shell.any() else 0. + report['median_CT_foreground_shell_contrast_HU'] = contrast + del outside,shell + if contrast < 80: + return base.copy(), dict(report, status='abstained', reason='insufficient_CT_contrast', changed_voxels=0) + try: + cores, centers = source_cores(anchors, affine, labels) + except ValueError as error: + return base.copy(), dict(report, status='abstained', reason=str(error), changed_voxels=0) + origin = centers.mean(0); _,_,vh = np.linalg.svd(centers-origin,full_matrices=False) + tangent = vh[0].copy() + if tangent[2] < 0: tangent = -tangent + anterior = np.array([0.,1.,0.])-tangent[1]*tangent + if np.linalg.norm(anterior) < .2: + return base.copy(), dict(report,status='abstained',reason='ambiguous_anterior_direction',changed_voxels=0) + anterior /= np.linalg.norm(anterior) + if np.any(np.diff((centers-origin)@tangent) < 5): + return base.copy(), dict(report,status='abstained',reason='intrinsic_core_order_invalid',changed_voxels=0) + marker_tiers = [dict() for _ in range(3)] + for index, body in enumerate(cores): + for tier, coordinates in enumerate(body['tiers']): + world = nib.affines.apply_affine(affine, coordinates) + local = coordinates-starts + inside = np.all((local>=0)&(local= -body['radius_mm'] + local = local[inside & front] + if len(local): local = local[d[tuple(local.T)] & (hu[tuple(local.T)]>=100)] + if len(local)*np.prod(spacing) < body['minimum_core_mm3']/3: + return base.copy(),dict(report,status='abstained',reason=f'insufficient_trusted_seed_{body["label"]}_tier_{tier}',changed_voxels=0) + marker_tiers[tier][body['label']] = [tuple(int(v) for v in q) for q in local] + depth = ndi.distance_transform_edt(d,sampling=spacing).astype(np.float32) + penalty = 1+np.maximum(0,150-hu)/150 + votes=[]; confidence = d.copy() + for tier,power in enumerate((1.,1.5,2.)): + print('global geodesic scale',tier,'power',power,'ROI',d.shape,flush=True) + costs = np.full(d.shape,np.inf,np.float32) + costs[d] = penalty[d]/(depth[d]+.5)**power + best = np.full(d.shape,np.inf,np.float64); second = best.copy(); owner=np.zeros(d.shape,np.uint8) + for label,seeds in marker_tiers[tier].items(): + solver=MCP_Geometric(costs,sampling=tuple(spacing)) + distances,_=solver.find_costs(seeds) + better=distances=.10 + confidence &= strong; votes.append(owner) + del best,second,owner,costs,unique,competing,strong + consensus=(votes[0]==votes[1])&(votes[0]==votes[2]) + certified=d & consensus & confidence & (votes[0]>0) + out=base.copy(); out[roi][certified]=votes[0][certified] + assert np.array_equal(out>0,base>0) + assert np.array_equal(out==23,base==23) and np.array_equal(out==24,base==24) + changed=int(np.count_nonzero(out!=base)) + return out,dict(report,status='applied' if changed else 'abstained', changed_voxels=changed, + source_body_centers_RAS_mm=centers.tolist(),native_ROI_shape=list(d.shape), + seed_counts_by_tier=[{str(k):len(v) for k,v in tier.items()} for tier in marker_tiers], + certified_foreground_voxels=int(certified.sum())) diff --git a/utils/_vertebrae_consensus/global_body_offset_gate.py b/utils/_vertebrae_consensus/global_body_offset_gate.py new file mode 100644 index 0000000..5c84364 --- /dev/null +++ b/utils/_vertebrae_consensus/global_body_offset_gate.py @@ -0,0 +1,60 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +import nibabel as nib + +import numpy as np + +from .component_identity_transfer import source_cores + +from .postprocessing_causal import NAMES + +def filter_proposal(base,anchors,proposal,affine,labels=tuple(range(1,23))): + if base.ndim!=3 or base.shape!=anchors.shape or base.shape!=proposal.shape: + raise ValueError('Matching3D arrays required') + for image in (base,anchors,proposal): + if not np.issubdtype(image.dtype,np.integer) or image.min()<0 or image.max()>24: + raise ValueError('Integer labels0..24 required') + if not np.array_equal(proposal>0,base>0):raise ValueError('Graph must preserve foreground') + if not np.array_equal(proposal==23,base==23) or not np.array_equal(proposal==24,base==24): + raise ValueError('Graph must preserve originalC1/C2 masks') + spacing=nib.affines.voxel_sizes(affine) + direction=affine[:3,:3]/spacing + if not np.allclose(direction.T@direction,np.eye(3),atol=1e-5):raise ValueError('Sheared grids unsupported') + report={'method':'Coherent body-numbering offset gates independent global graph', + 'reference_labels_used':False,'foreground_exact':True,'original_C1_C2_masks_exact':True, + 'required_adjacent_source_body_pairs':1,'same_nonzero_model_minus_source_offset':True, + 'required_erosion_scales':3,'dominant_ID_requires_strict_majority':True, + 'isolated_identity_conflicts_do_not_trigger':True, + 'selected_after_first_case_development_regression':True,'DSC':None} + try:cores,centers=source_cores(anchors,affine,labels) + except ValueError as error: + return base.copy(),dict(report,status='abstained',reason=str(error),changed_voxels=0) + body_rows=[]; offsets={} + for body in cores: + tiers=[] + for coords in body['tiers']: + values=base[tuple(coords.T)];counts=np.bincount(values,minlength=25) + foreground=int(counts[1:23].sum()) + dominant=int(counts[1:23].argmax()+1) if foreground else 0 + supported=foreground*np.prod(spacing)>=body['minimum_core_mm3']/3 + majority=supported and counts[dominant]>foreground/2 + tiers.append({'dominant_model_ID':dominant,'strict_majority':bool(majority), + 'foreground_core_voxels':foreground,'dominant_voxels':int(counts[dominant])}) + ids=[t['dominant_model_ID'] for t in tiers] + stable=all(t['strict_majority'] for t in tiers) and len(set(ids))==1 + offset=ids[0]-body['label'] if stable else None + offsets[body['label']]=offset + body_rows.append({'source_ID':body['label'],'scale_stable_dominant_offset':offset,'tiers':tiers}) + pairs=[] + for first,second in zip(labels,labels[1:]): + if second==first+1 and offsets[first] not in (None,0) and offsets[first]==offsets[second]: + pairs.append({'source_bodies':[first,second],'model_minus_source_ID_offset':offsets[first]}) + report.update(body_core_model_identity=body_rows,coherent_offset_pairs=pairs, + source_body_centers_RAS_mm=centers.tolist()) + if not pairs: + return base.copy(),dict(report,status='abstained',reason='no_coherent_multilevel_numbering_offset',changed_voxels=0) + changed=int(np.count_nonzero(base!=proposal)) + return proposal.copy(),dict(report,status='applied' if changed else 'abstained',changed_voxels=changed, + gate_passed=True,scope='All frozen global graph proposals accepted when coherent body-chain error detected') diff --git a/utils/_vertebrae_consensus/native_anchor_pipeline.py b/utils/_vertebrae_consensus/native_anchor_pipeline.py new file mode 100644 index 0000000..b2bb8c3 --- /dev/null +++ b/utils/_vertebrae_consensus/native_anchor_pipeline.py @@ -0,0 +1,57 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +import nibabel as nib + +import numpy as np + +from . import postprocessing_causal as caudal + +from . import cervicothoracic_causal as cervical + +def crop_band(base, affine, lo, hi): + positions = np.where((base >= lo) & (base <= hi)) + if not len(positions[0]): + raise ValueError('empty_anatomical_band') + roi = tuple(slice(max(0, int(v.min())-2), min(base.shape[i], int(v.max())+3)) + for i, v in enumerate(positions)) + local = affine.copy() + local[:3, 3] = nib.affines.apply_affine(affine, [s.start for s in roi]) + return roi, local + +def refine(raw, base, ct, affine): + if raw.shape != base.shape or ct.shape != base.shape or base.ndim != 3 or not np.isfinite(ct).all(): + raise ValueError('Matching finite 3D inputs required') + for labels in (raw, base): + if not np.issubdtype(labels.dtype, np.integer) or labels.min() < 0 or labels.max() > 24: + raise ValueError('Integer labels0..24 required') + spacing = nib.affines.voxel_sizes(affine) + direction = affine[:3, :3]/spacing + if not (np.allclose(np.abs(direction), np.eye(3), atol=1e-5) and direction[1, 1] > 0 and direction[2, 2] > 0): + raise ValueError('Native LAS/RAS required; no scanner-axis adapter applied') + out = base.copy() + target = (base >= 4) & (base <= 13) + mismatch = np.count_nonzero(target & (base != raw))/max(np.count_nonzero(target), 1) + stages = [] + for name, module, band in [('caudal', caudal, (3, 14)), ('cervical', cervical, (16, 23))]: + try: + if name == 'caudal' and mismatch < .02: + raise ValueError('no_body_repair_trigger') + roi, local = crop_band(out, affine, *band) + centers = module.body_anchors(out[roi], local) + result, report = module.refine(out[roi], ct[roi], local, centers) + out[roi] = result + report.update(status='applied' if report['changed_voxels'] else 'abstained', + body_centers_world_mm=centers.tolist()) + except ValueError as error: + report = {'status': 'abstained', 'reason': str(error), 'changed_voxels': 0} + stages.append({'stage': name, **report}) + assert np.array_equal(out > 0, base > 0) + assert np.array_equal(out == 24, base == 24) + assert np.all(out[base == 23] == 23) + return out, {'stages': stages, 'changed_voxels': int(np.count_nonzero(out != base)), + 'raw_baseline_disagreement_fraction': float(mismatch), + 'reference_labels_used': False, 'foreground_exact': True, + 'coordinate_adaptation': 'none; original native LAS/RAS indexing', + 'CT_boundary_stage_used': False, 'algorithm_parameters_changed': False} diff --git a/utils/_vertebrae_consensus/postprocessing_causal.py b/utils/_vertebrae_consensus/postprocessing_causal.py new file mode 100644 index 0000000..0ec5332 --- /dev/null +++ b/utils/_vertebrae_consensus/postprocessing_causal.py @@ -0,0 +1,112 @@ +"""Numerical routines from the submitted warmup; no cohort paths or CLI. + +Author: Hongkang Chu. Algorithm bodies are unchanged. See SOURCE_PROVENANCE.json. +""" +from __future__ import annotations + +import nibabel as nib + +import numpy as np + +from scipy import ndimage as ndi + +from skimage.segmentation import watershed + +NAMES = ['L5', 'L4', 'L3', 'L2', 'L1'] + [f'T{i}' for i in range(12, 0, -1)] + [f'C{i}' for i in range(7, 0, -1)] + +def body_anchors(base, affine): + """Derive body seeds directly; no experimental-report dependency.""" + spacing = nib.affines.voxel_sizes(affine) + centers = [] + for label in range(3, 15): + depth = ndi.distance_transform_edt(base == label, sampling=spacing) + components, count = ndi.label(depth >= 5.0, np.ones((3, 3, 3))) + sizes = np.bincount(components.ravel()); sizes[0] = 0 + if count == 0 or sizes.max() * np.prod(spacing) < 300: + raise ValueError(f'Insufficient anatomical body seed {label}') + center = ndi.center_of_mass(components == sizes.argmax()) + centers.append(nib.affines.apply_affine(affine, center)) + centers = np.array(centers) + if np.any(np.diff(centers[:, 2]) < 10) or np.any(np.diff(centers[:, 2]) > 55): + raise ValueError('Body anchors are not anatomically ordered') + return centers + +def sweep_one(base, hu, affine, centers, width_mm, root_depth_mm, min_hu=100): + spacing = nib.affines.voxel_sizes(affine) + # This experiment handles native axis-aligned superior-positive scans; + # other orientations abstain rather than silently reversing causal flow. + direction = affine[:3, :3] / spacing + if not (np.allclose(np.abs(direction), np.eye(3), atol=1e-5) and direction[2, 2] > 0 and direction[1, 1] > 0): + raise ValueError('Causal experiment requires native axial LAS/RAS, superior-positive') + x = affine[0, 3] + affine[0, 0] * np.arange(base.shape[0])[:, None] + y = affine[1, 3] + affine[1, 1] * np.arange(base.shape[1])[None, :] + z_world = affine[2, 3] + affine[2, 2] * np.arange(base.shape[2]) + center_z, center_x, center_y = centers[:, 2], centers[:, 0], centers[:, 1] + labels = np.arange(3, 15) + boundary_z = (center_z[1:] + center_z[:-1]) / 2 + previous = np.zeros(base.shape[:2], np.uint8) + out = base.copy() + for k in range(base.shape[2]-1, -1, -1): + z = z_world[k] + cx, cy = np.interp(z, center_z, center_x), np.interp(z, center_z, center_y) + corridor = (np.abs(x-cx) <= width_mm) & (y < cy-22) & (y >= cy-80) + foreground = (base[:, :, k] >= 3) & (base[:, :, k] <= 14) + domain = corridor & foreground & (hu[:, :, k] >= min_hu) + expected = int(labels[np.clip(np.searchsorted(boundary_z, z), 0, len(labels)-1)]) + # Roots enter through the anterior corridor edge only at their body + # level; long posterior tips cannot create an arbitrary new identity. + expected_bone = (base[:, :, k] == expected) & (hu[:, :, k] >= min_hu) + components, _ = ndi.label(expected_bone, np.ones((3, 3))) + body_cross_section = (np.abs(x-cx) <= 18) & (y >= cy-15) & (y <= cy+20) + body_components = np.unique(components[body_cross_section & expected_bone]) + body_components = body_components[body_components > 0] + body_connected = np.isin(components, body_components) if body_components.size else np.zeros(domain.shape, bool) + root_zone = domain & (y >= cy-root_depth_mm) & body_connected + markers = np.where(domain, previous, 0).astype(np.uint8) + markers[root_zone] = expected + # The top boundary is an unchanged anatomical anchor, never an edit. + if expected == 14: + markers[domain & (base[:, :, k] == 14)] = 14 + if markers.any(): + depth = ndi.distance_transform_edt(domain, sampling=spacing[:2]) + owner = watershed(-depth, markers=markers, mask=domain, connectivity=np.ones((3, 3))).astype(np.uint8) + else: + owner = np.zeros(domain.shape, np.uint8) + # Unreachable pieces retain baseline and do not seed subsequent slices. + editable = (base[:, :, k] >= 4) & (base[:, :, k] <= 13) + proposed = editable & domain & (owner >= 4) & (owner <= 13) + out[:, :, k][proposed] = owner[proposed] + previous = owner + return out + +def refine(base, hu, affine, centers, max_fraction=.10): + # Three independently specified corridors. Consensus belongs to the + # smallest common midline region; disagreement abstains. + settings = [(8., 34.), (10., 38.), (12., 42.)] + votes = [] + for width, root_depth in settings: + print(' causal corridor', width, 'mm; root depth', root_depth, 'mm', flush=True) + votes.append(sweep_one(base, hu, affine, centers, width, root_depth)) + owner = votes[0] + agrees = (owner == votes[1]) & (owner == votes[2]) + # A caudally drooping process may be captured one level below its true + # owner, so its corrected bottom-up id can increase by one. Opposite + # shifts are not justified by this experiment's causal evidence. + change = agrees & (owner.astype(int) - base.astype(int) == 1) + out = base.copy() + out[change] = owner[change] + capped = [] + for label in range(4, 14): + affected = (out != base) & ((out == label) | (base == label)) + if np.count_nonzero(affected) / max(np.count_nonzero(base == label), 1) > max_fraction: + capped.append(label) + out[np.isin(base, capped) | np.isin(out, capped)] = base[np.isin(base, capped) | np.isin(out, capped)] + assert np.array_equal(out > 0, base > 0) + assert all(np.array_equal(out == i, base == i) for i in list(range(1, 4)) + list(range(14, 25))) + return out, {'method': 'original three-corridor body-connected cranial-root-to-caudal sweep', + 'settings_width_root_depth_mm': settings, 'max_fraction': max_fraction, + 'changed_voxels': int(np.count_nonzero(out != base)), 'capped_labels': capped, + 'dsc': None, 'ground_truth_used': False, + 'foreground_exact': True, 'protected_labels_exact': True, + 'baseline_credit': 'Hao Yu / BodyMaps ShapeKit, unmodified engine', + 'anatomical_inspiration': 'Abhijit Das / ShapeKit-Pro, caudal spinous continuity; independent implementation'} diff --git a/utils/vertebrae_consensus.py b/utils/vertebrae_consensus.py new file mode 100644 index 0000000..8b96266 --- /dev/null +++ b/utils/vertebrae_consensus.py @@ -0,0 +1,294 @@ +"""Opt-in integration of Hongkang Chu's submitted multi-model warmup. + +Inputs are predictions, never reference annotations. Raw mode runs the frozen +CPU postprocessing chain; prepared mode replays its final boundary stage. +Neural inference is deliberately external to ShapeKit. +""" +import hashlib +import json +import logging +from pathlib import Path +import shutil +import tempfile +import time + +import nibabel as nib +import numpy as np + +from . import vertebrae_hao_engine as hao +from ._vertebrae_consensus import ( + native_anchor_pipeline as anchor_stage, + body_root_guided_consistency as identity_stage, + global_body_identity_graph as global_stage, + global_body_offset_gate as offset_stage, + consensus_boundary as boundary_stage, +) + +NAMES = tuple(hao.VERTEBRA_NAMES) +LOGGER = logging.getLogger(__name__) + + +def digest(path): + h = hashlib.sha256() + with Path(path).open('rb') as handle: + for block in iter(lambda: handle.read(1 << 20), b''): + h.update(block) + return h.hexdigest() + + +def disjoint(a, b): + return a != b and a not in b.parents and b not in a.parents + + +def check_grid(image, reference, name, metadata=False): + if (len(image.shape) != 3 or image.shape != reference.shape + or not np.allclose(image.affine, reference.affine, rtol=0, atol=1e-5) + or not np.allclose(image.header.get_zooms()[:3], + reference.header.get_zooms()[:3], rtol=0, atol=1e-5)): + raise ValueError(f'Native-grid mismatch: {name}; no resampling is performed') + if metadata: + if image.header.get_xyzt_units() != reference.header.get_xyzt_units(): + raise ValueError(f'Inconsistent mask units: {name}') + for kind in ('qform', 'sform'): + a, ac = getattr(image, 'get_' + kind)(coded=True) + b, bc = getattr(reference, 'get_' + kind)(coded=True) + if (ac != bc or (a is None) != (b is None) + or (a is not None and not np.allclose(a, b, rtol=0, atol=1e-5))): + raise ValueError(f'Inconsistent mask {kind}: {name}') + + +def check_physical_grid(image): + affine = image.affine + spacing = np.asarray(image.header.get_zooms()[:3], float) + if len(image.shape) != 3 or not np.isfinite(affine).all() or np.any(spacing <= 0): + raise ValueError('Finite 3D affine and positive spacing required') + direction = affine[:3, :3] / spacing + if not np.allclose(direction.T @ direction, np.eye(3), atol=1e-5): + raise ValueError('Sheared grids or inconsistent header spacing are unsupported') + + +def binary(image, name): + data = np.asanyarray(image.dataobj) + if data.ndim != 3 or not np.all((data == 0) | (data == 1)): + raise ValueError(f'Finite binary 3D mask required: {name}') + return data.astype(bool, copy=False) + + +def load_masks(case, subfolder='segmentations'): + """Use named masks, avoiding ambiguous combined-map label conventions.""" + folder = Path(case) / subfolder + unknown = [p.name for p in folder.glob('vertebrae_*.nii*') + if p.name not in {n + '.nii.gz' for n in NAMES}] + if unknown: + raise ValueError(f'Unsupported vertebra files (24-level contract): {unknown}') + reference, labels, missing, hashes = None, None, [], {} + for i, name in enumerate(NAMES, 1): + path = folder / (name + '.nii.gz') + if not path.is_file(): + missing.append(name) + continue + image = nib.load(path) + check_physical_grid(image) + mask = binary(image, name) + if reference is None: + reference = image + # NIfTI voxels arrive in Fortran order. Assemble in that order to + # avoid a strided whole-volume boolean assignment for every mask. + labels = np.zeros(image.shape, np.uint8, order='F') + check_grid(image, reference, name, metadata=True) + if np.any(mask & (labels > 0)): + raise ValueError(f'Overlapping vertebra masks: {name}') + np.copyto(labels, np.uint8(i), where=mask) + hashes[path.name] = digest(path) + if reference is None: + raise ValueError(f'No named vertebra masks in {folder}') + # Numerical stages receive the same C-contiguous labels as the original + # adapter; this conversion occurs once per model, rather than per mask. + return reference, np.ascontiguousarray(labels), missing, hashes + + +def save_volume(array, reference, path): + header = reference.header.copy() + header.set_data_dtype(array.dtype) + image = nib.Nifti1Image(array, reference.affine, header) + image.set_qform(*reference.get_qform(coded=True)) + image.set_sform(*reference.get_sform(coded=True)) + nib.save(image, path) + + +def run_pipeline(primary, suprem, v3, ct, reference, input_stage='raw', observer=None): + """Run unchanged numerical functions; raw v3 bypasses Hao by design.""" + if input_stage not in ('raw', 'prepared'): + raise ValueError('input_stage must be raw or prepared') + stages = [] + + def record(name, output, report, started): + stages.append(dict(stage=name, elapsed_seconds=time.monotonic()-started, **report)) + LOGGER.info('[ShapeKit-Consensus] %s: %s, %s changed voxels', + name, report.get('status'), report.get('changed_voxels')) + if observer is not None: + observer(name, output) + return output + + if input_stage == 'raw': + direction = reference.affine[:3, :3] / nib.affines.voxel_sizes(reference.affine) + if not (np.allclose(np.abs(direction), np.eye(3), atol=1e-5) + and direction[1, 1] > 0 and direction[2, 2] > 0): + raise ValueError('Raw pipeline requires native axis-aligned LAS/RAS; no orientation adapter') + spacing = np.asarray(reference.header.get_zooms()[:3], float) + + def run_hao(raw, name): + started = time.monotonic() + baseline = hao.run_v2(raw, spacing) + final, report, debug = hao.refine(raw, baseline, ct, reference.affine, + hao.Config(), progress=LOGGER.info) + del baseline, debug + return record(name, final, report, started) + + suprem_hao = run_hao(suprem, 'suprem_hao') + started = time.monotonic() + anchors, report = anchor_stage.refine(suprem, suprem_hao, ct, reference.affine) + anchors = record('suprem_anchors', anchors, report, started) + del suprem_hao + primary_hao = run_hao(primary, 'primary_hao') + started = time.monotonic() + incumbent, report = identity_stage.refine(primary_hao, ct, reference.affine, anchors) + incumbent = record('incumbent', incumbent, report, started) + del primary_hao + started = time.monotonic() + proposal, report = global_stage.refine(v3, anchors, ct, reference.affine) + proposal = record('v3_proposal', proposal, report, started) + started = time.monotonic() + aligned, report = offset_stage.filter_proposal(v3, anchors, proposal, reference.affine) + aligned = record('aligned_v3', aligned, report, started) + del proposal + else: + incumbent, anchors, aligned = primary, suprem, v3 + + started = time.monotonic() + final, report = boundary_stage.refine(incumbent, anchors, aligned, ct, + reference.affine, ct_guard=False) + final = record('boundary_agreement', final, report, started) + for label in (23, 24): + if not np.array_equal(final == label, incumbent == label): + raise AssertionError('Boundary stage changed C1/C2') + if np.any((incumbent > 0) & (final > 0) & (final != incumbent)): + raise AssertionError('Boundary stage changed a retained foreground identity') + return final, stages + + +def process_case(input_path, output_path, ct_path, class_map, + subfolder_name='segmentations', *, suprem_path, v3_path, + input_stage='raw', save_stages=False): + """Publish one case atomically after strict IO checks and full processing. + + Other organ masks are copied unchanged. Combined IDs follow class_map; + the numerical engine always uses 1=L5 through 24=C1. Full-resolution arrays + are written one at a time. A failed case never appears as completed output. + """ + started = time.monotonic() + source, output = Path(input_path).resolve(), Path(output_path).resolve() + roots = {'primary': source, 'suprem': Path(suprem_path).resolve(), + 'v3': Path(v3_path).resolve()} + if output.exists(): + raise FileExistsError(f'Refusing to overwrite: {output}') + if input_stage not in ('raw', 'prepared'): + raise ValueError('input_stage must be raw or prepared') + if Path(subfolder_name).name != subfolder_name or subfolder_name in ('', '.', '..'): + raise ValueError('subfolder_name must be one directory name') + if ct_path is None or not Path(ct_path).is_file(): + raise FileNotFoundError(f'Matching calibrated CT required: {ct_path}') + ct_path = Path(ct_path).resolve() + for path in [*roots.values(), ct_path]: + if not disjoint(output, path): + raise ValueError('Output must be disjoint from every source') + if (not class_map or any(not isinstance(k, int) or isinstance(k, bool) + or not 1 <= k <= 65535 for k in class_map) + or any(not isinstance(n, str) for n in class_map.values()) + or len(set(class_map.values())) != len(class_map)): + raise ValueError('class_map requires unique names and positive uint16 IDs') + name_to_id = {name: i for i, name in class_map.items()} + if not set(NAMES) <= name_to_id.keys(): + raise ValueError('class_map must include all 24 vertebra names') + images, arrays, missing, hashes = {}, {}, {}, {} + for role, path in roots.items(): + images[role], arrays[role], missing[role], hashes[role] = load_masks(path, subfolder_name) + reference = images['primary'] + for role, image in images.items(): + check_grid(image, reference, role) + if image.header.get_xyzt_units()[0] not in ('mm', 'unknown'): + raise ValueError(f'Prediction grid must use millimeter coordinates: {role}') + ct_image = nib.load(ct_path) + check_physical_grid(ct_image) + check_grid(ct_image, reference, 'CT') + if ct_image.header.get_xyzt_units()[0] != 'mm': + raise ValueError('CT must explicitly declare millimeter units') + # NIfTI scaling is applied before HU-dependent stages. Preserve mask header + # unit flags, including legacy unknown, only with a matching mm CT grid. + ct = ct_image.get_fdata(dtype=np.float32) + if not np.isfinite(ct).all(): + raise ValueError('Finite CT intensities required') + hashes['CT'] = digest(ct_path) + extras = [] + for path in sorted((source / subfolder_name).glob('*.nii.gz')): + if path.name[:-7] not in NAMES: + image = nib.load(path) + check_grid(image, reference, path.name, metadata=True) + binary(image, path.name) + extras.append(path) + hashes['primary'][path.name] = digest(path) + + output.parent.mkdir(parents=True, exist_ok=True) + staging_root = output.parent / '.incomplete' + staging_root.mkdir(exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=output.name+'-', dir=staging_root)) + + def observer(name, array): + if save_stages: + folder = staging / 'stages' + folder.mkdir(exist_ok=True) + save_volume(array, reference, folder / (name+'.nii.gz')) + + try: + final, stages = run_pipeline(arrays['primary'], arrays['suprem'], arrays['v3'], + ct, reference, input_stage, observer) + del ct, arrays + segdir = staging / subfolder_name + segdir.mkdir() + for path in extras: + shutil.copy2(path, segdir / path.name) + dtype = np.uint8 if max(class_map) <= 255 else np.uint16 + combined = np.zeros(final.shape, dtype=dtype) + for label_id, name in sorted(class_map.items()): + if name in NAMES: + mask = final == NAMES.index(name)+1 + save_volume(mask.astype(np.uint8), reference, segdir / (name+'.nii.gz')) + else: + path = segdir / (name+'.nii.gz') + if not path.is_file(): + continue + mask = binary(nib.load(path), name) + combined[mask] = label_id + save_volume(combined, reference, staging / 'combined_labels.nii.gz') + source_files = [Path(__file__), Path(hao.__file__), + *sorted(Path(__file__).with_name('_vertebrae_consensus').glob('*.py'))] + report = dict( + integration_status='complete', backend='shapekit_consensus', + case=source.name, input_stage=input_stage, ct_guard=False, + scope='CPU postprocessing only; upstream neural predictions are supplied', + missing_input_masks=missing, input_sha256=hashes, + numerical_source_sha256={p.name:digest(p) for p in source_files}, + label_map={str(i):name_to_id[n] for i,n in enumerate(NAMES,1)}, + prediction_units={r:list(im.header.get_xyzt_units()) for r,im in images.items()}, + physical_units_basis='Matching CT explicitly declares mm; prediction unit flags preserved', + stage_reports=stages, reference_labels_used=False, dsc=None, + limitations='Two-case development gain did not generalize to the additional three-case cohort', + runtime_seconds=time.monotonic()-started) + (staging / 'vertebrae_consensus_report.json').write_text(json.dumps(report,indent=2)+'\n') + if output.exists(): + raise FileExistsError(f'Output appeared during processing: {output}') + staging.rename(output) + except Exception: + LOGGER.exception('Incomplete case retained at %s', staging) + raise + return report