diff --git a/docs/architecture.md b/docs/architecture.md index 74689a906..197ac24a4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,7 +122,10 @@ benchmarks/harness/ Compatibility command entrypoints | Hardware | `infra/platforms`, `data/platform_packs` | Device selection, process exclusivity and independent backend/model evidence | The acceptance rule is: **a new model or hardware adapter must reuse the common -AB/BA loop, process cleanup and performance adjudication.** The extension +AB/BA loop, process cleanup and objective-specific adjudication.** Performance +remains the default; explicit correctness experiments require paired output +artifacts and validation without metrics or gain claims. See the +[correctness contract](operations.md#correctness-only-experiments). The extension regression test composes a new metric schema, non-Git source adapter and backend probe through those interfaces while running real subprocesses. Its numbers are synthetic contract fixtures, not accelerator measurements. diff --git a/docs/operations.md b/docs/operations.md index c223fd80b..9812b251d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -89,6 +89,30 @@ upstream instructions. - **Failure behavior:** ambiguous identity names the qualified choices; unavailable/stale revisions and missing/unsafe resources fail explicitly — no fallback to newer bytes. - **Example:** `inferrsi read calibrate-serving-simulation --skill-root src/inferrsi/data/skill_library --material` +## Correctness-only experiments + +Use `inferrsi plan` or `inferrsi run` with `--objective correctness` for paired +reference/candidate output validation. The default objective remains +`performance`; its positive finite metric and gain gates are unchanged. +`--repo-type nano_vllm` describes caller-supplied commands, not model support. + +Correctness trials retain alternating AB/BA order, source snapshots, deadlines +and process cleanup. Each command must write a nonempty JSON object to `{result}` +or `INFERRSI_RESULT_PATH`. No metric is extracted and no gain is computed. +The explicit `--validation-command` receives `{run_dir}` substitution so it can +read every fresh baseline/candidate trial and its output artifacts. It must +verify the frozen case coverage, model/input identities and numerical gates; +the generic runner only checks the presence of readable JSON objects. + +For correctness validators, exit 0 accepts the supplied checks, exit 1 rejects +them, and exit 2 declares incomplete evidence. Other nonzero exits preserve the +existing rejection behavior; validators must catch execution/dependency errors +and return 2 rather than confuse a crash with a completed rejection. Timeouts, +cancellation, missing outputs or validation, incomplete trial sets, and source +drift remain `INCONCLUSIVE`. Missing evidence takes precedence over rejection. +Correctness results cannot establish performance or service claims, and an +individual run does not replace campaign-level completeness checks. + ## Store candidates (Python) ### Create / Update — `KnowledgeStore.create` / `.update` diff --git a/evidence/qwen38-h20-fp8-text-phase1/patch/README.md b/evidence/qwen38-h20-fp8-text-phase1/patch/README.md new file mode 100644 index 000000000..e5dcab097 --- /dev/null +++ b/evidence/qwen38-h20-fp8-text-phase1/patch/README.md @@ -0,0 +1,4 @@ +# Source identities + +Record ``, `` and `` in the acceptance +issue and frozen runtime bundle. This preparation contains no revision pins. diff --git a/evidence/qwen38-h20-fp8-text-phase1/plan/README.md b/evidence/qwen38-h20-fp8-text-phase1/plan/README.md new file mode 100644 index 000000000..30581353e --- /dev/null +++ b/evidence/qwen38-h20-fp8-text-phase1/plan/README.md @@ -0,0 +1,49 @@ +# FP8 text engine and service preparation + +[Acceptance #57](https://github.com/InferMatrixLab/InferRSI/issues/57) owns this +separate FP8 campaign under [parent #53](https://github.com/InferMatrixLab/InferRSI/issues/53). +The historical BF16 preparation and evidence remain in their existing tree. + +Status: preparation only; INCONCLUSIVE. The workload is proposed, not frozen. +Authenticate the official Qwen/Qwen3.8-27B-FP8 checkpoint at ``. +Freeze its byte manifest, this workload, control/baseline/candidate revisions, +registered environment lock, exact GPU, validator sources, gates and budget in +the acceptance issue before any candidate experiment. No pins or numerical +gates are supplied by this template. + +The checkpoint uses dynamic e4m3 FP8 linear weights with 128 by 128 blocks and +BF16 activations; unquantized layers remain BF16. Reference and candidate must +retain and execute FP8 weights. Whole-checkpoint BF16 dequantization is excluded. +The reference additionally requires an immutable kernel-registry revision and +trusted source fingerprint for the prepared torch-cuda build of +kernels-community/finegrained-fp8, with offline caches and DeepGEMM disabled. + +The bounded `native_text_smoke` compares full-prefix Transformers teacher forcing +with the native hybrid-state engine through the common correctness loop. + +The accepted phase-1 scope also includes the public engine entrypoint and a +loopback-default HTTP text-completion service with non-streaming and SSE output. +The service supports a documented greedy-generation subset. Unsupported fields +and values must fail explicitly. Model calls may serialize while independent +requests interleave; concurrency support is not a throughput claim. + +The separate `native_service_functional` qualification compares public native +serial generation with HTTP/SSE serial and concurrent output. It must cover +normal completion, EOS and token limits, explicit cancellation, client disconnect, +malformed and over-limit requests, execution errors, shutdown and subsequent +reuse. Cancellation is cooperative at model-call boundaries. Every terminal path +must release request-owned KV, convolution and recurrent state and admission +capacity without affecting another request. + +Record ordered output tokens and SSE events, request IDs, terminal reasons, +overlapping admissions, state ownership, capacity and owned-process cleanup. +Logical request-state bytes are distinct from allocator-reserved GPU memory. +Freeze the service workload, protocol, source identities, admission limits and +all readiness/request/release/shutdown deadlines before candidate execution. + +CPU fake-engine socket tests establish protocol coverage only. Final service +qualification requires the real FP8 model and real HTTP sockets, separately from +the reference numerical comparison. Missing required coverage is INCONCLUSIVE. +Phase 1 includes L4 functional service evidence, with no performance gain, +broad task-quality, continuous batching, stochastic sampling, vision, MTP or +multi-GPU acceptance claim. diff --git a/evidence/qwen38-h20-fp8-text-phase1/plan/workload.json b/evidence/qwen38-h20-fp8-text-phase1/plan/workload.json new file mode 100644 index 000000000..d5c55ca7e --- /dev/null +++ b/evidence/qwen38-h20-fp8-text-phase1/plan/workload.json @@ -0,0 +1,58 @@ +{ + "schema_version": 1, + "model_id": "Qwen/Qwen3.8-27B-FP8", + "tokenization": "raw_text_no_special_tokens", + "cases": [ + { + "id": "english", + "prompt": "The purpose of a reproducible experiment is", + "continuation": " to make the methods and observations available so that another researcher can repeat the same procedure and compare the results.", + "max_steps": 8 + }, + { + "id": "chinese", + "prompt": "请解释为什么实验需要固定输入：", + "continuation": "固定输入可以控制变量，让不同实现的结果具有可比性，并帮助研究人员准确定位误差的来源。", + "max_steps": 8 + }, + { + "id": "code", + "prompt": "def add(a, b):\n", + "continuation": " return a + b\n\nassert add(2, 3) == 5\n", + "max_steps": 8 + }, + { + "id": "arithmetic", + "prompt": "Compute 17 + 25. Explain: ", + "continuation": "17 plus 25 equals 42 because 10 plus 20 is 30 and 7 plus 5 is 12.", + "max_steps": 8 + }, + { + "id": "longer", + "prompt": "A laboratory records the same experiment on several days. Each record contains the input, model revision, runtime version, device type, numerical precision, output, and validation result. Some experiments change a kernel, some change batching, and some change memory layout. A reliable comparison controls every unrelated variable. When an experiment is interrupted, the missing results cannot be treated as successful. Failed trials are retained because they can explain which assumptions were wrong. The team separates correctness tests from service benchmarks and measures latency only after checking output quality. Explain how these practices help: ", + "continuation": "They make each comparison reproducible, preserve the evidence needed to identify regressions, and prevent incomplete experiments from being reported as successful optimizations.", + "max_steps": 8 + }, + { + "id": "english_replay", + "prompt": "The purpose of a reproducible experiment is", + "continuation": " to make the methods and observations available so that another researcher can repeat the same procedure and compare the results.", + "max_steps": 8, + "replay_of": "english" + } + ], + "precision": { + "weight_dtype": "float8_e4m3fn", + "activation_dtype": "bfloat16", + "quantization": { + "quant_method": "fp8", + "activation_scheme": "dynamic", + "fmt": "e4m3", + "weight_block_size": [ + 128, + 128 + ] + }, + "dequantized": false + } +} diff --git a/evidence/qwen38-h20-fp8-text-phase1/results/README.md b/evidence/qwen38-h20-fp8-text-phase1/results/README.md new file mode 100644 index 000000000..2a49c4c98 --- /dev/null +++ b/evidence/qwen38-h20-fp8-text-phase1/results/README.md @@ -0,0 +1,8 @@ +# Results + +INCONCLUSIVE: no FP8 campaign execution or qualification evidence is included. +Retain every attempt and raw artifact after the acceptance issue freezes inputs. + +The accepted phase-1 scope includes public engine integration and HTTP/SSE +functional lifecycle validation. Implementation checks and real-model service +qualification must be reported separately; no service acceptance is recorded here. diff --git a/evidence/qwen38-h20-fp8-text-phase1/validator/README.md b/evidence/qwen38-h20-fp8-text-phase1/validator/README.md new file mode 100644 index 000000000..caa6082fd --- /dev/null +++ b/evidence/qwen38-h20-fp8-text-phase1/validator/README.md @@ -0,0 +1,17 @@ +# Validator preparation + +Freeze the example validator, profile helpers, trial adapters and offline kernel +binding with their SHA-256 hashes before execution. Numerical gates belong to +acceptance #57 and its frozen runtime records; this directory supplies no defaults. +Missing model, precision, kernel or source evidence is INCONCLUSIVE. Complete +numerical rejection is FAIL. Neither outcome may be relabelled as performance. + +Freeze the additional service validator, request fixtures and lifecycle protocol +separately. Require exact deterministic serial/HTTP/SSE/concurrent token and text +agreement, ordered terminal framing, actual overlapping request ownership, +isolated cancellation and evidence of state/capacity release and subsequent reuse. +An unobserved cancellation or overlap cannot pass because a request completed +early. Missing observations, identity or cleanup evidence are INCONCLUSIVE; +complete evidence of a protocol or isolation violation is FAIL. CPU fake-engine +results cannot qualify the real FP8 model service. Service equivalence does not +replace the independent reference-based numerical validation. diff --git a/evidence/qwen38-h20-text-phase1/patch/README.md b/evidence/qwen38-h20-text-phase1/patch/README.md new file mode 100644 index 000000000..800801b40 --- /dev/null +++ b/evidence/qwen38-h20-text-phase1/patch/README.md @@ -0,0 +1,9 @@ +# Source identity + +Implementation references are recorded in acceptance Issue #54 before execution. +The initial change adds an explicit correctness objective to InferRSI and a +reference-self-check example. It does not adapt nano-vllm to Qwen3.8. + +Freeze the control revision, baseline/candidate revisions, validator hashes and +dependency locks before measuring. Placeholder or dirty source identities cannot +support a published model/engine acceptance result. diff --git a/evidence/qwen38-h20-text-phase1/plan/README.md b/evidence/qwen38-h20-text-phase1/plan/README.md new file mode 100644 index 000000000..0f6615cfd --- /dev/null +++ b/evidence/qwen38-h20-text-phase1/plan/README.md @@ -0,0 +1,41 @@ +# Phase-1 acceptance preparation + +Parent: https://github.com/InferMatrixLab/InferRSI/issues/53 +Acceptance and numeric gates: https://github.com/InferMatrixLab/InferRSI/issues/54 + +Status: **INCONCLUSIVE; NOT FROZEN**. No model or engine acceptance has occurred. + +The first executable milestone compares Transformers full-prefix and incremental +cached execution on fixed raw-text teacher-forced tokens. It establishes a +reference self-check only. Both sides use the same checkpoint and tokenizer. +The independent nano-vllm-derived engine remains a separate incomplete milestone. +Do not describe this bootstrap as engine support, broad model quality or a speedup. + +## Freeze checklist + +- [ ] BF16 download finished; all required files present. +- [ ] Manifest anchored to immutable public model revision; actual weight and + config/tokenizer bytes verified against upstream identities. +- [ ] Exact baseline/candidate source and control/validator revisions recorded in + the acceptance Issue, including dependency versions and imported paths. +- [ ] Workload and validator hashes recorded; ensure each continuation produces + enough tokens for its requested steps without altering the frozen workload. +- [ ] Numerical gates, resource budget and supported scope frozen in the Issue. +- [ ] Actual single-GPU SKU, runtime and BF16 capability verified in allocation. + +`workload.json` is the proposed bounded smoke cohort: English, Chinese, code, +arithmetic, longer prose and a repeated request to detect state contamination. +It does not exercise chat templates, EOS stopping, stochastic sampling, batching, +preemption, prefix reuse, MTP, vision, long-context limits or task accuracy. +Those requirements must be covered before accepting the independent engine for +its declared feature set. Unsupported paths must fail explicitly. + +The intended numerical gates from the Issue are per-case NRMSE <= 0.01 (reference +RMS denominator floor 1e-8), top-1 agreement >= 0.99 and exact reset/replay equality. +They are campaign parameters supplied to the validator, never library defaults. + +The initial allocation budget is one GPU, 8 CPU cores, 128 GiB host RAM and +60 minutes; the paired experiment deadline is 45 minutes with two AB/BA +repetitions. Model intake verification may need a separately budgeted CPU job. +No optimization search has started. Do not relax gates after seeing candidates; +a changed protocol requires a new freeze and fresh references. diff --git a/evidence/qwen38-h20-text-phase1/plan/workload.json b/evidence/qwen38-h20-text-phase1/plan/workload.json new file mode 100644 index 000000000..79aae87dd --- /dev/null +++ b/evidence/qwen38-h20-text-phase1/plan/workload.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "model_id": "Qwen/Qwen3.8-27B", + "tokenization": "raw_text_no_special_tokens", + "cases": [ + {"id": "english", "prompt": "The purpose of a reproducible experiment is", "continuation": " to make the methods and observations available so that another researcher can repeat the same procedure and compare the results.", "max_steps": 8}, + {"id": "chinese", "prompt": "请解释为什么实验需要固定输入：", "continuation": "固定输入可以控制变量，让不同实现的结果具有可比性，并帮助研究人员准确定位误差的来源。", "max_steps": 8}, + {"id": "code", "prompt": "def add(a, b):\n", "continuation": " return a + b\n\nassert add(2, 3) == 5\n", "max_steps": 8}, + {"id": "arithmetic", "prompt": "Compute 17 + 25. Explain: ", "continuation": "17 plus 25 equals 42 because 10 plus 20 is 30 and 7 plus 5 is 12.", "max_steps": 8}, + {"id": "longer", "prompt": "A laboratory records the same experiment on several days. Each record contains the input, model revision, runtime version, device type, numerical precision, output, and validation result. Some experiments change a kernel, some change batching, and some change memory layout. A reliable comparison controls every unrelated variable. When an experiment is interrupted, the missing results cannot be treated as successful. Failed trials are retained because they can explain which assumptions were wrong. The team separates correctness tests from service benchmarks and measures latency only after checking output quality. Explain how these practices help: ", "continuation": "They make each comparison reproducible, preserve the evidence needed to identify regressions, and prevent incomplete experiments from being reported as successful optimizations.", "max_steps": 8}, + {"id": "english_replay", "prompt": "The purpose of a reproducible experiment is", "continuation": " to make the methods and observations available so that another researcher can repeat the same procedure and compare the results.", "max_steps": 8, "replay_of": "english"} + ] +} diff --git a/evidence/qwen38-h20-text-phase1/results/README.md b/evidence/qwen38-h20-text-phase1/results/README.md new file mode 100644 index 000000000..14a655593 --- /dev/null +++ b/evidence/qwen38-h20-text-phase1/results/README.md @@ -0,0 +1,11 @@ +# Results + +Verdict: **INCONCLUSIVE**. + +No GPU experiment has run for this campaign. CPU contract tests, when recorded, +qualify orchestration and rejection behavior only, not the model or accelerator. +The BF16 checkpoint was not available during initial setup. Model readiness must +be rechecked using the live downloader and byte-integrity preflight. + +Keep private raw runs outside tracked source. Publish only reviewed redacted +copies, preserving hashes, failed attempts, scope and verification limitations. diff --git a/evidence/qwen38-h20-text-phase1/validator/README.md b/evidence/qwen38-h20-text-phase1/validator/README.md new file mode 100644 index 000000000..fe80fbf28 --- /dev/null +++ b/evidence/qwen38-h20-text-phase1/validator/README.md @@ -0,0 +1,12 @@ +# Validator freeze + +The proposed validator is `examples/qwen38_text/validate.py`, paired with +`examples/qwen38_text/trial.py`. Before GPU execution, snapshot both files here +(or in the private campaign bundle), record their SHA256 plus source revision in +plan and acceptance Issue #54, and retain the complete model manifest/workload. +This directory is currently a preparation record, not a frozen validator bundle. + +Validation checks complete paired trials, matching identities, full-vocabulary +logit arrays, token coverage, finite values, file digests, numerical gates and +reset/replay consistency. Missing evidence returns exit 2; a completed numeric +rejection returns 1; scope-limited acceptance returns 0. diff --git a/examples/qwen38_text/README.md b/examples/qwen38_text/README.md new file mode 100644 index 000000000..f0e60de6e --- /dev/null +++ b/examples/qwen38_text/README.md @@ -0,0 +1,223 @@ +# Qwen3.8 text correctness bootstrap + +Parent [Issue #53](https://github.com/InferMatrixLab/InferRSI/issues/53) and +[acceptance #54](https://github.com/InferMatrixLab/InferRSI/issues/54) own the scope, +resource budget, pins, freeze and numerical gates. Preparation evidence is under +[`evidence/qwen38-h20-text-phase1`](../../evidence/qwen38-h20-text-phase1/plan/README.md). + +This example compares **Transformers full-prefix recomputation with Transformers +incremental cache**. It validates a bounded raw-text teacher-forcing reference +self-check. It does **not** implement a nano-vllm Qwen3.8 engine, validate general +model/task quality, or establish a service speedup. Engine adaptation remains a +separate milestone. No engine candidate may inherit this reference check's PASS. + +An explicit `native_text_smoke` scope additionally compares the trusted +Transformers full-prefix reference with the caller's native nano-vllm-derived +engine. It qualifies only the supplied raw-text teacher-forcing workload, +including declared replay cases. It is not whole-engine acceptance or a claim +about chat, batching, generation lifecycle, broad quality or performance. + +## Contract + +- `intake.py` verifies local model files against an immutable upstream revision + and produces a complete SHA256 manifest. It does not download weights. Run the + byte scan under a CPU Slurm allocation, after the existing download completes. +- `trial.py` verifies that manifest against actual file bytes before each model + load. It requires a single visible CUDA GPU of the exact requested SKU, + BF16, a local checkpoint and a local tokenizer. It loads with remote code + disabled and rejects missing/mismatched required parameters. + Hub kernels are explicitly disabled. This reference requires `fla` and + `causal_conv1d` to be absent, fixes full attention to eager, and records that + policy in every result. Eager attention alone does not disable optional + DeltaNet kernels in Transformers. An optimized kernel path needs its own + explicit candidate protocol and fresh reference qualification. +- Both trial modes produce full-vocabulary FP32 logits at every requested + teacher-forced position. No sampled token is fed back; no token-rate metric is + presented. Each request starts with fresh cache state. +- `validate.py` checks complete per-trial/case coverage, identity, dtype/device, + token sequences, logit shape/digest/finiteness, numerical gates and explicit + replay cases. Exit 0 means scoped acceptance, 1 a completed numerical rejection, + 2 incomplete or invalid evidence. Gate values have no defaults. +- `launch.py` uses the common InferRSI AB/BA runner with objective `correctness`. + It binds source snapshots to the actual control checkout. The `nano_vllm` + descriptor is the future integration route, not an assertion that nano-vllm + executed: trial implementation and qualification scope name Transformers. +- `slurm-run.sh` sets all writable caches/tmp inside the caller's authorized task + root and preserves Slurm device visibility. It refuses paths escaping that root. + Model execution requires an active allocation. `--dry-run` writes no campaign + freeze and is not authorization or qualification. + +The raw-text smoke does not test chat templates, stochastic sampling, EOS +stopping, batching, preemption, prefix reuse, MTP, vision or context limits. +Explicitly qualify these before claiming them for the dedicated engine. + +## Prepare and freeze + +Use a dedicated uv environment inside the authorized remote task root. Runtime +requirements are CUDA PyTorch, a Transformers version implementing +`Qwen3_5ForConditionalGeneration`, Accelerate, NumPy, Safetensors and tokenizer +requirements, plus this InferRSI checkout. Record exact versions/import origins, +freeze a dependency lock and register the environment in the H20 skill's registry. +Do not treat a checkpoint's `transformers_version` string as an installed runtime. + +The existing model downloader must finish first. Its exit status alone is not a +substitute for intake verification. Intake expects a model directory with +`config.json`, `tokenizer_config.json`, `tokenizer.json`, +`model.safetensors.index.json` and every indexed shard. Present optional tokenizer +and generation metadata are verified too. + +Example intake inside an authorized **CPU** allocation, with all paths supplied +at runtime and `TMPDIR`/cache variables already rooted in the task workspace: + +```bash +"$TASK_PYTHON" -m examples.qwen38_text.intake \ + --model-path "$MODEL_PATH" --model-revision "$MODEL_REVISION" \ + --output "$MANIFEST" +``` + +Before GPU execution, freeze the model manifest, workload, trial/validator source +hashes, baseline/candidate/control commits, dependency lock, GPU SKU, gates and +budget in acceptance #54 and the campaign's `plan/` / `validator/`. Copy the +validator files into the frozen bundle; do not overwrite historical freezes. + +## Execute + +Set runtime variables privately; placeholder paths below are templates: + +```bash +export TASK_ROOT='' +export CONTROL_REPO='' +export TASK_PYTHON='/bin/python' +MODEL_PATH='' +MODEL_REVISION='' +MANIFEST='' +WORKLOAD='' +DEVICE_MODEL='' +``` + +Set `REPETITIONS`, `MAX_MINUTES`, `MAX_NRMSE`, `MIN_TOP1`, `RMS_FLOOR`, +`CPU_COUNT`, `HOST_MEMORY` and `JOB_TIME` from the frozen acceptance plan. Submit +with explicit resources and output inside the authorized root: + +```bash +mkdir -p "$TASK_ROOT/artifacts/qwen38-phase1" +sbatch --partition=h20 --nodes=1 --ntasks=1 --gres=gpu:1 \ + --cpus-per-task="$CPU_COUNT" --mem="$HOST_MEMORY" --time="$JOB_TIME" \ + --chdir="$CONTROL_REPO" \ + --output="$TASK_ROOT/artifacts/qwen38-phase1/slurm-%j.log" \ + "$CONTROL_REPO/examples/qwen38_text/slurm-run.sh" \ + --repo "$CONTROL_REPO" --model-path "$MODEL_PATH" \ + --model-revision "$MODEL_REVISION" --manifest "$MANIFEST" \ + --workload "$WORKLOAD" --device-model "$DEVICE_MODEL" \ + --max-nrmse "$MAX_NRMSE" --min-top1 "$MIN_TOP1" --rms-floor "$RMS_FLOOR" \ + --repetitions "$REPETITIONS" --max-minutes "$MAX_MINUTES" \ + --output-dir "$TASK_ROOT/artifacts/qwen38-phase1/runs" +``` + +`sbatch` accepts the script directly; its contents set shell options and execute +the registered interpreter. Do not reuse unrelated allocations. Retain the job ID +and confirm release. A missing queue entry does not prove success: inspect the +job log, `status.json`, `evaluation.json`, validation result, all trial artifacts +and `report.html`. Missing or interrupted evidence remains INCONCLUSIVE. + +Raw InferRSI output includes operational paths and hardware identifiers. Keep it +in private task storage. Review and redact a separate copy before publication; +retain source/manifest hashes, failed attempts and verification limitations. + +## Native candidate execution + +After independently preparing and reviewing the native engine checkout, pass +`--qualification-scope native_text_smoke --candidate-repo ''` +to the same launcher/Slurm script. Keep `--repo` pointed at the actual control +checkout. Use a new output location and freeze both source identities plus the +same explicitly supplied model/workload/gate arguments before the run. + +The common InferRSI runner snapshots both checkouts and executes each side in +its own checkout. `native_trial.py` imports `load_checkpoint` and `RequestRunner` +from the requested engine; an already imported foreign package or an origin +escaping the checkout is rejected. The model class must originate in +`nanovllm.qwen38.model`. The adapter uses Transformers only for tokenizer +handling, and exports native teacher-forced logits through the same artifact +writer as the reference. Native execution is eager Torch with one GPU and no +tensor parallelism or CUDA Graphs. + +Native records retain control/engine source identities, imported-file hashes, +checkpoint loading coverage and the distinct implementation role. Validation +checks these against the common runner's locked and per-trial source snapshots, +checks execution directories, and reuses full-vocabulary FP32, input-token, +GPU/runtime, numerical-gate and replay checks. A cached Transformers trace +cannot substitute for the native candidate. A native smoke PASS cannot be +relabelled as reference self-check or promoted to full engine acceptance. + +These operational records contain private paths through the common runner; +keep originals in authorized private storage and publish reviewed redacted +copies only. Source-file verification during validation needs both checkouts +at the frozen revisions; absent or changed sources leave evidence incomplete. + +## CPU checks + +With the development environment and NumPy installed: + +```bash +python -m pytest tests/test_qwen38_trial.py tests/test_qwen38_validation.py \ + tests/test_qwen38_intake.py tests/test_qwen38_launch.py +ruff check src tests examples/qwen38_text +``` + +These tests use synthetic inputs and mock runtime objects, not model evidence. +No GPU work or model download is added to the default CI lanes. + +Native source evidence requires JSON objects for provenance, import records, +loading reports and source snapshots. Every declared, locked and before/after +identity must contain a lowercase hexadecimal Git object ID (40 or 64 digits) +and a 64-digit SHA-256 snapshot hash; imported-file hashes must be SHA-256. +Checkout paths and executed working directories must be nonempty strings. +Missing or malformed evidence writes an INCONCLUSIVE result and exits 2; +complete evidence rejected by the numerical gates still writes FAIL and exits 1. + +## Separate FP8 profile + +[Acceptance #57](https://github.com/InferMatrixLab/InferRSI/issues/57) owns FP8 +qualification; its separate preparation tree is +[`qwen38-h20-fp8-text-phase1`](../../evidence/qwen38-h20-fp8-text-phase1/plan/README.md). +BF16 remains the default and its prior preparation is unchanged. Explicitly pass +`--model-id Qwen/Qwen3.8-27B-FP8` to intake, launch, direct trials and validation. +FP8 manifests and workloads must carry the profile's explicit `precision` object. +The adapter supports only the official dynamic e4m3, 128 by 128 block profile: +FP8 linear weights, BF16 activations and unquantized layers, and FP32 recurrent +state. It rejects whole-model BF16 dequantization labelled as FP8. + +Prepare a separate registered environment with CUDA Torch, Transformers, +Accelerate, NumPy, Safetensors, Triton and `kernels`. The inspected Transformers +FP8 integration uses an external finegrained-fp8 kernel; `use_kernels=False` +does not disable quantizer-specific kernels. The adapter forces DeepGEMM off +and requires a previously prepared, immutable **kernel-registry** revision of +`kernels-community/finegrained-fp8`, with `matmul_2d`, `matmul_batched` and +`matmul_grouped` entrypoints in the `torch-cuda` build. The older model-registry +repository is not interchangeable. No version alias or online download may be +resolved during a trial. + +Before freezing, authenticate the kernel registry source/build and compute the +loaded package fingerprint using `fp8_reference.kernel_fingerprint(module)`. +This helper hashes all files in the package tree except Python bytecode; the +prepared package must contain its sibling kernel sources. Bind its expected +SHA-256 to the trusted preparation source, rather than treating a newly observed +runtime hash as authority. Freeze both `` and ``. +Set `HF_HUB_OFFLINE=1` and `TRANSFORMERS_OFFLINE=1` before process startup, and pass +`--fp8-kernel-revision '' --fp8-kernel-sha256 ''` +to launch (or to a direct reference trial and validator). Launch forwards these +only to the reference implementation and validator; the native candidate owns +its separate Triton kernel. Runtime kernel package paths are private provenance +and must be redacted before sharing. + +The reference verifies actual FP8 tensors and complete indexed module coverage. +Loaded BF16 block scales are explicitly, losslessly upcast to FP32; existing +FP32 scales remain unchanged. The result records this normalization and its +upcast count. Native validation checks actual FP8 tensors, the loader's declared +backend, successful per-layer FP8 calls and imported native kernel source. +Both sides record their precision and kernel policy, plus `kernels` and Triton +versions. Reference source fingerprints must remain identical across trials; +reference and candidate backend implementations intentionally differ. + +These contracts and CPU smoke tests are preparation, not FP8 model acceptance. +Freeze the new sources, environment, workload and gates before GPU experiments. diff --git a/examples/qwen38_text/SERVICE.md b/examples/qwen38_text/SERVICE.md new file mode 100644 index 000000000..49b2b7dea --- /dev/null +++ b/examples/qwen38_text/SERVICE.md @@ -0,0 +1,104 @@ +# Native service functional qualification + +`service_launch.py` uses the existing correctness RunSpec and paired runner. Its +`native_service_functional` scope compares the native public `LLM.generate` entry +with the same snapshotted engine's HTTP/SSE service. This is functional L4 evidence; +it provides no latency, throughput, SLO, or broad model-quality claim. The separate +trusted-reference teacher-forcing check remains required for model correctness. +CPU fixtures and fake servers exercise the scaffold only. Real GPU/model evidence +is pending until the accepted campaign's frozen protocol is actually completed. + +## Frozen protocol + +Supply a JSON object with these required fields, using acceptance-issue values: + +- `schema_version: 1`, `qualification_scope: "native_service_functional"`, + `model_id`, and the exact `precision` profile from `profiles.precision(model_id)`. +- `timeouts`: positive seconds for `startup`, `request`, `release`, `poll`, + `shutdown`, and `socket`. These are observation/cleanup budgets, not performance + gates. The surrounding RunSpec also has a frozen overall `max_minutes`. +- `limits`: positive integers `max_context_tokens`, `max_active_requests`, + `max_connections`, `max_body_bytes`, and `max_response_bytes`. Freeze sufficient + response space for all stream events and request text. At least two active slots + and two additional control connections are required by this protocol. +- `cases`: at least two objects with unique safe `id`, nonempty `prompt` (text or + token IDs), and positive `max_tokens`. Include both text and token-ID inputs and + suitable multilingual text in the campaign workload. +- `concurrent_case_ids`: distinct case IDs, including at least two distinct inputs, + within the active capacity. Admission overlap must actually be observed. +- `eos_case_id`: a declared case that must actually finish with `stop`; changing the + model's EOS configuration to force coverage is not permitted. Missing observed + EOS is `INCONCLUSIVE`. At least one other case must actually finish with + `length` at its requested token limit; otherwise runtime limit coverage is also + `INCONCLUSIVE`. +- `survivor_case_id`: a declared case whose exact serial output must survive + overlapping cancellation and disconnect of a separate request. Cancellation ACK + metadata must show the survivor active at the atomic cancellation point; the + disconnect terminal ledger must show it active when the victim is released. + Earlier overlap snapshots alone cannot satisfy this requirement. +- `lifecycle`: a safe `id`, `prompt`, and `max_tokens` sufficient to observe active + cancellation, disconnect, capacity saturation, active shutdown, and a second + model call for fault injection. Early natural completion is a coverage gap. +- `queued_case`: a safe `id`, `prompt`, and `max_tokens` for observing a second + admitted request cancelled before prefill. A race that already prefills the + target leaves queued coverage `INCONCLUSIVE`. + +All concrete prompts, limits, budgets, repetitions, revision pins, and numerical +quality gates belong in the acceptance issue and frozen campaign artifacts. The +adapter contains no defaults for timing gates and does not tune workloads during +candidate execution. + +Run inside an authorized single-GPU Slurm allocation, with authenticated model +intake already completed: + +```sh +python -m examples.qwen38_text.service_launch \ + --engine-repo '' --model-path '' \ + --model-id '' --model-revision '' \ + --manifest '' --workload '' \ + --device-model '' --max-minutes '' \ + --repetitions '' --output-dir '' +``` + +## Observations and verdicts + +The adapter records serial public/JSON/SSE tokens and decoded text, concurrent +admission and exact outputs, victim/survivor isolation, explicit cancel and real +socket disconnect, terminal ledgers, queued-before-prefill cancellation, saturated +capacity rejection with working control routes, invalid request/limit rejection, +active SIGTERM shutdown framing, descendant cleanup, and fresh-process generation. +SSE is consumed through connection EOF: missing terminal/DONE, duplicate DONE, and +trailing events cannot satisfy coverage. Concatenated deltas must equal the public +full decode, including Unicode boundaries. + +A separate owned process runs `service_fault.py`, a harness-only bootstrap. After +a normal warmup request, it lets the designated request complete one real model +call and injects one `RuntimeError` on its next call. The runner-state identity, +successful call count, trigger flag, and control source digest are recorded. Its +SSE error terminal, state release, and subsequent normal generation are required. +This is injected error-path evidence, not a claim that a natural GPU fault occurred. +The normal production server exposes no failure endpoint and is not patched. + +Each process reports model, actual GPU, precision, loader, runtime versions, and +validated import origins. FP8 runs additionally require every loaded FP8 linear +module to have a positive native call count and valid actual weight/scale dtypes; +module totals must agree. Restart and fault-process provenance is also checked. + +`runner_state_bytes` means logical live request tensors (KV, convolution, recurrent +state), not allocator-reserved VRAM. State accounting is sampled at completed model +boundaries; cancellation is cooperative between model calls. The frozen deadlines +bound observation without asserting an instantaneous GPU interrupt. + +Complete observed behavioral regressions are `FAIL`. Missing, malformed, timed-out, +wrong-source/device/precision, unobserved overlap/EOS, or incomplete cleanup evidence +is `INCONCLUSIVE`. Only complete protocol evidence can yield this limited scope's +`PASS`. Shared correctness evaluation performs no metric extraction or gain gating. + +## Private artifacts + +Raw ready/shutdown files, server logs, HTTP observation journals, and common-runner +operational snapshots may contain machine paths and stay in the private run root. +Before writing each final trial result, actual import paths/bytes are checked, then +origins become engine-relative; commands and listener addresses become placeholders. +Do not publish the raw run tree. Review/redact any additional artifacts before +publication under the repository privacy contract. diff --git a/examples/qwen38_text/fp8_reference.py b/examples/qwen38_text/fp8_reference.py new file mode 100644 index 000000000..426737670 --- /dev/null +++ b/examples/qwen38_text/fp8_reference.py @@ -0,0 +1,167 @@ +"""Offline, pinned Transformers FP8 reference binding and loaded-weight guards. + +Imports remain lazy so control-plane smoke requires no Torch or kernel package. +""" + +import copy +import hashlib +import json +import os +import re +from pathlib import Path + +KERNEL_REPO = "kernels-community/finegrained-fp8" + + +def kernel_fingerprint(kernel): + root = Path(kernel.__file__).absolute().parent.resolve(strict=True) + return _package_fingerprint(root) + + +def _package_fingerprint(root): + files = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or "__pycache__" in path.parts or path.suffix == ".pyc": + continue + resolved = path.resolve(strict=True) + # Hugging Face snapshots link individual files to this repository's blob cache. + cached_blob = (len(root.parents) >= 4 and root.parent.name == "build" + and root.parents[2].name == "snapshots" + and resolved.is_relative_to(root.parents[3] / "blobs")) + if not resolved.is_relative_to(root) and not cached_blob: + raise ValueError("Reference kernel source escapes package/cache") + files.append({"path": path.relative_to(root).as_posix(), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}) + if not files: + raise ValueError("Reference kernel source missing") + snapshot = hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest() + return {"files": files, "snapshot_sha256": snapshot} + + +def prepare_reference_kernel(revision, expected_sha256): + if (not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision) + or not isinstance(expected_sha256, str) + or not re.fullmatch(r"[0-9a-f]{64}", expected_sha256)): + raise ValueError("FP8 reference requires frozen kernel revision and source SHA-256") + if os.environ.get("HF_HUB_OFFLINE") != "1" or os.environ.get("TRANSFORMERS_OFFLINE") != "1": + raise ValueError("FP8 reference requires offline caches before process startup") + os.environ["TRANSFORMERS_DISABLE_DEEPGEMM_LINEAR"] = "1" + from transformers.integrations import finegrained_fp8, hub_kernels + + # Reject prior lazy loading: it may have resolved an unpinned version. + if (finegrained_fp8._FINEGRAINED_FP8 is not None + or hub_kernels._KERNEL_MODULE_MAPPING.get("finegrained-fp8") is not None): + raise ValueError("Reference kernel was loaded before pinning") + hub_kernels._HUB_KERNEL_MAPPING["finegrained-fp8"] = { + "repo_id": KERNEL_REPO, "revision": revision, + } + kernel = hub_kernels.lazy_load_kernel("finegrained-fp8") + if kernel is None or not all(callable(getattr(kernel, name, None)) for name in ( + "matmul_2d", "matmul_batched", "matmul_grouped", + )): + raise ValueError("Pinned reference kernel lacks required FP8 entrypoints") + root = Path(kernel.__file__).absolute().parent.resolve(strict=True) + if root.name != "torch-cuda": + raise ValueError("Reference kernel is not the prepared torch-cuda build") + if (len(root.parents) >= 3 and root.parents[2].name == "snapshots" + and root.parents[1].name != revision): + raise ValueError("Reference cache origin has wrong revision") + fingerprint = kernel_fingerprint(kernel) + if fingerprint["snapshot_sha256"] != expected_sha256: + raise ValueError("Reference kernel differs from frozen source") + finegrained_fp8.load_finegrained_fp8_kernel() + return {"repo_id": KERNEL_REPO, "repo_type": "kernel", + "revision": revision, "build": "torch-cuda", "package_root": str(root), **fingerprint} + + +def recheck_reference_kernel(record): + """Require the prepared package to remain unchanged through trial execution.""" + root = Path(record["package_root"]).resolve(strict=True) + if str(root) != record["package_root"]: + raise ValueError("Reference kernel package origin changed during execution") + fingerprint = _package_fingerprint(root) + if any(fingerprint[key] != record[key] for key in ("files", "snapshot_sha256")): + raise ValueError("Reference kernel source changed during execution") + + +def expected_fp8_modules(index): + """Derive text projections from authenticated checkpoint scale/weight pairs.""" + weights = index["weight_map"] + if (not isinstance(weights, dict) + or any(not isinstance(key, str) or not isinstance(value, str) + for key, value in weights.items())): + raise ValueError("Reference FP8 weight index is invalid") + suffix = ".weight_scale_inv" + names = {name[:-len(suffix)] for name in weights + if name.startswith("model.language_model.layers.") and name.endswith(suffix)} + if not names or any(name + ".weight" not in weights for name in names): + raise ValueError("Reference FP8 checkpoint projection coverage missing") + return names + + +def adapt_reference_config(config, index, module_names): + """Remove only nonexistent dense gate aliases that shadow indexed gate_proj. + + Transformers' skip matcher treats skip entries as unanchored regex prefixes. + The official dense checkpoint also contains MoE-only `.mlp.gate` aliases; + those incorrectly exclude `.mlp.gate_proj`. A real meta model's module tree + and the authenticated index must independently confirm each exact removal. + """ + expected = expected_fp8_modules(index) + layers = config["text_config"]["num_hidden_layers"] + if (config.get("model_type") != "qwen3_5" or type(layers) is not int or layers <= 0 + or not expected.issubset(module_names)): + raise ValueError("Reference configuration/module tree mismatch") + skip = config["quantization_config"]["modules_to_not_convert"] + if not isinstance(skip, list) or any(not isinstance(name, str) for name in skip): + raise ValueError("Reference exclusion list is invalid") + aliases = [f"model.language_model.layers.{layer}.mlp.gate" for layer in range(layers)] + if any(alias in module_names or skip.count(alias) != 1 or alias + "_proj" not in expected + for alias in aliases): + raise ValueError("Reference dense gate alias identity is not established") + adapted = copy.deepcopy(config) + adapted["quantization_config"]["modules_to_not_convert"] = [ + name for name in skip if name not in aliases] + + def identity(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), + allow_nan=False).encode()).hexdigest() + + return adapted, {"policy": "remove_nonexistent_dense_gate_aliases", "version": 1, + "original_config_sha256": identity(config), + "adapted_config_sha256": identity(adapted), + "config_identity_encoding": "sorted_compact_json_utf8", + "removed_exclusions": aliases, + "expected_fp8_modules": sorted(expected)} + + +def inspect_reference_fp8(model, torch, fp8_linear_type, expected_names): + quantizer = getattr(model, "hf_quantizer", None) + quant = getattr(quantizer, "quantization_config", None) + if (quant is None or getattr(quant, "dequantize", None) is not False + or getattr(quant, "activation_scheme", None) != "dynamic" + or list(getattr(quant, "weight_block_size", ()) or ()) != [128, 128]): + raise ValueError("Reference is dequantized or has unsupported FP8 execution") + names = set() + upcasts = 0 + for name, module in model.named_modules(): + if not isinstance(module, fp8_linear_type): + continue + names.add(name) + if module.weight_scale_inv.dtype == torch.bfloat16: + module.weight_scale_inv.data = module.weight_scale_inv.data.float() + upcasts += 1 + if (module.weight.dtype != torch.float8_e4m3fn + or module.weight_scale_inv.dtype != torch.float32 + or list(module.block_size or ()) != [128, 128] + or module.activation_scheme != "dynamic" + or module.activation_scale is not None): + raise ValueError("Reference FP8 linear has dequantized or mismatched tensors") + if (not isinstance(expected_names, (set, frozenset)) + or not expected_names or names != expected_names): + raise ValueError("Reference FP8 module coverage differs from checkpoint") + return {"execution": "transformers_fp8", "fp8_linear_modules": len(names), + "fp8_module_names": sorted(names), "dequantized": False, + "backend": "finegrained_fp8", "weight_dtype": "float8_e4m3fn", + "activation_dtype": "bfloat16", "scale_dtype": "float32", + "scale_normalization": "lossless_float32", "bf16_scale_upcasts": upcasts} diff --git a/examples/qwen38_text/intake.py b/examples/qwen38_text/intake.py new file mode 100644 index 000000000..70596ce4e --- /dev/null +++ b/examples/qwen38_text/intake.py @@ -0,0 +1,172 @@ +"""Authenticate an existing supported snapshot against an immutable public HF revision. + +No model files are downloaded or modified. Run the byte-intensive operation in +an allocated CPU job; the resulting manifest is input to the GPU reference trial. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import urllib.request +from pathlib import Path +from urllib.parse import urlsplit + +if __package__: + from .profiles import MODEL_IDS, precision, validate_config + from .trial import HEX40, HEX64, METADATA, MODEL_ID, REQUIRED, model_file +else: + from profiles import MODEL_IDS, precision, validate_config + from trial import HEX40, HEX64, METADATA, MODEL_ID, REQUIRED, model_file + +MAX_METADATA_BYTES = 8 * 1024 * 1024 +CHUNK_BYTES = 8 * 1024 * 1024 + + +def fetch_metadata(revision: str, model_id: str = MODEL_ID) -> tuple[dict, dict]: + if not isinstance(revision, str) or not HEX40.fullmatch(revision): + raise ValueError("Require an immutable lowercase 40-character model revision") + precision(model_id) + url = f"https://huggingface.co/api/models/{model_id}/revision/{revision}?blobs=true" + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=30) as response: + location = urlsplit(response.geturl()) + if location.scheme != "https" or location.netloc != "huggingface.co": + raise ValueError("Unexpected model metadata origin") + body = response.read(MAX_METADATA_BYTES + 1) + if len(body) > MAX_METADATA_BYTES: + raise ValueError("Model metadata response exceeds limit") + metadata = json.loads(body) + if metadata.get("sha") != revision or metadata.get("id") != model_id: + raise ValueError("Public model metadata identity does not match request") + return metadata, { + "source": url, + "response_sha256": hashlib.sha256(body).hexdigest(), + "verification": "HF revision; Git blob SHA-1 or LFS SHA-256; local SHA-256", + } + + +def authenticate_file(root: Path, name: str, sibling: dict, *, retain=False): + """Compute the local SHA-256 and authoritative digest in one streaming pass.""" + path = model_file(root, name) + size = sibling.get("size") + if type(size) is not int or size < 0: + raise ValueError("Public file metadata omits size") + if retain and size > MAX_METADATA_BYTES: + raise ValueError("Configuration or index exceeds parsing limit") + digest = hashlib.sha256() + lfs = sibling.get("lfs") + if lfs is not None: + expected = lfs.get("sha256") + if ( + not isinstance(expected, str) + or not HEX64.fullmatch(expected) + or lfs.get("size") != size + ): + raise ValueError("Invalid LFS digest or size") + authoritative = digest + else: + expected = sibling.get("blobId") + if not isinstance(expected, str) or not HEX40.fullmatch(expected): + raise ValueError("Missing public Git blob SHA-1") + authoritative = hashlib.sha1() + authoritative.update(f"blob {size}\0".encode("ascii")) + total = 0 + retained = [] + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(CHUNK_BYTES), b""): + total += len(chunk) + if total > size: + raise ValueError("Local file exceeds public size") + if authoritative is not digest: + authoritative.update(chunk) + digest.update(chunk) + if retain: + retained.append(chunk) + if total != size or authoritative.hexdigest() != expected: + raise ValueError("Local file differs from immutable public source") + return {"path": name, "size": total, "sha256": digest.hexdigest()}, b"".join(retained) + + +def create_manifest(root: Path, revision: str, output: Path, model_id: str = MODEL_ID) -> dict: + root = root.resolve(strict=True) + if not root.is_dir(): + raise ValueError("Model path must be a directory") + # Resolve the existing parent before writing. An output alias into the model + # tree is rejected just like a direct destination there. + parent = output.parent.resolve(strict=True) + target = parent / output.name + if target.is_relative_to(root) or target.exists() or target.is_symlink(): + raise ValueError("Output must be a new file outside the model directory") + metadata, provenance = fetch_metadata(revision, model_id) + # Also check here so injected/test transports cannot bypass source identity. + if metadata.get("sha") != revision or metadata.get("id") != model_id: + raise ValueError("Public model metadata identity does not match request") + siblings = metadata.get("siblings") + if not isinstance(siblings, list) or not siblings: + raise ValueError("Public metadata omits repository file list") + by_name = {} + for sibling in siblings: + name = sibling.get("rfilename") + if not isinstance(name, str) or name in by_name: + raise ValueError("Invalid or duplicate public file metadata") + by_name[name] = sibling + selected = REQUIRED | (METADATA & by_name.keys()) + # Existing tokenizer metadata must be authenticated even if absent upstream. + selected |= {name for name in METADATA if (root / name).exists()} + if not selected <= by_name.keys(): + raise ValueError("Public metadata omits required local model files") + records = {} + documents = {} + for name in sorted(selected): + record, content = authenticate_file( + root, + name, + by_name[name], + retain=name in {"config.json", "model.safetensors.index.json"}, + ) + records[name] = record + if content: + documents[name] = json.loads(content) + config = documents["config.json"] + validate_config(config, model_id) + weight_map = documents["model.safetensors.index.json"].get("weight_map") + if not isinstance(weight_map, dict) or not weight_map: + raise ValueError("Weight index must contain a nonempty weight map") + for name in sorted(set(weight_map.values())): + if not isinstance(name, str) or not name.endswith(".safetensors") or name not in by_name: + raise ValueError("Indexed shard is missing from public file metadata") + record, _content = authenticate_file(root, name, by_name[name]) + records[name] = record + manifest = { + "model_id": model_id, + "precision": precision(model_id), + "revision": revision, + "files": [records[name] for name in sorted(records)], + "provenance": provenance, + } + with target.open("x") as stream: + json.dump(manifest, stream, indent=2, allow_nan=False) + stream.write("\n") + return manifest + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-path", type=Path, required=True) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--model-id", choices=MODEL_IDS, default=MODEL_ID) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + create_manifest(args.model_path, args.model_revision, args.output, args.model_id) + except Exception as error: + print(f"Intake incomplete ({type(error).__name__}); no manifest accepted", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/qwen38_text/launch.py b/examples/qwen38_text/launch.py new file mode 100644 index 000000000..5f27ce951 --- /dev/null +++ b/examples/qwen38_text/launch.py @@ -0,0 +1,119 @@ +"""Run explicit reference or native text smoke through InferRSI in an allocation.""" + +import argparse +import json +import os +import sys +from pathlib import Path + +from examples.qwen38_text.profiles import BF16_MODEL_ID, FP8_MODEL_ID, MODEL_IDS +from examples.qwen38_text.trial import HEX40, load_workload +from inferrsi.harness.runner import run_experiment +from inferrsi.infra.inspection import inspect_environment +from inferrsi.runspec import RunSpec + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("repo", "model-path", "manifest", "workload", "output-dir"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--model-id", choices=MODEL_IDS, default=BF16_MODEL_ID) + parser.add_argument("--fp8-kernel-revision") + parser.add_argument("--fp8-kernel-sha256") + parser.add_argument("--device-model", required=True) + parser.add_argument("--qualification-scope", default="reference_self_check", + choices=("reference_self_check", "native_text_smoke")) + parser.add_argument("--candidate-repo", type=Path) + for name in ("max-nrmse", "min-top1", "rms-floor", "max-minutes"): + parser.add_argument("--" + name, required=True, type=float) + parser.add_argument("--repetitions", required=True, type=int) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + control_repo = Path(__file__).resolve().parents[2] + if args.repo.resolve() != control_repo: + parser.error("text smoke must snapshot its actual control checkout") + native = args.qualification_scope == "native_text_smoke" + if native and (args.candidate_repo is None or args.candidate_repo.resolve() == control_repo): + parser.error("native smoke requires a separate explicit engine checkout") + if not native and args.candidate_repo is not None: + parser.error("reference self-check cannot declare a native candidate checkout") + task_root = os.environ.get("TASK_ROOT") + if task_root and not args.output_dir.resolve().is_relative_to(Path(task_root).resolve()): + parser.error("output directory must stay inside the authorized task root") + if native and task_root and not args.candidate_repo.resolve().is_relative_to( + Path(task_root).resolve() + ): + parser.error("native engine checkout must stay inside the authorized task root") + if not HEX40.fullmatch(args.model_revision): + parser.error("model revision must be an immutable commit") + _, workload_hash = load_workload(args.workload, args.model_id) + fp8_args = () + if args.model_id == FP8_MODEL_ID: + from examples.qwen38_text.trial import HEX64 + if (not args.fp8_kernel_revision or not HEX40.fullmatch(args.fp8_kernel_revision) + or not args.fp8_kernel_sha256 or not HEX64.fullmatch(args.fp8_kernel_sha256)): + parser.error("FP8 requires immutable reference kernel revision and source digest") + fp8_args = ("--fp8-kernel-revision", args.fp8_kernel_revision, + "--fp8-kernel-sha256", args.fp8_kernel_sha256) + elif args.fp8_kernel_revision or args.fp8_kernel_sha256: + parser.error("BF16 cannot declare an FP8 reference kernel") + example = Path(__file__).resolve().parent + trial = (sys.executable, str(example / "trial.py"), + "--model-path", str(args.model_path.resolve()), + "--model-revision", args.model_revision, "--model-id", args.model_id, + "--manifest", str(args.manifest.resolve()), + "--workload", str(args.workload.resolve()), + "--device-model", args.device_model, "--output", "{result}") + validator = (sys.executable, str(example / "validate.py"), + "--run-dir", "{run_dir}", "--workload", str(args.workload.resolve()), + "--manifest", str(args.manifest.resolve()), + "--model-path", str(args.model_path.resolve()), + "--max-nrmse", str(args.max_nrmse), "--min-top1", str(args.min_top1), + "--rms-floor", str(args.rms_floor), "--output", "{result}") + baseline = trial + ("--mode", "full", "--qualification-scope", args.qualification_scope) + candidate = trial + ("--mode", "cached") + if native: + candidate = (sys.executable, str(example / "native_trial.py")) + trial[2:] + ( + "--engine-repo", str(args.candidate_repo.resolve()), + ) + baseline += fp8_args + if not native: + candidate += fp8_args + validator += ("--qualification-scope", args.qualification_scope, + "--model-id", args.model_id) + fp8_args + spec = RunSpec( + repo=str(args.repo.resolve()), repo_type="nano_vllm", backend="cuda", + candidate_repo=str(args.candidate_repo.resolve()) if native else None, + scenario="llm.text." + args.qualification_scope, + mode="code" if native else "config", objective="correctness", + baseline_command=baseline, candidate_command=candidate, validation_command=validator, + comparison_scope=args.qualification_scope, repetitions=args.repetitions, + max_minutes=args.max_minutes, workload_id=workload_hash, + model=args.model_id, model_revision=args.model_revision, + agent="codex", device_ids=tuple(filter(None, os.environ.get( + "CUDA_VISIBLE_DEVICES", "").split(","))), executor="local", + output_dir=str(args.output_dir.resolve()), + ) + spec.validate() + if args.dry_run: + # Contains operational paths: keep this preview in private task storage. + print(json.dumps(spec.to_dict(), indent=2)) + return 0 + if not os.environ.get("SLURM_JOB_ID"): + parser.error("model validation requires an active Slurm allocation") + if len(spec.device_ids) != 1: + parser.error("Slurm must expose exactly one allocated CUDA device") + environment = inspect_environment( + spec.repo, spec.repo_type, spec.backend, device_ids=spec.device_ids, + device_model=args.device_model, python=sys.executable, + ) + run_dir = run_experiment(spec, environment) + evaluation = json.loads((run_dir / "evaluation.json").read_text()) + print(json.dumps({"run_id": run_dir.name, "verdict": evaluation["verdict"], + "qualification_scope": args.qualification_scope})) + return {"PASS": 0, "FAIL": 1, "INCONCLUSIVE": 2}[evaluation["verdict"]] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/qwen38_text/native_trial.py b/examples/qwen38_text/native_trial.py new file mode 100644 index 000000000..f091ded45 --- /dev/null +++ b/examples/qwen38_text/native_trial.py @@ -0,0 +1,134 @@ +"""Bounded native text teacher-forcing smoke; no performance/engine acceptance claim.""" + +import argparse +import json +import sys +from pathlib import Path + +if __package__: + from . import trial + from .profiles import FP8_MODEL_ID, MODEL_IDS, precision, validate_native_loading + from .provenance import capture_sources, import_native +else: + import trial + from profiles import FP8_MODEL_ID, MODEL_IDS, precision, validate_native_loading + from provenance import capture_sources, import_native + + +NATIVE_POLICY = {"implementation": "native_torch", "full_attention": "eager", + "deltanet": "torch", "cuda_graph": False, "tensor_parallel_size": 1} + + +def run(args): + model_id = getattr(args, "model_id", trial.MODEL_ID) + manifest_hash = trial.verify_manifest( + args.model_path, args.manifest, args.model_revision, model_id, + ) + cases, workload_hash = trial.load_workload(args.workload, model_id) + config = json.loads(trial.model_file(args.model_path, "config.json").read_bytes()) + vocab_size = config["text_config"]["vocab_size"] + if type(vocab_size) is not int or vocab_size <= 0: + raise ValueError("Expected positive vocabulary size") + if args.output.exists(): + raise ValueError("Refusing to overwrite existing trial result") + import numpy as np + import torch + import transformers + from transformers import AutoTokenizer + + if not torch.cuda.is_available() or torch.cuda.device_count() != 1: + raise ValueError("Native trial requires exactly one visible CUDA GPU") + if not torch.cuda.is_bf16_supported() or torch.cuda.get_device_name(0) != args.device_model: + raise ValueError("Native trial GPU/precision differs from requested target") + tokenizer = AutoTokenizer.from_pretrained( + args.model_path, local_files_only=True, trust_remote_code=False, + ) + tokens = [trial.tokenize_case(tokenizer, case) for case in cases] + load_checkpoint, request_runner = import_native(args.engine_repo) + model, loading = load_checkpoint(args.model_path, device="cuda:0") + validate_native_loading(loading, model_id) + if model_id == FP8_MODEL_ID: + # Inspect actual tensors, not just the loader's precision label. + count = 0 + for module in model.modules(): + if (type(module).__module__ == "nanovllm.qwen38.quant" + and type(module).__name__ == "FP8Linear"): + count += 1 + if (module.weight.dtype != torch.float8_e4m3fn + or module.weight_scale_inv.dtype != torch.float32): + raise ValueError("Native FP8 linear was dequantized") + if count != loading["fp8_linear_modules"]: + raise ValueError("Native FP8 module coverage differs from loading report") + model.eval() + sources = capture_sources(args.engine_repo, excluded_run_dir=args.output.parents[2]) + model_module = type(model).__module__ + if model_module not in sources["engine"]["imports"]: + raise ValueError("Native model class did not originate in the engine checkout") + max_context = max(len(prompt) + len(continuation) for prompt, continuation in tokens) + engine = request_runner( + model, tokenizer, max_context_tokens=max_context, max_active_requests=1, + vocab_size=vocab_size, device="cuda:0", + ) + result = { + "schema_version": 1, "qualification_scope": "native_text_smoke", + "implementation": "nanovllm_qwen38", "model_id": model_id, + "model_revision": args.model_revision, "manifest_sha256": manifest_hash, + "workload_sha256": workload_hash, + "dtype": "fp8" if model_id == FP8_MODEL_ID else "bfloat16", + "precision": precision(model_id), "device_type": "cuda", + "device_count": 1, "device_model": torch.cuda.get_device_name(0), + "vocab_size": vocab_size, "tokenization": "raw_text_no_special_tokens", + "kernel_policy": ({**NATIVE_POLICY, "fp8_linear": "triton_block_fp8"} + if model_id == FP8_MODEL_ID else NATIVE_POLICY), + "source_provenance": sources, + "native_model_module": model_module, "loading": loading, + "versions": {"torch": torch.__version__, "transformers": transformers.__version__, + "numpy": np.__version__, "cuda": torch.version.cuda}, "cases": [], + } + + if model_id == FP8_MODEL_ID: + from importlib.metadata import version + result["versions"].update(kernels=version("kernels"), triton=version("triton")) + + def collect(prompt, continuation): + rows = engine.trace_teacher_forced(prompt, continuation) + if engine.active_request_ids: + raise ValueError("Native trace retained request state") + return [row.float().cpu().numpy() for row in rows] + + def finalize(record): + # Capture lazy kernel imports after actual execution, before serializing evidence. + record["source_provenance"] = capture_sources( + args.engine_repo, excluded_run_dir=args.output.parents[2], + ) + if model_id == FP8_MODEL_ID: + calls = [getattr(module, "native_fp8_calls", 0) for module in model.modules() + if type(module).__module__ == "nanovllm.qwen38.quant" + and type(module).__name__ == "FP8Linear"] + if (len(calls) != loading["fp8_linear_modules"] + or not all(type(value) is int and value > 0 for value in calls)): + raise ValueError("Native FP8 linears did not execute completely") + record["fp8_execution"] = {"modules": len(calls), "total_calls": sum(calls)} + + return trial.write_traces(result, args.output, cases, tokens, collect, finalize) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model-path", "manifest", "workload", "engine-repo", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--model-id", choices=MODEL_IDS, default=trial.MODEL_ID) + parser.add_argument("--device-model", required=True) + args = parser.parse_args() + try: + run(args) + except Exception as error: + print(f"Native trial incomplete ({type(error).__name__}); no qualification verdict", + file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/qwen38_text/profiles.py b/examples/qwen38_text/profiles.py new file mode 100644 index 000000000..76ff090ca --- /dev/null +++ b/examples/qwen38_text/profiles.py @@ -0,0 +1,68 @@ +"""Explicit checkpoint and execution precision profiles for the two supported models.""" + +import json + +BF16_MODEL_ID = "Qwen/Qwen3.8-27B" +FP8_MODEL_ID = "Qwen/Qwen3.8-27B-FP8" +MODEL_IDS = (BF16_MODEL_ID, FP8_MODEL_ID) +FP8_QUANTIZATION = {"quant_method": "fp8", "activation_scheme": "dynamic", "fmt": "e4m3", + "weight_block_size": [128, 128]} + + +def precision(model_id): + if model_id not in MODEL_IDS: + raise ValueError("Unsupported model identity") + if model_id == FP8_MODEL_ID: + return {"weight_dtype": "float8_e4m3fn", "activation_dtype": "bfloat16", + "quantization": dict(FP8_QUANTIZATION), "dequantized": False} + return {"weight_dtype": "bfloat16", "activation_dtype": "bfloat16", "quantization": None} + + +def validate_config(config, model_id): + precision(model_id) + if not isinstance(config, dict) or not isinstance(config.get("text_config"), dict): + raise ValueError("Invalid checkpoint configuration") + text = config["text_config"] + if (config.get("model_type") != "qwen3_5" + or config.get("architectures") != ["Qwen3_5ForConditionalGeneration"] + or text.get("model_type") != "qwen3_5_text" + or text.get("dtype", text.get("torch_dtype")) != "bfloat16" + or text.get("quantization_config")): + raise ValueError("Expected BF16 activation Qwen3.5-architecture checkpoint") + quant = config.get("quantization_config") + if model_id == BF16_MODEL_ID: + if quant: + raise ValueError("Expected unquantized BF16 checkpoint") + elif (not isinstance(quant, dict) + or any(json.dumps(quant.get(key), sort_keys=True) != json.dumps(value, sort_keys=True) + for key, value in FP8_QUANTIZATION.items()) + or quant.get("dequantize", False) is not False + or quant.get("scale_fmt", "float") != "float"): + raise ValueError("Expected dynamic e4m3 FP8 checkpoint with 128 by 128 blocks") + + +def check_precision(record, model_id): + expected = precision(model_id) + # Historical BF16 manifests/traces had no separate precision object. + actual = record.get("precision", expected if model_id == BF16_MODEL_ID else None) + if json.dumps(actual, sort_keys=True) != json.dumps(expected, sort_keys=True): + raise ValueError("Missing or mismatched execution precision identity") + + +def validate_native_loading(loading, model_id): + if not isinstance(loading, dict): + raise ValueError("Missing native loading report") + dtype = "fp8" if model_id == FP8_MODEL_ID else "bfloat16" + if (loading.get("scope") != "text_only" or loading.get("dtype") != dtype + or type(loading.get("loaded_parameters")) is not int + or loading["loaded_parameters"] <= 0): + raise ValueError("Native parameter loading coverage missing") + if model_id == FP8_MODEL_ID: + check_precision( + {"precision": {key: loading.get(key) for key in precision(model_id)}}, model_id, + ) + if (loading.get("execution") != "native_fp8" + or loading.get("backend") != "triton_block_fp8" + or type(loading.get("fp8_linear_modules")) is not int + or loading["fp8_linear_modules"] <= 0): + raise ValueError("Native FP8 execution coverage missing or dequantized") diff --git a/examples/qwen38_text/provenance.py b/examples/qwen38_text/provenance.py new file mode 100644 index 000000000..0b986853a --- /dev/null +++ b/examples/qwen38_text/provenance.py @@ -0,0 +1,63 @@ +"""Bind native smoke imports to explicitly snapshotted source checkouts.""" + +import hashlib +import importlib +import sys +from pathlib import Path + +CONTROL_REPO = Path(__file__).resolve().parents[2] + + +def import_records(prefix, root): + root = Path(root).resolve() + records = {} + for name, module in list(sys.modules.items()): + if name != prefix and not name.startswith(prefix + "."): + continue + for package_path in getattr(module, "__path__", ()): + if not Path(package_path).resolve().is_relative_to(root): + raise ValueError("Imported package search path escaped its source checkout") + origin = getattr(module, "__file__", None) + if origin is None: + continue # Namespace packages contain no executable source file. + path = Path(origin).resolve(strict=True) + if not path.is_relative_to(root): + raise ValueError("Imported module escaped its requested source checkout") + records[name] = {"path": path.relative_to(root).as_posix(), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + return records + + +def capture_sources(engine_repo=None, excluded_run_dir=None): + import_records("inferrsi", CONTROL_REPO) + sys.path.insert(0, str(CONTROL_REPO / "src")) + importlib.invalidate_caches() + from inferrsi.infra.targets import repository_identity + + roots = {"control": (CONTROL_REPO, "inferrsi")} + if engine_repo is not None: + roots["engine"] = (Path(engine_repo).resolve(), "nanovllm") + sources = {} + for label, (root, prefix) in roots.items(): + identity = repository_identity(root, excluded_run_dir) + if not identity.get("available"): + raise ValueError("Source checkout identity unavailable") + sources[label] = {"head": identity["head"], "snapshot_hash": identity["snapshot_hash"], + "imports": import_records(prefix, root)} + return sources + + +def import_native(engine_repo): + root = Path(engine_repo).resolve(strict=True) + if not (root / "nanovllm/qwen38/loader.py").is_file(): + raise ValueError("Requested engine checkout has no native checkpoint loader") + import_records("nanovllm", root) # Reject any already imported foreign package. + sys.path.insert(0, str(root)) + importlib.invalidate_caches() + loader = importlib.import_module("nanovllm.qwen38.loader") + runner = importlib.import_module("nanovllm.qwen38.runner") + records = import_records("nanovllm", root) + for name in (loader.__name__, runner.__name__): + if name not in records: + raise ValueError("Native entrypoint source origin unavailable") + return loader.load_checkpoint, runner.RequestRunner diff --git a/examples/qwen38_text/service_client.py b/examples/qwen38_text/service_client.py new file mode 100644 index 000000000..4b446d517 --- /dev/null +++ b/examples/qwen38_text/service_client.py @@ -0,0 +1,401 @@ +"""Real HTTP/SSE observations for the bounded native service functional protocol.""" + +import http.client +import json +import socket +import threading +import time + + +class EvidenceGap(ValueError): + """The client could not obtain the required complete observation.""" + + +class Client: + def __init__(self, host, port, timeout, max_response_bytes, journal=None): + if host != "127.0.0.1" or type(port) is not int or not 0 < port < 65536: + raise ValueError("Service validation only connects to its owned loopback endpoint") + self.host, self.port = host, port + self.timeout, self.max_bytes = timeout, max_response_bytes + self.journal = journal + self._journal_lock = threading.Lock() + + def retain(self, record): + if self.journal is not None: + with self._journal_lock, self.journal.open("a") as handle: + handle.write(json.dumps(record, ensure_ascii=False, allow_nan=False) + "\n") + + def request(self, method, path, payload=None, *, raw=None): + body = ( + raw + if raw is not None + else (json.dumps(payload).encode() if payload is not None else None) + ) + connection = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout) + try: + connection.request(method, path, body, {"Content-Type": "application/json"}) + response = connection.getresponse() + data = response.read(self.max_bytes + 1) + if len(data) > self.max_bytes: + raise EvidenceGap("Response exceeds frozen client observation limit") + record = { + "status": response.status, + "content_type": response.getheader("Content-Type"), + "body": json.loads(data), + } + self.retain( + { + "method": method, + "path": path, + "payload": payload, + "raw_bytes": len(raw) if raw is not None else None, + "response": record, + } + ) + return record + finally: + connection.close() + + def state(self): + response = self.request("GET", "/debug/state") + if response["status"] != 200 or not isinstance(response["body"], dict): + raise EvidenceGap("Service state observation unavailable") + return response["body"] + + def stream(self, payload): + return Stream(self, payload) + + +class Stream: + def __init__(self, client, payload): + self.client, self.payload = client, payload + self.first = threading.Event() + self.done = threading.Event() + self.connection = self.sock = None + self.disconnected = False + self.record = { + "request": payload, + "events": [], + "done_marker": False, + "client_disconnect": False, + "error": None, + } + self.thread = threading.Thread(target=self._read, daemon=True) + self.thread.start() + + def _read(self): + try: + self.connection = http.client.HTTPConnection( + self.client.host, + self.client.port, + timeout=self.client.timeout, + ) + self.connection.request( + "POST", + "/v1/completions", + json.dumps(self.payload), + {"Content-Type": "application/json"}, + ) + self.sock = self.connection.sock + response = self.connection.getresponse() + self.record.update( + status=response.status, content_type=response.getheader("Content-Type") + ) + if response.status != 200: + self.record["body"] = json.loads(response.read(self.client.max_bytes + 1)) + return + total = 0 + deadline = time.monotonic() + self.client.timeout + while True: + if time.monotonic() >= deadline: + raise EvidenceGap("SSE total observation deadline expired") + if self.sock is not None: + self.sock.settimeout(max(0.001, deadline - time.monotonic())) + line = response.readline(self.client.max_bytes + 1) + total += len(line) + if total > self.client.max_bytes: + raise EvidenceGap("SSE exceeds frozen client observation limit") + if not line: + break + if not line.strip(): + continue + if self.record["done_marker"]: + raise EvidenceGap("SSE data after DONE") + if not line.startswith(b"data: "): + raise EvidenceGap("Unexpected SSE framing") + data = line[6:].strip() + if data == b"[DONE]": + self.record["done_marker"] = True + continue + event = json.loads(data) + self.record["events"].append(event) + if any(choice.get("token_ids") for choice in event.get("choices", [])): + self.first.set() + except (OSError, ValueError, http.client.HTTPException) as error: + if not self.disconnected: + self.record["error"] = type(error).__name__ + finally: + if self.connection is not None: + self.connection.close() + self.client.retain({"stream": self.record}) + self.done.set() + + def wait_first(self): + deadline = time.monotonic() + self.client.timeout + while not self.first.is_set() and not self.done.is_set() and time.monotonic() < deadline: + self.first.wait(min(0.05, max(0, deadline - time.monotonic()))) + return self.first.is_set() + + def finish(self): + if not self.done.wait(self.client.timeout): + self.disconnect() + raise EvidenceGap("SSE request did not finish within frozen observation budget") + return self.record + + def disconnect(self): + self.disconnected = True + self.record["client_disconnect"] = True + if self.sock is not None: + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + if self.connection is not None: + self.connection.close() + + +def poll_state(client, predicate, timeout, poll_seconds): + deadline = time.monotonic() + timeout + observations = [] + while True: + state = client.state() + observations.append(state) + if predicate(state) or time.monotonic() >= deadline: + return observations + time.sleep(min(poll_seconds, max(0, deadline - time.monotonic()))) + + +def idle(state): + return ( + state.get("active_requests") == 0 + and state.get("runner_state_count") == 0 + and state.get("runner_state_bytes") == 0 + and state.get("queued_requests") == 0 + and state.get("available") == state.get("capacity") + ) + + +def payload(model_id, case, request_id, *, stream=False): + return { + "model": model_id, + "prompt": case["prompt"], + "max_tokens": case["max_tokens"], + "temperature": 0, + "stream": stream, + "request_id": request_id, + } + + +def observe_service(client, workload): + """Collect observations; the separate validator owns the verdict.""" + model_id, cases = workload["model_id"], workload["cases"] + timeouts, limits = workload["timeouts"], workload["limits"] + + def release(): + return poll_state(client, idle, timeouts["release"], timeouts["poll"]) + + result = {"initial_state": client.state(), "serial": [], "concurrent": [], "errors": []} + for case in cases: + json_request = payload(model_id, case, "json-" + case["id"]) + response = client.request("POST", "/v1/completions", json_request) + stream = client.stream(payload(model_id, case, "sse-" + case["id"], stream=True)) + result["serial"].append( + { + "id": case["id"], + "request": json_request, + "response": response, + "stream": stream.finish(), + "release": release(), + } + ) + + selected = {case["id"]: case for case in cases} + streams = [ + client.stream(payload(model_id, selected[name], f"concurrent-{index}", stream=True)) + for index, name in enumerate(workload["concurrent_case_ids"]) + ] + try: + ids = {stream.payload["request_id"] for stream in streams} + result["overlap"] = poll_state( + client, + lambda state: ids <= set(state.get("active_request_ids", [])), + timeouts["request"], + timeouts["poll"], + ) + for name, stream in zip(workload["concurrent_case_ids"], streams): + result["concurrent"].append({"id": name, "stream": stream.finish()}) + finally: + for stream in streams: + if not stream.done.is_set(): + stream.disconnect() + result["concurrent_release"] = release() + + lifecycle = workload["lifecycle"] + for reason in ("cancelled", "disconnected"): + request_id = "lifecycle-" + reason + stream = client.stream(payload(model_id, lifecycle, request_id, stream=True)) + survivor = None + try: + observed_first = stream.wait_first() + survivor = client.stream( + payload( + model_id, + selected[workload["survivor_case_id"]], + "survivor-" + reason, + stream=True, + ) + ) + overlap = poll_state( + client, + lambda state: ( + {request_id, "survivor-" + reason} <= set(state.get("active_request_ids", [])) + ), + timeouts["request"], + timeouts["poll"], + ) + before = client.state() + ack = None + if reason == "cancelled": + ack = client.request("POST", "/v1/cancel/" + request_id, {}) + else: + stream.disconnect() + record = stream.finish() + result[reason] = { + "observed_first": observed_first, + "before": before, + "ack": ack, + "stream": record, + "overlap": overlap, + "survivor": survivor.finish(), + "release": release(), + } + result[reason]["reuse"] = client.request( + "POST", + "/v1/completions", + payload(model_id, cases[0], "reuse-" + reason), + ) + result[reason]["reuse_release"] = release() + finally: + for active in (stream, survivor): + if active is not None and not active.done.is_set(): + active.disconnect() + + # Saturation checks control-plane availability while every active slot is occupied. + held = [ + client.stream(payload(model_id, lifecycle, f"capacity-{index}", stream=True)) + for index in range(limits["max_active_requests"]) + ] + try: + ids = {stream.payload["request_id"] for stream in held} + during = poll_state( + client, + lambda state: ids <= set(state.get("active_request_ids", [])), + timeouts["request"], + timeouts["poll"], + ) + overflow = client.request( + "POST", "/v1/completions", payload(model_id, cases[0], "overflow") + ) + duplicate = client.request( + "POST", "/v1/completions", payload(model_id, cases[0], "capacity-0") + ) + acknowledgements = [ + client.request("POST", "/v1/cancel/" + request_id, {}) for request_id in sorted(ids) + ] + result["capacity"] = { + "during": during, + "overflow": overflow, + "duplicate": duplicate, + "cancel_acknowledgements": acknowledgements, + "streams": [stream.finish() for stream in held], + "release": release(), + } + finally: + for stream in held: + if not stream.done.is_set(): + stream.disconnect() + + for case in error_probes(workload): + response = client.request( + "POST", + "/v1/completions", + case.get("payload"), + raw=case.get("raw", "").encode() if "raw" in case else None, + ) + result["errors"].append({"id": case["id"], "response": response, "release": release()}) + result["final_state"] = client.state() + return result + + +def observe_queued_cancel(client, workload): + """Require a real queued-before-prefill cancellation; races remain visible.""" + blocker = client.stream( + payload(workload["model_id"], workload["queued_case"], "queue-blocker", stream=True) + ) + target = None + try: + # Admission is observable before the first model call completes. + admitted = poll_state( + client, + lambda s: "queue-blocker" in s.get("active_request_ids", []), + workload["timeouts"]["request"], + workload["timeouts"]["poll"], + ) + target = client.stream( + payload(workload["model_id"], workload["lifecycle"], "queue-target", stream=True) + ) + during = poll_state( + client, + lambda s: ( + s.get("queued_requests", 0) > 0 + and "queue-target" in s.get("active_request_ids", []) + ), + workload["timeouts"]["request"], + workload["timeouts"]["poll"], + ) + ack = client.request("POST", "/v1/cancel/queue-target", {}) + client.request("POST", "/v1/cancel/queue-blocker", {}) + records = [blocker.finish(), target.finish()] + release = poll_state( + client, idle, workload["timeouts"]["release"], workload["timeouts"]["poll"] + ) + return { + "admitted": admitted, + "during": during, + "ack": ack, + "streams": records, + "release": release, + } + finally: + for stream in (blocker, target): + if stream is not None and not stream.done.is_set(): + stream.disconnect() + + +def error_probes(workload): + good = payload(workload["model_id"], workload["cases"][0], "error-probe") + return [ + {"id": "invalid_json", "raw": "{"}, + {"id": "unknown_field", "payload": {**good, "unsupported": True}}, + {"id": "sampling", "payload": {**good, "temperature": 1}}, + { + "id": "context_limit", + "payload": { + **good, + "max_tokens": 1, + "prompt": [0] * workload["limits"]["max_context_tokens"], + }, + }, + {"id": "output_limit", "payload": {**good, "max_tokens": 0}}, + {"id": "body_limit", "raw": " " * (workload["limits"]["max_body_bytes"] + 1)}, + ] diff --git a/examples/qwen38_text/service_fault.py b/examples/qwen38_text/service_fault.py new file mode 100644 index 000000000..81b2cac5d --- /dev/null +++ b/examples/qwen38_text/service_fault.py @@ -0,0 +1,56 @@ +"""Harness-only one-shot runtime fault; never installed into the production server. + +Run in its own owned server process. The designated request must complete one +real model call before its next call raises. Later requests use the original +forward function. Evidence explicitly labels this as fault injection. +""" + +import hashlib +import sys +from pathlib import Path + + +def install_fault(engine): + model = engine.runner.model + original = model.forward + provenance = engine.provenance + marker = { + "kind": "harness_injected_runtime_error", + "request_id": "injected-fault", + "successful_target_calls": 0, + "triggered": False, + "source": "examples/qwen38_text/service_fault.py", + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + } + + def forward(*args, **kwargs): + request = engine.runner._requests.get(marker["request_id"]) + target = request is not None and request.state is kwargs.get("state") + if target and marker["successful_target_calls"] == 1 and not marker["triggered"]: + marker["triggered"] = True + raise RuntimeError("Harness-injected generation failure") + output = original(*args, **kwargs) + if target: + marker["successful_target_calls"] += 1 + return output + + model.forward = forward + engine.provenance = lambda: {**provenance(), "fault_injection": dict(marker)} + return engine + + +def main(): + root = Path(sys.argv.pop(1)).resolve(strict=True) + sys.path.insert(0, str(root)) + import nanovllm + from nanovllm.qwen38 import server + + if not Path(nanovllm.__file__).resolve().is_relative_to(root): + raise ValueError("Wrong fault-probe engine origin") + original = nanovllm.LLM + nanovllm.LLM = lambda *args, **kwargs: install_fault(original(*args, **kwargs)) + server.main() + + +if __name__ == "__main__": + main() diff --git a/examples/qwen38_text/service_launch.py b/examples/qwen38_text/service_launch.py new file mode 100644 index 000000000..a672c90bd --- /dev/null +++ b/examples/qwen38_text/service_launch.py @@ -0,0 +1,116 @@ +"""Run public-entry versus HTTP functional qualification through the common runner.""" + +import argparse +import json +import os +import sys +from pathlib import Path + +from examples.qwen38_text.profiles import BF16_MODEL_ID, MODEL_IDS +from examples.qwen38_text.service_protocol import SCOPE, read_protocol +from examples.qwen38_text.trial import HEX40 +from inferrsi.harness.runner import run_experiment +from inferrsi.infra.inspection import inspect_environment +from inferrsi.runspec import RunSpec + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("engine-repo", "model-path", "manifest", "workload", "output-dir"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--model-id", choices=MODEL_IDS, default=BF16_MODEL_ID) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--device-model", required=True) + parser.add_argument("--max-minutes", type=float, required=True) + parser.add_argument("--repetitions", type=int, required=True) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + control = Path(__file__).resolve().parents[2] + engine = args.engine_repo.resolve(strict=True) + if engine == control or not HEX40.fullmatch(args.model_revision): + parser.error( + "service qualification requires a separate engine and immutable model revision" + ) + task_root = os.environ.get("TASK_ROOT") + if task_root and any( + not path.resolve().is_relative_to(Path(task_root).resolve()) + for path in (args.output_dir, engine) + ): + parser.error("service engine and output paths must stay inside the authorized task root") + _, workload_hash = read_protocol(args.workload, args.model_id) + example = Path(__file__).resolve().parent + common = ( + "--model-id", + args.model_id, + "--manifest", + str(args.manifest.resolve()), + "--workload", + str(args.workload.resolve()), + ) + trial = ( + sys.executable, + str(example / "service_trial.py"), + *common, + "--engine-repo", + str(engine), + "--model-path", + str(args.model_path.resolve()), + "--model-revision", + args.model_revision, + "--output", + "{result}", + ) + spec = RunSpec( + repo=str(control), + candidate_repo=str(engine), + repo_type="nano_vllm", + backend="cuda", + scenario="llm.text." + SCOPE, + comparison_scope=SCOPE, + objective="correctness", + mode="config", + baseline_command=trial + ("--mode", "public"), + candidate_command=trial + ("--mode", "http"), + validation_command=( + sys.executable, + str(example / "service_validate.py"), + *common, + "--run-dir", + "{run_dir}", + "--output", + "{result}", + ), + repetitions=args.repetitions, + max_minutes=args.max_minutes, + model=args.model_id, + model_revision=args.model_revision, + workload_id=workload_hash, + agent="codex", + executor="local", + device_ids=tuple(filter(None, os.environ.get("CUDA_VISIBLE_DEVICES", "").split(","))), + output_dir=str(args.output_dir.resolve()), + ) + spec.validate() + if args.dry_run: + print(json.dumps(spec.to_dict(), indent=2)) + return 0 + if not os.environ.get("SLURM_JOB_ID") or len(spec.device_ids) != 1: + parser.error("service qualification requires one visible GPU in an active Slurm allocation") + environment = inspect_environment( + spec.repo, + spec.repo_type, + spec.backend, + device_ids=spec.device_ids, + device_model=args.device_model, + python=sys.executable, + ) + run = run_experiment(spec, environment) + result = json.loads((run / "evaluation.json").read_text()) + print( + json.dumps({"run_id": run.name, "verdict": result["verdict"], "qualification_scope": SCOPE}) + ) + return {"PASS": 0, "FAIL": 1, "INCONCLUSIVE": 2}[result["verdict"]] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/qwen38_text/service_protocol.py b/examples/qwen38_text/service_protocol.py new file mode 100644 index 000000000..a4fad355a --- /dev/null +++ b/examples/qwen38_text/service_protocol.py @@ -0,0 +1,101 @@ +"""Frozen input contract for functional service validation, with no timing defaults.""" + +import hashlib +import json +import math +import re +from pathlib import Path + +from .profiles import check_precision + +SCOPE = "native_service_functional" +TIMEOUT_KEYS = {"startup", "request", "release", "poll", "shutdown", "socket"} +LIMIT_KEYS = { + "max_context_tokens", + "max_active_requests", + "max_connections", + "max_body_bytes", + "max_response_bytes", +} +ERROR_IDS = { + "invalid_json", + "unknown_field", + "sampling", + "context_limit", + "output_limit", + "body_limit", +} + + +def read_protocol(path, model_id): + content = Path(path).read_bytes() + value = json.loads(content) + if ( + not isinstance(value, dict) + or value.get("schema_version") != 1 + or value.get("qualification_scope") != SCOPE + or value.get("model_id") != model_id + ): + raise ValueError("Wrong service protocol identity") + check_precision(value, model_id) + for field, keys in (("timeouts", TIMEOUT_KEYS), ("limits", LIMIT_KEYS)): + items = value.get(field) + if not isinstance(items, dict) or set(items) != keys: + raise ValueError("Service protocol must freeze every timeout and limit") + for number in items.values(): + valid = type(number) is int if field == "limits" else type(number) in (int, float) + if not valid or not math.isfinite(number) or number <= 0: + raise ValueError("Service bounds must be positive finite numbers") + limits = value["limits"] + if ( + limits["max_active_requests"] < 2 + or limits["max_connections"] < limits["max_active_requests"] + 2 + ): + raise ValueError("Service scope requires concurrency and reserved control connections") + cases = value.get("cases") + if not isinstance(cases, list) or len(cases) < 2: + raise ValueError("Service scope requires multiple generation cases") + seen = set() + for case in cases + [value.get("lifecycle"), value.get("queued_case")]: + if not isinstance(case, dict): + raise ValueError("Service generation cases must be objects") + prompt = case.get("prompt") + if ( + not ( + isinstance(prompt, str) + and prompt + or isinstance(prompt, list) + and prompt + and all(type(token) is int and token >= 0 for token in prompt) + ) + or type(case.get("max_tokens")) is not int + or case["max_tokens"] <= 0 + ): + raise ValueError("Invalid service generation request") + for case in cases: + name = case.get("id") + if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", name) or name in seen: + raise ValueError("Service case IDs must be unique safe identifiers") + seen.add(name) + concurrent = value.get("concurrent_case_ids") + if ( + not isinstance(concurrent, list) + or len(concurrent) < 2 + or len(concurrent) > limits["max_active_requests"] + or any(not isinstance(name, str) or name not in seen for name in concurrent) + or len(set(concurrent)) != len(concurrent) + ): + raise ValueError("Invalid concurrent case coverage") + selected = {case["id"]: case for case in cases} + if len({json.dumps(selected[name]["prompt"], sort_keys=True) for name in concurrent}) < 2: + raise ValueError("Concurrent isolation requires distinct request inputs") + for key in ("lifecycle", "queued_case"): + name = value[key].get("id") + if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", name): + raise ValueError("Lifecycle case IDs must be explicit safe identifiers") + for key in ("eos_case_id", "survivor_case_id"): + if not isinstance(value.get(key), str) or value[key] not in seen: + raise ValueError("EOS and lifecycle survivor cases must be explicitly frozen") + if value["lifecycle"]["max_tokens"] < 2: + raise ValueError("Fault probe requires at least two model calls") + return value, hashlib.sha256(content).hexdigest() diff --git a/examples/qwen38_text/service_trial.py b/examples/qwen38_text/service_trial.py new file mode 100644 index 000000000..6e39c0859 --- /dev/null +++ b/examples/qwen38_text/service_trial.py @@ -0,0 +1,313 @@ +"""Owned native public-entry or HTTP service trial; no service performance metric.""" + +import argparse +import importlib +import json +import os +import sys +import time +from pathlib import Path + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + +# Invoke as a module from a snapshotted control checkout; engine imports are pinned below. +from examples.qwen38_text.profiles import BF16_MODEL_ID, MODEL_IDS, precision +from examples.qwen38_text.provenance import capture_sources, import_records +from examples.qwen38_text.service_client import ( + Client, + EvidenceGap, + idle, + observe_queued_cancel, + observe_service, + payload, + poll_state, +) +from examples.qwen38_text.service_protocol import SCOPE, read_protocol +from examples.qwen38_text.trial import verify_manifest +from inferrsi.infra.processes import launch_server, stop_server + + +def child_imports(info, engine_repo): + """Convert actual child-process origins to the shared relative source format.""" + import hashlib + + origins = info.get("module_origins") + if not isinstance(origins, dict) or not origins: + raise EvidenceGap("Child engine import provenance missing") + root = Path(engine_repo).resolve(strict=True) + records = {} + for name, record in origins.items(): + if not isinstance(record, dict) or not isinstance(record.get("path"), str): + raise EvidenceGap("Child engine import origin malformed") + path = Path(record["path"]).resolve(strict=True) + if ( + not (name == "nanovllm" or name.startswith("nanovllm.")) + or not path.is_relative_to(root) + or hashlib.sha256(path.read_bytes()).hexdigest() != record.get("sha256") + ): + raise EvidenceGap("Child engine source origin or bytes mismatch") + records[name] = {"path": path.relative_to(root).as_posix(), "sha256": record["sha256"]} + return records + + +def public_trial(args, workload, result): + root = args.engine_repo.resolve(strict=True) + import_records("nanovllm", root) + sys.path.insert(0, str(root)) + importlib.invalidate_caches() + from nanovllm import LLM, SamplingParams + + import_records("nanovllm", root) + engine = LLM( + str(args.model_path), + max_context_tokens=workload["limits"]["max_context_tokens"], + max_active_requests=workload["limits"]["max_active_requests"], + ) + result.update(initial_state=engine.stats(), cases=[]) + try: + for case in workload["cases"]: + output = engine.generate( + [case["prompt"]], + SamplingParams( + temperature=0, + max_tokens=case["max_tokens"], + ignore_eos=False, + ), + use_tqdm=False, + ) + result["cases"].append({"id": case["id"], "outputs": output, "state": engine.stats()}) + result["model_info"] = engine.provenance() + finally: + engine.close() + result["closed_state"] = engine.stats() + return result + + +def start_owned_server(args, workload, root, *, fault=False): + root.mkdir() + ready, final = root / "ready.json", root / "shutdown.json" + limits, timeouts = workload["limits"], workload["timeouts"] + command = [ + sys.executable, + "-m", + "nanovllm.qwen38.server", + "--model", + str(args.model_path), + "--host", + "127.0.0.1", + "--port", + "0", + "--ready-file", + str(ready), + "--shutdown-file", + str(final), + ] + if fault: + command[1:3] = [ + str(Path(__file__).with_name("service_fault.py")), + str(args.engine_repo.resolve()), + ] + for key in ("max_context_tokens", "max_active_requests", "max_connections", "max_body_bytes"): + command += ["--" + key.replace("_", "-"), str(limits[key])] + command += ["--socket-timeout", str(timeouts["socket"])] + environment = os.environ.copy() + # A module launched from this cwd must resolve the exact requested engine checkout. + environment["PYTHONPATH"] = str(args.engine_repo.resolve()) + with (root / "server.log").open("wb") as log: + process, ownership = launch_server(command, args.engine_repo, environment, log) + record = {"ownership": ownership, "command": command} + try: + deadline = time.monotonic() + timeouts["startup"] + while time.monotonic() < deadline: + if process.poll() is not None: + raise EvidenceGap("Owned server exited before readiness") + if ready.exists(): + try: + info = json.loads(ready.read_text()) + host, port = info["address"] + client = Client( + host, + port, + timeouts["request"], + limits["max_response_bytes"], + root / "http-observations.jsonl", + ) + health = client.request("GET", "/health") + if health["status"] == 200: + record.update(ready=info, health=health) + child_imports(info, args.engine_repo) + return process, client, record, final + except (OSError, ValueError, KeyError): + pass + time.sleep(timeouts["poll"]) + raise EvidenceGap("Owned server readiness deadline expired") + except BaseException: + record["cleanup"] = stop_server(process, ownership, timeouts["shutdown"]) + (root / "startup-failure.json").write_text(json.dumps(record, indent=2)) + raise + + +def stop_owned_server(process, record, final, timeout): + record["cleanup"] = stop_server(process, record["ownership"], timeout) + if final.is_file(): + record["closed_state"] = json.loads(final.read_text()) + + +def http_trial(args, workload, result): + process, client, server, final = start_owned_server( + args, workload, args.output.parent / "server" + ) + result["server"] = server + shutdown_stream = None + try: + result["http"] = observe_service(client, workload) + result["http"]["queued_cancel"] = observe_queued_cancel(client, workload) + result["model_info"] = client.state() + shutdown_stream = client.stream( + payload(workload["model_id"], workload["lifecycle"], "shutdown-active", stream=True) + ) + result["shutdown_first_token"] = shutdown_stream.wait_first() + result["shutdown_before"] = client.state() + finally: + stop_owned_server(process, server, final, workload["timeouts"]["shutdown"]) + if shutdown_stream is not None: + result["shutdown_stream"] = shutdown_stream.finish() + # A fresh process must recover readiness and the same generation after active shutdown. + process, client, restarted, final = start_owned_server( + args, + workload, + args.output.parent / "restart", + ) + result["restart"] = restarted + try: + restarted["response"] = client.request( + "POST", + "/v1/completions", + payload( + workload["model_id"], + workload["cases"][0], + "restart-check", + ), + ) + restarted["state"] = client.state() + finally: + stop_owned_server(process, restarted, final, workload["timeouts"]["shutdown"]) + process, client, fault, final = start_owned_server( + args, + workload, + args.output.parent / "fault", + fault=True, + ) + result["fault"] = fault + try: + fault["warmup"] = client.request( + "POST", + "/v1/completions", + payload( + workload["model_id"], + workload["cases"][0], + "fault-warmup", + ), + ) + stream = client.stream( + payload(workload["model_id"], workload["lifecycle"], "injected-fault", stream=True) + ) + fault["stream"] = stream.finish() + fault["release"] = poll_state( + client, idle, workload["timeouts"]["release"], workload["timeouts"]["poll"] + ) + fault["reuse"] = client.request( + "POST", + "/v1/completions", + payload( + workload["model_id"], + workload["cases"][0], + "fault-reuse", + ), + ) + fault["state"] = client.state() + finally: + stop_owned_server(process, fault, final, workload["timeouts"]["shutdown"]) + return result + + +def normalize_evidence(value, engine_repo): + """Validate raw origins before retaining only relative, publishable records.""" + if isinstance(value, list): + return [normalize_evidence(item, engine_repo) for item in value] + if not isinstance(value, dict): + return value + result = {} + for key, item in value.items(): + if key == "module_origins": + result[key] = child_imports(value, engine_repo) + elif key == "command": + result[key] = "" + elif key == "address": + result[key] = ["", ""] + else: + result[key] = normalize_evidence(item, engine_repo) + return result + + +def run(args): + workload, workload_hash = read_protocol(args.workload, args.model_id) + manifest_hash = verify_manifest( + args.model_path, args.manifest, args.model_revision, args.model_id + ) + if args.output.exists(): + raise ValueError("Refusing to overwrite service trial result") + args.output.parent.mkdir(parents=True, exist_ok=True) + result = { + "schema_version": 1, + "qualification_scope": SCOPE, + "mode": args.mode, + "model_id": args.model_id, + "precision": precision(args.model_id), + "model_revision": args.model_revision, + "manifest_sha256": manifest_hash, + "workload_sha256": workload_hash, + "complete": False, + } + try: + result["observations"] = {} + if args.mode == "public": + public_trial(args, workload, result["observations"]) + else: + http_trial(args, workload, result["observations"]) + result["source_provenance"] = capture_sources( + args.engine_repo, excluded_run_dir=args.output.parents[2] + ) + if args.mode == "http": + result["source_provenance"]["engine"]["imports"] = child_imports( + result["observations"]["model_info"], + args.engine_repo, + ) + result["complete"] = True + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: + result["gap"] = type(error).__name__ # Never echo private paths or exception strings. + result = normalize_evidence(result, args.engine_repo) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("engine-repo", "model-path", "manifest", "workload", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--model-id", choices=MODEL_IDS, default=BF16_MODEL_ID) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--mode", choices=("public", "http"), required=True) + args = parser.parse_args() + try: + run(args) + except Exception as error: + print(f"Service trial incomplete ({type(error).__name__})", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/qwen38_text/service_validate.py b/examples/qwen38_text/service_validate.py new file mode 100644 index 000000000..b968f3eec --- /dev/null +++ b/examples/qwen38_text/service_validate.py @@ -0,0 +1,731 @@ +"""Adjudicate complete native service functional observations; no latency claims.""" + +import argparse +import json +import sys +from pathlib import Path + +if not __package__: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + +from examples.qwen38_text.profiles import ( + BF16_MODEL_ID, + FP8_MODEL_ID, + MODEL_IDS, + check_precision, + validate_native_loading, +) +from examples.qwen38_text.service_client import EvidenceGap, idle, payload +from examples.qwen38_text.service_protocol import ERROR_IDS, SCOPE, read_protocol +from examples.qwen38_text.validate import digest, local_file, read_json, source_identity + + +def need(condition, message): + if not condition: + raise EvidenceGap(message) + + +def check(condition, message, failures): + if not condition: + failures.append(message) + + +def zero_state(state, failures, label, *, closed=False): + need(isinstance(state, dict), "Missing state observation") + for key in ( + "active_requests", + "capacity", + "available", + "queued_requests", + "runner_state_count", + "runner_state_bytes", + ): + need(type(state.get(key)) is int and state[key] >= 0, "Invalid state accounting") + need(isinstance(state.get("active_request_ids"), list), "Missing active request identities") + check( + idle(state) and not state["active_request_ids"], + label + ": state/capacity not released", + failures, + ) + check(state.get("closed") is closed, label + ": wrong closed state", failures) + + +def released(observations, failures, label): + need(isinstance(observations, list) and bool(observations), "Missing release observations") + zero_state(observations[-1], failures, label) + return observations[-1] + + +def terminal(state, request_id): + records = state.get("terminal_requests") + need(isinstance(records, list), "Missing terminal request ledger") + found = [record for record in records if record.get("request_id") == request_id] + need(len(found) == 1, "Terminal request coverage missing or ambiguous") + return found[0] + + +def decode_response(response, request_id, model_id): + need( + isinstance(response, dict) and response.get("status") == 200, + "Successful completion response missing", + ) + need( + str(response.get("content_type", "")).startswith("application/json"), + "Completion response content type missing", + ) + body = response["body"] + need( + body.get("id") == request_id + and body.get("request_id") == request_id + and body.get("model") == model_id, + "Response request/model identity mismatch", + ) + choices = body.get("choices") + need( + isinstance(choices, list) and len(choices) == 1 and choices[0].get("index") == 0, + "Completion choice coverage missing", + ) + item = choices[0] + need( + isinstance(item.get("token_ids"), list) + and all(type(token) is int and token >= 0 for token in item["token_ids"]) + and isinstance(item.get("text"), str), + "Completion tokens/text missing", + ) + usage = body.get("usage") + need( + isinstance(usage, dict) + and type(usage.get("prompt_tokens")) is int + and usage["prompt_tokens"] > 0, + "Completion token accounting missing", + ) + need( + usage.get("completion_tokens") == len(item["token_ids"]) + and usage.get("total_tokens") == usage["prompt_tokens"] + len(item["token_ids"]), + "Completion token accounting inconsistent", + ) + return {key: item[key] for key in ("text", "token_ids", "finish_reason")} + + +def decode_stream(stream, request_id, model_id, *, disconnected=False, injected_error=False): + need( + isinstance(stream, dict) and stream.get("status") == 200 and stream.get("error") is None, + "SSE transport incomplete", + ) + need( + str(stream.get("content_type", "")).startswith("text/event-stream"), + "Missing SSE content type", + ) + need(stream.get("request", {}).get("request_id") == request_id, "SSE request identity missing") + need(stream.get("client_disconnect") is disconnected, "Wrong SSE disconnect observation") + if not disconnected: + need(stream.get("done_marker") is True, "SSE terminal marker missing") + events = stream.get("events") + need(isinstance(events, list), "SSE events missing") + tokens, text, reason = [], "", None + error_events = 0 + for event in events: + need(isinstance(event, dict), "Malformed SSE event") + if "error" in event: + need( + injected_error + and reason is None + and error_events == 0 + and isinstance(event["error"], dict) + and event["error"].get("type") == "generation_error", + "Unexpected SSE error event", + ) + error_events += 1 + continue + need( + event.get("id") == request_id + and event.get("request_id") == request_id + and event.get("model") == model_id, + "SSE event request/model identity mismatch", + ) + choices = event.get("choices") + need( + isinstance(choices, list) and len(choices) == 1 and choices[0].get("index") == 0, + "SSE event choice coverage missing", + ) + item = choices[0] + need(reason is None, "SSE emitted data after terminal event") + need( + isinstance(item.get("text"), str) + and isinstance(item.get("token_ids"), list) + and all(type(token) is int and token >= 0 for token in item["token_ids"]), + "SSE token/delta coverage missing", + ) + tokens.extend(item["token_ids"]) + text += item["text"] + if item.get("finish_reason") is not None: + need( + not item["token_ids"] and not item["text"], "SSE terminal event contains token data" + ) + reason = item["finish_reason"] + if not disconnected: + need(reason is not None, "SSE terminal reason missing") + if injected_error: + need(error_events == 1, "Injected generation error event missing") + return {"token_ids": tokens, "text": text, "finish_reason": reason} + + +def compare(actual, expected, failures, label): + check(actual == expected, label + ": tokens, decoded text or terminal reason differ", failures) + + +def validate_sources(record, spec, locked, trial_root, side): + sources = record.get("source_provenance") + need( + isinstance(sources, dict) and set(sources) == {"control", "engine"}, + "Both source identities are required for both service roles", + ) + for label, target, repo in ( + ("control", "baseline", spec["repo"]), + ("engine", "candidate", spec["candidate_repo"]), + ): + identity = source_identity(locked[target], available=True) + source = source_identity(sources[label]) + need( + all(source[key] == identity[key] for key in ("head", "snapshot_hash")), + "Service source differs from locked checkout", + ) + imports = source.get("imports") + need(isinstance(imports, dict), "Service imports missing") + required = ( + {"inferrsi"} + if label == "control" + else { + "nanovllm", + "nanovllm.qwen38.engine", + "nanovllm.qwen38.model", + "nanovllm.qwen38.loader", + "nanovllm.qwen38.runner", + } + ) + if label == "engine" and side == "candidate": + required.add("nanovllm.qwen38.server") + if label == "engine" and record["model_id"] == FP8_MODEL_ID: + required |= {"nanovllm.qwen38.quant", "nanovllm.qwen38._triton_fp8"} + need(required <= set(imports), "Required service import origins missing") + prefix = "inferrsi" if label == "control" else "nanovllm" + for name, imported in imports.items(): + need(name == prefix or name.startswith(prefix + "."), "Wrong source namespace") + need(isinstance(imported, dict), "Malformed source import") + path = local_file(Path(repo), imported.get("path")) + need(digest(path) == imported.get("sha256"), "Service imported bytes differ") + check_record = read_json(trial_root / "source.json") + need(check_record.get("side") == side, "Wrong trial source check role") + for stage in ("before", "after"): + identity = source_identity(check_record.get(stage), available=True) + need( + all(identity[key] == locked[side][key] for key in ("head", "snapshot_hash")), + "Service source changed during trial", + ) + + +def model_origins(info, record, engine_repo): + origins = info.get("module_origins") + expected = record["source_provenance"]["engine"]["imports"] + need( + isinstance(origins, dict) and set(expected) <= set(origins), + "Per-process native module coverage missing", + ) + for name, item in origins.items(): + need( + isinstance(name, str) + and (name == "nanovllm" or name.startswith("nanovllm.")) + and isinstance(item, dict), + "Malformed per-process origin", + ) + path = local_file(Path(engine_repo), item.get("path")) + need(digest(path) == item.get("sha256"), "Per-process source bytes differ") + if name in expected: + need(item == expected[name], "Per-process origin differs from declared engine") + + +def model_info(info, model_id, device_model): + need( + isinstance(info, dict) and info.get("model_id") == model_id, + "Service model identity missing", + ) + need( + info.get("device_model") == device_model + and info.get("device_type") == "cuda" + and info.get("device_count") == 1, + "Service GPU qualification mismatch", + ) + need( + info.get("precision") == ("fp8" if model_id == FP8_MODEL_ID else "bfloat16"), + "Service precision mismatch", + ) + validate_native_loading(info.get("loader_report"), model_id) + versions = info.get("versions") + need( + isinstance(versions, dict) and versions.get("torch") and versions.get("transformers"), + "Service runtime versions missing", + ) + if model_id == FP8_MODEL_ID: + execution = info.get("fp8_execution") + count = info["loader_report"]["fp8_linear_modules"] + need( + isinstance(execution, dict) + and execution.get("modules") == count + and execution.get("executed_modules") == count + and type(execution.get("total_calls")) is int + and execution["total_calls"] >= count + and execution.get("weight_dtype") == "float8_e4m3fn" + and execution.get("scale_dtype") == "float32" + and execution.get("actual_dtypes_valid") is True + and isinstance(execution.get("per_module_calls"), dict) + and len(execution["per_module_calls"]) == count + and all(type(n) is int and n > 0 for n in execution["per_module_calls"].values()) + and sum(execution["per_module_calls"].values()) == execution["total_calls"], + "Actual service FP8 execution missing", + ) + need(bool(versions.get("triton")), "Triton runtime version missing") + return versions + + +def cleanup(server, failures, label): + ownership, result = server.get("ownership"), server.get("cleanup") + need( + isinstance(ownership, dict) and isinstance(result, dict), + "Service ownership/cleanup missing", + ) + need( + ownership.get("mode") == "inferrsi_trial_group" + and ownership.get("server_start_new_session") is False + and ownership.get("helper_pid") == ownership.get("process_group_id") + and ownership.get("helper_pid") == ownership.get("helper_process_group_id") + and type(ownership.get("server_pid")) is int, + "Service process ownership unverified", + ) + need( + result.get("mode") == ownership["mode"] + and result.get("process_group_id") == ownership["process_group_id"] + and result.get("status") == "completed" + and result.get("remaining_pids") == [], + "Owned service descendants were not confirmed released", + ) + check( + result.get("kill_sent") is False + and result.get("term_sent") is True + and result.get("server_returncode") == 0, + label + ": graceful shutdown failed", + failures, + ) + zero_state(server.get("closed_state"), failures, label, closed=True) + + +def functional(public, http, workload, failures): + model_id = workload["model_id"] + expected = {} + need(len(public["cases"]) == len(workload["cases"]), "Public entry case coverage missing") + zero_state(public["initial_state"], failures, "public initial") + for case, observed in zip(workload["cases"], public["cases"]): + need( + observed.get("id") == case["id"] and len(observed["outputs"]) == 1, + "Public entry case identity missing", + ) + result = observed["outputs"][0] + need( + isinstance(result.get("text"), str) + and isinstance(result.get("token_ids"), list) + and all(type(token) is int and token >= 0 for token in result["token_ids"]), + "Public entry generation missing", + ) + check( + 0 < len(result["token_ids"]) <= case["max_tokens"] + and result.get("finish_reason") in {"stop", "length"}, + "Public generation limit", + failures, + ) + check( + result.get("finish_reason") != "length" + or len(result["token_ids"]) == case["max_tokens"], + "Public length termination differs from requested budget", + failures, + ) + expected[case["id"]] = result + zero_state(observed["state"], failures, "public request") + need( + expected[workload["eos_case_id"]]["finish_reason"] == "stop", + "Frozen EOS fixture did not observe a stop; EOS coverage missing", + ) + need( + any(result["finish_reason"] == "length" for result in expected.values()), + "No runtime max-token termination observed", + ) + zero_state(public["closed_state"], failures, "public close", closed=True) + observations = http["http"] + zero_state(observations["initial_state"], failures, "HTTP initial") + need(len(observations["serial"]) == len(workload["cases"]), "HTTP serial coverage missing") + for case, observed in zip(workload["cases"], observations["serial"]): + name = case["id"] + need( + observed["id"] == name + and observed["request"] == payload(model_id, case, "json-" + name), + "HTTP serial input identity differs", + ) + compare( + decode_response(observed["response"], "json-" + name, model_id), + expected[name], + failures, + "HTTP JSON " + name, + ) + need( + observed["stream"]["request"] == payload(model_id, case, "sse-" + name, stream=True), + "HTTP SSE input identity differs", + ) + compare( + decode_stream(observed["stream"], "sse-" + name, model_id), + expected[name], + failures, + "HTTP SSE " + name, + ) + released(observed["release"], failures, "serial release") + concurrent = observations["concurrent"] + need( + len(concurrent) == len(workload["concurrent_case_ids"]), "Concurrent case coverage missing" + ) + ids = {f"concurrent-{index}" for index in range(len(concurrent))} + need( + any(ids <= set(state.get("active_request_ids", [])) for state in observations["overlap"]), + "Concurrent admission overlap was not observed", + ) + for index, (name, observed) in enumerate(zip(workload["concurrent_case_ids"], concurrent)): + need(observed["id"] == name, "Concurrent request identity missing") + case = next(case for case in workload["cases"] if case["id"] == name) + need( + observed["stream"]["request"] + == payload(model_id, case, f"concurrent-{index}", stream=True), + "Concurrent input identity differs", + ) + compare( + decode_stream(observed["stream"], f"concurrent-{index}", model_id), + expected[name], + failures, + "Concurrent " + name, + ) + released(observations["concurrent_release"], failures, "concurrent release") + first = workload["cases"][0]["id"] + for reason in ("cancelled", "disconnected"): + observed = observations[reason] + request_id = "lifecycle-" + reason + need( + observed["observed_first"] is True + and {request_id, "survivor-" + reason} <= set(observed["before"]["active_request_ids"]), + "Active lifecycle cancellation was not observed", + ) + survivor_id = "survivor-" + reason + need( + any( + {request_id, survivor_id} <= set(state.get("active_request_ids", [])) + for state in observed["overlap"] + ), + "Lifecycle isolation overlap missing", + ) + survivor_case = next( + case for case in workload["cases"] if case["id"] == workload["survivor_case_id"] + ) + need( + observed["survivor"]["request"] + == payload(model_id, survivor_case, survivor_id, stream=True), + "Survivor input mismatch", + ) + compare( + decode_stream(observed["survivor"], survivor_id, model_id), + expected[workload["survivor_case_id"]], + failures, + reason + " survivor isolation", + ) + stream = decode_stream( + observed["stream"], request_id, model_id, disconnected=reason == "disconnected" + ) + if reason == "cancelled": + need( + observed["ack"].get("status") == 200 + and observed["ack"].get("body", {}).get("cancelled") is True, + "Active cancellation acknowledgement missing", + ) + peers = observed["ack"]["body"].get("active_request_ids_before_cancel") + need( + isinstance(peers, list) and {request_id, survivor_id} <= set(peers), + "Survivor was not active at cancellation linearization", + ) + check( + stream["finish_reason"] == "cancelled", + "Acknowledged cancellation ignored", + failures, + ) + state = released(observed["release"], failures, reason + " release") + end = terminal(state, request_id) + if reason == "disconnected": + peers = end.get("active_peer_request_ids") + need( + isinstance(peers, list) and survivor_id in peers, + "Survivor was not active when disconnect released the victim", + ) + check(end.get("finish_reason") == reason, reason + " terminal reason differs", failures) + check(end.get("prefilled") is True, reason + " did not release a prefilled state", failures) + compare( + decode_response(observed["reuse"], "reuse-" + reason, model_id), + expected[first], + failures, + reason + " capacity reuse", + ) + released(observed["reuse_release"], failures, reason + " reuse release") + capacity = observations["capacity"] + capacity_ids = { + f"capacity-{index}" for index in range(workload["limits"]["max_active_requests"]) + } + need( + any( + capacity_ids <= set(state.get("active_request_ids", [])) for state in capacity["during"] + ), + "Full active capacity was not observed", + ) + check(capacity["overflow"].get("status") == 429, "Capacity overflow not rejected", failures) + check( + capacity["duplicate"].get("status") == 400, + "Duplicate active request not rejected", + failures, + ) + need( + len(capacity["cancel_acknowledgements"]) == len(capacity_ids), + "Saturation cancel probes missing", + ) + check( + all( + item.get("status") == 200 and item.get("body", {}).get("cancelled") is True + for item in capacity["cancel_acknowledgements"] + ), + "Control-plane cancel unavailable", + failures, + ) + need(len(capacity["streams"]) == len(capacity_ids), "Saturation streams missing") + for index, stream in enumerate(capacity["streams"]): + check( + decode_stream(stream, f"capacity-{index}", model_id)["finish_reason"] == "cancelled", + "Saturation cancellation ignored", + failures, + ) + released(capacity["release"], failures, "capacity release") + queued = observations["queued_cancel"] + need( + any( + state.get("queued_requests", 0) > 0 + and "queue-target" in state.get("active_request_ids", []) + for state in queued["during"] + ), + "Queued cancellation was not observed", + ) + need( + queued["ack"].get("status") == 200 + and queued["ack"].get("body", {}).get("cancelled") is True, + "Queued cancellation acknowledgement missing", + ) + need(len(queued["streams"]) == 2, "Queued stream coverage missing") + decode_stream(queued["streams"][0], "queue-blocker", model_id) + target = decode_stream(queued["streams"][1], "queue-target", model_id) + check( + target["finish_reason"] == "cancelled" and not target["token_ids"], + "Queued target produced tokens or ignored cancellation", + failures, + ) + state = released(queued["release"], failures, "queued release") + end = terminal(state, "queue-target") + need(end.get("prefilled") is False, "Cancellation raced with prefill; queued coverage missing") + check( + end.get("finish_reason") == "cancelled" and end.get("token_count") == 0, + "Queued cancellation allocated/executed state", + failures, + ) + errors = observations["errors"] + need( + len(errors) == len(ERROR_IDS) and {item["id"] for item in errors} == ERROR_IDS, + "Error/limit coverage missing", + ) + for item in errors: + check( + item["response"].get("status") == 400, item["id"] + ": wrong HTTP rejection", failures + ) + released(item["release"], failures, item["id"] + " release") + zero_state(observations["final_state"], failures, "HTTP final") + need( + http["shutdown_first_token"] is True + and "shutdown-active" in http["shutdown_before"]["active_request_ids"], + "Shutdown with active request was not observed", + ) + shutdown = decode_stream(http["shutdown_stream"], "shutdown-active", model_id) + check( + shutdown["finish_reason"] == "shutdown", "Active stream shutdown framing failed", failures + ) + cleanup(http["server"], failures, "active shutdown") + check( + terminal(http["server"]["closed_state"], "shutdown-active").get("finish_reason") + == "shutdown", + "Active request not closed during shutdown", + failures, + ) + compare( + decode_response(http["restart"]["response"], "restart-check", model_id), + expected[first], + failures, + "restart generation", + ) + zero_state(http["restart"]["state"], failures, "restart request") + cleanup(http["restart"], failures, "restart shutdown") + fault = http["fault"] + compare( + decode_response(fault["warmup"], "fault-warmup", model_id), + expected[first], + failures, + "fault-process warmup", + ) + injected = decode_stream(fault["stream"], "injected-fault", model_id, injected_error=True) + marker = fault["state"].get("fault_injection") + need( + isinstance(marker, dict) + and marker.get("kind") == "harness_injected_runtime_error" + and marker.get("request_id") == "injected-fault" + and marker.get("triggered") is True + and marker.get("successful_target_calls") == 1 + and marker.get("source") == "examples/qwen38_text/service_fault.py" + and marker.get("source_sha256") == digest(Path(__file__).with_name("service_fault.py")), + "Real-call fault injection coverage missing", + ) + check( + injected["finish_reason"] == "error" and len(injected["token_ids"]) == 1, + "Injected post-admission failure not terminated correctly", + failures, + ) + state = released(fault["release"], failures, "injected failure release") + check( + terminal(state, "injected-fault").get("finish_reason") == "error", + "Injected failure ledger missing", + failures, + ) + compare( + decode_response(fault["reuse"], "fault-reuse", model_id), + expected[first], + failures, + "injected failure reuse", + ) + zero_state(fault["state"], failures, "injected failure reuse") + cleanup(fault, failures, "fault-process shutdown") + + +def validate(run_dir, workload_path, manifest_path, model_id): + run_dir = Path(run_dir) + workload, workload_hash = read_protocol(workload_path, model_id) + manifest, spec = read_json(manifest_path), read_json(run_dir / "run.lock.json") + environment = read_json(run_dir / "environment.json") + locked = read_json(run_dir / "repositories.lock.json") + check_precision(manifest, model_id) + need( + spec.get("objective") == "correctness" + and spec.get("comparison_scope") == SCOPE + and spec.get("model") == model_id + and manifest.get("model_id") == model_id + and spec.get("model_revision") == manifest.get("revision") + and spec.get("workload_id") == workload_hash, + "Service RunSpec/input identity mismatch", + ) + repetitions = spec.get("repetitions") + need(type(repetitions) is int and repetitions > 0, "Invalid service repetition coverage") + failures, observations, versions = [], {}, None + for side, mode in (("baseline", "public"), ("candidate", "http")): + roots = sorted((run_dir / side).glob("trial-*")) + need( + [root.name for root in roots] == [f"trial-{n:03d}" for n in range(1, repetitions + 1)], + "Service trial coverage missing", + ) + observations[side] = [] + for root in roots: + record, execution = read_json(root / "result.json"), read_json(root / "execution.json") + need( + execution.get("status") == "completed" + and execution.get("returncode") == 0 + and record.get("complete") is True, + "Service execution incomplete", + ) + need( + Path(execution["cwd"]).resolve() + == Path(spec["repo" if side == "baseline" else "candidate_repo"]).resolve(), + "Wrong service trial cwd", + ) + identity = { + "schema_version": 1, + "qualification_scope": SCOPE, + "mode": mode, + "model_id": model_id, + "model_revision": manifest["revision"], + "manifest_sha256": digest(manifest_path), + "workload_sha256": workload_hash, + } + need( + all(record.get(key) == value for key, value in identity.items()), + "Wrong service identity", + ) + check_precision(record, model_id) + validate_sources(record, spec, locked, root, side) + observed = record["observations"] + model_origins(observed["model_info"], record, spec["candidate_repo"]) + current = model_info( + observed["model_info"], model_id, environment.get("requested_device_model") + ) + if side == "candidate": + for key in ("restart", "fault"): + model_origins(observed[key]["state"], record, spec["candidate_repo"]) + need( + model_info( + observed[key]["state"], + model_id, + environment.get("requested_device_model"), + ) + == current, + "Restart/fault process runtime differs", + ) + if versions is None: + versions = current + need(versions == current, "Service environment changed across trials") + observations[side].append(observed) + for public, http in zip(observations["baseline"], observations["candidate"]): + functional(public, http, workload, failures) + return { + "verdict": "FAIL" if failures else "PASS", + "qualification_scope": SCOPE, + "reasons": failures, + "limitations": [ + "L4 functional service scope only", + "No performance or SLO claim", + "Model correctness remains a separate teacher-forcing check", + ], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("run-dir", "workload", "manifest", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--model-id", choices=MODEL_IDS, default=BF16_MODEL_ID) + args = parser.parse_args() + try: + result = validate(args.run_dir, args.workload, args.manifest, args.model_id) + except (OSError, ValueError, KeyError, TypeError, AttributeError) as error: + result = { + "verdict": "INCONCLUSIVE", + "qualification_scope": SCOPE, + "reasons": [f"Incomplete service evidence ({type(error).__name__})"], + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + print(json.dumps(result)) + return {"PASS": 0, "FAIL": 1, "INCONCLUSIVE": 2}[result["verdict"]] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/qwen38_text/slurm-run.sh b/examples/qwen38_text/slurm-run.sh new file mode 100755 index 000000000..665b3c11a --- /dev/null +++ b/examples/qwen38_text/slurm-run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Submit with explicit partition, GPU/CPU/memory/time and private output paths. +set -euo pipefail +: "${SLURM_JOB_ID:?Run under a Slurm allocation}" +: "${TASK_ROOT:?Set the authorized remote task root}" +: "${CONTROL_REPO:?Set the frozen InferRSI checkout}" +: "${TASK_PYTHON:?Set the registered uv environment interpreter}" +TASK_ROOT=$(realpath -e "$TASK_ROOT") +export TASK_ROOT +CONTROL_REPO=$(realpath -e "$CONTROL_REPO") +TASK_PYTHON=$(realpath -s -e "$TASK_PYTHON") +for task_path in "$CONTROL_REPO" "$TASK_PYTHON"; do + case "$task_path" in "$TASK_ROOT"/*) ;; *) echo 'Task path escapes root' >&2; exit 2;; esac +done +export TMPDIR="$TASK_ROOT/tmp/qwen38-$SLURM_JOB_ID" +export XDG_CACHE_HOME="$TASK_ROOT/cache/qwen38" +export UV_CACHE_DIR="$XDG_CACHE_HOME/uv" +export HF_HOME="$XDG_CACHE_HOME/huggingface" +export KERNELS_CACHE="${KERNELS_CACHE:-$HF_HOME/hub}" +export TORCH_HOME="$XDG_CACHE_HOME/torch" +export TRITON_CACHE_DIR="$XDG_CACHE_HOME/triton" +export VLLM_CACHE_ROOT="$XDG_CACHE_HOME/vllm" +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +for task_path in "$TMPDIR" "$XDG_CACHE_HOME" "$UV_CACHE_DIR" "$HF_HOME" "$KERNELS_CACHE" \ + "$TORCH_HOME" "$TRITON_CACHE_DIR" "$VLLM_CACHE_ROOT"; do + case "$(realpath -m "$task_path")" in + "$TASK_ROOT"/*) mkdir -p "$task_path" ;; + *) echo 'Cache path escapes root' >&2; exit 2 ;; + esac +done +cd "$CONTROL_REPO" +export PYTHONPATH="$CONTROL_REPO/src:$CONTROL_REPO" +exec "$TASK_PYTHON" -m examples.qwen38_text.launch "$@" diff --git a/examples/qwen38_text/trial.py b/examples/qwen38_text/trial.py new file mode 100644 index 000000000..3b947bd73 --- /dev/null +++ b/examples/qwen38_text/trial.py @@ -0,0 +1,427 @@ +"""GPU-only reference self-check: recomputation versus Transformers cache. + +This trial does not implement or qualify the nano-vllm derived engine. Optional +GPU dependencies are imported only after input integrity checks. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import importlib.util +import json +import re +import sys +from pathlib import Path, PurePosixPath + +if __package__: + from .profiles import ( + BF16_MODEL_ID, + FP8_MODEL_ID, + MODEL_IDS, + check_precision, + precision, + validate_config, + ) +else: + from profiles import ( + BF16_MODEL_ID, + FP8_MODEL_ID, + MODEL_IDS, + check_precision, + precision, + validate_config, + ) + +MODEL_ID = BF16_MODEL_ID +HEX40 = re.compile(r"[0-9a-f]{40}") +HEX64 = re.compile(r"[0-9a-f]{64}") +CASE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*") +METADATA = { + "config.json", + "tokenizer_config.json", + "tokenizer.json", + "generation_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "chat_template.jinja", +} +REQUIRED = { + "config.json", + "tokenizer_config.json", + "tokenizer.json", + "model.safetensors.index.json", +} +TRIAL_STAGES = frozenset({ + "setup", "integrity", "workload", "kernel", "imports", "device", "tokenization", + "config_adaptation", "checkpoint_load", "loading_info", "fp8_inspection", + "result_metadata", "logits", "finalization", +}) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def model_file(root: Path, name: str) -> Path: + """Resolve only canonical relative names whose final target stays in root.""" + if not isinstance(name, str) or not name or "\\" in name: + raise ValueError("Invalid manifest file name") + relative = PurePosixPath(name) + if relative.is_absolute() or ".." in relative.parts or str(relative) != name: + raise ValueError("Manifest file name must be a canonical relative path") + resolved = (root / name).resolve(strict=True) + if not resolved.is_relative_to(root.resolve()) or not resolved.is_file(): + raise ValueError("Manifest file escapes model directory or is not a file") + return resolved + + +def verify_manifest(root: Path, manifest_path: Path, revision: str, + model_id: str = MODEL_ID) -> str: + """Read and hash every frozen file; size/mtime are never integrity evidence.""" + manifest_bytes = manifest_path.read_bytes() + manifest = json.loads(manifest_bytes) + if not HEX40.fullmatch(revision): + raise ValueError("Model revision must be a lowercase 40-character commit") + if manifest.get("model_id") != model_id or manifest.get("revision") != revision: + raise ValueError("Manifest model identity differs from requested identity") + check_precision(manifest, model_id) + records = manifest.get("files") + if not isinstance(records, list) or not records: + raise ValueError("Manifest files must be nonempty") + covered = set() + for record in records: + name, digest, size = record.get("path"), record.get("sha256"), record.get("size") + if not isinstance(digest, str) or not HEX64.fullmatch(digest): + raise ValueError("Every manifest entry requires SHA-256") + if type(size) is not int or size < 0 or name in covered: + raise ValueError("Invalid size or duplicate manifest entry") + path = model_file(root, name) + if path.stat().st_size != size or sha256_file(path) != digest: + raise ValueError("Model file integrity mismatch") + covered.add(name) + if not REQUIRED <= covered: + raise ValueError("Manifest omits required model/tokenizer/index metadata") + for name in METADATA: + if (root / name).exists() and name not in covered: + raise ValueError("Manifest omits present tokenizer or generation metadata") + index = json.loads(model_file(root, "model.safetensors.index.json").read_bytes()) + weight_map = index.get("weight_map") + if not isinstance(weight_map, dict) or not weight_map: + raise ValueError("Weight index must contain a nonempty weight map") + for name in weight_map.values(): + model_file(root, name) + if not name.endswith(".safetensors") or name not in covered: + raise ValueError("Manifest omits an indexed safetensors shard") + config = json.loads(model_file(root, "config.json").read_bytes()) + validate_config(config, model_id) + return hashlib.sha256(manifest_bytes).hexdigest() + + +def load_workload(path: Path, model_id: str = MODEL_ID) -> tuple[list[dict], str]: + workload_bytes = path.read_bytes() + workload = json.loads(workload_bytes) + cases = workload.get("cases") + if ( + workload.get("schema_version") != 1 + or workload.get("model_id") != model_id + or not isinstance(cases, list) + or not cases + ): + raise ValueError("Invalid or empty workload") + check_precision(workload, model_id) + seen = set() + for case in cases: + name = case.get("id") + if ( + not isinstance(name, str) + or not CASE_ID.fullmatch(name) + or name in seen + or not isinstance(case.get("prompt"), str) + or not case["prompt"] + or not isinstance(case.get("continuation"), str) + or not case["continuation"] + or type(case.get("max_steps")) is not int + or case["max_steps"] <= 0 + ): + raise ValueError("Invalid or duplicate workload case") + seen.add(name) + return cases, hashlib.sha256(workload_bytes).hexdigest() + + +def tokenize_case(tokenizer, case: dict) -> tuple[list[int], list[int]]: + prompt = tokenizer.encode(case["prompt"], add_special_tokens=False) + continuation = tokenizer.encode(case["continuation"], add_special_tokens=False) + steps = case["max_steps"] + if not prompt or len(continuation) < steps: + raise ValueError("Empty tokenized prompt or insufficient teacher-forcing tokens") + return prompt, continuation[:steps] + + +def collect_logits(model, torch, prompt: list[int], continuation: list[int], mode: str): + """Each row predicts one fixed continuation token; never feed sampled tokens.""" + if mode not in {"full", "cached"}: + raise ValueError("Unknown reference mode") + rows = [] + cache = None + with torch.inference_mode(): + for step in range(len(continuation)): + ids = prompt + continuation[:step] + if mode == "cached" and step: + ids = continuation[step - 1 : step] + inputs = torch.tensor([ids], dtype=torch.long, device="cuda:0") + outputs = model( + input_ids=inputs, + use_cache=mode == "cached", + past_key_values=cache, + logits_to_keep=1, + return_dict=True, + ) + row = outputs.logits[0, -1].detach().float().cpu().numpy() + rows.append(row) + if mode == "cached": + cache = outputs.past_key_values + if cache is None: + raise ValueError("Cached mode did not return cache state") + return rows + + +def kernel_policy() -> dict: + """This reference deliberately fixes Torch fallback, not optional kernels.""" + optional = { + name: importlib.util.find_spec(name) is not None for name in ("fla", "causal_conv1d") + } + if any(optional.values()): + raise ValueError("Reference self-check requires optional DeltaNet kernels to be absent") + return {"hub_kernels": False, "full_attention": "eager", "optional_modules": optional} + + +def run(args: argparse.Namespace) -> dict: + args._trial_stage = "setup" + scope = getattr(args, "qualification_scope", "reference_self_check") + if scope not in {"reference_self_check", "native_text_smoke"}: + raise ValueError("Unknown qualification scope") + if scope == "native_text_smoke" and args.mode != "full": + raise ValueError("Native smoke reference must use full-prefix recomputation") + model_id = getattr(args, "model_id", MODEL_ID) + args._trial_stage = "integrity" + manifest_hash = verify_manifest(args.model_path, args.manifest, args.model_revision, model_id) + args._trial_stage = "workload" + cases, workload_hash = load_workload(args.workload, model_id) + config = json.loads(model_file(args.model_path, "config.json").read_bytes()) + vocab_size = config["text_config"]["vocab_size"] + if type(vocab_size) is not int or vocab_size <= 0: + raise ValueError("Expected positive text vocabulary size") + if args.output.exists(): + raise ValueError("Refusing to overwrite an existing trial result") + args._trial_stage = "kernel" + policy = kernel_policy() + reference_kernel = None + if model_id == FP8_MODEL_ID: + if __package__: + from .fp8_reference import prepare_reference_kernel + else: + from fp8_reference import prepare_reference_kernel + reference_kernel = prepare_reference_kernel( + args.fp8_kernel_revision, args.fp8_kernel_sha256, + ) + policy = {**policy, "fp8_linear": "finegrained_fp8", "deepgemm": False} + # CPU-only smoke checks must not import optional GPU packages. + args._trial_stage = "imports" + import numpy as np + import torch + import transformers + from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration + + args._trial_stage = "device" + if not torch.cuda.is_available() or torch.cuda.device_count() != 1: + raise ValueError("Trial requires exactly one visible CUDA GPU") + if not torch.cuda.is_bf16_supported(): + raise ValueError("Visible CUDA GPU does not support BF16") + if torch.cuda.get_device_name(0) != args.device_model: + raise ValueError("Visible CUDA GPU differs from frozen device model") + args._trial_stage = "tokenization" + tokenizer = AutoTokenizer.from_pretrained( + args.model_path, + local_files_only=True, + trust_remote_code=False, + ) + tokens = [tokenize_case(tokenizer, case) for case in cases] + load_options = {} + adaptation = None + expected_names = None + if model_id == FP8_MODEL_ID: + args._trial_stage = "config_adaptation" + if __package__: + from .fp8_reference import adapt_reference_config, expected_fp8_modules + else: + from fp8_reference import adapt_reference_config, expected_fp8_modules + from transformers import Qwen3_5Config + index = json.loads(model_file(args.model_path, "model.safetensors.index.json").read_bytes()) + expected_names = expected_fp8_modules(index) + meta_config = Qwen3_5Config.from_dict(copy.deepcopy(config)) + meta_config._attn_implementation = "eager" + with torch.device("meta"): + skeleton = Qwen3_5ForConditionalGeneration(meta_config) + if any(parameter.device.type != "meta" for parameter in skeleton.parameters()): + raise ValueError("Reference configuration inspection allocated real parameters") + adapted, adaptation = adapt_reference_config( + config, index, set(dict(skeleton.named_modules()))) + del skeleton + load_options["config"] = Qwen3_5Config.from_dict(adapted) + args._trial_stage = "checkpoint_load" + model, loading = Qwen3_5ForConditionalGeneration.from_pretrained( + args.model_path, + dtype=torch.bfloat16, + device_map={"": "cuda:0"}, + local_files_only=True, + trust_remote_code=False, + use_safetensors=True, + attn_implementation="eager", + use_kernels=False, + output_loading_info=True, + **load_options, + ) + args._trial_stage = "loading_info" + if any(loading.get(key) for key in ("missing_keys", "mismatched_keys", "error_msgs")): + raise ValueError("Reference checkpoint did not load all required parameters") + if any(name.startswith(("model.language_model.", "lm_head.")) + for name in loading.get("unexpected_keys", ())): + raise ValueError("Reference checkpoint has unexpected text parameters") + fp8_loading = None + if model_id == FP8_MODEL_ID: + args._trial_stage = "fp8_inspection" + if __package__: + from .fp8_reference import inspect_reference_fp8 + else: + from fp8_reference import inspect_reference_fp8 + from transformers.integrations.finegrained_fp8 import FP8Linear + fp8_loading = inspect_reference_fp8(model, torch, FP8Linear, expected_names) + fp8_loading["config_adaptation"] = adaptation + args._trial_stage = "result_metadata" + model.eval() + result = { + "schema_version": 1, + "qualification_scope": scope, + "implementation": "transformers_" + args.mode, + "model_id": model_id, + "model_revision": args.model_revision, + "manifest_sha256": manifest_hash, + "workload_sha256": workload_hash, + "dtype": "fp8" if model_id == FP8_MODEL_ID else "bfloat16", + "precision": precision(model_id), + "device_type": "cuda", + "device_count": 1, + "device_model": torch.cuda.get_device_name(0), + "vocab_size": vocab_size, + "tokenization": "raw_text_no_special_tokens", + "kernel_policy": policy, + "versions": { + "torch": torch.__version__, + "transformers": transformers.__version__, + "numpy": np.__version__, + "cuda": torch.version.cuda, + }, + "cases": [], + } + if reference_kernel is not None: + from importlib.metadata import version + result["versions"].update(kernels=version("kernels"), triton=version("triton")) + result["reference_kernel"] = reference_kernel + result["fp8_loading"] = fp8_loading + if scope == "native_text_smoke": + if __package__: + from .provenance import capture_sources + else: + from provenance import capture_sources + result["source_provenance"] = capture_sources(excluded_run_dir=args.output.parents[2]) + + def finalize(record): + args._trial_stage = "finalization" + if reference_kernel is not None: + if __package__: + from .fp8_reference import recheck_reference_kernel + else: + from fp8_reference import recheck_reference_kernel + recheck_reference_kernel(record["reference_kernel"]) + + args._trial_stage = "logits" + return write_traces( + result, args.output, cases, tokens, + lambda prompt, continuation: collect_logits(model, torch, prompt, continuation, args.mode), + finalize, + ) + + +def write_traces(result, output, cases, tokens, collect, finalize=None): + """Write the shared full-vocabulary observation format for either implementation.""" + import numpy as np + + vocab_size = result["vocab_size"] + output.parent.mkdir(parents=True, exist_ok=True) + for case, (prompt, continuation) in zip(cases, tokens): + rows = collect(prompt, continuation) + logits = np.stack(rows) + if logits.shape != (case["max_steps"], vocab_size) or not np.isfinite(logits).all(): + raise ValueError("Model produced invalid logits") + logits_path = output.with_name(output.stem + "-" + case["id"] + ".npz") + with logits_path.open("xb") as stream: + np.savez(stream, logits=logits) + result["cases"].append( + { + "id": case["id"], + "input_ids": prompt, + "continuation_ids": continuation, + "logits_file": logits_path.name, + "logits_sha256": sha256_file(logits_path), + "greedy_ids": logits.argmax(axis=1).tolist(), + } + ) + if finalize is not None: + finalize(result) + with output.open("x") as stream: + json.dump(result, stream, indent=2, allow_nan=False) + stream.write("\n") + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-path", type=Path, required=True) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--model-id", choices=MODEL_IDS, default=MODEL_ID) + parser.add_argument("--fp8-kernel-revision") + parser.add_argument("--fp8-kernel-sha256") + parser.add_argument("--device-model", required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--workload", type=Path, required=True) + parser.add_argument("--mode", choices=("full", "cached"), required=True) + parser.add_argument("--qualification-scope", default="reference_self_check", + choices=("reference_self_check", "native_text_smoke")) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + run(args) + except Exception as error: + # Exceptions from libraries/filesystems can contain personal runtime paths. + stage = getattr(args, "_trial_stage", "setup") + if stage not in TRIAL_STAGES: + stage = "setup" + print( + json.dumps({"status": "incomplete", "stage": stage, + "error_type": type(error).__name__, "qualification_verdict": None}), + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/qwen38_text/validate.py b/examples/qwen38_text/validate.py new file mode 100644 index 000000000..b85f39dc1 --- /dev/null +++ b/examples/qwen38_text/validate.py @@ -0,0 +1,472 @@ +"""Validate scoped text traces; missing evidence exits 2, numeric rejection 1. + +Reference self-check is the default; native smoke requires distinct source and +implementation evidence. Neither scope establishes general quality/performance. +""" + +import argparse +import copy +import hashlib +import json +import math +import re +import zipfile +from pathlib import Path + +if __package__: + from .profiles import ( + BF16_MODEL_ID, + FP8_MODEL_ID, + MODEL_IDS, + check_precision, + validate_config, + validate_native_loading, + ) +else: + from profiles import ( + BF16_MODEL_ID, + FP8_MODEL_ID, + MODEL_IDS, + check_precision, + validate_config, + validate_native_loading, + ) + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def read_json(path): + value = json.loads(Path(path).read_text()) + if not isinstance(value, dict): + raise ValueError("expected a JSON object") + return value + + +def require_object(value): + if not isinstance(value, dict): + raise ValueError("evidence field must be a JSON object") + return value + + +def require_string(value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("evidence field must be a nonempty string") + return value + + +def require_digest(value, pattern=r"[0-9a-f]{64}"): + if not isinstance(value, str) or re.fullmatch(pattern, value) is None: + raise ValueError("missing or malformed source digest") + return value + + +def source_identity(value, *, available=False): + value = require_object(value) + # Git supports SHA-1 and SHA-256 object IDs; snapshot digests are SHA-256. + require_digest(value.get("head"), r"(?:[0-9a-f]{40}|[0-9a-f]{64})") + require_digest(value.get("snapshot_hash")) + if available and value.get("available") is not True: + raise ValueError("source snapshot unavailable") + return value + + +def local_file(root, name): + if not isinstance(name, str) or not name or Path(name).is_absolute(): + raise ValueError("artifact must use a relative path") + path = (root / name).resolve() + if not path.is_relative_to(root.resolve()) or not path.is_file(): + raise ValueError("artifact missing or escapes its trial") + return path + + +def token_ids(value): + return isinstance(value, list) and bool(value) and all( + type(item) is int and item >= 0 for item in value + ) + + +REFERENCE_POLICY = {"hub_kernels": False, "full_attention": "eager", + "optional_modules": {"fla": False, "causal_conv1d": False}} +NATIVE_POLICY = {"implementation": "native_torch", "full_attention": "eager", + "deltanet": "torch", "cuda_graph": False, "tensor_parallel_size": 1} + + +def validate_sources(record, spec, locked, root, side, model_id=BF16_MODEL_ID): + sources = require_object(record.get("source_provenance")) + required = {"control"} if side == "baseline" else {"control", "engine"} + if set(sources) != required: + raise ValueError("missing or extra source provenance") + for label in required: + target = "baseline" if label == "control" else "candidate" + identity = source_identity(locked.get(target), available=True) + source = source_identity(sources[label]) + if any( + source.get(key) != identity.get(key) for key in ("head", "snapshot_hash") + ): + raise ValueError("trial source identity differs from frozen checkout") + source_root = Path(spec["repo"] if label == "control" else spec["candidate_repo"]) + imports = require_object(source.get("imports")) + required_modules = {"inferrsi", "inferrsi.infra.targets.git"} if label == "control" else { + "nanovllm", "nanovllm.qwen38.loader", "nanovllm.qwen38.runner", "nanovllm.qwen38.model", + } + if label == "engine" and model_id == FP8_MODEL_ID: + required_modules |= {"nanovllm.qwen38.quant", "nanovllm.qwen38._triton_fp8"} + if not required_modules <= set(imports): + raise ValueError("required source import origins missing") + prefix = "inferrsi" if label == "control" else "nanovllm" + for name, imported in imports.items(): + require_string(name) + imported = require_object(imported) + require_digest(imported.get("sha256")) + if name != prefix and not name.startswith(prefix + "."): + raise ValueError("unexpected import provenance namespace") + path = local_file(source_root, imported.get("path")) + if digest(path) != imported["sha256"]: + raise ValueError("imported source file changed or has wrong digest") + if side == "candidate" and record.get("native_model_module") != "nanovllm.qwen38.model": + raise ValueError("candidate model is not the native engine class") + if side == "candidate": + validate_native_loading(record.get("loading"), model_id) + if model_id == FP8_MODEL_ID: + execution = require_object(record.get("fp8_execution")) + count = record["loading"]["fp8_linear_modules"] + if (type(execution.get("modules")) is not int or execution["modules"] != count + or type(execution.get("total_calls")) is not int + or execution["total_calls"] < count): + raise ValueError("Native FP8 execution evidence incomplete") + source_check = read_json(root / "source.json") + if source_check.get("side") != side: + raise ValueError("trial source check has wrong role") + for stage in ("before", "after"): + checked = source_identity(source_check.get(stage), available=True) + if any( + checked[key] != locked[side][key] + for key in ("head", "snapshot_hash") + ): + raise ValueError("trial source changed during execution") + + +def validate_reference_kernel(record, revision, snapshot): + kernel = require_object(record.get("reference_kernel")) + if (kernel.get("repo_id") != "kernels-community/finegrained-fp8" + or kernel.get("repo_type") != "kernel" + or kernel.get("revision") != revision or kernel.get("snapshot_sha256") != snapshot): + raise ValueError("Reference kernel differs from frozen provenance") + package_root = require_string(kernel.get("package_root")) + if (kernel.get("build") != "torch-cuda" or not Path(package_root).is_absolute() + or Path(package_root).name != "torch-cuda"): + raise ValueError("Reference kernel build origin missing") + files = kernel.get("files") + if not isinstance(files, list) or not files: + raise ValueError("Reference kernel file coverage missing") + names = set() + for item in files: + require_object(item) + name = require_string(item.get("path")) + if Path(name).is_absolute() or ".." in Path(name).parts or name in names: + raise ValueError("Invalid reference kernel file identity") + names.add(name) + require_digest(item.get("sha256")) + observed = hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest() + if observed != snapshot: + raise ValueError("Reference kernel fingerprint mismatch") + loading = require_object(record.get("fp8_loading")) + expected = {"execution": "transformers_fp8", "dequantized": False, + "backend": "finegrained_fp8", "weight_dtype": "float8_e4m3fn", + "activation_dtype": "bfloat16", "scale_dtype": "float32", + "scale_normalization": "lossless_float32"} + if (loading.get("dequantized") is not False + or any(loading.get(key) != value for key, value in expected.items()) + or type(loading.get("fp8_linear_modules")) is not int + or loading["fp8_linear_modules"] <= 0 + or type(loading.get("bf16_scale_upcasts")) is not int + or not 0 <= loading["bf16_scale_upcasts"] <= loading["fp8_linear_modules"]): + raise ValueError("Reference FP8 execution coverage missing or dequantized") + return kernel + + +def reference_adaptation_contract(model_path, manifest): + """Independently derive identities from frozen-manifest-authenticated metadata.""" + if model_path is None: + raise ValueError("FP8 validation requires checkpoint metadata") + if __package__: + from .trial import model_file + else: + from trial import model_file + entries = {} + for entry in manifest["files"]: + if entry["path"] in entries: + raise ValueError("Duplicate manifest identity") + entries[entry["path"]] = entry + metadata = {} + for name in ("config.json", "model.safetensors.index.json"): + path = model_file(Path(model_path), name) + item = entries[name] + payload = path.read_bytes() + if hashlib.sha256(payload).hexdigest() != item["sha256"] or len(payload) != item["size"]: + raise ValueError("Reference metadata differs from frozen manifest") + metadata[name] = require_object(json.loads(payload)) + config = metadata["config.json"] + validate_config(config, FP8_MODEL_ID) + weights = require_object(metadata["model.safetensors.index.json"]["weight_map"]) + if any(not isinstance(key, str) or not isinstance(value, str) + for key, value in weights.items()): + raise ValueError("Reference weight index is invalid") + suffix = ".weight_scale_inv" + names = sorted(name[:-len(suffix)] for name in weights + if name.startswith("model.language_model.layers.") and name.endswith(suffix)) + if not names or any(name + ".weight" not in weights for name in names): + raise ValueError("Reference metadata lacks FP8 pairs") + layers = config["text_config"]["num_hidden_layers"] + if type(layers) is not int or layers <= 0: + raise ValueError("Reference metadata layer count invalid") + aliases = [f"model.language_model.layers.{layer}.mlp.gate" for layer in range(layers)] + skip = config["quantization_config"]["modules_to_not_convert"] + if (not isinstance(skip, list) or any(not isinstance(name, str) for name in skip) + or any(skip.count(alias) != 1 or alias + "_proj" not in names for alias in aliases)): + raise ValueError("Reference metadata gate alias mismatch") + adapted = copy.deepcopy(config) + adapted["quantization_config"]["modules_to_not_convert"] = [ + name for name in skip if name not in aliases] + def identity(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), + allow_nan=False).encode()).hexdigest() + return {"policy": "remove_nonexistent_dense_gate_aliases", "version": 1, + "original_config_sha256": identity(config), "adapted_config_sha256": identity(adapted), + "config_identity_encoding": "sorted_compact_json_utf8", + "removed_exclusions": aliases, "expected_fp8_modules": names} + + +def validate_adaptation_record(loading, contract): + if (loading.get("config_adaptation") != contract + or loading.get("fp8_module_names") != contract["expected_fp8_modules"] + or loading.get("fp8_linear_modules") != len(contract["expected_fp8_modules"])): + raise ValueError("Reference adaptation/module identity differs from authenticated metadata") + + +def validate(run_dir, workload_path, manifest_path, max_nrmse, min_top1, rms_floor, + qualification_scope="reference_self_check", model_id=BF16_MODEL_ID, + fp8_kernel_revision=None, fp8_kernel_sha256=None, model_path=None): + import numpy as np + + for number in (max_nrmse, min_top1, rms_floor): + if not math.isfinite(number): + raise ValueError("gates must be finite") + if max_nrmse < 0 or not 0 <= min_top1 <= 1 or rms_floor <= 0: + raise ValueError("invalid numerical gates") + run_dir = Path(run_dir) + workload = read_json(workload_path) + manifest = read_json(manifest_path) + spec = read_json(run_dir / "run.lock.json") + environment_record = read_json(run_dir / "environment.json") + if qualification_scope not in {"reference_self_check", "native_text_smoke"}: + raise ValueError("unknown qualification scope") + native = qualification_scope == "native_text_smoke" + fp8 = model_id == FP8_MODEL_ID + for document in (manifest, workload): + check_precision(document, model_id) + if fp8: + require_digest(fp8_kernel_revision, r"[0-9a-f]{40}") + require_digest(fp8_kernel_sha256) + adaptation_contract = reference_adaptation_contract(model_path, manifest) if fp8 else None + if spec.get("model", None if fp8 else model_id) != model_id: + raise ValueError("RunSpec model identity differs from requested profile") + if native: + if spec.get("comparison_scope") != qualification_scope or not spec.get("candidate_repo"): + raise ValueError("native smoke requires an explicit native RunSpec and engine checkout") + for key in ("repo", "candidate_repo"): + require_string(spec.get(key)) + if Path(spec["candidate_repo"]).resolve() == Path(spec["repo"]).resolve(): + raise ValueError("control and native engine checkouts must be distinct") + locked = read_json(run_dir / "repositories.lock.json") + for side in ("baseline", "candidate"): + source_identity(locked.get(side), available=True) + elif spec.get("comparison_scope") == "native_text_smoke": + raise ValueError("native evidence cannot be relabelled as a reference self-check") + if spec.get("objective") != "correctness": + raise ValueError("reference self-check requires correctness objective") + count = spec.get("repetitions") + if type(count) is not int or count < 1: + raise ValueError("invalid repetition count") + cases = workload["cases"] + expected = {case["id"]: case for case in cases} + if not expected or len(expected) != len(cases): + raise ValueError("workload case IDs must be unique and nonempty") + for case in cases: + if type(case["max_steps"]) is not int or case["max_steps"] < 1: + raise ValueError("invalid workload steps") + identities = { + "schema_version": 1, + "qualification_scope": qualification_scope, + "model_id": model_id, + "model_revision": manifest["revision"], + "manifest_sha256": digest(manifest_path), + "workload_sha256": digest(workload_path), + "dtype": "fp8" if fp8 else "bfloat16", + "device_type": "cuda", + "device_count": 1, + "tokenization": "raw_text_no_special_tokens", + } + if manifest.get("model_id") != identities["model_id"]: + raise ValueError("wrong model manifest") + if workload.get("model_id") != identities["model_id"]: + raise ValueError("wrong model workload") + if spec.get("model_revision") != manifest["revision"]: + raise ValueError("RunSpec model revision differs from manifest") + if spec.get("workload_id") != identities["workload_sha256"]: + raise ValueError("RunSpec workload hash differs from workload") + traces = {} + environment = None + reference_kernel = None + candidate_role = "nanovllm_qwen38" if native else "transformers_cached" + for side, implementation in (("baseline", "transformers_full"), ("candidate", candidate_role)): + wanted = {f"trial-{index:03d}" for index in range(1, count + 1)} + found = {path.name for path in (run_dir / side).glob("trial-*")} + if found != wanted: + raise ValueError("missing or extra trial coverage") + traces[side] = [] + for trial_name in sorted(wanted): + root = run_dir / side / trial_name + record = read_json(root / "result.json") + execution = read_json(root / "execution.json") + if execution.get("status") != "completed" or execution.get("returncode") != 0: + raise ValueError("trial execution did not complete") + if any(record.get(key) != value for key, value in identities.items()): + raise ValueError("trace identity, precision or scope mismatch") + if record.get("implementation") != implementation: + raise ValueError("incorrect reference implementation role") + check_precision(record, model_id) + policy = NATIVE_POLICY if native and side == "candidate" else REFERENCE_POLICY + if fp8: + policy = ({**policy, "fp8_linear": "triton_block_fp8"} + if native and side == "candidate" else + {**policy, "fp8_linear": "finegrained_fp8", "deepgemm": False}) + if not (native and side == "candidate"): + kernel = validate_reference_kernel( + record, fp8_kernel_revision, fp8_kernel_sha256, + ) + validate_adaptation_record(record["fp8_loading"], adaptation_contract) + if reference_kernel is None: + reference_kernel = kernel + elif reference_kernel != kernel: + raise ValueError("Reference FP8 kernel changed across trials") + if record.get("kernel_policy") != policy: + raise ValueError("incorrect implementation kernel policy") + if native: + expected_repo = spec["repo"] if side == "baseline" else spec["candidate_repo"] + require_string(execution.get("cwd")) + if Path(execution["cwd"]).resolve() != Path(expected_repo).resolve(): + raise ValueError("trial did not execute in its snapshotted checkout") + validate_sources(record, spec, locked, root, side, model_id) + vocab = record.get("vocab_size") + if type(vocab) is not int or vocab < 1: + raise ValueError("missing vocabulary size") + current = {key: record.get(key) for key in ("device_model", "versions", "vocab_size")} + if not current["device_model"] or not isinstance(current["versions"], dict): + raise ValueError("missing environment qualification") + if current["device_model"] != environment_record.get("requested_device_model"): + raise ValueError("actual GPU differs from requested qualification target") + version_keys = ("torch", "transformers", "kernels", "triton") if fp8 else ( + "torch", "transformers") + if not all(current["versions"].get(key) for key in version_keys): + raise ValueError("missing runtime versions") + if environment is None: + environment = current + elif current != environment: + raise ValueError("environment changed across trials") + outputs = record["cases"] + by_id = {case["id"]: case for case in outputs} + if len(by_id) != len(outputs) or set(by_id) != set(expected): + raise ValueError("missing, duplicate or unexpected case") + loaded = {} + for case_id, case in by_id.items(): + steps = expected[case_id]["max_steps"] + if not token_ids(case.get("input_ids")) or not token_ids( + case.get("continuation_ids") + ) or len(case["continuation_ids"]) != steps: + raise ValueError("invalid token coverage") + if any(token >= vocab for token in case["input_ids"] + case["continuation_ids"]): + raise ValueError("recorded token ID exceeds vocabulary") + artifact = local_file(root, case["logits_file"]) + if digest(artifact) != case["logits_sha256"]: + raise ValueError("logit artifact hash mismatch") + with np.load(artifact, allow_pickle=False) as data: + if data["logits"].dtype != np.float32: + raise ValueError("logit observations must be float32") + logits = data["logits"].astype(np.float64) + if logits.shape != (steps, vocab) or not np.isfinite(logits).all(): + raise ValueError("incomplete or nonfinite logit array") + if case.get("greedy_ids") != logits.argmax(axis=1).tolist(): + raise ValueError("greedy observations disagree with logits") + loaded[case_id] = (case, logits) + traces[side].append(loaded) + measurements, failed = [], [] + for index in range(count): + for case_id in expected: + a, reference = traces["baseline"][index][case_id] + b, candidate = traces["candidate"][index][case_id] + if any(a[key] != b[key] for key in ("input_ids", "continuation_ids")): + raise ValueError("paired trials used different tokens") + rms = float(np.sqrt(np.mean(reference * reference))) + nrmse = float(np.sqrt(np.mean((candidate - reference) ** 2))) / max(rms, rms_floor) + top1 = float(np.mean(reference.argmax(axis=1) == candidate.argmax(axis=1))) + measurements.append({"trial": index + 1, "case": case_id, + "nrmse": nrmse, "top1_agreement": top1}) + if nrmse > max_nrmse or top1 < min_top1: + failed.append(f"trial {index + 1}, case {case_id}: numerical gate rejected") + # Explicit replay_of pairs detect state contamination within each trial. + for side, trials in traces.items(): + for index, trial in enumerate(trials): + for case_id, requested in expected.items(): + if "replay_of" not in requested: + continue + original = requested["replay_of"] + if original not in trial: + raise ValueError("replay source case missing") + a, logits_a = trial[original] + b, logits_b = trial[case_id] + if any(a[key] != b[key] for key in ("input_ids", "continuation_ids")): + raise ValueError("replay cases did not use identical tokens") + if not np.array_equal(logits_a, logits_b): + failed.append(f"{side} trial {index + 1}: reset/replay differs") + return {"verdict": "FAIL" if failed else "PASS", "reasons": failed, + "qualification_scope": qualification_scope, "measurements": measurements, + "gates": {"max_nrmse": max_nrmse, "min_top1": min_top1, "rms_floor": rms_floor}, + "limitations": [ + "No whole-engine acceptance" if native else "No dedicated engine validated", + "No performance claim", + "Bounded raw-text teacher-forcing smoke, not broad task quality"]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-id", choices=MODEL_IDS, default=BF16_MODEL_ID) + parser.add_argument("--fp8-kernel-revision") + parser.add_argument("--fp8-kernel-sha256") + parser.add_argument("--model-path", type=Path) + for name in ("run-dir", "workload", "manifest", "output"): + parser.add_argument("--" + name, required=True, type=Path) + for name in ("max-nrmse", "min-top1", "rms-floor"): + parser.add_argument("--" + name, required=True, type=float) + parser.add_argument("--qualification-scope", default="reference_self_check", + choices=("reference_self_check", "native_text_smoke")) + args = parser.parse_args() + try: + result = validate(args.run_dir, args.workload, args.manifest, + args.max_nrmse, args.min_top1, args.rms_floor, args.qualification_scope, + args.model_id, args.fp8_kernel_revision, args.fp8_kernel_sha256, + args.model_path) + except (OSError, ValueError, KeyError, TypeError, ImportError, zipfile.BadZipFile) as exc: + # Do not echo exception strings, which may contain private runtime paths. + result = {"verdict": "INCONCLUSIVE", "qualification_scope": args.qualification_scope, + "reasons": [f"Invalid or missing validation evidence ({type(exc).__name__})"]} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + print(json.dumps(result)) + return {"PASS": 0, "FAIL": 1, "INCONCLUSIVE": 2}[result["verdict"]] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 30712463b..1f3bd488d 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -53,6 +53,10 @@ def parser() -> argparse.ArgumentParser: "--scenario", default="external_benchmark", help="Caller-declared scenario label" ) child.add_argument("--mode", choices=("config", "code", "semantic"), default="config") + child.add_argument( + "--objective", choices=("performance", "correctness"), default="performance", + help="Correctness compares paired outputs without a performance metric or claim", + ) child.add_argument( "--agent", choices=("manual", "codex", "external"), @@ -251,6 +255,7 @@ def command(value: str | None) -> tuple[str, ...] | None: backend=args.backend, scenario=args.scenario, mode=args.mode, + objective=args.objective, baseline_command=command(args.baseline_command), candidate_command=command(args.candidate_command), validation_command=command(args.validation_command), diff --git a/src/inferrsi/catalog.py b/src/inferrsi/catalog.py index d27c9fe9a..d9266085e 100644 --- a/src/inferrsi/catalog.py +++ b/src/inferrsi/catalog.py @@ -7,7 +7,9 @@ from importlib.resources import files from typing import Any -REPO_TYPES = ("vllm", "vllm_omni", "afd_plugin", "vllm_rlt", "strata", "sglang", "sglang_omni") +REPO_TYPES = ( + "vllm", "vllm_omni", "afd_plugin", "vllm_rlt", "strata", "sglang", "sglang_omni", "nano_vllm", +) # Learning scope does not imply that a repository has a runnable target adapter. # Legacy repository-pack records retain their contribution tier; an independent # peer card can describe a different learning use case for the same source repo. diff --git a/src/inferrsi/data/repo_packs/nano_vllm/pack.json b/src/inferrsi/data/repo_packs/nano_vllm/pack.json new file mode 100644 index 000000000..9de6c3fe0 --- /dev/null +++ b/src/inferrsi/data/repo_packs/nano_vllm/pack.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "id": "nano_vllm", + "source_repository": "GeeeekExplorer/nano-vllm", + "execution_support": "caller_supplied_external_commands", + "scenarios": ["llm.correctness", "llm.decode", "llm.serving"], + "backend_compatibility": { + "cuda": "unverified", + "rocm": "unverified", + "ascend": "unverified" + }, + "notes": [ + "This descriptor provides repository routing, not a built-in model launcher or accelerator qualification.", + "Freeze actual source, model, dependencies and hardware before validation. Derived engines retain their own source identity." + ] +} diff --git a/src/inferrsi/harness/evaluation.py b/src/inferrsi/harness/evaluation.py index 9ff4cb605..6301ad830 100644 --- a/src/inferrsi/harness/evaluation.py +++ b/src/inferrsi/harness/evaluation.py @@ -44,12 +44,22 @@ def evaluate( for index, trial in enumerate(trials, 1): if trial.get("status") != "completed": incomplete.append(f"{label} trial {index}: {trial.get('status')}") - if trial.get("metric") is None: + if spec.objective == "correctness" and not trial.get("output_verified"): + incomplete.append( + f"{label} trial {index}: {trial.get('output_error') or 'output missing'}" + ) + if spec.objective == "performance" and trial.get("metric") is None: incomplete.append( f"{label} trial {index}: {trial.get('metric_error') or 'metric missing'}" ) if validation is None: incomplete.append("quality validation command was not supplied") + elif ( + spec.objective == "correctness" + and validation.get("status") == "failed" + and validation.get("returncode") == 2 + ): + incomplete.append("quality validation: incomplete evidence (exit 2)") elif validation.get("status") == "failed": failed.append(f"quality validation: {validation.get('status')}") elif validation.get("status") != "completed": @@ -57,12 +67,15 @@ def evaluate( if integrity is None or not integrity.get("verified"): incomplete.append((integrity or {}).get("reason", "source integrity was not verified")) - before, after = _summary(baseline), _summary(candidate) + before, after = ( + (_summary(baseline), _summary(candidate)) + if spec.objective == "performance" else (None, None) + ) improvement = None all_metrics = len(baseline) == len(candidate) == spec.repetitions and all( t.get("status") == "completed" and t.get("metric") is not None for t in baseline + candidate ) - if all_metrics: + if spec.objective == "performance" and all_metrics: improvement = (after["median"] - before["median"]) / before["median"] * 100 if spec.direction == "minimize": improvement *= -1 @@ -80,17 +93,27 @@ def evaluate( verdict = Verdict.INCONCLUSIVE if incomplete else Verdict.FAIL if failed else Verdict.PASS return { "verdict": verdict.value, + "objective": spec.objective, + "claim_scope": ( + "Supplied correctness validator only; no performance claim." + if spec.objective == "correctness" else spec.comparison_scope + ), "reasons": failed + incomplete, - "metric_key": spec.metric_key, - "metric_unit": spec.metric_unit, + "metric_key": spec.metric_key if spec.objective == "performance" else None, + "metric_unit": spec.metric_unit if spec.objective == "performance" else None, "comparison_scope": spec.comparison_scope, - "direction": spec.direction, - "min_improvement_pct": spec.min_improvement_pct, + "direction": spec.direction if spec.objective == "performance" else None, + "min_improvement_pct": ( + spec.min_improvement_pct if spec.objective == "performance" else None + ), "baseline": before, "candidate": after, "improvement_pct": improvement, - "measurement_method": "Alternating AB/BA sequential trials; median " - "and range are descriptive only.", + "measurement_method": ( + "Alternating AB/BA sequential output trials; no performance measurement." + if spec.objective == "correctness" else + "Alternating AB/BA sequential trials; median and range are descriptive only." + ), "quality_contract": "Exit zero covers only the supplied validation " "command; inspect its logs and coverage.", "source_integrity": integrity, diff --git a/src/inferrsi/harness/runner.py b/src/inferrsi/harness/runner.py index a59707337..ed1d52def 100644 --- a/src/inferrsi/harness/runner.py +++ b/src/inferrsi/harness/runner.py @@ -51,10 +51,21 @@ def event(phase: str, **details: object) -> None: } def execute_checked( - command: tuple[str, ...], label: str, trial_dir: Path, metric_key: str | None + command: tuple[str, ...], label: str, trial_dir: Path, metric_key: str | None, + *, require_output: bool = False, ) -> dict: source_before = snapshot(repos[label], run_dir) record = execute(command, repos[label], trial_dir, deadline, metric_key) + if require_output: + try: + result = json.loads(Path(record["result_path"]).read_text(encoding="utf-8")) + if not isinstance(result, dict) or not result: + raise ValueError("trial result must be a nonempty JSON object") + record["output_verified"] = True + except (OSError, ValueError, TypeError, KeyError) as exc: + record["output_verified"] = False + record["output_error"] = str(exc) + _write_json(trial_dir / "execution.json", record) source_after = snapshot(repos[label], run_dir) check = { "path": str(trial_dir), @@ -73,12 +84,16 @@ def execute_checked( commands = {"baseline": spec.baseline_command, "candidate": spec.candidate_command} for index in range(spec.repetitions): for label in ("baseline", "candidate") if index % 2 == 0 else ("candidate", "baseline"): - event("measurement", side=label, trial=index + 1) + event( + "correctness_trial" if spec.objective == "correctness" else "measurement", + side=label, trial=index + 1, + ) record = execute_checked( commands[label], label, run_dir / label / f"trial-{index + 1:03d}", - spec.metric_key, + spec.metric_key if spec.objective == "performance" else None, + require_output=spec.objective == "correctness", ) (baseline if label == "baseline" else candidate).append(record) if record["status"] == "cancelled": @@ -92,7 +107,8 @@ def execute_checked( if spec.validation_command is not None and state == "completed": event("quality_validation") validation = execute_checked( - spec.validation_command, "candidate", run_dir / "validation", None + tuple(arg.replace("{run_dir}", str(run_dir)) for arg in spec.validation_command), + "candidate", run_dir / "validation", None, ) if validation["status"] != "completed": state = ( diff --git a/src/inferrsi/runspec.py b/src/inferrsi/runspec.py index 45f2c0b84..62909e69c 100644 --- a/src/inferrsi/runspec.py +++ b/src/inferrsi/runspec.py @@ -15,6 +15,7 @@ class RunSpec: backend: str = "cpu" scenario: str = "external_benchmark" mode: str = "config" + objective: str = "performance" validation_command: tuple[str, ...] | None = None metric_key: str = "metrics.latency_ms" metric_unit: str = "ms" @@ -73,6 +74,10 @@ def validate(self) -> None: raise ValueError("direction must be minimize or maximize") if self.mode not in {"config", "code", "semantic"}: raise ValueError("mode must be config, code or semantic") + if self.objective not in {"performance", "correctness"}: + raise ValueError("objective must be performance or correctness") + if self.objective == "correctness" and self.min_improvement_pct != 0: + raise ValueError("correctness objective cannot declare a performance gain threshold") if ( isinstance(self.repetitions, bool) or not isinstance(self.repetitions, int) diff --git a/tests/test_cli_catalog.py b/tests/test_cli_catalog.py index cc9eeb030..b27fb6a75 100644 --- a/tests/test_cli_catalog.py +++ b/tests/test_cli_catalog.py @@ -273,6 +273,19 @@ def test_dry_run_is_not_execution(self): self.assertEqual(result["execution"], "not started") self.assertFalse(self.output_dir.exists()) + def test_correctness_cli_and_nano_vllm_repository_routing(self): + arguments = self.arguments.copy() + arguments[arguments.index("--repo-type") + 1] = "nano_vllm" + arguments.extend(["--objective", "correctness"]) + code, result = self.invoke(arguments) + self.assertEqual(code, 0) + directory = Path(result["run_dir"]) + evaluation = json.loads((directory / "evaluation.json").read_text()) + self.assertEqual(evaluation["objective"], "correctness") + self.assertIsNone(evaluation["improvement_pct"]) + environment = json.loads((directory / "environment.json").read_text()) + self.assertEqual(environment["repository_pack"]["id"], "nano_vllm") + def test_missing_validation_is_inconclusive(self): arguments = self.arguments.copy() start = arguments.index("--validation-command") diff --git a/tests/test_evidence_inventory.py b/tests/test_evidence_inventory.py index 2d6d00843..5a4270d9d 100644 --- a/tests/test_evidence_inventory.py +++ b/tests/test_evidence_inventory.py @@ -145,6 +145,9 @@ def test_checked_in_inventory(): repo = Path(__file__).resolve().parents[1] result = validate_inventory(repo) assert result["errors"] == [] - assert len(result["bundles"]) == 7 + assert len(result["bundles"]) == 9 + assert {"qwen38-h20-text-phase1", "qwen38-h20-fp8-text-phase1"} <= { + item["name"] for item in result["bundles"] + } assert result["checked_hashes"] == 5 assert result["unavailable"] # Historical raw artifacts are not in this checkout. diff --git a/tests/test_qwen38_fp8_reference.py b/tests/test_qwen38_fp8_reference.py new file mode 100644 index 000000000..65078cc97 --- /dev/null +++ b/tests/test_qwen38_fp8_reference.py @@ -0,0 +1,267 @@ +"""FP8 execution guards without GPU/model imports.""" + +import sys +import types +from types import SimpleNamespace as NS + +import pytest + +from examples.qwen38_text import fp8_reference +from examples.qwen38_text.profiles import FP8_MODEL_ID, precision, validate_native_loading + + +@pytest.fixture +def reference_modules(): + class Linear: + weight = NS(dtype="fp8") + weight_scale_inv = NS(dtype="fp32") + block_size = [128, 128] + activation_scheme = "dynamic" + activation_scale = None + + module = Linear() + quant = NS(dequantize=False, activation_scheme="dynamic", weight_block_size=[128, 128]) + model = NS(hf_quantizer=NS(quantization_config=quant), + named_modules=lambda: [("projection", module)]) + torch = NS(float8_e4m3fn="fp8", float32="fp32", bfloat16="bf16") + return model, torch, Linear, module, quant + + +def test_actual_fp8_tensor_coverage_is_required(reference_modules): + model, torch, linear, module, _quant = reference_modules + report = fp8_reference.inspect_reference_fp8(model, torch, linear, {"projection"}) + assert report["dequantized"] is False + module.weight = NS(dtype="bf16") + with pytest.raises(ValueError, match="dequantized"): + fp8_reference.inspect_reference_fp8(model, torch, linear, {"projection"}) + + +@pytest.mark.parametrize("field,value", [("dequantize", True), ("activation_scheme", "static"), + ("weight_block_size", [64, 64])]) +def test_reference_quantizer_fallback_is_rejected(reference_modules, field, value): + model, torch, linear, _module, quant = reference_modules + setattr(quant, field, value) + with pytest.raises(ValueError): + fp8_reference.inspect_reference_fp8(model, torch, linear, {"projection"}) + + +def test_partial_fp8_model_is_rejected(reference_modules): + model, torch, linear, _module, _quant = reference_modules + with pytest.raises(ValueError, match="coverage"): + fp8_reference.inspect_reference_fp8(model, torch, linear, {"projection", "missing"}) + + +def test_same_count_wrong_fp8_projection_is_rejected(reference_modules): + model, torch, linear, _module, _quant = reference_modules + with pytest.raises(ValueError, match="coverage"): + fp8_reference.inspect_reference_fp8(model, torch, linear, {"different_projection"}) + + +@pytest.fixture +def dense_alias_config(): + gate = "model.language_model.layers.0.mlp.gate" + projection = gate + "_proj" + config = {"model_type": "qwen3_5", "text_config": {"num_hidden_layers": 1}, + "quantization_config": {"modules_to_not_convert": [gate, "lm_head", "model.visual"]}} + index = {"weight_map": {projection + ".weight": "layer.safetensors", + projection + ".weight_scale_inv": "layer.safetensors"}} + return config, index, {projection, "lm_head", "model.visual"}, gate, projection + + +def test_dense_alias_repair_preserves_original_and_true_exclusions(dense_alias_config): + import copy + import re + + config, index, names, gate, projection = dense_alias_config + original = copy.deepcopy(config) + # The installed Transformers matcher accepts a regex prefix without a boundary. + assert re.match(gate, projection) + adapted, record = fp8_reference.adapt_reference_config(config, index, names) + assert config == original + assert adapted["quantization_config"]["modules_to_not_convert"] == ["lm_head", "model.visual"] + assert record["removed_exclusions"] == [gate] + assert record["expected_fp8_modules"] == [projection] + assert record["original_config_sha256"] != record["adapted_config_sha256"] + assert record == fp8_reference.adapt_reference_config(config, index, names)[1] + + +def test_installed_transformers_dense_gate_prefix_regression(dense_alias_config): + """Exercise the real installed matcher/replacement, without weights or a GPU.""" + torch = pytest.importorskip("torch") + pytest.importorskip("transformers") + from transformers.integrations.finegrained_fp8 import FP8Linear, replace_with_fp8_linear + from transformers.utils.quantization_config import FineGrainedFP8Config + + config, index, _names, _gate, projection = dense_alias_config + def skeleton(): + root = torch.nn.Module() + parent = root + for component in projection.split(".")[:-1]: + child = torch.nn.Module() + parent.add_module(component, child) + parent = child + with torch.device("meta"): + parent.add_module("gate_proj", torch.nn.Linear(128, 128, bias=False)) + return root + + original = skeleton() + quant = FineGrainedFP8Config(activation_scheme="dynamic", weight_block_size=[128, 128]) + replace_with_fp8_linear( + original, config["quantization_config"]["modules_to_not_convert"], quant, True) + assert type(original.get_submodule(projection)) is torch.nn.Linear + repaired = skeleton() + adapted, _record = fp8_reference.adapt_reference_config( + config, index, set(dict(repaired.named_modules()))) + replace_with_fp8_linear( + repaired, adapted["quantization_config"]["modules_to_not_convert"], quant, True) + assert isinstance(repaired.get_submodule(projection), FP8Linear) + assert repaired.get_submodule(projection).weight.device.type == "meta" + + +@pytest.mark.parametrize("change", ["real_gate", "missing_projection", "missing_weight", + "missing_scale", "missing_alias", "duplicate_alias"]) +def test_alias_repair_requires_model_and_checkpoint_agreement(dense_alias_config, change): + config, index, names, gate, projection = dense_alias_config + if change == "real_gate": + names.add(gate) + elif change == "missing_projection": + names.remove(projection) + elif change in {"missing_weight", "missing_scale"}: + suffix = ".weight" if change == "missing_weight" else ".weight_scale_inv" + del index["weight_map"][projection + suffix] + elif change == "missing_alias": + config["quantization_config"]["modules_to_not_convert"].remove(gate) + else: + config["quantization_config"]["modules_to_not_convert"].append(gate) + with pytest.raises(ValueError): + fp8_reference.adapt_reference_config(config, index, names) + + +@pytest.fixture +def offline_kernel(tmp_path, monkeypatch): + source = tmp_path / "torch-cuda/__init__.py" + source.parent.mkdir() + source.write_text("# synthetic kernel package\n") + kernel = NS(__file__=str(source), **{name: lambda: None for name in + ("matmul_2d", "matmul_batched", "matmul_grouped")}) + fingerprint = fp8_reference.kernel_fingerprint(kernel) + hub = NS(_KERNEL_MODULE_MAPPING={}, _HUB_KERNEL_MAPPING={}) + def load(name): + assert hub._HUB_KERNEL_MAPPING[name] == { + "repo_id": fp8_reference.KERNEL_REPO, "revision": "a" * 40, + } + assert __import__("os").environ["TRANSFORMERS_DISABLE_DEEPGEMM_LINEAR"] == "1" + return kernel + hub.lazy_load_kernel = load + fine = NS(_FINEGRAINED_FP8=None, load_finegrained_fp8_kernel=lambda: None) + package = types.ModuleType("transformers.integrations") + package.finegrained_fp8, package.hub_kernels = fine, hub + monkeypatch.setitem(sys.modules, "transformers", types.ModuleType("transformers")) + monkeypatch.setitem(sys.modules, "transformers.integrations", package) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + return fingerprint["snapshot_sha256"], fine, kernel + + +def test_exact_offline_kernel_pin_and_content(offline_kernel): + digest, _fine, _kernel = offline_kernel + result = fp8_reference.prepare_reference_kernel("a" * 40, digest) + assert result["snapshot_sha256"] == digest and result["repo_type"] == "kernel" + with pytest.raises(ValueError, match="frozen source"): + fp8_reference.prepare_reference_kernel("a" * 40, "b" * 64) + + +def test_online_unpinned_and_preloaded_kernels_rejected(offline_kernel, monkeypatch): + digest, fine, _kernel = offline_kernel + with pytest.raises(ValueError, match="frozen"): + fp8_reference.prepare_reference_kernel("main", digest) + fine._FINEGRAINED_FP8 = object() + with pytest.raises(ValueError, match="before pinning"): + fp8_reference.prepare_reference_kernel("a" * 40, digest) + monkeypatch.delenv("HF_HUB_OFFLINE") + with pytest.raises(ValueError, match="offline"): + fp8_reference.prepare_reference_kernel("a" * 40, digest) + + +def test_native_dequantization_cannot_be_labelled_fp8(): + loading = {"scope": "text_only", "dtype": "fp8", "loaded_parameters": 3, + **precision(FP8_MODEL_ID), "execution": "native_fp8", + "backend": "triton_block_fp8", "fp8_linear_modules": 1} + validate_native_loading(loading, FP8_MODEL_ID) + loading["dequantized"] = True + with pytest.raises(ValueError, match="precision"): + validate_native_loading(loading, FP8_MODEL_ID) + + +def test_reference_scale_upcast_is_explicit_and_recorded(reference_modules): + model, torch, linear, module, _quant = reference_modules + + class Scale: + dtype = "bf16" + @property + def data(self): + return self + @data.setter + def data(self, value): + self.dtype = value.dtype + def float(self): + return NS(dtype="fp32") + + module.weight_scale_inv = Scale() + result = fp8_reference.inspect_reference_fp8(model, torch, linear, {"projection"}) + assert result["bf16_scale_upcasts"] == 1 + assert result["scale_dtype"] == "float32" + assert module.weight.dtype == "fp8" + + +def test_kernel_snapshot_symlinks_cover_own_blob_cache_only(tmp_path): + repo = tmp_path / "kernels--synthetic" + root = repo / "snapshots" / ("a" * 40) / "build/torch-cuda" + root.mkdir(parents=True) + blobs = repo / "blobs" + blobs.mkdir() + blob = blobs / "source" + blob.write_text("# cached synthetic kernel\n") + init = root / "__init__.py" + init.symlink_to(blob) + module = NS(__file__=str(init)) + result = fp8_reference.kernel_fingerprint(module) + assert result["files"][0]["path"] == "__init__.py" + outside = tmp_path / "other.py" + outside.write_text("# unrelated source\n") + (root / "escape.py").symlink_to(outside) + with pytest.raises(ValueError, match="escapes"): + fp8_reference.kernel_fingerprint(module) + + +@pytest.mark.parametrize("change", ["modify", "delete", "add"]) +def test_reference_kernel_drift_during_trial_prevents_result(offline_kernel, tmp_path, change): + from pathlib import Path + + np = pytest.importorskip("numpy") + + from examples.qwen38_text.trial import write_traces + + digest, _fine, kernel = offline_kernel + prepared = fp8_reference.prepare_reference_kernel("a" * 40, digest) + fp8_reference.recheck_reference_kernel(prepared) + output = tmp_path / "trial/result.json" + result = {"vocab_size": 2, "cases": [], "reference_kernel": prepared} + + def collect(_prompt, _continuation): + source = Path(kernel.__file__) + if change == "modify": + source.write_text("# changed after preparation\n") + elif change == "delete": + source.unlink() + else: + source.with_name("late_import.py").write_text("# new source\n") + return [np.array([0., 1.], dtype=np.float32)] + + with pytest.raises(ValueError, match="source"): + write_traces(result, output, [{"id": "one", "max_steps": 1}], [([0], [1])], + collect, lambda record: fp8_reference.recheck_reference_kernel( + record["reference_kernel"])) + assert not output.exists() + # Preserve partial logits for diagnosis; they cannot substitute for result evidence. + assert output.with_name("result-one.npz").exists() diff --git a/tests/test_qwen38_intake.py b/tests/test_qwen38_intake.py new file mode 100644 index 000000000..9abef5641 --- /dev/null +++ b/tests/test_qwen38_intake.py @@ -0,0 +1,219 @@ +"""Offline intake contract checks using synthetic local files and API responses.""" + +import hashlib +import io +import json + +import pytest + +from examples.qwen38_text import intake, trial + +REVISION = "a" * 40 + + +def sibling(path): + content = path.read_bytes() + record = {"rfilename": path.name, "size": len(content)} + if path.suffix == ".safetensors": + record["lfs"] = {"size": len(content), "sha256": hashlib.sha256(content).hexdigest()} + else: + record["blobId"] = hashlib.sha1( + f"blob {len(content)}\0".encode() + content, + ).hexdigest() + return record + + +@pytest.fixture +def snapshot(tmp_path, monkeypatch): + root = tmp_path / "model" + root.mkdir() + config = { + "model_type": "qwen3_5", + "architectures": ["Qwen3_5ForConditionalGeneration"], + "text_config": {"model_type": "qwen3_5_text", "dtype": "bfloat16"}, + } + files = { + "config.json": config, + "tokenizer.json": {}, + "tokenizer_config.json": {}, + "model.safetensors.index.json": { + "weight_map": {"a": "part.safetensors", "b": "part.safetensors"} + }, + } + for name, value in files.items(): + (root / name).write_text(json.dumps(value)) + (root / "part.safetensors").write_bytes(b"synthetic weights") + response = { + "id": trial.MODEL_ID, + "sha": REVISION, + "siblings": [sibling(path) for path in sorted(root.iterdir())], + } + monkeypatch.setattr(intake, "fetch_metadata", + lambda revision, model_id=trial.MODEL_ID: (response, {"source": "test"})) + return root, response, tmp_path / "manifest.json" + + +def test_public_identity_manifest_and_single_weight_pass(snapshot, monkeypatch): + root, _response, output = snapshot + calls = [] + authenticate = intake.authenticate_file + + def record_call(*args, **kwargs): + calls.append(args[1]) + return authenticate(*args, **kwargs) + + monkeypatch.setattr(intake, "authenticate_file", record_call) + result = intake.create_manifest(root, REVISION, output) + assert result == json.loads(output.read_text()) + assert result["revision"] == REVISION and result["model_id"] == trial.MODEL_ID + assert calls.count("part.safetensors") == 1 + assert trial.verify_manifest(root, output, REVISION) == trial.sha256_file(output) + assert str(root) not in output.read_text() + + +@pytest.mark.parametrize("name", ["part.safetensors", "tokenizer.json"]) +def test_same_length_tamper_fails(snapshot, name): + root, _response, output = snapshot + path = root / name + original = path.read_bytes() + path.write_bytes(b"X" + original[1:]) + with pytest.raises(ValueError, match="differs"): + intake.create_manifest(root, REVISION, output) + assert not output.exists() + + +def test_missing_local_shard_fails(snapshot): + root, _response, output = snapshot + (root / "part.safetensors").unlink() + with pytest.raises(FileNotFoundError): + intake.create_manifest(root, REVISION, output) + assert not output.exists() + + +def test_missing_remote_shard_metadata_fails(snapshot): + root, response, output = snapshot + response["siblings"] = [ + r for r in response["siblings"] if not r["rfilename"].endswith("safetensors") + ] + with pytest.raises(ValueError, match="missing"): + intake.create_manifest(root, REVISION, output) + + +def test_revision_mismatch_fails(snapshot): + root, response, output = snapshot + response["sha"] = "b" * 40 + with pytest.raises(ValueError, match="identity"): + intake.create_manifest(root, REVISION, output) + + +def test_metadata_available_upstream_must_exist_locally(snapshot): + root, response, output = snapshot + extra = root / "added_tokens.json" + extra.write_text("{}") + response["siblings"].append(sibling(extra)) + extra.unlink() + with pytest.raises(FileNotFoundError): + intake.create_manifest(root, REVISION, output) + + +def test_output_is_exclusive_and_outside_model(snapshot, tmp_path): + root, _response, output = snapshot + with pytest.raises(ValueError, match="outside"): + intake.create_manifest(root, REVISION, root / "manifest.json") + output.write_text("keep existing") + with pytest.raises(ValueError, match="new file"): + intake.create_manifest(root, REVISION, output) + assert output.read_text() == "keep existing" + alias = tmp_path / "model-alias" + alias.symlink_to(root, target_is_directory=True) + with pytest.raises(ValueError, match="outside"): + intake.create_manifest(root, REVISION, alias / "manifest.json") + output.unlink() + output.symlink_to(tmp_path / "missing") + with pytest.raises(ValueError, match="new file"): + intake.create_manifest(root, REVISION, output) + + +def test_model_symlink_escape_rejected(snapshot, tmp_path): + root, _response, output = snapshot + path = root / "part.safetensors" + outside = tmp_path / "outside.safetensors" + path.rename(outside) + path.symlink_to(outside) + with pytest.raises(ValueError, match="escapes"): + intake.create_manifest(root, REVISION, output) + + +def test_authenticated_index_cannot_use_traversal(snapshot): + root, response, output = snapshot + index = root / "model.safetensors.index.json" + index.write_text(json.dumps({"weight_map": {"a": "../part.safetensors"}})) + response["siblings"] = [ + sibling(index) if r["rfilename"] == index.name else r for r in response["siblings"] + ] + response["siblings"].append( + {"rfilename": "../part.safetensors", "size": 1, "lfs": {"size": 1, "sha256": "b" * 64}} + ) + with pytest.raises(ValueError, match="relative"): + intake.create_manifest(root, REVISION, output) + + +class Response(io.BytesIO): + def geturl(self): + return "https://huggingface.co/api/models/Qwen/Qwen3.8-27B" + + +def test_fetch_immutable_endpoint_and_bounded_response(monkeypatch): + body = json.dumps({"id": trial.MODEL_ID, "sha": REVISION}).encode() + + def open_response(request, *, timeout): + assert request.full_url.endswith(f"/revision/{REVISION}?blobs=true") + assert timeout == 30 + return Response(body) + + monkeypatch.setattr(intake.urllib.request, "urlopen", open_response) + metadata, provenance = intake.fetch_metadata(REVISION) + assert metadata["sha"] == REVISION + assert provenance["response_sha256"] == hashlib.sha256(body).hexdigest() + monkeypatch.setattr(intake, "MAX_METADATA_BYTES", 2) + with pytest.raises(ValueError, match="limit"): + intake.fetch_metadata(REVISION) + + +@pytest.mark.parametrize("revision", ["main", "a" * 39, "../bad", "A" * 40]) +def test_revision_validation_precedes_network(revision, monkeypatch): + def forbidden(*_args, **_kwargs): + raise AssertionError("network must not be called") + + monkeypatch.setattr(intake.urllib.request, "urlopen", forbidden) + with pytest.raises(ValueError, match="immutable"): + intake.fetch_metadata(revision) + + +def test_fp8_intake_authenticates_separate_model_identity(snapshot): + from examples.qwen38_text.profiles import FP8_MODEL_ID, FP8_QUANTIZATION, precision + + root, response, output = snapshot + config_path = root / "config.json" + config = json.loads(config_path.read_text()) + config["quantization_config"] = FP8_QUANTIZATION + config_path.write_text(json.dumps(config)) + response["id"] = FP8_MODEL_ID + response["siblings"] = [sibling(path) for path in sorted(root.iterdir())] + result = intake.create_manifest(root, REVISION, output, FP8_MODEL_ID) + assert result["precision"] == precision(FP8_MODEL_ID) + + +@pytest.mark.parametrize("change", [ + {"quant_method": "compressed-tensors"}, {"fmt": "e5m2"}, {"activation_scheme": "static"}, + {"weight_block_size": [64, 64]}, {"dequantize": True}, {"scale_fmt": "ue8m0"}, +]) +def test_fp8_profile_rejects_other_quantization(snapshot, change): + from examples.qwen38_text.profiles import FP8_MODEL_ID, FP8_QUANTIZATION, validate_config + + config = json.loads((snapshot[0] / "config.json").read_text()) + config["quantization_config"] = {**FP8_QUANTIZATION, **change} + with pytest.raises(ValueError, match="FP8"): + validate_config(config, FP8_MODEL_ID) + with pytest.raises(ValueError, match="unquantized"): + validate_config(config, trial.MODEL_ID) diff --git a/tests/test_qwen38_launch.py b/tests/test_qwen38_launch.py new file mode 100644 index 000000000..a3e02f460 --- /dev/null +++ b/tests/test_qwen38_launch.py @@ -0,0 +1,107 @@ +import json +import sys +from pathlib import Path + +import pytest + +from examples.qwen38_text import launch + + +def args(tmp_path): + workload = tmp_path / "workload.json" + workload.write_text(json.dumps({"schema_version": 1, "model_id": "Qwen/Qwen3.8-27B", + "cases": [{"id": "one", "prompt": "abc", + "continuation": "def", "max_steps": 1}]})) + return ["launch", "--repo", str(Path(launch.__file__).resolve().parents[2]), + "--model-path", str(tmp_path), "--manifest", str(tmp_path / "manifest.json"), + "--model-revision", "a" * 40, "--workload", str(workload), + "--output-dir", str(tmp_path / "results"), "--device-model", "Synthetic GPU", + "--max-nrmse", "0.01", "--min-top1", "0.99", "--rms-floor", "1e-8", + "--max-minutes", "1", "--repetitions", "1"] + + +def test_preview_retains_source_and_allocation(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3") + monkeypatch.setattr(sys, "argv", args(tmp_path) + ["--dry-run"]) + assert launch.main() == 0 + spec = json.loads(capsys.readouterr().out) + assert spec["device_ids"] == ["3"] + assert spec["objective"] == "correctness" + assert spec["comparison_scope"] == "reference_self_check" + assert "{run_dir}" in spec["validation_command"] + assert not (tmp_path / "results").exists() + + +def test_unrelated_source_cannot_be_snapshotted(tmp_path, monkeypatch): + argv = args(tmp_path) + argv[argv.index("--repo") + 1] = str(tmp_path) + monkeypatch.setattr(sys, "argv", argv + ["--dry-run"]) + with pytest.raises(SystemExit) as exc: + launch.main() + assert exc.value.code == 2 + + +def test_execution_requires_slurm(tmp_path, monkeypatch): + monkeypatch.delenv("SLURM_JOB_ID", raising=False) + monkeypatch.setattr(sys, "argv", args(tmp_path)) + with pytest.raises(SystemExit) as exc: + launch.main() + assert exc.value.code == 2 + + +def test_output_cannot_escape_task_root(tmp_path, monkeypatch): + monkeypatch.setenv("TASK_ROOT", str(tmp_path / "authorized")) + monkeypatch.setattr(sys, "argv", args(tmp_path) + ["--dry-run"]) + with pytest.raises(SystemExit) as exc: + launch.main() + assert exc.value.code == 2 + + +def test_native_preview_binds_distinct_control_and_candidate(tmp_path, monkeypatch, capsys): + engine = tmp_path / "engine" + engine.mkdir() + monkeypatch.setattr(sys, "argv", args(tmp_path) + [ + "--qualification-scope", "native_text_smoke", "--candidate-repo", str(engine), "--dry-run", + ]) + assert launch.main() == 0 + spec = json.loads(capsys.readouterr().out) + assert spec["candidate_repo"] == str(engine) + assert spec["comparison_scope"] == "native_text_smoke" + assert spec["baseline_command"][-4:] == ["--mode", "full", "--qualification-scope", + "native_text_smoke"] + assert spec["candidate_command"][1].endswith("native_trial.py") + assert "--mode" not in spec["candidate_command"] + assert spec["candidate_command"][-2:] == ["--engine-repo", str(engine)] + + +@pytest.mark.parametrize("extra", [ + ["--qualification-scope", "native_text_smoke"], + ["--candidate-repo", "."], +]) +def test_native_role_requires_explicit_consistent_scope(tmp_path, monkeypatch, extra): + monkeypatch.setattr(sys, "argv", args(tmp_path) + extra + ["--dry-run"]) + with pytest.raises(SystemExit) as exc: + launch.main() + assert exc.value.code == 2 + + +def test_fp8_preview_propagates_explicit_profile_and_frozen_kernel(tmp_path, monkeypatch, capsys): + from examples.qwen38_text.profiles import FP8_MODEL_ID, precision + + argv = args(tmp_path) + path = tmp_path / "workload.json" + workload = json.loads(path.read_text()) + workload.update(model_id=FP8_MODEL_ID, precision=precision(FP8_MODEL_ID)) + path.write_text(json.dumps(workload)) + argv += ["--model-id", FP8_MODEL_ID, "--fp8-kernel-revision", "b" * 40, + "--fp8-kernel-sha256", "c" * 64, "--dry-run"] + monkeypatch.setattr(sys, "argv", argv) + assert launch.main() == 0 + spec = json.loads(capsys.readouterr().out) + assert spec["model"] == FP8_MODEL_ID + for key in ("baseline_command", "candidate_command", "validation_command"): + assert "--model-id" in spec[key] and FP8_MODEL_ID in spec[key] + assert "--fp8-kernel-revision" in spec[key] and "b" * 40 in spec[key] + monkeypatch.setattr(sys, "argv", argv[:-5] + ["--dry-run"]) + with pytest.raises(SystemExit): + launch.main() diff --git a/tests/test_qwen38_native_adapter.py b/tests/test_qwen38_native_adapter.py new file mode 100644 index 000000000..65d189f71 --- /dev/null +++ b/tests/test_qwen38_native_adapter.py @@ -0,0 +1,213 @@ +"""Native adapter import binding and role contracts; no model/GPU execution.""" + +import argparse +import json +import subprocess +import sys +import types +from pathlib import Path + +import pytest + +from examples.qwen38_text import native_trial, provenance, trial + + +def test_preloaded_foreign_engine_is_rejected_before_import(tmp_path, monkeypatch): + loader = tmp_path / "engine/nanovllm/qwen38/loader.py" + loader.parent.mkdir(parents=True) + loader.write_text("raise AssertionError('must not execute')\n") + foreign = tmp_path / "foreign.py" + foreign.write_text("# foreign installed package\n") + module = types.ModuleType("nanovllm") + module.__file__ = str(foreign) + monkeypatch.setitem(sys.modules, "nanovllm", module) + with pytest.raises(ValueError, match="escaped"): + provenance.import_native(tmp_path / "engine") + + +def test_import_origin_symlink_cannot_escape_checkout(tmp_path, monkeypatch): + root = tmp_path / "engine" + root.mkdir() + outside = tmp_path / "outside.py" + outside.write_text("# wrong source\n") + link = root / "module.py" + link.symlink_to(outside) + module = types.ModuleType("nanovllm") + module.__file__ = str(link) + monkeypatch.setitem(sys.modules, "nanovllm", module) + with pytest.raises(ValueError, match="escaped"): + provenance.import_records("nanovllm", root) + + +def test_foreign_namespace_search_path_is_rejected(tmp_path, monkeypatch): + root = tmp_path / "engine" + root.mkdir() + module = types.ModuleType("nanovllm.qwen38") + module.__path__ = [str(tmp_path / "foreign")] + monkeypatch.setitem(sys.modules, "nanovllm.qwen38", module) + with pytest.raises(ValueError, match="search path"): + provenance.import_records("nanovllm", root) + + +def test_explicit_engine_import_wins_over_unrelated_installed_package(tmp_path): + engine, foreign = tmp_path / "engine", tmp_path / "installed" + for root in (engine, foreign): + package = root / "nanovllm/qwen38" + package.mkdir(parents=True) + (root / "nanovllm/__init__.py").write_text("") + (package / "__init__.py").write_text("") + (package / "loader.py").write_text("def load_checkpoint(*args): return 'selected'\n") + (package / "runner.py").write_text("class RequestRunner: pass\n") + code = ( + "import json,sys; " + "from examples.qwen38_text.provenance import import_native,import_records; " + "sys.path.insert(0,sys.argv[2]); load,runner=import_native(sys.argv[1]); " + "print(json.dumps(import_records('nanovllm',sys.argv[1])))" + ) + completed = subprocess.run( + [sys.executable, "-c", code, str(engine), str(foreign)], + cwd=Path(__file__).resolve().parents[1], capture_output=True, text=True, check=True, + ) + records = json.loads(completed.stdout) + assert records["nanovllm.qwen38.loader"]["path"] == "nanovllm/qwen38/loader.py" + assert len(records["nanovllm.qwen38.runner"]["sha256"]) == 64 + + +def test_cached_reference_cannot_claim_native_baseline_scope(): + args = argparse.Namespace(qualification_scope="native_text_smoke", mode="cached") + with pytest.raises(ValueError, match="full-prefix"): + trial.run(args) + + +@pytest.fixture +def native_runtime(tmp_path, monkeypatch): + np = pytest.importorskip("numpy") + model_path = tmp_path / "model" + model_path.mkdir() + (model_path / "config.json").write_text(json.dumps({"text_config": {"vocab_size": 3}})) + cases = [{"id": "one", "prompt": "prompt", "continuation": "next", "max_steps": 1}] + monkeypatch.setattr(trial, "verify_manifest", lambda *_: "a" * 64) + monkeypatch.setattr(trial, "load_workload", lambda *_: (cases, "b" * 64)) + tokenizer = types.SimpleNamespace(encode=lambda text, **_: [1] if text == "prompt" else [2]) + cuda = types.SimpleNamespace(is_available=lambda: True, device_count=lambda: 1, + is_bf16_supported=lambda: True, + get_device_name=lambda _: "Synthetic GPU") + torch = types.SimpleNamespace(cuda=cuda, __version__="test", + version=types.SimpleNamespace(cuda="test")) + transformers = types.SimpleNamespace( + __version__="test", + AutoTokenizer=types.SimpleNamespace(from_pretrained=lambda *_a, **_kw: tokenizer), + ) # No Transformers model API: candidate execution cannot call one. + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setitem(sys.modules, "transformers", transformers) + calls = [] + model = type("NativeModel", (), {"__module__": "nanovllm.qwen38.model", + "eval": lambda self: self})() + + def load(path, *, device): + calls.append((path, device)) + return model, {"scope": "text_only", "dtype": "bfloat16", "loaded_parameters": 3} + + class Row: + def float(self): + return self + + def cpu(self): + return self + + def numpy(self): + return np.array([0.0, 1.0, 2.0], dtype=np.float32) + + class Runner: + active_request_ids = () + + def __init__(self, loaded, selected_tokenizer, **kwargs): + assert loaded is model and selected_tokenizer is tokenizer + assert kwargs["vocab_size"] == 3 and kwargs["max_context_tokens"] == 2 + + def trace_teacher_forced(self, prompt, continuation): + assert prompt == [1] and continuation == [2] + return [Row()] + + monkeypatch.setattr(native_trial, "import_native", lambda *_: (load, Runner)) + monkeypatch.setattr(native_trial, "capture_sources", lambda *_a, **_kw: { + "engine": {"imports": {"nanovllm.qwen38.model": {}}}, "control": {}, + }) + args = argparse.Namespace(model_path=model_path, manifest=tmp_path / "manifest", + workload=tmp_path / "workload", model_revision="a" * 40, + engine_repo=tmp_path / "engine", device_model="Synthetic GPU", + output=tmp_path / "run/candidate/trial-001/result.json") + return args, cuda, calls + + +def test_native_serialization_uses_native_loader_and_trace_only(native_runtime): + args, _cuda, calls = native_runtime + result = native_trial.run(args) + assert calls == [(args.model_path, "cuda:0")] + assert result["implementation"] == "nanovllm_qwen38" + assert result["qualification_scope"] == "native_text_smoke" + assert result["kernel_policy"] == native_trial.NATIVE_POLICY + assert result["cases"][0]["greedy_ids"] == [2] + assert result == json.loads(args.output.read_text()) + with pytest.raises(ValueError, match="overwrite"): + native_trial.run(args) + + +@pytest.mark.parametrize("count", [0, 2]) +def test_native_gpu_guard_prevents_loading(native_runtime, count): + args, cuda, calls = native_runtime + cuda.device_count = lambda: count + with pytest.raises(ValueError, match="exactly one"): + native_trial.run(args) + assert not calls and not args.output.exists() + + +def test_native_gpu_model_guard_prevents_loading(native_runtime): + args, _cuda, calls = native_runtime + args.device_model = "Different GPU" + with pytest.raises(ValueError, match="target"): + native_trial.run(args) + assert not calls + + +@pytest.mark.parametrize("actual_dtype", ["fp8", "bf16"]) +def test_native_fp8_trial_checks_actual_tensors_and_calls( + native_runtime, monkeypatch, actual_dtype, +): + from importlib import metadata + + from examples.qwen38_text.profiles import FP8_MODEL_ID, precision + + args, _cuda, _calls = native_runtime + args.model_id = FP8_MODEL_ID + torch = sys.modules["torch"] + torch.float8_e4m3fn, torch.float32 = "fp8", "fp32" + original_load, original_runner = native_trial.import_native(args.engine_repo) + model, _report = original_load(args.model_path, device="cuda:0") + module = type("FP8Linear", (), {"__module__": "nanovllm.qwen38.quant"})() + module.weight = types.SimpleNamespace(dtype=actual_dtype) + module.weight_scale_inv = types.SimpleNamespace(dtype="fp32") + module.native_fp8_calls = 0 + model.modules = lambda: [module] + report = {"scope": "text_only", "dtype": "fp8", "loaded_parameters": 3, + **precision(FP8_MODEL_ID), "execution": "native_fp8", "backend": "triton_block_fp8", + "fp8_linear_modules": 1} + + class Runner(original_runner): + def trace_teacher_forced(self, prompt, continuation): + rows = super().trace_teacher_forced(prompt, continuation) + module.native_fp8_calls += 1 + return rows + + monkeypatch.setattr(native_trial, "import_native", lambda *_: ( + lambda *_a, **_kw: (model, report), Runner, + )) + monkeypatch.setattr(metadata, "version", lambda _: "test") + if actual_dtype == "bf16": + with pytest.raises(ValueError, match="dequantized"): + native_trial.run(args) + assert not args.output.exists() and module.native_fp8_calls == 0 + else: + result = native_trial.run(args) + assert result["dtype"] == "fp8" and result["precision"] == precision(FP8_MODEL_ID) + assert result["fp8_execution"] == {"modules": 1, "total_calls": 1} diff --git a/tests/test_qwen38_service.py b/tests/test_qwen38_service.py new file mode 100644 index 000000000..7866ec41f --- /dev/null +++ b/tests/test_qwen38_service.py @@ -0,0 +1,703 @@ +"""CPU protocol regressions only; synthetic observations never qualify a model/GPU.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from examples.qwen38_text.profiles import BF16_MODEL_ID, precision +from examples.qwen38_text.service_client import Client, EvidenceGap, payload +from examples.qwen38_text.service_fault import install_fault +from examples.qwen38_text.service_protocol import ERROR_IDS, SCOPE, read_protocol +from examples.qwen38_text.service_trial import child_imports, normalize_evidence +from examples.qwen38_text.service_validate import decode_stream, digest, functional + + +@pytest.fixture +def protocol(): + return { + "schema_version": 1, + "qualification_scope": SCOPE, + "model_id": BF16_MODEL_ID, + "precision": precision(BF16_MODEL_ID), + "timeouts": dict.fromkeys( + ("startup", "request", "release", "poll", "shutdown", "socket"), 2 + ), + "limits": { + "max_context_tokens": 128, + "max_active_requests": 2, + "max_connections": 4, + "max_body_bytes": 4096, + "max_response_bytes": 65536, + }, + "cases": [ + {"id": "eos", "prompt": "hello", "max_tokens": 4}, + {"id": "unicode", "prompt": [2, 3], "max_tokens": 3}, + ], + "concurrent_case_ids": ["eos", "unicode"], + "eos_case_id": "eos", + "survivor_case_id": "unicode", + "lifecycle": {"id": "long", "prompt": "continue", "max_tokens": 100}, + "queued_case": {"id": "prefill", "prompt": "long prefill", "max_tokens": 100}, + } + + +@pytest.mark.parametrize( + "mutation", + [ + lambda p: p.update(concurrent_case_ids=["eos", "eos"]), + lambda p: p["cases"][1].update(prompt="hello"), + lambda p: p.update(eos_case_id=None), + lambda p: p["queued_case"].update(id="../bad"), + lambda p: p["timeouts"].pop("shutdown"), + lambda p: p["limits"].update(max_connections=2), + ], +) +def test_protocol_rejects_incomplete_or_masked_coverage(protocol, tmp_path, mutation): + mutation(protocol) + path = tmp_path / "protocol.json" + path.write_text(json.dumps(protocol)) + with pytest.raises(ValueError): + read_protocol(path, BF16_MODEL_ID) + + +def state(*active, closed=False, terminal=()): + return { + "active_request_ids": list(active), + "active_requests": len(active), + "capacity": 2, + "available": 2 - len(active), + "queued_requests": 0, + "runner_state_count": len(active), + "runner_state_bytes": 32 * len(active), + "closed": closed, + "terminal_requests": list(terminal), + } + + +def ending(name, reason, prefilled=True, count=1): + return { + "request_id": name, + "finish_reason": reason, + "prefilled": prefilled, + "token_count": count, + "active_peer_request_ids": ["survivor-disconnected"], + } + + +def output(reason="length"): + return {"text": "中🙂�", "token_ids": [4, 5, 6], "finish_reason": reason} + + +def event(name, text, tokens, reason=None): + return { + "id": name, + "request_id": name, + "model": BF16_MODEL_ID, + "choices": [{"index": 0, "text": text, "token_ids": tokens, "finish_reason": reason}], + } + + +def stream(name, result=None, case=None, disconnected=False): + result = result or output() + events = [event(name, result["text"], result["token_ids"])] + if not disconnected: + events.append(event(name, "", [], result["finish_reason"])) + return { + "request": payload( + BF16_MODEL_ID, case or {"prompt": "x", "max_tokens": 3}, name, stream=True + ), + "status": 200, + "content_type": "text/event-stream", + "events": events, + "done_marker": not disconnected, + "client_disconnect": disconnected, + "error": None, + } + + +def response(name, result=None): + result = result or output() + body = event(name, result["text"], result["token_ids"], result["finish_reason"]) + body["usage"] = { + "prompt_tokens": 2, + "completion_tokens": len(result["token_ids"]), + "total_tokens": 2 + len(result["token_ids"]), + } + return {"status": 200, "content_type": "application/json", "body": body} + + +def ack(name): + return { + "status": 200, + "body": { + "request_id": name, + "cancelled": True, + "active_request_ids_before_cancel": [name, "survivor-cancelled"], + }, + } + + +def server(closed=None): + return { + "ownership": { + "mode": "inferrsi_trial_group", + "server_pid": 12, + "helper_pid": 11, + "process_group_id": 11, + "helper_process_group_id": 11, + "server_start_new_session": False, + }, + "cleanup": { + "mode": "inferrsi_trial_group", + "process_group_id": 11, + "status": "completed", + "remaining_pids": [], + "kill_sent": False, + "term_sent": True, + "server_returncode": 0, + }, + "closed_state": closed or state(closed=True), + } + + +@pytest.fixture +def observations(protocol): + public = {"initial_state": state(), "closed_state": state(closed=True), "cases": []} + obs = { + "initial_state": state(), + "serial": [], + "concurrent": [], + "concurrent_release": [state()], + "overlap": [state("concurrent-0", "concurrent-1")], + "final_state": state(), + } + for index, case in enumerate(protocol["cases"]): + name = case["id"] + result = output("stop" if name == "eos" else "length") + public["cases"].append({"id": name, "outputs": [result], "state": state()}) + obs["serial"].append( + { + "id": name, + "request": payload(BF16_MODEL_ID, case, "json-" + name), + "response": response("json-" + name, result), + "stream": stream("sse-" + name, result, case), + "release": [state()], + } + ) + obs["concurrent"].append( + {"id": name, "stream": stream(f"concurrent-{index}", result, case)} + ) + for reason in ("cancelled", "disconnected"): + name = "lifecycle-" + reason + obs[reason] = { + "observed_first": True, + "before": state(name, "survivor-" + reason), + "overlap": [state(name, "survivor-" + reason)], + "ack": ack(name), + "stream": stream(name, output(reason), protocol["lifecycle"], reason == "disconnected"), + "survivor": stream("survivor-" + reason, case=protocol["cases"][1]), + "release": [state(terminal=[ending(name, reason)])], + "reuse": response("reuse-" + reason, output("stop")), + "reuse_release": [state()], + } + obs["capacity"] = { + "during": [state("capacity-0", "capacity-1")], + "overflow": {"status": 429}, + "duplicate": {"status": 400}, + "cancel_acknowledgements": [ack(f"capacity-{i}") for i in range(2)], + "streams": [stream(f"capacity-{i}", output("cancelled")) for i in range(2)], + "release": [state()], + } + queued = state("queue-blocker", "queue-target") + queued["queued_requests"] = 1 + obs["queued_cancel"] = { + "during": [queued], + "ack": ack("queue-target"), + "streams": [ + stream("queue-blocker", output("cancelled")), + stream("queue-target", {"text": "", "token_ids": [], "finish_reason": "cancelled"}), + ], + "release": [state(terminal=[ending("queue-target", "cancelled", False, 0)])], + } + obs["errors"] = [ + {"id": name, "response": {"status": 400}, "release": [state()]} for name in ERROR_IDS + ] + http = { + "http": obs, + "shutdown_first_token": True, + "shutdown_before": state("shutdown-active"), + "shutdown_stream": stream("shutdown-active", output("shutdown")), + "server": server(state(closed=True, terminal=[ending("shutdown-active", "shutdown")])), + "restart": { + **server(), + "response": response("restart-check", output("stop")), + "state": state(), + }, + } + fault_stream = stream( + "injected-fault", {"text": "中", "token_ids": [4], "finish_reason": "error"} + ) + fault_stream["events"].insert(1, {"error": {"type": "generation_error"}}) + fault_state = state() + fault_state["fault_injection"] = { + "kind": "harness_injected_runtime_error", + "request_id": "injected-fault", + "triggered": True, + "successful_target_calls": 1, + "source": "examples/qwen38_text/service_fault.py", + "source_sha256": digest( + Path(__file__).parents[1] / "examples/qwen38_text/service_fault.py" + ), + } + http["fault"] = { + **server(), + "warmup": response("fault-warmup", output("stop")), + "stream": fault_stream, + "release": [state(terminal=[ending("injected-fault", "error")])], + "reuse": response("fault-reuse", output("stop")), + "state": fault_state, + } + return public, http + + +def test_complete_synthetic_functional_contract(protocol, observations): + failures = [] + functional(*observations, protocol, failures) + assert failures == [] + + +@pytest.mark.parametrize( + "mutation", + [ + lambda p, h: h["http"]["cancelled"]["survivor"]["events"][-1]["choices"][0].update( + finish_reason="cancelled" + ), + lambda p, h: h["http"]["serial"][0]["stream"]["events"][0]["choices"][0].update( + text="corrupt" + ), + lambda p, h: h["http"]["errors"][0]["response"].update(status=200), + lambda p, h: h["http"]["cancelled"]["release"][-1].update(runner_state_bytes=32), + lambda p, h: h["shutdown_stream"]["events"][-1]["choices"][0].update(finish_reason="stop"), + ], +) +def test_complete_behavior_regression_fails(protocol, observations, mutation): + mutation(*observations) + failures = [] + functional(*observations, protocol, failures) + assert failures + + +@pytest.mark.parametrize( + "mutation", + [ + lambda p, h: h["http"].update(overlap=[]), + lambda p, h: h["http"]["cancelled"].update(overlap=[]), + lambda p, h: p["cases"][0]["outputs"][0].update(finish_reason="length"), + lambda p, h: h["shutdown_stream"].update(done_marker=False), + lambda p, h: h["fault"]["state"]["fault_injection"].update(triggered=False), + lambda p, h: h["http"]["queued_cancel"]["release"][0]["terminal_requests"][0].update( + prefilled=True + ), + ], +) +def test_missing_observation_is_gap(protocol, observations, mutation): + mutation(*observations) + with pytest.raises(EvidenceGap): + functional(*observations, protocol, []) + + +@pytest.fixture +def socket_server(): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + name = body["request_id"] + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + events = [ + event(name, "中", [4]), + event(name, "🙂�", [5]), + event(name, "", [], "stop"), + "[DONE]", + ] + if name == "trailing": + events.append("[DONE]") + if name == "incomplete": + events = events[:-2] + for item in events: + self.wfile.write( + ( + "data: " + + (item if isinstance(item, str) else json.dumps(item, ensure_ascii=False)) + + "\n\n" + ).encode() + ) + self.wfile.flush() + self.close_connection = True + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield Client("127.0.0.1", server.server_port, 2, 65536) + server.shutdown() + server.server_close() + thread.join() + + +def test_real_socket_unicode_and_terminal_framing(socket_server): + record = socket_server.stream({"request_id": "valid"}).finish() + result = decode_stream(record, "valid", BF16_MODEL_ID) + assert result == {"text": "中🙂�", "token_ids": [4, 5], "finish_reason": "stop"} + + +@pytest.mark.parametrize("name", ["trailing", "incomplete"]) +def test_real_socket_missing_or_duplicate_done_rejected(socket_server, name): + record = socket_server.stream({"request_id": name}).finish() + with pytest.raises(EvidenceGap): + decode_stream(record, name, BF16_MODEL_ID) + + +def test_fault_hook_is_request_specific_once_and_reusable(): + calls = [] + target = object() + model = SimpleNamespace(forward=lambda *a, **k: calls.append(k["state"]) or "logits") + engine = SimpleNamespace( + runner=SimpleNamespace(model=model, _requests={}), provenance=lambda: {} + ) + install_fault(engine) + assert model.forward(state=object()) == "logits" + engine.runner._requests["injected-fault"] = SimpleNamespace(state=target) + assert model.forward(state=target) == "logits" + with pytest.raises(RuntimeError, match="Harness-injected"): + model.forward(state=target) + engine.runner._requests.clear() + assert model.forward(state=object()) == "logits" + assert len(calls) == 3 and engine.provenance()["fault_injection"]["triggered"] + + +def test_child_origin_validation_precedes_redaction(tmp_path): + root = tmp_path / "engine" + root.mkdir() + path = root / "module.py" + path.write_text("# fixture") + raw = { + "module_origins": {"nanovllm": {"path": str(path), "sha256": digest(path)}}, + "command": ["python", str(tmp_path / "private-model")], + "address": ["127.0.0.1", 9999], + } + normalized = normalize_evidence(raw, root) + assert normalized["module_origins"]["nanovllm"]["path"] == "module.py" + assert str(tmp_path) not in json.dumps(normalized) + path.write_text("# changed") + with pytest.raises(EvidenceGap): + child_imports(raw, root) + + +@pytest.fixture +def bundle(tmp_path, protocol, observations): + import copy + + def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + run, workload, manifest = ( + tmp_path / name for name in ("run", "workload.json", "manifest.json") + ) + write(workload, protocol) + write(manifest, {"model_id": BF16_MODEL_ID, "revision": "a" * 40}) + sources = {} + for label, namespace, modules in ( + ("control", "inferrsi", [""]), + ( + "engine", + "nanovllm", + [ + "", + ".qwen38.engine", + ".qwen38.model", + ".qwen38.runner", + ".qwen38.loader", + ".qwen38.server", + ], + ), + ): + imports = {} + for suffix in modules: + name = namespace + suffix + relative = name.replace(".", "/") + ("/__init__.py" if not suffix else ".py") + path = tmp_path / label / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# synthetic source\n") + imports[name] = {"path": relative, "sha256": digest(path)} + sources[label] = { + "head": ("a" if label == "control" else "b") * 40, + "snapshot_hash": ("c" if label == "control" else "d") * 64, + "imports": imports, + } + locked = { + side: {"available": True, **sources[label]} + for side, label in (("baseline", "control"), ("candidate", "engine")) + } + spec = { + "objective": "correctness", + "comparison_scope": SCOPE, + "model": BF16_MODEL_ID, + "model_revision": "a" * 40, + "workload_id": digest(workload), + "repetitions": 1, + "repo": str(tmp_path / "control"), + "candidate_repo": str(tmp_path / "engine"), + } + write(run / "run.lock.json", spec) + write(run / "repositories.lock.json", locked) + write(run / "environment.json", {"requested_device_model": "Synthetic GPU"}) + info = { + "model_id": BF16_MODEL_ID, + "device_model": "Synthetic GPU", + "device_type": "cuda", + "device_count": 1, + "precision": "bfloat16", + "loader_report": {"scope": "text_only", "dtype": "bfloat16", "loaded_parameters": 1}, + "versions": {"torch": "synthetic", "transformers": "synthetic"}, + "module_origins": sources["engine"]["imports"], + } + for side, mode, obs in zip(("baseline", "candidate"), ("public", "http"), observations): + obs["model_info"] = copy.deepcopy(info) + if side == "candidate": + for key in ("restart", "fault"): + obs[key]["state"].update(copy.deepcopy(info)) + root = run / side / "trial-001" + write( + root / "result.json", + { + "schema_version": 1, + "qualification_scope": SCOPE, + "mode": mode, + "model_id": BF16_MODEL_ID, + "precision": precision(BF16_MODEL_ID), + "complete": True, + "model_revision": "a" * 40, + "manifest_sha256": digest(manifest), + "workload_sha256": digest(workload), + "source_provenance": sources, + "observations": obs, + }, + ) + write( + root / "execution.json", + { + "status": "completed", + "returncode": 0, + "cwd": spec["repo" if side == "baseline" else "candidate_repo"], + }, + ) + write(root / "source.json", {"side": side, "before": locked[side], "after": locked[side]}) + return run, workload, manifest + + +@pytest.mark.parametrize( + "kind,expected", [("valid", 0), ("tokens", 1), ("missing", 2), ("null", 2), ("source", 2)] +) +def test_validator_subprocess_written_verdict(bundle, kind, expected): + import subprocess + import sys + + run, workload, manifest = bundle + path = run / "candidate/trial-001/result.json" + data = json.loads(path.read_text()) + if kind == "tokens": + data["observations"]["http"]["serial"][0]["stream"]["events"][0]["choices"][0]["text"] = ( + "bad" + ) + elif kind == "missing": + data["observations"]["http"]["overlap"] = [] + elif kind == "null": + data["source_provenance"] = None + elif kind == "source": + data["source_provenance"]["engine"].pop("head") + path.write_text(json.dumps(data)) + output_path = run / "validation.json" + script = Path(__file__).parents[1] / "examples/qwen38_text/service_validate.py" + result = subprocess.run( + [ + sys.executable, + str(script), + "--run-dir", + str(run), + "--workload", + str(workload), + "--manifest", + str(manifest), + "--output", + str(output_path), + ], + capture_output=True, + text=True, + ) + assert result.returncode == expected, result.stderr + result.stdout + result = json.loads(output_path.read_text()) + assert result["verdict"] == {0: "PASS", 1: "FAIL", 2: "INCONCLUSIVE"}[expected] + assert str(run.parent) not in output_path.read_text() + + +def test_synthetic_service_uses_common_correctness_loop(bundle, tmp_path): + import sys + + from inferrsi.harness.runner import run_experiment + from inferrsi.runspec import RunSpec + + fixture, workload, manifest = bundle + frozen = json.loads((fixture / "run.lock.json").read_text()) + locked = json.loads((fixture / "repositories.lock.json").read_text()) + + def command(side): + return ( + sys.executable, + "-c", + "import shutil,sys;shutil.copyfile(sys.argv[1],sys.argv[2])", + str(fixture / side / "trial-001/result.json"), + "{result}", + ) + + script = Path(__file__).parents[1] / "examples/qwen38_text/service_validate.py" + spec = RunSpec( + repo=frozen["repo"], + candidate_repo=frozen["candidate_repo"], + objective="correctness", + comparison_scope=SCOPE, + repetitions=1, + max_minutes=1, + model=BF16_MODEL_ID, + model_revision="a" * 40, + workload_id=digest(workload), + baseline_command=command("baseline"), + candidate_command=command("candidate"), + output_dir=str(tmp_path / "outputs"), + validation_command=( + sys.executable, + str(script), + "--run-dir", + "{run_dir}", + "--workload", + str(workload), + "--manifest", + str(manifest), + "--output", + "{result}", + ), + ) + run = run_experiment( + spec, + {"requested_device_model": "Synthetic GPU"}, + snapshot=lambda path, *_: locked[ + "baseline" if str(path) == frozen["repo"] else "candidate" + ], + ) + result = json.loads((run / "evaluation.json").read_text()) + assert result["verdict"] == "PASS", result + assert result["improvement_pct"] is None + + +@pytest.fixture +def lifecycle_socket(): + cancelled, disconnected = threading.Event(), threading.Event() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + self.send_response(200) + self.send_header("Connection", "close") + if self.path.startswith("/v1/cancel/"): + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(ack("held")["body"]).encode()) + self.wfile.flush() + cancelled.set() + else: + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + self.wfile.write( + ("data: " + json.dumps(event("held", "中", [4])) + "\n\n").encode() + ) + self.wfile.flush() + import select + import socket + + while not cancelled.wait(0.005): + if select.select([self.connection], [], [], 0)[0]: + if not self.connection.recv(1, socket.MSG_PEEK): + disconnected.set() + break + else: + self.wfile.write( + ( + "data: " + + json.dumps(event("held", "", [], "cancelled")) + + "\n\ndata: [DONE]\n\n" + ).encode() + ) + self.wfile.flush() + self.close_connection = True + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield Client("127.0.0.1", server.server_port, 2, 65536), disconnected + cancelled.set() + server.shutdown() + server.server_close() + thread.join() + + +def test_real_socket_cancel_control_and_terminal(lifecycle_socket): + client, _ = lifecycle_socket + held = client.stream({"request_id": "held"}) + assert held.wait_first() + assert client.request("POST", "/v1/cancel/held", {})["body"]["cancelled"] + record = held.finish() + assert decode_stream(record, "held", BF16_MODEL_ID)["finish_reason"] == "cancelled" + + +def test_real_socket_disconnect_reaches_peer(lifecycle_socket): + client, disconnected = lifecycle_socket + held = client.stream({"request_id": "held"}) + assert held.wait_first() + held.disconnect() + record = held.finish() + assert disconnected.wait(2) + assert decode_stream(record, "held", BF16_MODEL_ID, disconnected=True)["token_ids"] == [4] + + +@pytest.mark.parametrize("reason", ["cancelled", "disconnected"]) +def test_survivor_finished_before_lifecycle_linearization_is_gap(protocol, observations, reason): + observed = observations[1]["http"][reason] + if reason == "cancelled": + observed["ack"]["body"]["active_request_ids_before_cancel"] = ["lifecycle-cancelled"] + else: + observed["release"][0]["terminal_requests"][0]["active_peer_request_ids"] = [] + with pytest.raises(EvidenceGap, match="Survivor was not active"): + functional(*observations, protocol, []) + + +def test_missing_runtime_length_coverage_is_gap(protocol, observations): + for case in observations[0]["cases"]: + case["outputs"][0]["finish_reason"] = "stop" + with pytest.raises(EvidenceGap, match="max-token"): + functional(*observations, protocol, []) diff --git a/tests/test_qwen38_trial.py b/tests/test_qwen38_trial.py new file mode 100644 index 000000000..d5a124b82 --- /dev/null +++ b/tests/test_qwen38_trial.py @@ -0,0 +1,401 @@ +"""CPU contract tests; these do not claim model or GPU qualification.""" + +import hashlib +import json +from contextlib import nullcontext +from types import SimpleNamespace + +import pytest + +from examples.qwen38_text import trial + +REVISION = "a" * 40 + + +def write_json(path, value): + path.write_text(json.dumps(value)) + + +@pytest.fixture +def model_bundle(tmp_path): + model = tmp_path / "model" + model.mkdir() + config = { + "model_type": "qwen3_5", + "architectures": ["Qwen3_5ForConditionalGeneration"], + "text_config": {"model_type": "qwen3_5_text", "dtype": "bfloat16", "vocab_size": 2}, + } + write_json(model / "config.json", config) + for name in ("tokenizer_config.json", "tokenizer.json"): + write_json(model / name, {}) + write_json( + model / "model.safetensors.index.json", {"weight_map": {"weight": "part.safetensors"}} + ) + (model / "part.safetensors").write_bytes(b"synthetic weight bytes") + manifest = tmp_path / "manifest.json" + records = [ + {"path": item.name, "sha256": trial.sha256_file(item), "size": item.stat().st_size} + for item in sorted(model.iterdir()) + ] + write_json(manifest, {"model_id": trial.MODEL_ID, "revision": REVISION, "files": records}) + return model, manifest + + +def test_integrity_reads_bytes_even_when_size_unchanged(model_bundle): + model, manifest = model_bundle + assert trial.verify_manifest(model, manifest, REVISION) == trial.sha256_file(manifest) + weight = model / "part.safetensors" + original = weight.read_bytes() + weight.write_bytes(b"X" + original[1:]) + with pytest.raises(ValueError, match="integrity mismatch"): + trial.verify_manifest(model, manifest, REVISION) + + +def test_manifest_must_cover_index_and_tokenizer(model_bundle): + model, manifest = model_bundle + value = json.loads(manifest.read_text()) + value["files"] = [row for row in value["files"] if row["path"] != "part.safetensors"] + write_json(manifest, value) + with pytest.raises(ValueError, match="indexed"): + trial.verify_manifest(model, manifest, REVISION) + value["files"] = [row for row in value["files"] if row["path"] != "tokenizer.json"] + write_json(manifest, value) + with pytest.raises(ValueError, match="required"): + trial.verify_manifest(model, manifest, REVISION) + + +def test_extra_tokenizer_metadata_requires_hash(model_bundle): + model, manifest = model_bundle + write_json(model / "added_tokens.json", {}) + with pytest.raises(ValueError, match="metadata"): + trial.verify_manifest(model, manifest, REVISION) + + +def test_symlink_and_traversal_cannot_escape_model(tmp_path): + root = tmp_path / "model" + root.mkdir() + outside = tmp_path / "outside" + outside.write_text("synthetic") + (root / "link").symlink_to(outside) + for name in ("../outside", "/outside", "link", "a/../link"): + with pytest.raises(ValueError): + trial.model_file(root, name) + + +def test_wrong_identity_and_quantization_rejected(model_bundle): + model, manifest = model_bundle + with pytest.raises(ValueError, match="identity"): + trial.verify_manifest(model, manifest, "b" * 40) + config_path = model / "config.json" + config = json.loads(config_path.read_text()) + config["quantization_config"] = {"quant_method": "fp8"} + write_json(config_path, config) + value = json.loads(manifest.read_text()) + for row in value["files"]: + if row["path"] == "config.json": + row.update(size=config_path.stat().st_size, sha256=trial.sha256_file(config_path)) + write_json(manifest, value) + with pytest.raises(ValueError, match="unquantized"): + trial.verify_manifest(model, manifest, REVISION) + + +def test_workload_validation_and_raw_bytes_hash(tmp_path): + path = tmp_path / "workload.json" + case = {"id": "plain", "prompt": "P", "continuation": "ABC", "max_steps": 2} + value = {"schema_version": 1, "model_id": trial.MODEL_ID, "cases": [case]} + write_json(path, value) + cases, digest = trial.load_workload(path) + assert cases == [case] + assert digest == hashlib.sha256(path.read_bytes()).hexdigest() + value["cases"].append(case) + write_json(path, value) + with pytest.raises(ValueError, match="duplicate"): + trial.load_workload(path) + value["cases"] = [{**case, "max_steps": True}] + write_json(path, value) + with pytest.raises(ValueError): + trial.load_workload(path) + + +def test_tokenization_never_adds_special_tokens_and_requires_enough(): + def encode(text, *, add_special_tokens): + assert add_special_tokens is False + return list(text.encode()) + + tokenizer = SimpleNamespace(encode=encode) + case = {"prompt": "P", "continuation": "ABC", "max_steps": 2} + assert trial.tokenize_case(tokenizer, case) == ([80], [65, 66]) + with pytest.raises(ValueError, match="insufficient"): + trial.tokenize_case(tokenizer, {**case, "max_steps": 4}) + + +class FakeRow: + def detach(self): + return self + + def float(self): + return self + + def cpu(self): + return self + + def numpy(self): + return pytest.importorskip("numpy").array([1.0, 3.0], dtype="float32") + + +class FakeLogits: + def __getitem__(self, key): + assert key == (0, -1) + return FakeRow() + + +@pytest.mark.parametrize( + "mode,expected", + [ + ("full", [[10, 11], [10, 11, 20], [10, 11, 20, 21]]), + ("cached", [[10, 11], [20], [21]]), + ], +) +def test_teacher_forcing_prefix_and_cache_transition(mode, expected): + calls = [] + caches = [] + + def model(**kwargs): + calls.append(kwargs["input_ids"][0]) + assert kwargs["use_cache"] == (mode == "cached") + assert kwargs["past_key_values"] == (caches[-1] if caches and mode == "cached" else None) + cache = object() + caches.append(cache) + return SimpleNamespace(logits=FakeLogits(), past_key_values=cache) + + def tensor(value, *, dtype, device): + assert dtype == "long" and device == "cuda:0" + return value + + torch = SimpleNamespace(tensor=tensor, long="long", inference_mode=nullcontext) + rows = trial.collect_logits(model, torch, [10, 11], [20, 21, 22], mode) + assert calls == expected + assert len(rows) == 3 + assert all(row.dtype == "float32" for row in rows) + # A second request starts from empty recurrent/KV state even in cached mode. + caches.clear() + trial.collect_logits(model, torch, [30], [40], mode) + assert calls[-1] == [30] + + +def test_failure_message_does_not_disclose_runtime_paths(monkeypatch, capsys): + def fail(_args): + raise OSError("/private/example-user/model: permission denied") + + monkeypatch.setattr(trial, "run", fail) + result = trial.main( + [ + "--model-path", + "/private/example-user/model", + "--model-revision", + REVISION, + "--device-model", + "TEST GPU", + "--manifest", + "manifest", + "--workload", + "workload", + "--mode", + "full", + "--output", + "result", + ] + ) + assert result == 1 + output = capsys.readouterr() + assert "OSError" in output.err and "example-user" not in output.err + assert json.loads(output.err)["stage"] == "setup" + + +@pytest.mark.parametrize("stage,expected", [("checkpoint_load", "checkpoint_load"), + ("fp8_inspection", "fp8_inspection"), + ("/private/example-user", "setup")]) +def test_failure_stage_is_bounded_and_safe(monkeypatch, capsys, stage, expected): + def fail(args): + args._trial_stage = stage + raise ValueError("/private/example-user: injected library detail") + monkeypatch.setattr(trial, "run", fail) + assert trial.main(["--model-path", "model", "--model-revision", REVISION, + "--device-model", "TEST GPU", "--manifest", "manifest", + "--workload", "workload", "--mode", "full", "--output", "result"]) == 1 + stderr = capsys.readouterr().err + assert json.loads(stderr) == {"status": "incomplete", "stage": expected, + "error_type": "ValueError", "qualification_verdict": None} + assert "example-user" not in stderr + + +@pytest.fixture +def fake_runtime(monkeypatch): + """Dependency doubles exercise serialization and guards, not GPU inference.""" + import sys + + np = pytest.importorskip("numpy") + calls = [] + + def forward(**kwargs): + calls.append(kwargs) + return SimpleNamespace(logits=FakeLogits(), past_key_values=object()) + + model = type( + "FakeModel", + (), + {"eval": lambda self: self, "__call__": lambda self, **kwargs: forward(**kwargs)}, + )() + + def load_model(_path, **kwargs): + assert kwargs["dtype"] == "bfloat16" + assert kwargs["device_map"] == {"": "cuda:0"} + assert kwargs["local_files_only"] and not kwargs["trust_remote_code"] + assert kwargs["use_kernels"] is False + return model, {"missing_keys": [], "mismatched_keys": [], "error_msgs": []} + + tokenizer = SimpleNamespace(encode=lambda text, **_kwargs: list(text.encode())) + cuda = SimpleNamespace( + is_available=lambda: True, + device_count=lambda: 1, + is_bf16_supported=lambda: True, + get_device_name=lambda _index: "TEST GPU", + ) + torch = SimpleNamespace( + cuda=cuda, + long="long", + bfloat16="bfloat16", + __version__="test", + version=SimpleNamespace(cuda="test"), + inference_mode=nullcontext, + tensor=lambda value, **_kwargs: value, + ) + transformers = SimpleNamespace( + __version__="test", + AutoTokenizer=SimpleNamespace(from_pretrained=lambda *_a, **_k: tokenizer), + Qwen3_5ForConditionalGeneration=SimpleNamespace(from_pretrained=load_model), + ) + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setitem(sys.modules, "transformers", transformers) + return cuda, calls, np + + +def trial_args(model_bundle, tmp_path): + model, manifest = model_bundle + workload = tmp_path / "workload.json" + write_json( + workload, + { + "schema_version": 1, + "model_id": trial.MODEL_ID, + "cases": [ + {"id": "simple", "prompt": "P", "continuation": "ABC", "max_steps": 2}, + ], + }, + ) + return SimpleNamespace( + model_path=model, + manifest=manifest, + model_revision=REVISION, + device_model="TEST GPU", + workload=workload, + mode="cached", + output=tmp_path / "result.json", + ) + + +def test_result_artifacts_are_linked_and_labeled_self_check(model_bundle, tmp_path, fake_runtime): + _cuda, calls, np = fake_runtime + args = trial_args(model_bundle, tmp_path) + result = trial.run(args) + assert result["qualification_scope"] == "reference_self_check" + assert result["implementation"] == "transformers_cached" + assert result["tokenization"] == "raw_text_no_special_tokens" + assert result["vocab_size"] == 2 and result["device_model"] == "TEST GPU" + assert result["workload_sha256"] == trial.sha256_file(args.workload) + assert result == json.loads(args.output.read_text()) + assert str(tmp_path) not in args.output.read_text() + case = result["cases"][0] + assert case["continuation_ids"] == [65, 66] + assert case["greedy_ids"] == [1, 1] + logits_path = args.output.parent / case["logits_file"] + assert trial.sha256_file(logits_path) == case["logits_sha256"] + with np.load(logits_path, allow_pickle=False) as archive: + assert archive["logits"].shape == (2, 2) + assert archive["logits"].dtype == np.float32 + assert len(calls) == 2 + with pytest.raises(ValueError, match="overwrite"): + trial.run(args) + + +def test_unexpected_text_scale_is_rejected_at_loading_info(model_bundle, tmp_path, fake_runtime): + import sys + + loader = sys.modules["transformers"].Qwen3_5ForConditionalGeneration + original = loader.from_pretrained + def unexpected(*args, **kwargs): + model, loading = original(*args, **kwargs) + loading["unexpected_keys"] = [ + "model.language_model.layers.0.mlp.gate_proj.weight_scale_inv"] + return model, loading + loader.from_pretrained = unexpected + args = trial_args(model_bundle, tmp_path) + with pytest.raises(ValueError, match="unexpected text"): + trial.run(args) + assert args._trial_stage == "loading_info" + assert not args.output.exists() + + +@pytest.mark.parametrize("count", [0, 2]) +def test_gpu_count_guard_before_model_execution(model_bundle, tmp_path, fake_runtime, count): + cuda, calls, _np = fake_runtime + cuda.device_count = lambda: count + args = trial_args(model_bundle, tmp_path) + with pytest.raises(ValueError, match="exactly one"): + trial.run(args) + assert not args.output.exists() and not calls + + +def test_gpu_model_mismatch_rejected(model_bundle, tmp_path, fake_runtime): + _cuda, calls, _np = fake_runtime + args = trial_args(model_bundle, tmp_path) + args.device_model = "OTHER GPU" + with pytest.raises(ValueError, match="frozen device model"): + trial.run(args) + assert not args.output.exists() and not calls + + +def test_optional_kernel_cannot_silently_change_reference(monkeypatch): + monkeypatch.setattr(trial.importlib.util, "find_spec", lambda name: object()) + with pytest.raises(ValueError, match="optional DeltaNet kernels"): + trial.kernel_policy() + + +def test_fp8_manifest_and_workload_are_separate(model_bundle, tmp_path): + from examples.qwen38_text.profiles import FP8_MODEL_ID, FP8_QUANTIZATION, precision + + model, manifest = model_bundle + config = json.loads((model / "config.json").read_text()) + config["quantization_config"] = FP8_QUANTIZATION + write_json(model / "config.json", config) + document = json.loads(manifest.read_text()) + document.update(model_id=FP8_MODEL_ID, precision=precision(FP8_MODEL_ID)) + for record in document["files"]: + path = model / record["path"] + record.update(size=path.stat().st_size, sha256=trial.sha256_file(path)) + write_json(manifest, document) + assert trial.verify_manifest(model, manifest, REVISION, FP8_MODEL_ID) + with pytest.raises(ValueError, match="identity"): + trial.verify_manifest(model, manifest, REVISION) + workload = tmp_path / "workload.json" + write_json(workload, {"schema_version": 1, "model_id": FP8_MODEL_ID, + "precision": precision(FP8_MODEL_ID), + "cases": [{"id": "one", "prompt": "abc", "continuation": "def", + "max_steps": 1}]}) + assert len(trial.load_workload(workload, FP8_MODEL_ID)[0]) == 1 + with pytest.raises(ValueError): + trial.load_workload(workload) + document.pop("precision") + write_json(manifest, document) + with pytest.raises(ValueError, match="precision"): + trial.verify_manifest(model, manifest, REVISION, FP8_MODEL_ID) diff --git a/tests/test_qwen38_validation.py b/tests/test_qwen38_validation.py new file mode 100644 index 000000000..bc922f4dc --- /dev/null +++ b/tests/test_qwen38_validation.py @@ -0,0 +1,561 @@ +"""Synthetic evidence tests; these never qualify Qwen or a GPU.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from examples.qwen38_text.validate import digest, validate + +np = pytest.importorskip("numpy") + + +@pytest.fixture +def trace_bundle(tmp_path): + def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + workload = tmp_path / "workload.json" + manifest = tmp_path / "manifest.json" + run = tmp_path / "run" + cases = [{"id": "first", "max_steps": 2}, + {"id": "replay", "max_steps": 2, "replay_of": "first"}] + write(workload, {"model_id": "Qwen/Qwen3.8-27B", "cases": cases}) + write(manifest, {"model_id": "Qwen/Qwen3.8-27B", "revision": "a" * 40}) + write(run / "run.lock.json", {"objective": "correctness", "repetitions": 1, + "model_revision": "a" * 40, "workload_id": digest(workload)}) + write(run / "environment.json", {"requested_device_model": "Synthetic GPU"}) + for side, implementation in (("baseline", "transformers_full"), + ("candidate", "transformers_cached")): + root = run / side / "trial-001" + root.mkdir(parents=True) + outputs = [] + for case in cases: + path = root / (case["id"] + ".npz") + np.savez(path, logits=np.array([[1., 2., 3.], [3., 1., 2.]], dtype=np.float32)) + outputs.append({"id": case["id"], "input_ids": [1, 2], + "continuation_ids": [2, 0], "greedy_ids": [2, 0], + "logits_file": path.name, "logits_sha256": digest(path)}) + write(root / "result.json", { + "schema_version": 1, "qualification_scope": "reference_self_check", + "implementation": implementation, "model_id": "Qwen/Qwen3.8-27B", + "model_revision": "a" * 40, "manifest_sha256": digest(manifest), + "workload_sha256": digest(workload), "dtype": "bfloat16", "device_type": "cuda", + "device_count": 1, "tokenization": "raw_text_no_special_tokens", "vocab_size": 3, + "kernel_policy": {"hub_kernels": False, "full_attention": "eager", + "optional_modules": {"fla": False, "causal_conv1d": False}}, + "device_model": "Synthetic GPU", "versions": {"torch": "test", "transformers": "test"}, + "cases": outputs}) + write(root / "execution.json", {"status": "completed", "returncode": 0}) + return run, workload, manifest + + +def check(bundle): + return validate(*bundle, max_nrmse=0.01, min_top1=0.99, rms_floor=1e-8) + + +def mutate_case(bundle, callback): + path = bundle[0] / "candidate/trial-001/result.json" + result = json.loads(path.read_text()) + callback(result) + path.write_text(json.dumps(result)) + + +def test_complete_synthetic_evidence_is_scope_limited(trace_bundle): + result = check(trace_bundle) + assert result["verdict"] == "PASS" + assert result["qualification_scope"] == "reference_self_check" + assert "No dedicated engine validated" in result["limitations"] + + +@pytest.mark.parametrize("mutation", [ + lambda r: r["cases"].pop(), + lambda r: r["cases"].append(r["cases"][0]), + lambda r: r.update(model_revision="b" * 40), + lambda r: r.update(implementation="nano_vllm"), + lambda r: r["cases"][0].update(input_ids=[42]), + lambda r: r["cases"][0].update(logits_file="../../outside.npz"), + lambda r: r["cases"][0].update(greedy_ids=[0, 0]), + lambda r: r.update(dtype="float16"), + lambda r: r["kernel_policy"]["optional_modules"].update(fla=True), +]) +def test_missing_mismatched_and_wrong_scope_evidence_rejected(trace_bundle, mutation): + mutate_case(trace_bundle, mutation) + with pytest.raises(ValueError): + check(trace_bundle) + + +def test_tampered_array_rejected(trace_bundle): + path = trace_bundle[0] / "candidate/trial-001/first.npz" + np.savez(path, logits=np.zeros((2, 3))) + with pytest.raises(ValueError, match="hash"): + check(trace_bundle) + + +def test_complete_numerical_regression_is_fail(trace_bundle): + root = trace_bundle[0] / "candidate/trial-001" + def alter(result): + for case in result["cases"]: + path = root / case["logits_file"] + np.savez(path, logits=np.array([[3., 2., 1.], [1., 2., 3.]], dtype=np.float32)) + case.update(logits_sha256=digest(path), greedy_ids=[0, 2]) + mutate_case(trace_bundle, alter) + assert check(trace_bundle)["verdict"] == "FAIL" + + +def test_nonfinite_is_incomplete_even_with_matching_hash(trace_bundle): + root = trace_bundle[0] / "candidate/trial-001" + def alter(result): + path = root / result["cases"][0]["logits_file"] + np.savez(path, logits=np.full((2, 3), np.nan, dtype=np.float32)) + result["cases"][0]["logits_sha256"] = digest(path) + mutate_case(trace_bundle, alter) + with pytest.raises(ValueError, match="nonfinite"): + check(trace_bundle) + + +def test_replay_change_rejected_even_with_loose_numerical_gates(trace_bundle): + root = trace_bundle[0] / "candidate/trial-001" + def alter(result): + path = root / result["cases"][1]["logits_file"] + np.savez(path, logits=np.array([[1., 2., 3.01], [3., 1., 2.]], dtype=np.float32)) + result["cases"][1]["logits_sha256"] = digest(path) + mutate_case(trace_bundle, alter) + result = check(trace_bundle) + assert result["verdict"] == "FAIL" + assert any("reset/replay" in reason for reason in result["reasons"]) + + +def test_synthetic_traces_complete_real_subprocess_loop(trace_bundle, tmp_path): + from inferrsi.harness.runner import run_experiment + from inferrsi.runspec import RunSpec + + fixture, workload, manifest = trace_bundle + repo = tmp_path / "source" + repo.mkdir() + emit = ( + "import pathlib,shutil,sys; " + "src=pathlib.Path(sys.argv[1]); dst=pathlib.Path(sys.argv[2]); " + "shutil.copyfile(src/'result.json',dst); " + "[shutil.copyfile(p,dst.parent/p.name) for p in src.glob('*.npz')]" + ) + def command(side): + return (sys.executable, "-c", emit, str(fixture / side / "trial-001"), "{result}") + + validator = Path(__file__).resolve().parents[1] / "examples/qwen38_text/validate.py" + spec = RunSpec( + repo=str(repo), objective="correctness", repetitions=1, max_minutes=1, + model="Qwen/Qwen3.8-27B", model_revision="a" * 40, workload_id=digest(workload), + baseline_command=command("baseline"), candidate_command=command("candidate"), + validation_command=(sys.executable, str(validator), "--run-dir", "{run_dir}", + "--workload", str(workload), "--manifest", str(manifest), + "--max-nrmse", "0.01", "--min-top1", "0.99", + "--rms-floor", "1e-8", "--output", "{result}"), + output_dir=str(tmp_path / "outputs"), + ) + result_dir = run_experiment( + spec, {"requested_device_model": "Synthetic GPU"}, + snapshot=lambda *_: {"available": True, "snapshot_hash": "synthetic-source"}, + ) + evaluation = json.loads((result_dir / "evaluation.json").read_text()) + assert evaluation["verdict"] == "PASS" + assert evaluation["improvement_pct"] is None + assert (result_dir / "report.html").exists() + quality = json.loads((result_dir / "validation/result.json").read_text()) + assert quality["qualification_scope"] == "reference_self_check" + + +@pytest.fixture +def native_bundle(trace_bundle, tmp_path): + from examples.qwen38_text.validate import NATIVE_POLICY + + run, workload, manifest = trace_bundle + control, engine = tmp_path / "control", tmp_path / "engine" + sources = {} + for label, root, modules in ( + ("control", control, {"inferrsi": "src/inferrsi/__init__.py", + "inferrsi.infra.targets.git": "src/inferrsi/infra/targets/git.py"}), + ("engine", engine, {"nanovllm": "nanovllm/__init__.py", + **{f"nanovllm.qwen38.{name}": f"nanovllm/qwen38/{name}.py" + for name in ("loader", "runner", "model")}}), + ): + imports = {} + for name, relative in modules.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# synthetic source fixture\n") + imports[name] = {"path": relative, "sha256": digest(path)} + sources[label] = {"head": ("a" if label == "control" else "b") * 40, + "snapshot_hash": ("c" if label == "control" else "d") * 64, + "imports": imports} + spec = json.loads((run / "run.lock.json").read_text()) + spec.update(repo=str(control), candidate_repo=str(engine), comparison_scope="native_text_smoke") + (run / "run.lock.json").write_text(json.dumps(spec)) + locked = {} + for side, label in (("baseline", "control"), ("candidate", "engine")): + locked[side] = {key: sources[label][key] for key in ("head", "snapshot_hash")} + locked[side]["available"] = True + (run / "repositories.lock.json").write_text(json.dumps(locked)) + for side, cwd in (("baseline", control), ("candidate", engine)): + root = run / side / "trial-001" + record = json.loads((root / "result.json").read_text()) + record.update(qualification_scope="native_text_smoke", + source_provenance={"control": sources["control"]}) + if side == "candidate": + record.update(implementation="nanovllm_qwen38", kernel_policy=NATIVE_POLICY, + native_model_module="nanovllm.qwen38.model", + loading={"scope": "text_only", "dtype": "bfloat16", + "loaded_parameters": 5}) + record["source_provenance"]["engine"] = sources["engine"] + (root / "result.json").write_text(json.dumps(record)) + (root / "execution.json").write_text(json.dumps( + {"status": "completed", "returncode": 0, "cwd": str(cwd)})) + (root / "source.json").write_text(json.dumps( + {"side": side, "before": locked[side], "after": locked[side]})) + return run, workload, manifest + + +def check_native(bundle): + return validate(*bundle, max_nrmse=0.01, min_top1=0.99, rms_floor=1e-8, + qualification_scope="native_text_smoke") + + +def test_native_smoke_cannot_be_relabelled_reference(native_bundle): + result = check_native(native_bundle) + assert result["verdict"] == "PASS" + assert result["qualification_scope"] == "native_text_smoke" + assert "No whole-engine acceptance" in result["limitations"] + with pytest.raises(ValueError, match="relabelled"): + check(native_bundle) + + +@pytest.mark.parametrize("mutation", [ + lambda r: r.update(implementation="transformers_cached"), + lambda r: r.update(qualification_scope="reference_self_check"), + lambda r: r.update(native_model_module="transformers.fake"), + lambda r: r["source_provenance"].pop("engine"), + lambda r: r["source_provenance"]["engine"].update(head="c" * 40), + lambda r: r["source_provenance"]["control"].update(snapshot_hash="changed"), + lambda r: r["source_provenance"]["engine"]["imports"].pop("nanovllm.qwen38.model"), + lambda r: r["source_provenance"]["engine"]["imports"]["nanovllm"].update(path="../other.py"), + lambda r: r["source_provenance"]["engine"]["imports"]["nanovllm"].update(sha256="0" * 64), + lambda r: r["loading"].update(loaded_parameters=0), + lambda r: r["kernel_policy"].update(deltanet="unknown"), +]) +def test_native_rejects_role_source_import_and_loading_mismatch(native_bundle, mutation): + mutate_case(native_bundle, mutation) + with pytest.raises(ValueError): + check_native(native_bundle) + + +def test_native_rejects_wrong_execution_checkout_and_trial_source_drift(native_bundle): + root = native_bundle[0] / "candidate/trial-001" + execution = json.loads((root / "execution.json").read_text()) + good = execution["cwd"] + execution["cwd"] = str(root) + (root / "execution.json").write_text(json.dumps(execution)) + with pytest.raises(ValueError, match="snapshotted"): + check_native(native_bundle) + execution["cwd"] = good + (root / "execution.json").write_text(json.dumps(execution)) + source = json.loads((root / "source.json").read_text()) + source["after"]["snapshot_hash"] = "f" * 64 + (root / "source.json").write_text(json.dumps(source)) + with pytest.raises(ValueError, match="source changed"): + check_native(native_bundle) + + +@pytest.mark.parametrize("fp8", [False, True]) +@pytest.mark.parametrize("malformed", [False, True]) +def test_native_synthetic_traces_use_distinct_repos_in_common_loop( + native_bundle, tmp_path, malformed, fp8, +): + from inferrsi.harness.runner import run_experiment + from inferrsi.runspec import RunSpec + + fixture, workload, manifest = native_bundle + fp8_args = () + model_id = "Qwen/Qwen3.8-27B" + if fp8: + kernel = upgrade_fp8_bundle(native_bundle) + model_id = "Qwen/Qwen3.8-27B-FP8" + fp8_args = ("--model-id", model_id, "--fp8-kernel-revision", kernel["revision"], + "--fp8-kernel-sha256", kernel["snapshot_sha256"], + "--model-path", str(manifest.parent / "model-metadata")) + if malformed: + mutate_case(native_bundle, lambda r: r.update(loading=None)) + declared = json.loads((fixture / "run.lock.json").read_text()) + locked = json.loads((fixture / "repositories.lock.json").read_text()) + emit = ( + "import pathlib,shutil,sys; src=pathlib.Path(sys.argv[1]); dst=pathlib.Path(sys.argv[2]); " + "shutil.copyfile(src/'result.json',dst); " + "[shutil.copyfile(p,dst.parent/p.name) for p in src.glob('*.npz')]" + ) + def command(side): + return (sys.executable, "-c", emit, str(fixture / side / "trial-001"), "{result}") + + validator = Path(__file__).resolve().parents[1] / "examples/qwen38_text/validate.py" + spec = RunSpec( + repo=declared["repo"], candidate_repo=declared["candidate_repo"], + objective="correctness", comparison_scope="native_text_smoke", repetitions=1, + model=model_id, model_revision="a" * 40, workload_id=digest(workload), max_minutes=1, + baseline_command=command("baseline"), candidate_command=command("candidate"), + validation_command=(sys.executable, str(validator), "--run-dir", "{run_dir}", + "--workload", str(workload), "--manifest", str(manifest), + "--max-nrmse", "0.01", "--min-top1", "0.99", "--rms-floor", "1e-8", + "--qualification-scope", "native_text_smoke", + "--output", "{result}") + fp8_args, + output_dir=str(tmp_path / "native-outputs"), + ) + run = run_experiment(spec, {"requested_device_model": "Synthetic GPU"}, snapshot=lambda p, *_: + locked["baseline" if p == Path(spec.repo) else "candidate"]) + result = json.loads((run / "evaluation.json").read_text()) + assert result["verdict"] == ("INCONCLUSIVE" if malformed else "PASS") + assert result["improvement_pct"] is None + quality = json.loads((run / "validation/result.json").read_text()) + assert quality["qualification_scope"] == "native_text_smoke" + + +def native_validator_subprocess(bundle, extra=()): + run, workload, manifest = bundle + output = run / "standalone-validation.json" + validator = Path(__file__).resolve().parents[1] / "examples/qwen38_text/validate.py" + completed = subprocess.run( + [sys.executable, str(validator), "--run-dir", str(run), "--workload", str(workload), + "--manifest", str(manifest), "--max-nrmse", "0.01", "--min-top1", "0.99", + "--rms-floor", "1e-8", "--qualification-scope", "native_text_smoke", + "--output", str(output), *extra], capture_output=True, text=True, check=False, + ) + result = json.loads(output.read_text()) + assert json.loads(completed.stdout) == result + assert not completed.stderr + assert str(run.parent) not in completed.stdout + return completed.returncode, result + + +# Each path is traversed independently so every schema boundary is exercised. +MALFORMED_FIELDS = [ + ("candidate/trial-001/result.json", ("loading",)), + ("candidate/trial-001/result.json", ("source_provenance",)), + *[("candidate/trial-001/result.json", ("source_provenance", label) + suffix) + for label in ("control", "engine") + for suffix in ((), ("imports",), ("head",), ("snapshot_hash",))], + *[("candidate/trial-001/result.json", ("source_provenance", "engine", "imports", + "nanovllm") + suffix) + for suffix in ((), ("path",), ("sha256",))], + *[("repositories.lock.json", (side,) + suffix) + for side in ("baseline", "candidate") + for suffix in ((), ("head",), ("snapshot_hash",), ("available",))], + *[(f"{side}/trial-001/source.json", (stage,) + suffix) + for side in ("baseline", "candidate") for stage in ("before", "after") + for suffix in ((), ("head",), ("snapshot_hash",), ("available",))], + *[("run.lock.json", (key,)) for key in ("repo", "candidate_repo")], + *[(f"{side}/trial-001/execution.json", ("cwd",)) for side in ("baseline", "candidate")], +] + + +@pytest.mark.parametrize("file,fields", MALFORMED_FIELDS) +@pytest.mark.parametrize("value", [None, [], "", "missing"]) +def test_native_malformed_evidence_writes_inconclusive(native_bundle, file, fields, value): + path = native_bundle[0] / file + record = json.loads(path.read_text()) + target = record + for field in fields[:-1]: + target = target[field] + if value == "missing": + target.pop(fields[-1]) + else: + target[fields[-1]] = value + path.write_text(json.dumps(record)) + code, result = native_validator_subprocess(native_bundle) + assert code == 2 + assert result["verdict"] == "INCONCLUSIVE" + assert result["reasons"] == ["Invalid or missing validation evidence (ValueError)"] + + +@pytest.mark.parametrize("value", [None, "missing", "not-a-digest"]) +def test_native_matching_missing_identities_cannot_pass(native_bundle, value): + def strip(node): + if isinstance(node, dict): + for key in list(node): + if key in {"head", "snapshot_hash"}: + if value == "missing": + del node[key] + else: + node[key] = value + else: + strip(node[key]) + elif isinstance(node, list): + for item in node: + strip(item) + + for path in native_bundle[0].rglob("*.json"): + record = json.loads(path.read_text()) + strip(record) + path.write_text(json.dumps(record)) + code, result = native_validator_subprocess(native_bundle) + assert code == 2 and result["verdict"] == "INCONCLUSIVE" + + +def test_native_complete_numerical_failure_still_exits_one(native_bundle): + root = native_bundle[0] / "candidate/trial-001" + + def alter(record): + for case in record["cases"]: + path = root / case["logits_file"] + np.savez(path, logits=np.array([[3., 2., 1.], [1., 2., 3.]], dtype=np.float32)) + case.update(logits_sha256=digest(path), greedy_ids=[0, 2]) + + mutate_case(native_bundle, alter) + code, result = native_validator_subprocess(native_bundle) + assert code == 1 and result["verdict"] == "FAIL" + assert all("numerical gate rejected" in reason for reason in result["reasons"]) + + +def upgrade_fp8_bundle(bundle): + import hashlib + + from examples.qwen38_text.profiles import FP8_MODEL_ID, precision + + run, workload, manifest = bundle + profile = precision(FP8_MODEL_ID) + from examples.qwen38_text.fp8_reference import adapt_reference_config + model_path = manifest.parent / "model-metadata" + model_path.mkdir() + gate = "model.language_model.layers.0.mlp.gate" + projection = gate + "_proj" + config = {"model_type": "qwen3_5", "architectures": ["Qwen3_5ForConditionalGeneration"], + "text_config": {"model_type": "qwen3_5_text", "dtype": "bfloat16", + "num_hidden_layers": 1}, + "quantization_config": {**profile["quantization"], + "modules_to_not_convert": [gate, "lm_head"]}} + index = {"weight_map": {projection + ".weight": "layer.safetensors", + projection + ".weight_scale_inv": "layer.safetensors"}} + files_metadata = [] + for name, document in (("config.json", config), ("model.safetensors.index.json", index)): + path = model_path / name + path.write_text(json.dumps(document)) + files_metadata.append({"path": name, "sha256": digest(path), "size": path.stat().st_size}) + _adapted, adaptation = adapt_reference_config(config, index, {projection}) + for path in (workload, manifest): + record = json.loads(path.read_text()) + record.update(model_id=FP8_MODEL_ID, precision=profile) + if path == manifest: + record["files"] = files_metadata + path.write_text(json.dumps(record)) + spec = json.loads((run / "run.lock.json").read_text()) + spec.update(model=FP8_MODEL_ID, workload_id=digest(workload)) + (run / "run.lock.json").write_text(json.dumps(spec)) + files = [{"path": "__init__.py", "sha256": "a" * 64}] + kernel = {"repo_id": "kernels-community/finegrained-fp8", "repo_type": "kernel", + "revision": "b" * 40, "files": files, "build": "torch-cuda", + "package_root": "/synthetic/kernel/build/torch-cuda", + "snapshot_sha256": hashlib.sha256( + json.dumps(files, sort_keys=True).encode()).hexdigest()} + for side in ("baseline", "candidate"): + path = run / side / "trial-001/result.json" + record = json.loads(path.read_text()) + record.update(model_id=FP8_MODEL_ID, dtype="fp8", precision=profile, + manifest_sha256=digest(manifest), workload_sha256=digest(workload)) + record["versions"].update(kernels="test", triton="test") + if side == "baseline": + record["kernel_policy"].update(fp8_linear="finegrained_fp8", deepgemm=False) + record["reference_kernel"] = kernel + record["fp8_loading"] = {"execution": "transformers_fp8", "dequantized": False, + "backend": "finegrained_fp8", "weight_dtype": "float8_e4m3fn", + "activation_dtype": "bfloat16", "fp8_linear_modules": 1, + "scale_dtype": "float32", "bf16_scale_upcasts": 1, + "scale_normalization": "lossless_float32", + "config_adaptation": adaptation, + "fp8_module_names": [projection]} + else: + record["kernel_policy"].update(fp8_linear="triton_block_fp8") + record["fp8_execution"] = {"modules": 1, "total_calls": 4} + engine_root = Path(spec["candidate_repo"]) + for name in ("quant", "_triton_fp8"): + source = engine_root / f"nanovllm/qwen38/{name}.py" + source.write_text("# synthetic FP8 source\n") + record["source_provenance"]["engine"]["imports"][f"nanovllm.qwen38.{name}"] = { + "path": source.relative_to(engine_root).as_posix(), "sha256": digest(source), + } + record["loading"].update(dtype="fp8", **profile, execution="native_fp8", + backend="triton_block_fp8", fp8_linear_modules=1) + path.write_text(json.dumps(record)) + return kernel + + +@pytest.mark.parametrize("mutation", [ + lambda r: r.update(dtype="bfloat16"), + lambda r: r.update(precision=None), + lambda r: r["precision"].update(dequantized=True), + lambda r: r["loading"].update(dequantized=True), + lambda r: r["loading"].update(fp8_linear_modules=0), + lambda r: r["loading"].update(backend="bf16_linear"), +]) +def test_fp8_complete_identity_and_no_dequantized_candidate(native_bundle, mutation): + kernel = upgrade_fp8_bundle(native_bundle) + extra = ("--model-id", "Qwen/Qwen3.8-27B-FP8", "--fp8-kernel-revision", kernel["revision"], + "--fp8-kernel-sha256", kernel["snapshot_sha256"], + "--model-path", str(native_bundle[2].parent / "model-metadata")) + code, result = native_validator_subprocess(native_bundle, extra) + assert code == 0 and result["verdict"] == "PASS" + mutate_case(native_bundle, mutation) + code, result = native_validator_subprocess(native_bundle, extra) + assert code == 2 and result["verdict"] == "INCONCLUSIVE" + + +@pytest.mark.parametrize("mutation", [ + lambda r: r["reference_kernel"].update(revision="c" * 40), + lambda r: r["reference_kernel"].update(repo_type="model"), + lambda r: r["reference_kernel"]["files"][0].update(sha256="d" * 64), + lambda r: r["fp8_loading"].update(dequantized=True), + lambda r: r.update(fp8_loading=None), + lambda r: r["kernel_policy"].update(deepgemm=True), + lambda r: r["fp8_loading"].update(fp8_module_names=["same_count_wrong_module"]), + lambda r: r["fp8_loading"]["config_adaptation"].update(removed_exclusions=[]), + lambda r: r["fp8_loading"]["config_adaptation"].update(adapted_config_sha256="f" * 64), +]) +def test_fp8_reference_kernel_and_loading_must_match_frozen_profile(native_bundle, mutation): + kernel = upgrade_fp8_bundle(native_bundle) + path = native_bundle[0] / "baseline/trial-001/result.json" + record = json.loads(path.read_text()) + mutation(record) + path.write_text(json.dumps(record)) + extra = ("--model-id", "Qwen/Qwen3.8-27B-FP8", "--fp8-kernel-revision", kernel["revision"], + "--fp8-kernel-sha256", kernel["snapshot_sha256"], + "--model-path", str(native_bundle[2].parent / "model-metadata")) + code, result = native_validator_subprocess(native_bundle, extra) + assert code == 2 and result["verdict"] == "INCONCLUSIVE" + + +@pytest.mark.parametrize("mutation", ["no_path", "missing", "tamper", "escape", "malformed"]) +def test_fp8_reference_requires_authenticated_metadata(native_bundle, mutation, tmp_path): + kernel = upgrade_fp8_bundle(native_bundle) + model_path = native_bundle[2].parent / "model-metadata" + metadata = model_path / "config.json" + extra = ("--model-id", "Qwen/Qwen3.8-27B-FP8", "--fp8-kernel-revision", kernel["revision"], + "--fp8-kernel-sha256", kernel["snapshot_sha256"]) + if mutation != "no_path": + extra += ("--model-path", str(model_path)) + if mutation == "missing": + metadata.unlink() + elif mutation == "tamper": + metadata.write_text(metadata.read_text() + " ") + elif mutation == "escape": + outside = tmp_path / "outside-config.json" + outside.write_bytes(metadata.read_bytes()) + metadata.unlink() + metadata.symlink_to(outside) + elif mutation == "malformed": + index = model_path / "model.safetensors.index.json" + index.write_text(json.dumps({"weight_map": [1]})) + manifest = json.loads(native_bundle[2].read_text()) + for item in manifest["files"]: + if item["path"] == index.name: + item.update(sha256=digest(index), size=index.stat().st_size) + native_bundle[2].write_text(json.dumps(manifest)) + code, result = native_validator_subprocess(native_bundle, extra) + assert code == 2 and result["verdict"] == "INCONCLUSIVE" diff --git a/tests/test_runtime.py b/tests/test_runtime.py index a5781fae1..9f76e8576 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -121,6 +121,83 @@ def test_missing_quality_evidence_is_inconclusive(self): self.assertEqual(evaluation["verdict"], "INCONCLUSIVE") self.assertTrue(any("quality" in reason for reason in evaluation["reasons"])) + def test_correctness_compares_fresh_outputs_without_performance_metrics(self): + output = ( + sys.executable, "-c", + "import pathlib,sys; pathlib.Path(sys.argv[1]).write_text('{\"tokens\":[1,2]}')", + "{result}", + ) + validator = ( + sys.executable, "-c", + "import json,pathlib,sys; root=pathlib.Path(sys.argv[1]); " + "outputs=[json.loads(p.read_text()) for p in root.glob('*/trial-*/result.json')]; " + "assert len(outputs)==4; assert all(x=={'tokens':[1,2]} for x in outputs)", + "{run_dir}", + ) + path, result = self.run_spec( + objective="correctness", baseline_command=output, + candidate_command=output, validation_command=validator, + ) + self.assertEqual(result["verdict"], "PASS") + self.assertIsNone(result["improvement_pct"]) + self.assertIsNone(result["metric_key"]) + self.assertIsNone(result["baseline"]) + self.assertIn("no performance claim", result["claim_scope"]) + events = [json.loads(line) for line in (path / "events.jsonl").read_text().splitlines()] + self.assertEqual( + [event["side"] for event in events if event["phase"] == "correctness_trial"], + ["baseline", "candidate", "candidate", "baseline"], + ) + + def test_correctness_requires_readable_nonempty_output_objects(self): + for content in (None, "broken", "[]", "{}", "null"): + command = (sys.executable, "-c", "pass") if content is None else ( + sys.executable, "-c", + "import pathlib,sys; pathlib.Path(sys.argv[1]).write_text(sys.argv[2])", + "{result}", content, + ) + with self.subTest(content=content): + _, result = self.run_spec( + objective="correctness", candidate_command=command, repetitions=1, + ) + self.assertEqual(result["verdict"], "INCONCLUSIVE") + self.assertTrue(any("candidate trial" in reason for reason in result["reasons"])) + + def test_correctness_validation_rejection_and_incomplete_exit_are_distinct(self): + for code, verdict in ((0, "PASS"), (1, "FAIL"), (2, "INCONCLUSIVE")): + with self.subTest(code=code): + _, result = self.run_spec( + objective="correctness", repetitions=1, + validation_command=(sys.executable, "-c", f"raise SystemExit({code})"), + ) + self.assertEqual(result["verdict"], verdict) + _, result = self.run_spec( + validation_command=(sys.executable, "-c", "raise SystemExit(2)"), + ) + self.assertEqual(result["verdict"], "FAIL") # Existing performance contract. + + def test_correctness_missing_validation_and_source_drift_are_inconclusive(self): + _, result = self.run_spec(objective="correctness", validation_command=None, repetitions=1) + self.assertEqual(result["verdict"], "INCONCLUSIVE") + mutation = ( + sys.executable, "-c", + "import pathlib,sys; pathlib.Path('source.txt').write_text('drift'); " + "pathlib.Path(sys.argv[1]).write_text('{\"tokens\":[1]}')", + "{result}", + ) + _, result = self.run_spec( + objective="correctness", candidate_command=mutation, repetitions=1, + ) + self.assertEqual(result["verdict"], "INCONCLUSIVE") + self.assertFalse(result["source_integrity"]["verified"]) + + def test_correctness_trial_timeout_preserves_incomplete_evidence(self): + _, result = self.run_spec( + objective="correctness", repetitions=1, max_minutes=0.01, + candidate_command=(sys.executable, "-c", "import time; time.sleep(30)"), + ) + self.assertEqual(result["verdict"], "INCONCLUSIVE") + def test_tracked_source_mutation_is_detected_inside_output_directory(self): source = self.repo / "src" / "implementation.py" source.parent.mkdir() @@ -347,6 +424,8 @@ def test_contract_rejects_invalid_inputs_before_allocating_run(self): {"min_improvement_pct": -1}, {"baseline_command": "echo unsafe"}, {"metric_key": "metrics..latency"}, + {"objective": "unknown"}, + {"objective": "correctness", "min_improvement_pct": 1}, {"output_dir": str(self.repo)}, ): with self.subTest(changes=changes), self.assertRaises(ValueError): diff --git a/tools/evidence_inventory.json b/tools/evidence_inventory.json index db28b9004..88ff7390d 100644 --- a/tools/evidence_inventory.json +++ b/tools/evidence_inventory.json @@ -43,6 +43,14 @@ "qwen3-tts-ascend-import": { "kind": "import", "manifests": [], "external": ["Original campaign validators and artifacts remain at the upstream source described in plan/README.md."] + }, + "qwen38-h20-text-phase1": { + "kind": "campaign", "manifests": [], + "external": ["Historical BF16 preparation only; no frozen model acceptance or committed execution artifacts."] + }, + "qwen38-h20-fp8-text-phase1": { + "kind": "campaign", "manifests": [], + "external": ["FP8 preparation only; authenticated intake and CPU preparation records are private. GPU qualification and final freeze remain pending."] } } }