From 1eba8b2c4ceb72cae853a7691ebad39a983811e8 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Fri, 6 Nov 2026 22:00:22 +0800 Subject: [PATCH 1/3] feat: support shared YAML task input Use one versioned contract for CLI and Agent task previews and pass complete tasks to the existing RunSpec workflow. Verify ordinary wheel installs and preserve flag-only and bundled plugin behavior. Agent-assisted-by: Codex / unknown [effort=unknown] (author) Signed-off-by: zengchuang --- .github/workflows/ci-ready.yml | 5 +- docs/contributing.md | 10 +- docs/knowledge-operations.md | 7 +- docs/operations.md | 15 +- docs/quickstart.md | 17 +- docs/task-input.md | 73 ++++ pyproject.toml | 9 +- src/inferrsi/arguments.py | 216 +++++++----- src/inferrsi/cli.py | 60 +++- src/inferrsi/data/task_input/schema-v1.json | 43 +++ src/inferrsi/data/task_input/template-v1.yaml | 25 ++ src/inferrsi/harness/runner.py | 7 + src/inferrsi/infra/inspection.py | 3 + src/inferrsi/inspection.py | 2 + src/inferrsi/runspec.py | 4 +- src/inferrsi/skill_catalog.py | 2 +- src/inferrsi/task_input.py | 326 ++++++++++++++++++ tests/check_task_installed.py | 36 ++ tests/fixtures/task-v1.yaml | 26 ++ tests/test_knowledge_fresh_install.py | 15 +- tests/test_knowledge_install.py | 3 +- tests/test_task_input.py | 181 ++++++++++ tools/plugin_bundle.py | 1 + 23 files changed, 959 insertions(+), 127 deletions(-) create mode 100644 docs/task-input.md create mode 100644 src/inferrsi/data/task_input/schema-v1.json create mode 100644 src/inferrsi/data/task_input/template-v1.yaml create mode 100644 src/inferrsi/task_input.py create mode 100644 tests/check_task_installed.py create mode 100644 tests/fixtures/task-v1.yaml create mode 100644 tests/test_task_input.py diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index aeda080ff..0ec683ba3 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -30,10 +30,11 @@ jobs: run: | python -m venv "$RUNNER_TEMP/wheel-env" wheels=("$RUNNER_TEMP"/dist/*.whl) - "$RUNNER_TEMP/wheel-env/bin/python" -m pip install "${wheels[0]}[skills]" - cp tests/check_installed.py "$RUNNER_TEMP/check_installed.py" + "$RUNNER_TEMP/wheel-env/bin/python" -m pip install "${wheels[0]}" + cp tests/check_installed.py tests/check_task_installed.py tests/fixtures/task-v1.yaml "$RUNNER_TEMP/" cd "$RUNNER_TEMP" env -u PYTHONPATH "$RUNNER_TEMP/wheel-env/bin/python" -I check_installed.py + env -u PYTHONPATH "$RUNNER_TEMP/wheel-env/bin/python" -I check_task_installed.py full-check: strategy: diff --git a/docs/contributing.md b/docs/contributing.md index 27319aaf8..f7a0708e7 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -221,11 +221,11 @@ Keep the existing four lanes: | [Merge](../.github/workflows/ci-merge.yml) | Manifest check and lightweight CPU smoke on main | | [Nightly](../.github/workflows/ci-nightly.yml) | Scheduled lint/tests, evidence layout and label drift | -Ready CI also builds the source distribution and wheel, installs the wheel with -the optional `skills` parser in a fresh virtual environment, and runs -`tests/check_installed.py` outside the checkout with `PYTHONPATH` unset and Python -isolated mode enabled. This checks packaged knowledge, skill bodies, licenses, -helpers and CLI reads without installing any inference engine. +Ready CI builds the source distribution and wheel and installs the wheel with +its core dependencies in a fresh virtual environment. It runs +`tests/check_installed.py` and `tests/check_task_installed.py` outside the checkout +with `PYTHONPATH` unset and Python isolated mode enabled. The checks cover packaged +knowledge, skills, task schema and template, and read-only CLI operations. The ready test and installed-package jobs cover the declared minimum Python 3.10 alongside Python 3.11. This matrix does not certify later Python releases or accelerator runtime compatibility. diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md index e3a5833cf..d41a0b3cc 100644 --- a/docs/knowledge-operations.md +++ b/docs/knowledge-operations.md @@ -32,8 +32,7 @@ inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/ inferrsi read calibrate-serving-simulation --skill-root src/inferrsi/data/skill_library ``` -The `--skill-root` selections require the `skills` extra (`python -m pip install -'.[skills]'`); package-only collect/read work with the base install. +Both packaged and checkout skill reads work with the base install. Every item has an owning `authority` (`package`, `checkout` or `store`) and a qualified `ref` of the form `:`. Public IDs are unchanged and may be used @@ -225,8 +224,8 @@ to narrow results. An explicit ID such as Package repo/backend/scenario filters still exclude checkout cases; inspect the returned applicability rather than assuming those filters qualify a case. -For explicit CLI resource-path resolution from the installed library (optional -`skills` extra required), or a custom library selected with `--root`: +The CLI resolves resource paths from the installed library or a custom library +selected with `--root`. ```bash inferrsi skills resource inference-optimization-casebook cases/qwen-ascend-rejected-directions.md diff --git a/docs/operations.md b/docs/operations.md index 39abbed32..f6922b6d2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -121,7 +121,7 @@ those resources retain their repository owner and source attribution. ### Discover/read packaged or custom skills — `skills list` / `show` / `resource` / `reference` -- **Inputs:** `list` reads frontmatter and sidecar metadata; `show ` selects one instruction body; `resource ` and `reference ` resolve one path. The default is the installed library's registered skills; an explicit `--root` selects a custom flat tree or a categorized library with `skill-index.json`. The optional `skills` extra supplies the YAML parser. +- **Inputs:** `list` reads frontmatter and sidecar metadata; `show ` selects one instruction body; `resource ` and `reference ` resolve one path. The default is the installed library's registered skills; an explicit `--root` selects a custom flat tree or a categorized library with `skill-index.json`. - **Scope:** complete packaged skills under `data/skill_library/`, or the explicitly selected custom root. The 21 English engine and research-router skills remain accessible through catalog records and library navigation, separately from the registry. - **Output:** metadata listing, one instruction body, or a resolved path. - **Side effects:** none. `resource`/`reference` do not load or execute the target. @@ -190,9 +190,18 @@ normal scoped PR path. ## Experiments +### Read shared YAML input — `task schema`, `task template`, `task validate`, `task preview` + +- **Inputs** The packaged schema/template, or a v1 YAML task path and optional preview overrides. +- **Output** Schema or template, validation result, or normalized values and required-field blockers for plan/run. Use `--json` for Agent handoff. +- **Side effects** Read-only. These commands run no probes or subprocesses and write nothing to the workspace or experiment directory. +- **Failure behavior** Exit 2 with a field or location for invalid input. A valid draft can have missing execution inputs. + +See the [shared YAML task guide](task-input.md) for paths, merging and the CLI/Agent workflow. + ### Inspect/prepare — `doctor`, `plan`, `--dry-run` -- **Inputs:** repository/backend/device flags; `plan` takes the full experiment specification. +- **Inputs:** repository/backend/device flags; `plan` takes the full experiment specification as flags or `--task` YAML. - **Scope:** environment inspection and specification preview. - **Output:** environment facts, or a preview of the RunSpec. - **Side effects:** `doctor` may invoke bounded read-only probes; `plan`/`--dry-run` persist nothing. @@ -205,7 +214,7 @@ compatibility. ### Execute/inspect — `run`, `status`, `report` -- **Inputs:** the full experiment specification (`run`); a run ID and output directory (`status`/`report`). +- **Inputs:** the full experiment specification as flags or `--task` YAML with `--confirm-acceptance` (`run`); a run ID and output directory (`status`/`report`). - **Scope:** caller-supplied benchmark, candidate and validation commands under the experiment contract. - **Output:** a run ID and evidence directory; recorded progress; an HTML report. - **Side effects:** `run` executes the supplied commands and owns child-process cleanup; `report` writes HTML into the run directory. diff --git a/docs/quickstart.md b/docs/quickstart.md index 24d2fda79..bc3aa8491 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,6 +1,7 @@ # Quickstart -This release uses CLI args and a **local executor on a prepared machine**. Install it directly on the remote machine where the target engine already works. It does not set up models, CUDA/ROCm/CANN, services, SSH workers or autonomous Agents. +Use CLI flags or a [shared YAML task](task-input.md) with the **local executor +on a prepared machine**. Install InferRSI where the target engine already works. ## Install and smoke @@ -14,11 +15,7 @@ inferrsi --help inferrsi doctor --repo . --repo-type vllm_omni --backend cpu ``` -Package-only knowledge commands (`knowledge`, `context`, package `collect`/`read`) -work with this base install. Reading skill libraries — the `skills` command and -any `--skill-root` selection — additionally requires -`python -m pip install '.[skills]'` (PyYAML); without it those operations fail -with that install hint instead of a traceback. +The base install supports packaged knowledge, skill-library reads and YAML tasks. For the maintained model examples, see [Qwen B300](qwen-image-2.1-turbo-b300.md) and [Qwen Ascend](qwen-image-2.1-turbo-ascend.md). Continue below for the generic @@ -53,7 +50,7 @@ entry point, effect class and boundary: ```bash inferrsi knowledge search 'Qwen-Image-2.1' --repo-type vllm_omni --backend ascend inferrsi context --repo-type vllm_omni --backend cuda --output context.md -inferrsi skills list --json # requires python -m pip install '.[skills]' +inferrsi skills list --json inferrsi read inferrsi.ascend.skills.ascend_graph_mode --material ``` @@ -61,7 +58,7 @@ Normalized Collect, exact-revision reads and resource access are available as CLI commands and as the Python `KnowledgeReader` ([contract](knowledge-operations.md)); authorized local candidate Create/Update remains a Python `KnowledgeStore` operation ([contract](knowledge-store.md)). -The `--skill-root` examples below require the `skills` extra from above: +To read from a checkout's skill root: ```bash inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/inferrsi/data/skill_library @@ -88,8 +85,10 @@ examples above explicitly adds that root's skill and case IDs. ## The actual configuration interface ```text +task: schema | template | validate TASK.yaml | preview TASK.yaml --json doctor: --repo --repo-type --backend --device-ids --device-model --python -plan/run: all doctor flags, plus +plan/run: --task TASK.yaml (--confirm-acceptance to execute) or the existing flags + all doctor flags, plus --candidate-repo --scenario --mode --agent --executor local --baseline-command --candidate-command --validation-command --metric-key --metric-unit --comparison-scope --direction diff --git a/docs/task-input.md b/docs/task-input.md new file mode 100644 index 000000000..107401c1a --- /dev/null +++ b/docs/task-input.md @@ -0,0 +1,73 @@ +# Shared YAML tasks + +Use a v1 task file to give the CLI and an Agent the same input. A normal +InferRSI install includes the parser, [v1 schema](../src/inferrsi/data/task_input/schema-v1.json) +and [YAML template](../src/inferrsi/data/task_input/template-v1.yaml). + +```bash +inferrsi task template > task.yaml +inferrsi task schema +inferrsi task validate task.yaml +inferrsi task preview task.yaml --json +``` + +An Agent can read or fill in the same template. Have it run the same validation +and JSON preview and review the effective values with you before proceeding. +A valid draft can omit any section or leave a field as null. The preview lists +missing fields by dotted path and sets `complete` when all required inputs are +present. Validation and preview read the file without invoking processes, +inspecting the runtime, or writing workspace or experiment files. Invalid input +exits with status 2. + +| Section | Required for `plan` and `run` | +| --- | --- | +| `optimization` | Repository, model, framework (the CLI's repository type), workload, objective and SIAS plane | +| `environment` | Backend, baseline and candidate argv, benchmark repetitions and time budget | +| `acceptance` | Issue URL or local plan path, full baseline Git object ID, metric key, unit, direction and agreed minimum improvement percentage | + +Use the schema for exact field names, types and CLI mappings. Supply any device +selectors or quality inputs the accepted plan requires. Set +`acceptance.require_quality: true` when the quality command is mandatory. +`acceptance.constraints` records additional plan requirements for review. +The existing runner evaluates its configured metric and validation command. + +- Use a single UTF-8 YAML document up to 64 KiB. Use plain mappings and argv + lists of strings. Duplicate and unknown keys, aliases, anchors and explicit + tags are errors. Numeric inputs use decimal notation, booleans use + `true`/`false`, and strings that resemble these values should be quoted. +- YAML filesystem paths are relative to the file. Paths in explicit CLI flags + are relative to the caller's working directory. With a task file, the default + output directory is `runs` next to the YAML. A bare Python executable name is + resolved through the existing runtime lookup. Script paths and other argv + tokens keep the existing per-checkout working-directory semantics. Model + identifiers and HTTP(S) acceptance references stay as given. +- In preview, plan and run, explicit mapped CLI flags override YAML for + optimization and environment inputs. Built-in defaults fill omitted optional + fields. Acceptance settings come from YAML. A conflicting CLI acceptance + flag is rejected. Existing flag-only commands use their usual defaults. +- Keep credentials in the existing runtime channels. Review the YAML, + normalized preview and experiment records before sharing them. + +After reviewing the exact commands and the effective values against the +accepted frozen gates, inspect the prepared environment and explicitly run +the experiment: + +```bash +inferrsi plan --task task.yaml +inferrsi run --task task.yaml --confirm-acceptance +``` + +`plan` and `run --dry-run` use the existing environment probes, including +the configured Python interpreter. Task-file operations check the baseline +checkout's HEAD against the pinned object ID. A local acceptance reference +must point to an existing file. Pass `--confirm-acceptance` after checking +the effective values against the referenced frozen gates. InferRSI records +the confirmation and effective task. Review of the issue or plan is performed +by the caller or Agent. The [operation guide](operations.md#experiments) and +[evidence rules](contributing.md#evidence-and-claims) apply to the result. + +The DSH draft format is tracked separately under +[#152](https://github.com/InferMatrixLab/InferRSI/issues/152). Check legacy +DSH files against the core schema and migrate incompatible fields before +using them with these commands. DSH integration uses the core contract and +shared fixtures when it advertises core v1 support. diff --git a/pyproject.toml b/pyproject.toml index 50a9c792d..2a14afc61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "CLI-first reproducible inference experiments and source-grounded readme = "README.md" license = "Apache-2.0" requires-python = ">=3.10" -dependencies = [] +dependencies = ["PyYAML>=6.0"] [project.scripts] inferrsi = "inferrsi.cli:main" @@ -23,7 +23,7 @@ where = ["src"] namespaces = false [tool.setuptools.package-data] -inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE", "data/dashboard/*.html"] +inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE", "data/dashboard/*.html", "data/task_input/*.yaml"] [tool.setuptools_scm] version_file = "src/inferrsi/_version.py" @@ -34,8 +34,9 @@ initial_version = "0.1.0" prerelease = true [project.optional-dependencies] -skills = ["PyYAML>=6.0"] -dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0"] +# Keep the former extra as an alias for existing installs. +skills = [] +dev = ["pytest>=7.4", "ruff>=0.6"] image = ["numpy>=1.24", "Pillow>=10", "scikit-image>=0.21"] [tool.pytest.ini_options] diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 577c91a90..966d18545 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -1,4 +1,4 @@ -"""The user configuration surface is argparse, not task files.""" +"""CLI argument declarations and RunSpec construction.""" from __future__ import annotations @@ -12,13 +12,55 @@ from .catalog import BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES from .runspec import RunSpec +TARGET_DEFAULTS = { + "repo": ".", "repo_type": "vllm_omni", "backend": "cpu", "device_ids": "", + "device_model": None, "python": sys.executable, +} +EXPERIMENT_DEFAULTS = { + **TARGET_DEFAULTS, + "candidate_repo": None, + "scenario": "external_benchmark", + "mode": "config", + "agent": "manual", + "executor": "local", + "validation_command": None, + "metric_key": "metrics.latency_ms", + "require_trial_check": (), + "metric_unit": "ms", + "comparison_scope": "external_command", + "direction": "minimize", + "repetitions": 3, + "min_improvement_pct": 0.0, + "max_minutes": 120.0, + "workload_id": None, + "model": None, + "model_revision": None, + "output_dir": "runs", +} + + +class _ArgumentParser(argparse.ArgumentParser): + def parse_args(self, args=None, namespace=None): + parsed = super().parse_args(args, namespace) + if parsed.command in ("plan", "run") and not hasattr(parsed, "task"): + if getattr(parsed, "confirm_acceptance", False): + self.error("--confirm-acceptance requires --task") + required = ("baseline_command", "candidate_command") + missing = [ + f"--{name.replace('_', '-')}" for name in required if not hasattr(parsed, name) + ] + if missing: + self.error("the following arguments are required: " + ", ".join(missing)) + return argparse.Namespace(**(EXPERIMENT_DEFAULTS | vars(parsed))) + return parsed + def _common(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--repo", default=".", help="Prepared target Git checkout") - parser.add_argument("--repo-type", choices=REPO_TYPES, default="vllm_omni") - parser.add_argument("--backend", choices=BACKENDS, default="cpu") + parser.add_argument("--repo", help="Prepared target Git checkout") + parser.add_argument("--repo-type", choices=REPO_TYPES) + parser.add_argument("--backend", choices=BACKENDS) parser.add_argument( - "--device-ids", default="", + "--device-ids", help=( "Comma-separated caller-assigned device IDs; " "unsupported by MLX unified-memory discovery" @@ -29,83 +71,101 @@ def _common(parser: argparse.ArgumentParser) -> None: help="Expected CUDA device model or Apple Silicon chip (case-insensitive match)", ) parser.add_argument( - "--python", default=sys.executable, help="Python used for target package inspection" + "--python", help="Python used for target package inspection" ) +def _experiment_options(child: argparse.ArgumentParser) -> None: + _common(child) + child.add_argument( + "--candidate-repo", help="Candidate checkout; defaults to the baseline checkout" + ) + child.add_argument( + "--scenario", help="Caller-declared scenario label" + ) + child.add_argument("--mode", choices=("config", "code", "semantic")) + child.add_argument( + "--agent", + choices=("manual", "codex", "external"), + help="Handoff label only; no Agent is launched", + ) + child.add_argument( + "--executor", + choices=("local",), + help="Install this CLI on the remote machine for this first release", + ) + child.add_argument( + "--baseline-command", + help="Quoted argv command; {result} is replaced with a fresh output path", + ) + child.add_argument("--candidate-command") + child.add_argument( + "--validation-command", + help="Explicit quality/correctness command; missing evidence is INCONCLUSIVE", + ) + child.add_argument( + "--metric-key", + help="Flat or dotted key in benchmark JSON", + ) + child.add_argument( + "--require-trial-check", action="append", metavar="DOTTED_KEY", + help="Require a literal true qualification field in every benchmark JSON; repeatable", + ) + child.add_argument("--metric-unit") + child.add_argument("--comparison-scope") + child.add_argument("--direction", choices=("minimize", "maximize")) + child.add_argument("--repetitions", type=int) + child.add_argument("--min-improvement-pct", type=float) + child.add_argument( + "--max-minutes", + type=float, + help="Experiment deadline, at most seven days", + ) + child.add_argument("--workload-id", help="Caller-declared fixed workload identity") + child.add_argument("--model", help="Model identity/path recorded in the experiment") + child.add_argument("--model-revision") + child.add_argument("--output-dir") + + def parser() -> argparse.ArgumentParser: - root = argparse.ArgumentParser( + root = _ArgumentParser( description="CLI-first reproducible experiments and tuning knowledge" ) - root.add_argument("--version", action="version", version=__version__) commands = root.add_subparsers(dest="command", required=True) + root.add_argument("--version", action="version", version=__version__) doctor = commands.add_parser( "doctor", help="Inspect a prepared local target without installing dependencies" ) _common(doctor) + doctor.set_defaults(**TARGET_DEFAULTS) for name in ("plan", "run"): child = commands.add_parser( name, - help="Freeze an external-command experiment" + help="Preview an external-command experiment" if name == "plan" else "Measure baseline and candidate commands", + argument_default=argparse.SUPPRESS, ) - _common(child) - child.add_argument( - "--candidate-repo", help="Candidate checkout; defaults to the baseline checkout" - ) - child.add_argument( - "--scenario", default="external_benchmark", help="Caller-declared scenario label" - ) - child.add_argument("--mode", choices=("config", "code", "semantic"), default="config") - child.add_argument( - "--agent", - choices=("manual", "codex", "external"), - default="manual", - help="Handoff label only; no Agent is launched", - ) - child.add_argument( - "--executor", - choices=("local",), - default="local", - help="Install this CLI on the remote machine for this first release", - ) - child.add_argument( - "--baseline-command", - required=True, - help="Quoted argv command; {result} is replaced with a fresh output path", - ) - child.add_argument("--candidate-command", required=True) - child.add_argument( - "--validation-command", - help="Explicit quality/correctness command; missing evidence is INCONCLUSIVE", - ) - child.add_argument( - "--metric-key", - default="metrics.latency_ms", - help="Flat or dotted key in benchmark JSON", - ) - child.add_argument( - "--require-trial-check", action="append", default=[], metavar="DOTTED_KEY", - help="Require a literal true qualification field in every benchmark JSON; repeatable", - ) - child.add_argument("--metric-unit", default="ms") - child.add_argument("--comparison-scope", default="external_command") - child.add_argument("--direction", choices=("minimize", "maximize"), default="minimize") - child.add_argument("--repetitions", type=int, default=3) - child.add_argument("--min-improvement-pct", type=float, default=0.0) - child.add_argument( - "--max-minutes", - type=float, - default=120.0, - help="Experiment deadline, at most seven days", - ) - child.add_argument("--workload-id", help="Caller-declared fixed workload identity") - child.add_argument("--model", help="Model identity/path recorded in the experiment") - child.add_argument("--model-revision") - child.add_argument("--output-dir", default="runs") - child.add_argument("--dry-run", action="store_true") + _experiment_options(child) + child.add_argument("--task", help="Read a versioned YAML task") + child.add_argument("--dry-run", action="store_true", default=False) + if name == "run": + child.add_argument( + "--confirm-acceptance", action="store_true", + help="Confirm the effective YAML task was checked against the frozen gates", + ) + + task = commands.add_parser("task", help="Read or inspect shared YAML tasks") + actions = task.add_subparsers(dest="task_action", required=True) + for name in ("validate", "preview"): + child = actions.add_parser(name, argument_default=argparse.SUPPRESS) + child.add_argument("task", help="Path to the YAML task") + child.add_argument("--json", action="store_true", default=False) + if name == "preview": + _experiment_options(child) + actions.add_parser("schema", help="Print the packaged v1 schema") + actions.add_parser("template", help="Print the packaged v1 YAML template") for name in ("status", "report"): child = commands.add_parser( @@ -283,7 +343,7 @@ def parser() -> argparse.ArgumentParser: collect.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select an explicit skills root", ) collect.add_argument( "--store", @@ -302,7 +362,7 @@ def parser() -> argparse.ArgumentParser: read.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select an explicit skills root", ) read.add_argument( "--store", @@ -322,7 +382,7 @@ def parser() -> argparse.ArgumentParser: resource.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select an explicit skills root", ) resource.add_argument( "--store", @@ -351,14 +411,16 @@ def device_ids(args: argparse.Namespace) -> tuple[str, ...]: return values +def command_argv(value: str | list[str] | None) -> tuple[str, ...] | None: + if value is None: + return None + parsed = tuple(shlex.split(value, posix=True)) if isinstance(value, str) else tuple(value) + if not parsed: + raise ValueError("Commands cannot be empty") + return parsed + + def make_spec(args: argparse.Namespace) -> RunSpec: - def command(value: str | None) -> tuple[str, ...] | None: - if value is None: - return None - parsed = tuple(shlex.split(value, posix=True)) - if not parsed: - raise ValueError("Commands cannot be empty") - return parsed spec = RunSpec( repo=str(Path(args.repo).resolve()), @@ -367,9 +429,9 @@ def command(value: str | None) -> tuple[str, ...] | None: backend=args.backend, scenario=args.scenario, mode=args.mode, - baseline_command=command(args.baseline_command), - candidate_command=command(args.candidate_command), - validation_command=command(args.validation_command), + baseline_command=command_argv(args.baseline_command), + candidate_command=command_argv(args.candidate_command), + validation_command=command_argv(args.validation_command), metric_key=args.metric_key, required_trial_checks=tuple(args.require_trial_check), metric_unit=args.metric_unit, diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 6736ba47e..587bc8e97 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -5,6 +5,7 @@ import json import sqlite3 import sys +from dataclasses import replace from importlib.resources import files from pathlib import Path @@ -18,8 +19,8 @@ from .research import research_knowledge -def _print(value: object) -> None: - print(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False)) +def _print(value: object, *, ensure_ascii: bool = False) -> None: + print(json.dumps(value, ensure_ascii=ensure_ascii, indent=2, allow_nan=False)) def _knowledge_reader(args) -> KnowledgeReader: @@ -116,9 +117,33 @@ def main(argv: list[str] | None = None) -> int: argument_parser = parser() args = argument_parser.parse_args(argv) try: + if args.command == "task": + if args.task_action in ("schema", "template"): + name = "schema-v1.json" if args.task_action == "schema" else "template-v1.yaml" + resource = files("inferrsi").joinpath("data", "task_input", name) + print(resource.read_text(encoding="utf-8"), end="") + return 0 + from .task_input import normalize_task + + task = normalize_task(args.task, vars(args)) + if args.task_action == "validate": + if args.json: + _print({"valid": True}) + else: + print("Valid task YAML") + elif args.json: + _print(task.preview(), ensure_ascii=True) + else: + for section, fields in task.task.items(): + if isinstance(fields, dict): + for key, value in fields.items(): + print(f"{section}.{key} = {json.dumps(value)}") + else: + print(f"{section} = {fields}") + for field in task.missing: + print(f"Required for plan/run: {field}") + return 0 if args.command == "skills": - # Optional dependencies load only for the skills command; the core - # CLI stays independent of them. from .skill_catalog import ( get_skill, skill_instructions, @@ -357,6 +382,21 @@ def main(argv: list[str] | None = None) -> int: _print({"record": str(path)}) return 0 + task = None + if args.command in ("plan", "run") and hasattr(args, "task"): + from .task_input import normalize_task + + task = normalize_task(args.task, vars(args)) + args = task.execution_args(args) + if args.command == "run" and not args.dry_run and not getattr( + args, "confirm_acceptance", False + ): + raise ValueError( + "acceptance.reference: review the effective task against the frozen gates " + "and pass --confirm-acceptance to run" + ) + spec = replace(task.make_spec(args), revision=task.task["acceptance"]["baseline"]) + environment = inspect_environment( args.repo, args.repo_type, @@ -364,6 +404,7 @@ def main(argv: list[str] | None = None) -> int: device_ids=device_ids(args), device_model=args.device_model, python=args.python, + baseline_revision=spec.revision if task else "", ) environment["invocation"] = argv if argv is not None else sys.argv[1:] if args.command == "doctor": @@ -375,7 +416,10 @@ def main(argv: list[str] | None = None) -> int: ].get("environment") return 0 if available and known else 1 - spec = make_spec(args) + if task is None: + spec = make_spec(args) + else: + environment["task"] = task.task if args.command == "plan" or args.dry_run: _print( { @@ -383,7 +427,8 @@ def main(argv: list[str] | None = None) -> int: "environment": environment, "execution": "not started", "validation": "unverified", - } + }, + ensure_ascii=task is not None, ) return 0 if not environment["repository"].get("available"): @@ -407,7 +452,8 @@ def main(argv: list[str] | None = None) -> int: "run_dir": str(directory), "report": str(directory / "report.html"), "status": status, - } + }, + ensure_ascii=task is not None, ) return ( 0 if status.get("verdict") == "PASS" else (1 if status.get("verdict") == "FAIL" else 2) diff --git a/src/inferrsi/data/task_input/schema-v1.json b/src/inferrsi/data/task_input/schema-v1.json new file mode 100644 index 000000000..d2811c1c0 --- /dev/null +++ b/src/inferrsi/data/task_input/schema-v1.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "sections": { + "optimization": { + "repository": {"type": "path", "cli_argument": "--repo", "required_for_plan_run": true}, + "candidate_repository": {"type": "path", "cli_argument": "--candidate-repo"}, + "model": {"type": "text", "cli_argument": "--model", "required_for_plan_run": true}, + "model_revision": {"type": "text", "cli_argument": "--model-revision"}, + "framework": {"type": "framework", "cli_argument": "--repo-type", "required_for_plan_run": true}, + "workload": {"type": "text", "cli_argument": "--workload-id", "required_for_plan_run": true}, + "objective": {"type": "text", "required_for_plan_run": true}, + "plane": {"type": "choice", "choices": ["harness", "model", "infra"], "required_for_plan_run": true}, + "scenario": {"type": "text", "cli_argument": "--scenario"}, + "mode": {"type": "choice", "choices": ["config", "code", "semantic"], "cli_argument": "--mode"}, + "agent": {"type": "choice", "choices": ["manual", "codex", "external"], "cli_argument": "--agent"} + }, + "environment": { + "backend": {"type": "backend", "cli_argument": "--backend", "required_for_plan_run": true}, + "device_ids": {"type": "ids", "cli_argument": "--device-ids"}, + "device_model": {"type": "text", "cli_argument": "--device-model"}, + "python": {"type": "interpreter", "cli_argument": "--python"}, + "executor": {"type": "choice", "choices": ["local"], "cli_argument": "--executor"}, + "baseline_command": {"type": "argv", "cli_argument": "--baseline-command", "required_for_plan_run": true}, + "candidate_command": {"type": "argv", "cli_argument": "--candidate-command", "required_for_plan_run": true}, + "quality_command": {"type": "argv", "cli_argument": "--validation-command"}, + "repetitions": {"type": "positive_integer", "cli_argument": "--repetitions", "required_for_plan_run": true}, + "max_minutes": {"type": "minutes", "cli_argument": "--max-minutes", "required_for_plan_run": true}, + "output_directory": {"type": "path", "cli_argument": "--output-dir"} + }, + "acceptance": { + "reference": {"type": "reference", "required_for_plan_run": true}, + "baseline": {"type": "revision", "required_for_plan_run": true}, + "metric_key": {"type": "metric", "cli_argument": "--metric-key", "required_for_plan_run": true}, + "metric_unit": {"type": "text", "cli_argument": "--metric-unit", "required_for_plan_run": true}, + "direction": {"type": "choice", "choices": ["minimize", "maximize"], "cli_argument": "--direction", "required_for_plan_run": true}, + "min_improvement_pct": {"type": "threshold", "cli_argument": "--min-improvement-pct", "required_for_plan_run": true}, + "comparison_scope": {"type": "text", "cli_argument": "--comparison-scope"}, + "required_trial_checks": {"type": "checks", "cli_argument": "--require-trial-check"}, + "require_quality": {"type": "boolean"}, + "constraints": {"type": "text"} + } + } +} diff --git a/src/inferrsi/data/task_input/template-v1.yaml b/src/inferrsi/data/task_input/template-v1.yaml new file mode 100644 index 000000000..4b5948f2a --- /dev/null +++ b/src/inferrsi/data/task_input/template-v1.yaml @@ -0,0 +1,25 @@ +# Fill the required fields before plan/run. Null is a draft placeholder. +# Keep credentials in the existing runtime channels. +schema_version: 1 +optimization: + repository: . + model: null + framework: null + workload: null + objective: null + plane: null # harness, model, or infra +environment: + backend: null # cpu, cuda, rocm, ascend, or mlx + baseline_command: null # YAML argv list, including "{result}" where needed + candidate_command: null + quality_command: null + repetitions: null + max_minutes: null +acceptance: + reference: null # HTTP(S) acceptance issue URL or a local plan path + baseline: null # Full pinned Git revision as a quoted string + metric_key: null + metric_unit: null + direction: null # minimize or maximize + min_improvement_pct: null + constraints: null diff --git a/src/inferrsi/harness/runner.py b/src/inferrsi/harness/runner.py index a15c6b2a1..12d69be26 100644 --- a/src/inferrsi/harness/runner.py +++ b/src/inferrsi/harness/runner.py @@ -62,10 +62,16 @@ def record_status(phase: str, **fields: object) -> None: "candidate": Path(spec.candidate_repo or spec.repo).resolve(), } + def check_baseline_revision(identity: dict) -> None: + if spec.revision and identity.get("head") != spec.revision: + raise ValueError("Baseline Git checkout HEAD differs from the pinned RunSpec revision") + def execute_checked( command: tuple[str, ...], label: str, trial_dir: Path, metric_key: str | None ) -> dict: source_before = snapshot(repos[label], run_dir) + if label == "baseline": + check_baseline_revision(source_before) record = execute(command, repos[label], trial_dir, deadline, metric_key) if metric_key is not None and spec.required_trial_checks: record["trial_checks"] = collect_trial_checks(record, spec.required_trial_checks) @@ -161,6 +167,7 @@ def terminate() -> None: event("source_snapshot") before = {label: snapshot(repo, run_dir) for label, repo in repos.items()} _write_json(run_dir / "repositories.lock.json", before) + check_baseline_revision(before["baseline"]) commands = {"baseline": spec.baseline_command, "candidate": spec.candidate_command} for index in range(spec.repetitions): for label in ("baseline", "candidate") if index % 2 == 0 else ("candidate", "baseline"): diff --git a/src/inferrsi/infra/inspection.py b/src/inferrsi/infra/inspection.py index 597d3185a..f1c0acf3e 100644 --- a/src/inferrsi/infra/inspection.py +++ b/src/inferrsi/infra/inspection.py @@ -21,10 +21,13 @@ def inspect_environment( device_ids: tuple[str, ...] = (), device_model: str | None = None, python: str | None = None, + baseline_revision: str = "", command: Callable[[list[str]], dict] = probe_command, snapshot: Callable[[Path], dict] = repository_identity, ) -> dict: repository = snapshot(Path(repo).resolve()) + if baseline_revision and repository.get("head") != baseline_revision: + raise ValueError("acceptance.baseline: Git checkout HEAD differs from the pinned revision") python = python or sys.executable package_code = ( "import json,sys,importlib.metadata as m\n" diff --git a/src/inferrsi/inspection.py b/src/inferrsi/inspection.py index e7079b826..83a388d6c 100644 --- a/src/inferrsi/inspection.py +++ b/src/inferrsi/inspection.py @@ -12,6 +12,7 @@ def inspect_environment( device_ids: tuple[str, ...] = (), device_model: str | None = None, python: str | None = None, + baseline_revision: str = "", ) -> dict: from .execution import repository_identity @@ -22,6 +23,7 @@ def inspect_environment( device_ids=device_ids, device_model=device_model, python=python, + baseline_revision=baseline_revision, command=_command, snapshot=repository_identity, ) diff --git a/src/inferrsi/runspec.py b/src/inferrsi/runspec.py index 161408204..685ddeafb 100644 --- a/src/inferrsi/runspec.py +++ b/src/inferrsi/runspec.py @@ -4,6 +4,8 @@ from dataclasses import asdict, dataclass from pathlib import Path +MAX_EXPERIMENT_MINUTES = 7 * 24 * 60 + @dataclass(frozen=True) class RunSpec: @@ -109,7 +111,7 @@ def validate(self) -> None: raise ValueError(f"{label} must be finite") if self.max_minutes <= 0 or self.min_improvement_pct < 0: raise ValueError("max_minutes must be positive and min_improvement_pct nonnegative") - if self.max_minutes > 7 * 24 * 60: + if self.max_minutes > MAX_EXPERIMENT_MINUTES: raise ValueError("max_minutes cannot exceed seven days in this release") if not self.metric_key or any(not part for part in self.metric_key.split(".")): raise ValueError("metric_key must be a nonempty dotted JSON key") diff --git a/src/inferrsi/skill_catalog.py b/src/inferrsi/skill_catalog.py index ad3ad25a2..d4ffb477b 100644 --- a/src/inferrsi/skill_catalog.py +++ b/src/inferrsi/skill_catalog.py @@ -7,7 +7,7 @@ try: import yaml -except ImportError as exc: # The base install stays PyYAML-free by contract. +except ImportError as exc: # Plugin and direct-source runtimes may omit PyYAML. raise ValueError( "Reading checkout skills requires the 'skills' extra (PyYAML); install it " "from your InferRSI checkout, replacing the placeholder path: " diff --git a/src/inferrsi/task_input.py b/src/inferrsi/task_input.py new file mode 100644 index 000000000..05424d67d --- /dev/null +++ b/src/inferrsi/task_input.py @@ -0,0 +1,326 @@ +"""Load and normalize the packaged v1 task contract without probing the target.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from urllib.parse import urlsplit + +from .arguments import EXPERIMENT_DEFAULTS, command_argv, make_spec +from .catalog import BACKENDS, REPO_TYPES +from .runspec import MAX_EXPERIMENT_MINUTES, RunSpec + +try: + import yaml + from yaml.events import AliasEvent, DocumentStartEvent + from yaml.nodes import MappingNode, ScalarNode, SequenceNode +except ImportError as exc: + raise ValueError( + "YAML task input requires PyYAML. Install the standard InferRSI package." + ) from exc + +_MAX_BYTES = 64 * 1024 +_INVALID_TEXT = re.compile(r"[\x00\uD800-\uDFFF]") +_SCALAR_TAG = "tag:yaml.org,2002:" +_SCALAR_TAGS = {_SCALAR_TAG + kind for kind in ("str", "null", "int", "float", "bool")} + + +class _TaskLoader(yaml.SafeLoader): + # Use a small set of unambiguous scalar spellings instead of YAML 1.1's + # dates, sexagesimal numbers and implicit yes/no/on/off booleans. + yaml_implicit_resolvers = {} + + def compose_document(self): + event = self.peek_event() + if isinstance(event, DocumentStartEvent) and (event.version or event.tags): + raise ValueError("YAML directives are unsupported in task files") + return super().compose_document() + + def compose_node(self, parent, index): + event = self.peek_event() + if isinstance(event, AliasEvent) or event.anchor or event.tag: + mark = event.start_mark + raise ValueError( + f"YAML line {mark.line + 1}: anchors, aliases and tags are unsupported" + ) + return super().compose_node(parent, index) + + +for _kind, _pattern in { + "null": r"^(?:null|~)?$", + "bool": r"^(?:true|false)$", + "int": r"^[+-]?(?:0|[1-9][0-9]*)$", + "float": r"^[+-]?(?:[0-9]+[.][0-9]*|[.][0-9]+)(?:[eE][+-]?[0-9]+)?$" + r"|^[+-]?[0-9]+[eE][+-]?[0-9]+$", +}.items(): + _TaskLoader.add_implicit_resolver(_SCALAR_TAG + _kind, re.compile(_pattern), None) + + +def _scalar(loader: _TaskLoader, node: ScalarNode, field: str): + if node.tag not in _SCALAR_TAGS: + raise ValueError(f"{field}: use a string or a plain numeric, boolean or null value") + try: + value = loader.construct_object(node) + except (ValueError, OverflowError): + raise ValueError(f"{field}: invalid scalar") from None + if isinstance(value, str) and _INVALID_TEXT.search(value): + raise ValueError(f"{field}: contains an invalid Unicode character") + return value + + +def _mapping(node, field: str) -> dict: + if not isinstance(node, MappingNode): + raise ValueError(f"{field}: expected a mapping") + result = {} + for key, value in node.value: + if ( + not isinstance(key, ScalarNode) + or key.tag != _SCALAR_TAG + "str" + or _INVALID_TEXT.search(key.value) + ): + raise ValueError(f"{field}: field names must be strings") + name = key.value + if name in result: + prefix = "" if field == "task" else f"{field}." + raise ValueError(f"{prefix}{name}: duplicate field") + result[name] = value + return result + + +def _read_task(path: Path, schema: dict) -> dict: + if not path.is_file(): + raise ValueError("task: expected a readable YAML file") + with path.open("rb") as stream: + raw = stream.read(_MAX_BYTES + 1) + if len(raw) > _MAX_BYTES: + raise ValueError("task: YAML exceeds 64 KiB") + loader = None + try: + loader = _TaskLoader(raw.decode("utf-8-sig")) + root = _mapping(loader.get_single_node(), "task") + version = root.pop("schema_version", None) + if not isinstance(version, ScalarNode): + raise ValueError("schema_version: expected integer 1") + number = _scalar(loader, version, "schema_version") + if type(number) is not int or number != schema["schema_version"]: + raise ValueError("schema_version: expected integer 1") + result = {} + for section, node in root.items(): + if section not in schema["sections"]: + raise ValueError(f"{section}: unknown section") + values = {} + for key, field_node in _mapping(node, section).items(): + field = f"{section}.{key}" + if key not in schema["sections"][section]: + raise ValueError(f"{field}: unknown field") + if isinstance(field_node, SequenceNode): + value = [ + _scalar(loader, item, f"{field}[{index}]") + if isinstance(item, ScalarNode) else None + for index, item in enumerate(field_node.value) + ] + elif isinstance(field_node, ScalarNode): + value = _scalar(loader, field_node, field) + else: + raise ValueError(f"{field}: expected a scalar or list") + if value is not None: + values[key] = value + result[section] = values + return result + except UnicodeError: + raise ValueError("task: expected UTF-8 YAML") from None + except yaml.YAMLError as exc: + mark = getattr(exc, "problem_mark", None) + location = f" at line {mark.line + 1}, column {mark.column + 1}" if mark else "" + raise ValueError(f"task: invalid YAML{location}") from None + except RecursionError: + raise ValueError("task: YAML is nested too deeply") from None + finally: + if loader is not None: + loader.dispose() + + +def _text(value, field: str) -> str: + if not isinstance(value, str) or not value.strip() or _INVALID_TEXT.search(value): + raise ValueError(f"{field}: expected a nonempty string") + return value.strip() + + +def _dotted_key(value, field: str) -> str: + key = _text(value, field) + if any(not part or part != part.strip() for part in key.split(".")): + raise ValueError(f"{field}: expected a dotted JSON key") + return key + + +def _path(value: str, field: str, base: Path) -> str: + try: + return str((base / value).resolve()) + except (OSError, RuntimeError, ValueError): + raise ValueError(f"{field}: invalid filesystem path") from None + + +def _normalize(value, rule: dict, field: str, base: Path, *, cli: bool = False): + kind = rule["type"] + if kind == "boolean": + if type(value) is not bool: + raise ValueError(f"{field}: expected a boolean") + return value + if kind == "positive_integer": + if type(value) is not int or value < 1: + raise ValueError(f"{field}: expected a positive integer") + return value + if kind in ("minutes", "threshold"): + # Convert only finite JSON-representable numbers. A YAML integer may be huge. + try: + number = float(value) if type(value) in (int, float) else math.nan + except OverflowError: + number = math.nan + if kind == "minutes" and not 0 < number <= MAX_EXPERIMENT_MINUTES: + raise ValueError(f"{field}: expected minutes greater than zero and at most seven days") + if kind == "threshold" and (not math.isfinite(number) or number < 0): + raise ValueError(f"{field}: expected a finite nonnegative percentage") + return number + if kind in ("argv", "ids", "checks"): + if cli and kind == "argv": + try: + value = list(command_argv(value)) + except ValueError: + raise ValueError(f"{field}: expected a nonempty quoted argv command") from None + elif cli and kind == "ids": + value = [item.strip() for item in value.split(",") if item.strip()] + if not isinstance(value, list) or any( + not isinstance(item, str) or _INVALID_TEXT.search(item) for item in value + ): + raise ValueError(f"{field}: expected a list of strings") + if kind == "argv": + if not value or not value[0].strip(): + raise ValueError(f"{field}: expected a nonempty executable as the first argv item") + return value + if kind == "checks": + value = [_dotted_key(item, field) for item in value] + else: + value = [_text(item, field) for item in value] + if any("," in item for item in value): + raise ValueError(f"{field}: individual device IDs cannot contain commas") + if len(value) != len(set(value)): + raise ValueError(f"{field}: values must be unique") + return value + + value = _text(value, field) + if kind in ("choice", "framework", "backend"): + choices = {"framework": REPO_TYPES, "backend": BACKENDS}.get(kind, rule.get("choices")) + if value not in choices: + raise ValueError(f"{field}: expected one of {', '.join(choices)}") + if kind == "revision": + if not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", value): + raise ValueError(f"{field}: expected a full Git object ID") + return value.lower() + if kind == "metric": + return _dotted_key(value, field) + if kind == "reference" and re.match(r"(?i)https?:", value): + try: + url = urlsplit(value) + valid = ( + url.hostname and url.username is None and url.password is None + and url.port != 0 and not url.query + ) + if valid and not any(ord(char) < 32 or char.isspace() for char in value): + return value + except ValueError: + pass + raise ValueError(f"{field}: expected an HTTP(S) issue URL or local plan path") + if kind == "reference" and "://" in value: + raise ValueError(f"{field}: only HTTP(S) issue URLs or local plan paths are supported") + if kind in ("path", "reference") or ( + kind == "interpreter" and (Path(value).drive or "/" in value or "\\" in value) + ): + return _path(value, field, base) + return value + + +@dataclass(frozen=True) +class TaskInput: + task: dict + missing: tuple[str, ...] + arguments: dict + + def preview(self) -> dict: + return {"task": self.task, "complete": not self.missing, "missing": self.missing} + + def execution_args(self, args: argparse.Namespace) -> argparse.Namespace: + if self.missing: + raise ValueError("task: required fields for plan/run: " + ", ".join(self.missing)) + reference = self.task["acceptance"]["reference"] + if not re.match(r"(?i)https?://", reference) and not Path(reference).is_file(): + raise ValueError("acceptance.reference: local plan must be an existing file") + return argparse.Namespace(**(vars(args) | self.arguments)) + + def make_spec(self, args: argparse.Namespace) -> RunSpec: + try: + spec = make_spec(args) + except ValueError as exc: + # Scalar and argv validation is complete at this point. RunSpec + # still checks filesystem-dependent invariants before any probe. + field = { + "repo": "optimization.repository", + "candidate_repo": "optimization.candidate_repository", + "output_dir": "environment.output_directory", + }.get(str(exc).partition(" ")[0], "task") + raise ValueError(f"{field}: {exc}") from None + output = Path(spec.output_dir) + if any(path.exists() and not path.is_dir() for path in (output, *output.parents)): + raise ValueError("environment.output_directory: expected a directory path") + return spec + + +def normalize_task(path: str | Path, overrides: dict | None = None) -> TaskInput: + """Normalize YAML and explicitly supplied CLI flags, without probing or writing.""" + schema_resource = files("inferrsi").joinpath("data", "task_input", "schema-v1.json") + schema = json.loads(schema_resource.read_text(encoding="utf-8")) + path = Path(path) + yaml_base = path.absolute().parent + raw = _read_task(path, schema) + overrides = overrides or {} + task = {"schema_version": schema["schema_version"]} + missing = [] + arguments = {} + for section, fields in schema["sections"].items(): + normalized = {} + task[section] = normalized + for key, rule in fields.items(): + field = f"{section}.{key}" + flag = rule.get("cli_argument") + arg = flag[2:].replace("-", "_") if flag else None + value = raw.get(section, {}).get(key) + if value is not None: + value = _normalize(value, rule, field, yaml_base) + if arg in overrides: + override = _normalize(overrides[arg], rule, field, Path.cwd(), cli=True) + if section == "acceptance" and override != value: + raise ValueError(f"{field}: {flag} conflicts with the task file") + value = override + if value is None: + if rule.get("required_for_plan_run"): + missing.append(field) + elif arg in EXPERIMENT_DEFAULTS: + default = EXPERIMENT_DEFAULTS[arg] + value = ( + _path(default, field, yaml_base) if rule["type"] == "path" and default + else [] if rule["type"] in ("ids", "checks") + else default + ) + if value is not None: + normalized[key] = value + if arg: + arguments[arg] = ",".join(value) if rule["type"] == "ids" else value + if task["acceptance"].get("require_quality") and "quality_command" not in task["environment"]: + missing.append("environment.quality_command") + if task["environment"].get("backend") == "mlx" and task["environment"].get("device_ids"): + raise ValueError("environment.device_ids: unsupported for the mlx backend") + return TaskInput(task, tuple(missing), arguments) diff --git a/tests/check_task_installed.py b/tests/check_task_installed.py new file mode 100644 index 000000000..2ab94144b --- /dev/null +++ b/tests/check_task_installed.py @@ -0,0 +1,36 @@ +"""Exercise the public task commands against an installed wheel without extras.""" + +import json +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + + +def command(*args, cwd): + return subprocess.check_output( + [sys.executable, "-I", "-m", "inferrsi", "task", *args], + cwd=cwd, + encoding="utf-8", + ) + + +fixture = Path(__file__).with_name("task-v1.yaml") +with TemporaryDirectory() as directory: + root = Path(directory).resolve() + path = root / fixture.name + path.write_bytes(fixture.read_bytes()) + assert json.loads(command("schema", cwd=root))["schema_version"] == 1 + template = root / "template.yaml" + template.write_text(command("template", cwd=root), encoding="utf-8") + assert json.loads(command("validate", template.name, "--json", cwd=root)) == {"valid": True} + assert json.loads(command("validate", path.name, "--json", cwd=root)) == {"valid": True} + preview = json.loads(command("preview", path.name, "--json", cwd=root)) + assert preview["complete"] and preview["missing"] == [] + assert preview["task"]["optimization"]["repository"] == str(root / "checkout") + assert preview["task"]["environment"]["backend"] == "cuda" + assert preview["task"]["environment"]["repetitions"] == 2 + assert preview["task"]["environment"]["max_minutes"] == 1.5 + assert preview["task"]["acceptance"]["min_improvement_pct"] == 7.5 + assert set(root.iterdir()) == {path, template} + assert path.read_bytes() == fixture.read_bytes() diff --git a/tests/fixtures/task-v1.yaml b/tests/fixtures/task-v1.yaml new file mode 100644 index 000000000..030a3c00c --- /dev/null +++ b/tests/fixtures/task-v1.yaml @@ -0,0 +1,26 @@ +# Synthetic input for read-only tests. Commands and revision are test placeholders. +schema_version: 1 +optimization: + repository: ./checkout + model: test-model + framework: vllm + workload: fixed-synthetic-input + objective: Verify task normalization + plane: harness +environment: + backend: cuda + device_ids: ["2"] + baseline_command: [fixture-benchmark, baseline, "{result}"] + candidate_command: [fixture-benchmark, candidate, "{result}"] + quality_command: [fixture-quality] + repetitions: 2 + max_minutes: 1.5 + output_directory: ./results +acceptance: + reference: ./frozen-plan.md + baseline: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + metric_key: metrics.latency_ms + metric_unit: ms + direction: minimize + min_improvement_pct: 7.5 + require_quality: true diff --git a/tests/test_knowledge_fresh_install.py b/tests/test_knowledge_fresh_install.py index 05b31aba9..69e30574a 100644 --- a/tests/test_knowledge_fresh_install.py +++ b/tests/test_knowledge_fresh_install.py @@ -1,4 +1,4 @@ -"""Fresh-venv install smoke: base install package-only, skills extra unlocks the checkout chain. +"""Fresh-venv install smoke for package and checkout knowledge reads. Opt-in because it builds a real venv and installs from this source tree (minutes of I/O, network access for the build-isolation backend, and an @@ -66,7 +66,7 @@ def _cli(venv_python: Path, *args: str) -> subprocess.CompletedProcess: ) -def test_base_install_package_only_then_skills_extra_chain(): +def test_base_install_supports_package_and_checkout_reads(): workspace = _short_workspace() try: venv = workspace / "venv" @@ -79,21 +79,12 @@ def test_base_install_package_only_then_skills_extra_chain(): assert created.returncode == 0, created.stderr python = _venv_python(venv) - _pip(python, "install", str(REPO)) # base only: no PyYAML, no pytest/ruff + _pip(python, "install", str(REPO)) package_only = _cli(python, "collect", "--limit", "2", "--json") assert package_only.returncode == 0, package_only.stderr assert json.loads(package_only.stdout)["items"] - blocked = _cli( - python, "collect", "--id", f"checkout:{SOURCE}", "--skill-root", str(SKILLS) - ) - assert blocked.returncode == 2 - assert "PyYAML" in blocked.stderr and "[skills]" in blocked.stderr - assert "Traceback" not in blocked.stderr - - _pip(python, "install", f"{REPO}[skills]") - collected = _cli( python, "collect", "--id", f"checkout:{SOURCE}", "--skill-root", str(SKILLS), "--json" ) diff --git a/tests/test_knowledge_install.py b/tests/test_knowledge_install.py index 21a061d40..9c3d45173 100644 --- a/tests/test_knowledge_install.py +++ b/tests/test_knowledge_install.py @@ -37,8 +37,7 @@ def _run(args: list[str], env: dict, *, cwd: Path = REPO) -> subprocess.Complete @pytest.fixture def without_yaml(tmp_path): - # Shadow PyYAML with a failing module: exactly the base install without - # the skills extra, without touching the test environment itself. + # Emulate a direct-source or bundled plugin runtime without PyYAML. blocked = tmp_path / "blocked" blocked.mkdir() (blocked / "yaml.py").write_text( diff --git a/tests/test_task_input.py b/tests/test_task_input.py new file mode 100644 index 000000000..b8e46bf9e --- /dev/null +++ b/tests/test_task_input.py @@ -0,0 +1,181 @@ +import json +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from inferrsi import cli, inspection +from inferrsi.task_input import normalize_task + +FIXTURE = Path(__file__).parent / "fixtures" / "task-v1.yaml" + + +@pytest.fixture +def task_file(tmp_path): + path = tmp_path / "inputs" / "task.yaml" + path.parent.mkdir() + path.write_bytes(FIXTURE.read_bytes()) + return path + + +def test_preview_and_validation_are_read_only(task_file, tmp_path, monkeypatch, capsys): + def unexpected(*args, **kwargs): + pytest.fail("read-only task command attempted to probe or execute") + + monkeypatch.setattr(cli, "inspect_environment", unexpected) + monkeypatch.setattr(subprocess, "Popen", unexpected) + monkeypatch.chdir(tmp_path) + before = sorted(tmp_path.rglob("*")) + assert cli.main(["task", "validate", str(task_file), "--json"]) == 0 + assert json.loads(capsys.readouterr().out) == {"valid": True} + assert cli.main(["task", "preview", str(task_file), "--json"]) == 0 + preview = json.loads(capsys.readouterr().out) + assert preview["task"] == normalize_task(task_file).task + assert preview["complete"] and not preview["missing"] + assert preview["task"]["optimization"]["repository"] == str(task_file.parent / "checkout") + assert preview["task"]["environment"]["backend"] == "cuda" + assert preview["task"]["environment"]["repetitions"] == 2 + assert preview["task"]["environment"]["max_minutes"] == 1.5 + assert preview["task"]["acceptance"]["min_improvement_pct"] == 7.5 + assert preview["task"]["acceptance"]["reference"] == str(task_file.parent / "frozen-plan.md") + assert sorted(tmp_path.rglob("*")) == before + + +def test_preview_uses_explicit_flags_and_respects_frozen_fields(task_file, tmp_path, monkeypatch, + capsys): + monkeypatch.chdir(tmp_path) + command = ["task", "preview", str(task_file), "--json"] + assert cli.main([ + *command, "--backend=cpu", "--device-ids=", "--repo", "cli-checkout", + "--repetitions", "5", "--min-improvement-pct", "7.5", + ]) == 0 + task = json.loads(capsys.readouterr().out)["task"] + assert task["environment"]["backend"] == "cpu" + assert task["environment"]["device_ids"] == [] + assert task["environment"]["repetitions"] == 5 + assert task["optimization"]["repository"] == str(tmp_path / "cli-checkout") + assert task["acceptance"]["min_improvement_pct"] == 7.5 + with pytest.raises(SystemExit) as exc: + cli.main([*command, "--min-improvement-pct", "0"]) + assert exc.value.code == 2 + assert "acceptance.min_improvement_pct" in capsys.readouterr().err + + +@pytest.mark.parametrize("command", ["plan", "run"]) +def test_incomplete_task_stops_before_probes(task_file, monkeypatch, capsys, command): + task_file.write_text("schema_version: 1\noptimization:\n model: null\n", encoding="utf-8") + assert cli.main(["task", "preview", str(task_file), "--json"]) == 0 + preview = json.loads(capsys.readouterr().out) + assert not preview["complete"] + assert {"optimization.model", "environment.backend", "acceptance.reference"} <= set( + preview["missing"] + ) + + def unexpected(*args, **kwargs): + pytest.fail("incomplete input reached an operation") + + monkeypatch.setattr(cli, "inspect_environment", unexpected) + monkeypatch.setattr(cli, "run_experiment", unexpected) + with pytest.raises(SystemExit) as exc: + cli.main([command, "--task", str(task_file)]) + assert exc.value.code == 2 + assert "optimization.model" in capsys.readouterr().err + assert list(task_file.parent.iterdir()) == [task_file] + + +@pytest.mark.parametrize(("source", "message"), [ + ("schema_version: true\n", "schema_version"), + ("schema_version: 2\n", "schema_version"), + ("schema_version: 1\nschema_version: 1\n", "schema_version: duplicate"), + ("schema_version: 1\nenvironment: null\n", "environment: expected a mapping"), + ("schema_version: 1\nenvironment: {backend: gpu}\n", "environment.backend"), + ("schema_version: 1\nenvironment: {backend: cpu, backend: cuda}\n", + "environment.backend: duplicate"), + ("schema_version: 1\nenvironment: {maxRounds: 2}\n", "environment.maxRounds"), + ("schema_version: 1\nenvironment: {repetitions: 2.0}\n", "environment.repetitions"), + ("schema_version: 1\nenvironment: {max_minutes: 1e999}\n", "environment.max_minutes"), + ("schema_version: 1\nenvironment: {device_ids: ['0', '0']}\n", "environment.device_ids"), + ("schema_version: 1\nenvironment: {baseline_command: [null]}\n", + "environment.baseline_command"), + ("schema_version: 1\nacceptance: {baseline: HEAD}\n", "acceptance.baseline"), + ("schema_version: 1\nacceptance: {reference: 'https://user:secret@example.invalid/issues/1'}\n", + "acceptance.reference"), + ("schema_version: 1\noptimization: {model: &alias model}\n", "anchors, aliases and tags"), + ("schema_version: 1\noptimization: {model: !!str model}\n", "anchors, aliases and tags"), + ('schema_version: 1\noptimization: {objective: "\\uD800"}\n', "optimization.objective"), + ('schema_version: 1\nacceptance: {reference: "https://@example.invalid/1"}\n', + "acceptance.reference"), + ('schema_version: 1\nacceptance: {reference: "https://example.invalid:abc/1"}\n', + "acceptance.reference"), + ("schema_version: 1\n# \0\n", "invalid YAML"), + ("schema_version: 1\n---\nschema_version: 1\n", "invalid YAML"), +]) +def test_invalid_yaml_has_a_field_or_location(task_file, source, message): + task_file.write_text(source, encoding="utf-8") + with pytest.raises(ValueError, match=message) as exc: + normalize_task(task_file, {"backend": "cpu"}) + assert "user:secret" not in str(exc.value) + + +def test_plan_and_explicit_run_share_task_and_keep_source_pin(task_file, capsys, monkeypatch): + repo = task_file.parent / "checkout" + repo.mkdir() + + def git(*args): + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() + + git("init", "-q") + git("-c", "commit.gpgsign=false", "-c", "user.name=Fixture", + "-c", "user.email=fixture@example.invalid", + "commit", "--allow-empty", "-qm", "synthetic") + head = git("rev-parse", "HEAD") + task = yaml.safe_load(task_file.read_text(encoding="utf-8")) + task["acceptance"]["baseline"] = head + (task_file.parent / "frozen-plan.md").write_text("Synthetic plan fixture", encoding="utf-8") + script = ( + "from pathlib import Path; import sys; " + "Path(sys.argv[1]).write_text('{\"metrics\":{\"latency_ms\":' + sys.argv[2] + '}}')" + ) + task["environment"].update( + baseline_command=[sys.executable, "-c", script, "{result}", "100"], + candidate_command=[sys.executable, "-c", script, "{result}", "90"], + quality_command=[sys.executable, "-c", "pass"], + device_ids=[], + ) + task_file.write_text(yaml.safe_dump(task), encoding="utf-8") + options = ["--task", str(task_file), "--backend", "cpu"] + assert cli.main(["plan", *options]) == 0 + plan = json.loads(capsys.readouterr().out) + spec = plan["run_spec"] + assert spec["revision"] == head + assert spec["backend"] == "cpu" + assert spec["repetitions"] == 2 and spec["max_minutes"] == 1.5 + assert spec["min_improvement_pct"] == 7.5 and spec["model"] == "test-model" + output = task_file.parent / "results" + assert not output.exists() + with pytest.raises(SystemExit) as exc: + cli.main(["run", *options]) + assert exc.value.code == 2 + assert "--confirm-acceptance" in capsys.readouterr().err + assert not output.exists() + assert cli.main(["run", *options, "--confirm-acceptance"]) == 0 + result = json.loads(capsys.readouterr().out) + run = Path(result["run_dir"]) + assert run.parent == output + assert json.loads((run / "run.lock.json").read_text()) == spec + assert json.loads((run / "environment.json").read_text())["task"] == plan["environment"]["task"] + + task["acceptance"]["baseline"] = "0" * 40 + task_file.write_text(yaml.safe_dump(task), encoding="utf-8") + + def unexpected(*args, **kwargs): + pytest.fail("revision mismatch reached a runtime probe") + + monkeypatch.setattr(inspection, "_command", unexpected) + with pytest.raises(SystemExit) as exc: + cli.main(["run", *options, "--confirm-acceptance"]) + assert exc.value.code == 2 + assert "acceptance.baseline" in capsys.readouterr().err + assert list(output.iterdir()) == [run] diff --git a/tools/plugin_bundle.py b/tools/plugin_bundle.py index f90c1ee91..0c87add48 100644 --- a/tools/plugin_bundle.py +++ b/tools/plugin_bundle.py @@ -56,6 +56,7 @@ def bundle_runtime(source: Path, runtime: Path, *, plugin_version: str, and relative.parts[-3:-1] == ("references", "licenses") ) or (path.suffix == ".html" and relative.parent.parts == ("data", "dashboard")) + or (path.suffix == ".yaml" and relative.parent.parts == ("data", "task_input")) or (path.name == "upstream-NOTICE" and relative.parent.name == "references") ) if path.suffix != ".py" and not package_data: From e76189ed9889870d94b520dfba267a2760c583a2 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Fri, 6 Nov 2026 22:43:39 +0800 Subject: [PATCH 2/3] fix: update checkout skills PyYAML install hint Direct-source users now see the standard package install command when the core YAML dependency is missing. Keep the checkout and lightweight-controller error paths covered. Agent-assisted-by: Codex / unknown [effort=unknown] (author) Signed-off-by: zengchuang --- src/inferrsi/skill_catalog.py | 6 +++--- tests/test_knowledge_install.py | 7 ++++--- tests/test_skill_eval_optional_dependencies.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/inferrsi/skill_catalog.py b/src/inferrsi/skill_catalog.py index d4ffb477b..28a62e2ad 100644 --- a/src/inferrsi/skill_catalog.py +++ b/src/inferrsi/skill_catalog.py @@ -9,9 +9,9 @@ import yaml except ImportError as exc: # Plugin and direct-source runtimes may omit PyYAML. raise ValueError( - "Reading checkout skills requires the 'skills' extra (PyYAML); install it " - "from your InferRSI checkout, replacing the placeholder path: " - "python -m pip install '[skills]'" + "Reading checkout skills requires PyYAML. Install the standard InferRSI " + "package from your checkout, replacing the placeholder path: " + "python -m pip install ''" ) from exc from .catalog import BACKENDS diff --git a/tests/test_knowledge_install.py b/tests/test_knowledge_install.py index 9c3d45173..f8631cc2a 100644 --- a/tests/test_knowledge_install.py +++ b/tests/test_knowledge_install.py @@ -65,8 +65,8 @@ def test_skill_root_without_pyyaml_fails_with_install_hint(without_yaml): ) assert result.returncode == 2 assert "Traceback" not in result.stderr - assert "[skills]" in result.stderr and "PyYAML" in result.stderr - assert "[skills]" in result.stderr + assert "requires PyYAML" in result.stderr + assert "python -m pip install ''" in result.stderr def test_install_hint_is_actionable_outside_the_repository(without_yaml, tmp_path): @@ -78,7 +78,8 @@ def test_install_hint_is_actionable_outside_the_repository(without_yaml, tmp_pat ) assert result.returncode == 2 assert "Traceback" not in result.stderr - assert "[skills]" in result.stderr and "placeholder" in result.stderr + assert "python -m pip install ''" in result.stderr + assert "placeholder" in result.stderr def test_checkout_chain_collect_read_resource_through_real_cli(tmp_path): diff --git a/tests/test_skill_eval_optional_dependencies.py b/tests/test_skill_eval_optional_dependencies.py index 3a243090c..60148f5b2 100644 --- a/tests/test_skill_eval_optional_dependencies.py +++ b/tests/test_skill_eval_optional_dependencies.py @@ -71,7 +71,7 @@ def test_selected_snapshot_still_requires_yaml_before_side_effects(self): try: selected_snapshot(root, 'a' * 40, 'library', 'selected', destination) except ValueError as error: - assert "'skills' extra (PyYAML)" in str(error), str(error) + assert 'requires PyYAML' in str(error), str(error) else: raise AssertionError('Selected snapshot silently skipped the skills parser') git.assert_not_called() From 24ef10a5062ccd6d005f43658216fa01e0fc9599 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Mon, 9 Nov 2026 14:46:26 +0800 Subject: [PATCH 3/3] fix: keep shared task input outside the knowledge library Restore the knowledge operations guide, the existing skill loader hint, and its assertions. The shared YAML task contract still uses PyYAML as a core dependency and validates the standard wheel without extras. Agent-assisted-by: Codex (scope correction and validation) Signed-off-by: zengchuang --- docs/knowledge-operations.md | 7 ++++--- src/inferrsi/skill_catalog.py | 8 ++++---- tests/test_knowledge_install.py | 10 +++++----- tests/test_skill_eval_optional_dependencies.py | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md index d41a0b3cc..e3a5833cf 100644 --- a/docs/knowledge-operations.md +++ b/docs/knowledge-operations.md @@ -32,7 +32,8 @@ inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/ inferrsi read calibrate-serving-simulation --skill-root src/inferrsi/data/skill_library ``` -Both packaged and checkout skill reads work with the base install. +The `--skill-root` selections require the `skills` extra (`python -m pip install +'.[skills]'`); package-only collect/read work with the base install. Every item has an owning `authority` (`package`, `checkout` or `store`) and a qualified `ref` of the form `:`. Public IDs are unchanged and may be used @@ -224,8 +225,8 @@ to narrow results. An explicit ID such as Package repo/backend/scenario filters still exclude checkout cases; inspect the returned applicability rather than assuming those filters qualify a case. -The CLI resolves resource paths from the installed library or a custom library -selected with `--root`. +For explicit CLI resource-path resolution from the installed library (optional +`skills` extra required), or a custom library selected with `--root`: ```bash inferrsi skills resource inference-optimization-casebook cases/qwen-ascend-rejected-directions.md diff --git a/src/inferrsi/skill_catalog.py b/src/inferrsi/skill_catalog.py index 28a62e2ad..ad3ad25a2 100644 --- a/src/inferrsi/skill_catalog.py +++ b/src/inferrsi/skill_catalog.py @@ -7,11 +7,11 @@ try: import yaml -except ImportError as exc: # Plugin and direct-source runtimes may omit PyYAML. +except ImportError as exc: # The base install stays PyYAML-free by contract. raise ValueError( - "Reading checkout skills requires PyYAML. Install the standard InferRSI " - "package from your checkout, replacing the placeholder path: " - "python -m pip install ''" + "Reading checkout skills requires the 'skills' extra (PyYAML); install it " + "from your InferRSI checkout, replacing the placeholder path: " + "python -m pip install '[skills]'" ) from exc from .catalog import BACKENDS diff --git a/tests/test_knowledge_install.py b/tests/test_knowledge_install.py index f8631cc2a..21a061d40 100644 --- a/tests/test_knowledge_install.py +++ b/tests/test_knowledge_install.py @@ -37,7 +37,8 @@ def _run(args: list[str], env: dict, *, cwd: Path = REPO) -> subprocess.Complete @pytest.fixture def without_yaml(tmp_path): - # Emulate a direct-source or bundled plugin runtime without PyYAML. + # Shadow PyYAML with a failing module: exactly the base install without + # the skills extra, without touching the test environment itself. blocked = tmp_path / "blocked" blocked.mkdir() (blocked / "yaml.py").write_text( @@ -65,8 +66,8 @@ def test_skill_root_without_pyyaml_fails_with_install_hint(without_yaml): ) assert result.returncode == 2 assert "Traceback" not in result.stderr - assert "requires PyYAML" in result.stderr - assert "python -m pip install ''" in result.stderr + assert "[skills]" in result.stderr and "PyYAML" in result.stderr + assert "[skills]" in result.stderr def test_install_hint_is_actionable_outside_the_repository(without_yaml, tmp_path): @@ -78,8 +79,7 @@ def test_install_hint_is_actionable_outside_the_repository(without_yaml, tmp_pat ) assert result.returncode == 2 assert "Traceback" not in result.stderr - assert "python -m pip install ''" in result.stderr - assert "placeholder" in result.stderr + assert "[skills]" in result.stderr and "placeholder" in result.stderr def test_checkout_chain_collect_read_resource_through_real_cli(tmp_path): diff --git a/tests/test_skill_eval_optional_dependencies.py b/tests/test_skill_eval_optional_dependencies.py index 60148f5b2..3a243090c 100644 --- a/tests/test_skill_eval_optional_dependencies.py +++ b/tests/test_skill_eval_optional_dependencies.py @@ -71,7 +71,7 @@ def test_selected_snapshot_still_requires_yaml_before_side_effects(self): try: selected_snapshot(root, 'a' * 40, 'library', 'selected', destination) except ValueError as error: - assert 'requires PyYAML' in str(error), str(error) + assert "'skills' extra (PyYAML)" in str(error), str(error) else: raise AssertionError('Selected snapshot silently skipped the skills parser') git.assert_not_called()