diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index aeda080ff..0ec683ba3 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -30,10 +30,11 @@ jobs: run: | python -m venv "$RUNNER_TEMP/wheel-env" wheels=("$RUNNER_TEMP"/dist/*.whl) - "$RUNNER_TEMP/wheel-env/bin/python" -m pip install "${wheels[0]}[skills]" - cp tests/check_installed.py "$RUNNER_TEMP/check_installed.py" + "$RUNNER_TEMP/wheel-env/bin/python" -m pip install "${wheels[0]}" + cp tests/check_installed.py tests/check_task_installed.py tests/fixtures/task-v1.yaml "$RUNNER_TEMP/" cd "$RUNNER_TEMP" env -u PYTHONPATH "$RUNNER_TEMP/wheel-env/bin/python" -I check_installed.py + env -u PYTHONPATH "$RUNNER_TEMP/wheel-env/bin/python" -I check_task_installed.py full-check: strategy: diff --git a/docs/contributing.md b/docs/contributing.md index 27319aaf8..f7a0708e7 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -221,11 +221,11 @@ Keep the existing four lanes: | [Merge](../.github/workflows/ci-merge.yml) | Manifest check and lightweight CPU smoke on main | | [Nightly](../.github/workflows/ci-nightly.yml) | Scheduled lint/tests, evidence layout and label drift | -Ready CI also builds the source distribution and wheel, installs the wheel with -the optional `skills` parser in a fresh virtual environment, and runs -`tests/check_installed.py` outside the checkout with `PYTHONPATH` unset and Python -isolated mode enabled. This checks packaged knowledge, skill bodies, licenses, -helpers and CLI reads without installing any inference engine. +Ready CI builds the source distribution and wheel and installs the wheel with +its core dependencies in a fresh virtual environment. It runs +`tests/check_installed.py` and `tests/check_task_installed.py` outside the checkout +with `PYTHONPATH` unset and Python isolated mode enabled. The checks cover packaged +knowledge, skills, task schema and template, and read-only CLI operations. The ready test and installed-package jobs cover the declared minimum Python 3.10 alongside Python 3.11. This matrix does not certify later Python releases or accelerator runtime compatibility. diff --git a/docs/operations.md b/docs/operations.md index 39abbed32..f6922b6d2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -121,7 +121,7 @@ those resources retain their repository owner and source attribution. ### Discover/read packaged or custom skills — `skills list` / `show` / `resource` / `reference` -- **Inputs:** `list` reads frontmatter and sidecar metadata; `show ` selects one instruction body; `resource ` and `reference ` resolve one path. The default is the installed library's registered skills; an explicit `--root` selects a custom flat tree or a categorized library with `skill-index.json`. The optional `skills` extra supplies the YAML parser. +- **Inputs:** `list` reads frontmatter and sidecar metadata; `show ` selects one instruction body; `resource ` and `reference ` resolve one path. The default is the installed library's registered skills; an explicit `--root` selects a custom flat tree or a categorized library with `skill-index.json`. - **Scope:** complete packaged skills under `data/skill_library/`, or the explicitly selected custom root. The 21 English engine and research-router skills remain accessible through catalog records and library navigation, separately from the registry. - **Output:** metadata listing, one instruction body, or a resolved path. - **Side effects:** none. `resource`/`reference` do not load or execute the target. @@ -190,9 +190,18 @@ normal scoped PR path. ## Experiments +### Read shared YAML input — `task schema`, `task template`, `task validate`, `task preview` + +- **Inputs** The packaged schema/template, or a v1 YAML task path and optional preview overrides. +- **Output** Schema or template, validation result, or normalized values and required-field blockers for plan/run. Use `--json` for Agent handoff. +- **Side effects** Read-only. These commands run no probes or subprocesses and write nothing to the workspace or experiment directory. +- **Failure behavior** Exit 2 with a field or location for invalid input. A valid draft can have missing execution inputs. + +See the [shared YAML task guide](task-input.md) for paths, merging and the CLI/Agent workflow. + ### Inspect/prepare — `doctor`, `plan`, `--dry-run` -- **Inputs:** repository/backend/device flags; `plan` takes the full experiment specification. +- **Inputs:** repository/backend/device flags; `plan` takes the full experiment specification as flags or `--task` YAML. - **Scope:** environment inspection and specification preview. - **Output:** environment facts, or a preview of the RunSpec. - **Side effects:** `doctor` may invoke bounded read-only probes; `plan`/`--dry-run` persist nothing. @@ -205,7 +214,7 @@ compatibility. ### Execute/inspect — `run`, `status`, `report` -- **Inputs:** the full experiment specification (`run`); a run ID and output directory (`status`/`report`). +- **Inputs:** the full experiment specification as flags or `--task` YAML with `--confirm-acceptance` (`run`); a run ID and output directory (`status`/`report`). - **Scope:** caller-supplied benchmark, candidate and validation commands under the experiment contract. - **Output:** a run ID and evidence directory; recorded progress; an HTML report. - **Side effects:** `run` executes the supplied commands and owns child-process cleanup; `report` writes HTML into the run directory. diff --git a/docs/quickstart.md b/docs/quickstart.md index 24d2fda79..bc3aa8491 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,6 +1,7 @@ # Quickstart -This release uses CLI args and a **local executor on a prepared machine**. Install it directly on the remote machine where the target engine already works. It does not set up models, CUDA/ROCm/CANN, services, SSH workers or autonomous Agents. +Use CLI flags or a [shared YAML task](task-input.md) with the **local executor +on a prepared machine**. Install InferRSI where the target engine already works. ## Install and smoke @@ -14,11 +15,7 @@ inferrsi --help inferrsi doctor --repo . --repo-type vllm_omni --backend cpu ``` -Package-only knowledge commands (`knowledge`, `context`, package `collect`/`read`) -work with this base install. Reading skill libraries — the `skills` command and -any `--skill-root` selection — additionally requires -`python -m pip install '.[skills]'` (PyYAML); without it those operations fail -with that install hint instead of a traceback. +The base install supports packaged knowledge, skill-library reads and YAML tasks. For the maintained model examples, see [Qwen B300](qwen-image-2.1-turbo-b300.md) and [Qwen Ascend](qwen-image-2.1-turbo-ascend.md). Continue below for the generic @@ -53,7 +50,7 @@ entry point, effect class and boundary: ```bash inferrsi knowledge search 'Qwen-Image-2.1' --repo-type vllm_omni --backend ascend inferrsi context --repo-type vllm_omni --backend cuda --output context.md -inferrsi skills list --json # requires python -m pip install '.[skills]' +inferrsi skills list --json inferrsi read inferrsi.ascend.skills.ascend_graph_mode --material ``` @@ -61,7 +58,7 @@ Normalized Collect, exact-revision reads and resource access are available as CLI commands and as the Python `KnowledgeReader` ([contract](knowledge-operations.md)); authorized local candidate Create/Update remains a Python `KnowledgeStore` operation ([contract](knowledge-store.md)). -The `--skill-root` examples below require the `skills` extra from above: +To read from a checkout's skill root: ```bash inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/inferrsi/data/skill_library @@ -88,8 +85,10 @@ examples above explicitly adds that root's skill and case IDs. ## The actual configuration interface ```text +task: schema | template | validate TASK.yaml | preview TASK.yaml --json doctor: --repo --repo-type --backend --device-ids --device-model --python -plan/run: all doctor flags, plus +plan/run: --task TASK.yaml (--confirm-acceptance to execute) or the existing flags + all doctor flags, plus --candidate-repo --scenario --mode --agent --executor local --baseline-command --candidate-command --validation-command --metric-key --metric-unit --comparison-scope --direction diff --git a/docs/task-input.md b/docs/task-input.md new file mode 100644 index 000000000..107401c1a --- /dev/null +++ b/docs/task-input.md @@ -0,0 +1,73 @@ +# Shared YAML tasks + +Use a v1 task file to give the CLI and an Agent the same input. A normal +InferRSI install includes the parser, [v1 schema](../src/inferrsi/data/task_input/schema-v1.json) +and [YAML template](../src/inferrsi/data/task_input/template-v1.yaml). + +```bash +inferrsi task template > task.yaml +inferrsi task schema +inferrsi task validate task.yaml +inferrsi task preview task.yaml --json +``` + +An Agent can read or fill in the same template. Have it run the same validation +and JSON preview and review the effective values with you before proceeding. +A valid draft can omit any section or leave a field as null. The preview lists +missing fields by dotted path and sets `complete` when all required inputs are +present. Validation and preview read the file without invoking processes, +inspecting the runtime, or writing workspace or experiment files. Invalid input +exits with status 2. + +| Section | Required for `plan` and `run` | +| --- | --- | +| `optimization` | Repository, model, framework (the CLI's repository type), workload, objective and SIAS plane | +| `environment` | Backend, baseline and candidate argv, benchmark repetitions and time budget | +| `acceptance` | Issue URL or local plan path, full baseline Git object ID, metric key, unit, direction and agreed minimum improvement percentage | + +Use the schema for exact field names, types and CLI mappings. Supply any device +selectors or quality inputs the accepted plan requires. Set +`acceptance.require_quality: true` when the quality command is mandatory. +`acceptance.constraints` records additional plan requirements for review. +The existing runner evaluates its configured metric and validation command. + +- Use a single UTF-8 YAML document up to 64 KiB. Use plain mappings and argv + lists of strings. Duplicate and unknown keys, aliases, anchors and explicit + tags are errors. Numeric inputs use decimal notation, booleans use + `true`/`false`, and strings that resemble these values should be quoted. +- YAML filesystem paths are relative to the file. Paths in explicit CLI flags + are relative to the caller's working directory. With a task file, the default + output directory is `runs` next to the YAML. A bare Python executable name is + resolved through the existing runtime lookup. Script paths and other argv + tokens keep the existing per-checkout working-directory semantics. Model + identifiers and HTTP(S) acceptance references stay as given. +- In preview, plan and run, explicit mapped CLI flags override YAML for + optimization and environment inputs. Built-in defaults fill omitted optional + fields. Acceptance settings come from YAML. A conflicting CLI acceptance + flag is rejected. Existing flag-only commands use their usual defaults. +- Keep credentials in the existing runtime channels. Review the YAML, + normalized preview and experiment records before sharing them. + +After reviewing the exact commands and the effective values against the +accepted frozen gates, inspect the prepared environment and explicitly run +the experiment: + +```bash +inferrsi plan --task task.yaml +inferrsi run --task task.yaml --confirm-acceptance +``` + +`plan` and `run --dry-run` use the existing environment probes, including +the configured Python interpreter. Task-file operations check the baseline +checkout's HEAD against the pinned object ID. A local acceptance reference +must point to an existing file. Pass `--confirm-acceptance` after checking +the effective values against the referenced frozen gates. InferRSI records +the confirmation and effective task. Review of the issue or plan is performed +by the caller or Agent. The [operation guide](operations.md#experiments) and +[evidence rules](contributing.md#evidence-and-claims) apply to the result. + +The DSH draft format is tracked separately under +[#152](https://github.com/InferMatrixLab/InferRSI/issues/152). Check legacy +DSH files against the core schema and migrate incompatible fields before +using them with these commands. DSH integration uses the core contract and +shared fixtures when it advertises core v1 support. diff --git a/pyproject.toml b/pyproject.toml index 50a9c792d..2a14afc61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "CLI-first reproducible inference experiments and source-grounded readme = "README.md" license = "Apache-2.0" requires-python = ">=3.10" -dependencies = [] +dependencies = ["PyYAML>=6.0"] [project.scripts] inferrsi = "inferrsi.cli:main" @@ -23,7 +23,7 @@ where = ["src"] namespaces = false [tool.setuptools.package-data] -inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE", "data/dashboard/*.html"] +inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE", "data/dashboard/*.html", "data/task_input/*.yaml"] [tool.setuptools_scm] version_file = "src/inferrsi/_version.py" @@ -34,8 +34,9 @@ initial_version = "0.1.0" prerelease = true [project.optional-dependencies] -skills = ["PyYAML>=6.0"] -dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0"] +# Keep the former extra as an alias for existing installs. +skills = [] +dev = ["pytest>=7.4", "ruff>=0.6"] image = ["numpy>=1.24", "Pillow>=10", "scikit-image>=0.21"] [tool.pytest.ini_options] diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 577c91a90..966d18545 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -1,4 +1,4 @@ -"""The user configuration surface is argparse, not task files.""" +"""CLI argument declarations and RunSpec construction.""" from __future__ import annotations @@ -12,13 +12,55 @@ from .catalog import BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES from .runspec import RunSpec +TARGET_DEFAULTS = { + "repo": ".", "repo_type": "vllm_omni", "backend": "cpu", "device_ids": "", + "device_model": None, "python": sys.executable, +} +EXPERIMENT_DEFAULTS = { + **TARGET_DEFAULTS, + "candidate_repo": None, + "scenario": "external_benchmark", + "mode": "config", + "agent": "manual", + "executor": "local", + "validation_command": None, + "metric_key": "metrics.latency_ms", + "require_trial_check": (), + "metric_unit": "ms", + "comparison_scope": "external_command", + "direction": "minimize", + "repetitions": 3, + "min_improvement_pct": 0.0, + "max_minutes": 120.0, + "workload_id": None, + "model": None, + "model_revision": None, + "output_dir": "runs", +} + + +class _ArgumentParser(argparse.ArgumentParser): + def parse_args(self, args=None, namespace=None): + parsed = super().parse_args(args, namespace) + if parsed.command in ("plan", "run") and not hasattr(parsed, "task"): + if getattr(parsed, "confirm_acceptance", False): + self.error("--confirm-acceptance requires --task") + required = ("baseline_command", "candidate_command") + missing = [ + f"--{name.replace('_', '-')}" for name in required if not hasattr(parsed, name) + ] + if missing: + self.error("the following arguments are required: " + ", ".join(missing)) + return argparse.Namespace(**(EXPERIMENT_DEFAULTS | vars(parsed))) + return parsed + def _common(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--repo", default=".", help="Prepared target Git checkout") - parser.add_argument("--repo-type", choices=REPO_TYPES, default="vllm_omni") - parser.add_argument("--backend", choices=BACKENDS, default="cpu") + parser.add_argument("--repo", help="Prepared target Git checkout") + parser.add_argument("--repo-type", choices=REPO_TYPES) + parser.add_argument("--backend", choices=BACKENDS) parser.add_argument( - "--device-ids", default="", + "--device-ids", help=( "Comma-separated caller-assigned device IDs; " "unsupported by MLX unified-memory discovery" @@ -29,83 +71,101 @@ def _common(parser: argparse.ArgumentParser) -> None: help="Expected CUDA device model or Apple Silicon chip (case-insensitive match)", ) parser.add_argument( - "--python", default=sys.executable, help="Python used for target package inspection" + "--python", help="Python used for target package inspection" ) +def _experiment_options(child: argparse.ArgumentParser) -> None: + _common(child) + child.add_argument( + "--candidate-repo", help="Candidate checkout; defaults to the baseline checkout" + ) + child.add_argument( + "--scenario", help="Caller-declared scenario label" + ) + child.add_argument("--mode", choices=("config", "code", "semantic")) + child.add_argument( + "--agent", + choices=("manual", "codex", "external"), + help="Handoff label only; no Agent is launched", + ) + child.add_argument( + "--executor", + choices=("local",), + help="Install this CLI on the remote machine for this first release", + ) + child.add_argument( + "--baseline-command", + help="Quoted argv command; {result} is replaced with a fresh output path", + ) + child.add_argument("--candidate-command") + child.add_argument( + "--validation-command", + help="Explicit quality/correctness command; missing evidence is INCONCLUSIVE", + ) + child.add_argument( + "--metric-key", + help="Flat or dotted key in benchmark JSON", + ) + child.add_argument( + "--require-trial-check", action="append", metavar="DOTTED_KEY", + help="Require a literal true qualification field in every benchmark JSON; repeatable", + ) + child.add_argument("--metric-unit") + child.add_argument("--comparison-scope") + child.add_argument("--direction", choices=("minimize", "maximize")) + child.add_argument("--repetitions", type=int) + child.add_argument("--min-improvement-pct", type=float) + child.add_argument( + "--max-minutes", + type=float, + help="Experiment deadline, at most seven days", + ) + child.add_argument("--workload-id", help="Caller-declared fixed workload identity") + child.add_argument("--model", help="Model identity/path recorded in the experiment") + child.add_argument("--model-revision") + child.add_argument("--output-dir") + + def parser() -> argparse.ArgumentParser: - root = argparse.ArgumentParser( + root = _ArgumentParser( description="CLI-first reproducible experiments and tuning knowledge" ) - root.add_argument("--version", action="version", version=__version__) commands = root.add_subparsers(dest="command", required=True) + root.add_argument("--version", action="version", version=__version__) doctor = commands.add_parser( "doctor", help="Inspect a prepared local target without installing dependencies" ) _common(doctor) + doctor.set_defaults(**TARGET_DEFAULTS) for name in ("plan", "run"): child = commands.add_parser( name, - help="Freeze an external-command experiment" + help="Preview an external-command experiment" if name == "plan" else "Measure baseline and candidate commands", + argument_default=argparse.SUPPRESS, ) - _common(child) - child.add_argument( - "--candidate-repo", help="Candidate checkout; defaults to the baseline checkout" - ) - child.add_argument( - "--scenario", default="external_benchmark", help="Caller-declared scenario label" - ) - child.add_argument("--mode", choices=("config", "code", "semantic"), default="config") - child.add_argument( - "--agent", - choices=("manual", "codex", "external"), - default="manual", - help="Handoff label only; no Agent is launched", - ) - child.add_argument( - "--executor", - choices=("local",), - default="local", - help="Install this CLI on the remote machine for this first release", - ) - child.add_argument( - "--baseline-command", - required=True, - help="Quoted argv command; {result} is replaced with a fresh output path", - ) - child.add_argument("--candidate-command", required=True) - child.add_argument( - "--validation-command", - help="Explicit quality/correctness command; missing evidence is INCONCLUSIVE", - ) - child.add_argument( - "--metric-key", - default="metrics.latency_ms", - help="Flat or dotted key in benchmark JSON", - ) - child.add_argument( - "--require-trial-check", action="append", default=[], metavar="DOTTED_KEY", - help="Require a literal true qualification field in every benchmark JSON; repeatable", - ) - child.add_argument("--metric-unit", default="ms") - child.add_argument("--comparison-scope", default="external_command") - child.add_argument("--direction", choices=("minimize", "maximize"), default="minimize") - child.add_argument("--repetitions", type=int, default=3) - child.add_argument("--min-improvement-pct", type=float, default=0.0) - child.add_argument( - "--max-minutes", - type=float, - default=120.0, - help="Experiment deadline, at most seven days", - ) - child.add_argument("--workload-id", help="Caller-declared fixed workload identity") - child.add_argument("--model", help="Model identity/path recorded in the experiment") - child.add_argument("--model-revision") - child.add_argument("--output-dir", default="runs") - child.add_argument("--dry-run", action="store_true") + _experiment_options(child) + child.add_argument("--task", help="Read a versioned YAML task") + child.add_argument("--dry-run", action="store_true", default=False) + if name == "run": + child.add_argument( + "--confirm-acceptance", action="store_true", + help="Confirm the effective YAML task was checked against the frozen gates", + ) + + task = commands.add_parser("task", help="Read or inspect shared YAML tasks") + actions = task.add_subparsers(dest="task_action", required=True) + for name in ("validate", "preview"): + child = actions.add_parser(name, argument_default=argparse.SUPPRESS) + child.add_argument("task", help="Path to the YAML task") + child.add_argument("--json", action="store_true", default=False) + if name == "preview": + _experiment_options(child) + actions.add_parser("schema", help="Print the packaged v1 schema") + actions.add_parser("template", help="Print the packaged v1 YAML template") for name in ("status", "report"): child = commands.add_parser( @@ -283,7 +343,7 @@ def parser() -> argparse.ArgumentParser: collect.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select an explicit skills root", ) collect.add_argument( "--store", @@ -302,7 +362,7 @@ def parser() -> argparse.ArgumentParser: read.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select an explicit skills root", ) read.add_argument( "--store", @@ -322,7 +382,7 @@ def parser() -> argparse.ArgumentParser: resource.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select an explicit skills root", ) resource.add_argument( "--store", @@ -351,14 +411,16 @@ def device_ids(args: argparse.Namespace) -> tuple[str, ...]: return values +def command_argv(value: str | list[str] | None) -> tuple[str, ...] | None: + if value is None: + return None + parsed = tuple(shlex.split(value, posix=True)) if isinstance(value, str) else tuple(value) + if not parsed: + raise ValueError("Commands cannot be empty") + return parsed + + def make_spec(args: argparse.Namespace) -> RunSpec: - def command(value: str | None) -> tuple[str, ...] | None: - if value is None: - return None - parsed = tuple(shlex.split(value, posix=True)) - if not parsed: - raise ValueError("Commands cannot be empty") - return parsed spec = RunSpec( repo=str(Path(args.repo).resolve()), @@ -367,9 +429,9 @@ def command(value: str | None) -> tuple[str, ...] | None: backend=args.backend, scenario=args.scenario, mode=args.mode, - baseline_command=command(args.baseline_command), - candidate_command=command(args.candidate_command), - validation_command=command(args.validation_command), + baseline_command=command_argv(args.baseline_command), + candidate_command=command_argv(args.candidate_command), + validation_command=command_argv(args.validation_command), metric_key=args.metric_key, required_trial_checks=tuple(args.require_trial_check), metric_unit=args.metric_unit, diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 6736ba47e..587bc8e97 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -5,6 +5,7 @@ import json import sqlite3 import sys +from dataclasses import replace from importlib.resources import files from pathlib import Path @@ -18,8 +19,8 @@ from .research import research_knowledge -def _print(value: object) -> None: - print(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False)) +def _print(value: object, *, ensure_ascii: bool = False) -> None: + print(json.dumps(value, ensure_ascii=ensure_ascii, indent=2, allow_nan=False)) def _knowledge_reader(args) -> KnowledgeReader: @@ -116,9 +117,33 @@ def main(argv: list[str] | None = None) -> int: argument_parser = parser() args = argument_parser.parse_args(argv) try: + if args.command == "task": + if args.task_action in ("schema", "template"): + name = "schema-v1.json" if args.task_action == "schema" else "template-v1.yaml" + resource = files("inferrsi").joinpath("data", "task_input", name) + print(resource.read_text(encoding="utf-8"), end="") + return 0 + from .task_input import normalize_task + + task = normalize_task(args.task, vars(args)) + if args.task_action == "validate": + if args.json: + _print({"valid": True}) + else: + print("Valid task YAML") + elif args.json: + _print(task.preview(), ensure_ascii=True) + else: + for section, fields in task.task.items(): + if isinstance(fields, dict): + for key, value in fields.items(): + print(f"{section}.{key} = {json.dumps(value)}") + else: + print(f"{section} = {fields}") + for field in task.missing: + print(f"Required for plan/run: {field}") + return 0 if args.command == "skills": - # Optional dependencies load only for the skills command; the core - # CLI stays independent of them. from .skill_catalog import ( get_skill, skill_instructions, @@ -357,6 +382,21 @@ def main(argv: list[str] | None = None) -> int: _print({"record": str(path)}) return 0 + task = None + if args.command in ("plan", "run") and hasattr(args, "task"): + from .task_input import normalize_task + + task = normalize_task(args.task, vars(args)) + args = task.execution_args(args) + if args.command == "run" and not args.dry_run and not getattr( + args, "confirm_acceptance", False + ): + raise ValueError( + "acceptance.reference: review the effective task against the frozen gates " + "and pass --confirm-acceptance to run" + ) + spec = replace(task.make_spec(args), revision=task.task["acceptance"]["baseline"]) + environment = inspect_environment( args.repo, args.repo_type, @@ -364,6 +404,7 @@ def main(argv: list[str] | None = None) -> int: device_ids=device_ids(args), device_model=args.device_model, python=args.python, + baseline_revision=spec.revision if task else "", ) environment["invocation"] = argv if argv is not None else sys.argv[1:] if args.command == "doctor": @@ -375,7 +416,10 @@ def main(argv: list[str] | None = None) -> int: ].get("environment") return 0 if available and known else 1 - spec = make_spec(args) + if task is None: + spec = make_spec(args) + else: + environment["task"] = task.task if args.command == "plan" or args.dry_run: _print( { @@ -383,7 +427,8 @@ def main(argv: list[str] | None = None) -> int: "environment": environment, "execution": "not started", "validation": "unverified", - } + }, + ensure_ascii=task is not None, ) return 0 if not environment["repository"].get("available"): @@ -407,7 +452,8 @@ def main(argv: list[str] | None = None) -> int: "run_dir": str(directory), "report": str(directory / "report.html"), "status": status, - } + }, + ensure_ascii=task is not None, ) return ( 0 if status.get("verdict") == "PASS" else (1 if status.get("verdict") == "FAIL" else 2) diff --git a/src/inferrsi/data/task_input/schema-v1.json b/src/inferrsi/data/task_input/schema-v1.json new file mode 100644 index 000000000..d2811c1c0 --- /dev/null +++ b/src/inferrsi/data/task_input/schema-v1.json @@ -0,0 +1,43 @@ +{ + "schema_version": 1, + "sections": { + "optimization": { + "repository": {"type": "path", "cli_argument": "--repo", "required_for_plan_run": true}, + "candidate_repository": {"type": "path", "cli_argument": "--candidate-repo"}, + "model": {"type": "text", "cli_argument": "--model", "required_for_plan_run": true}, + "model_revision": {"type": "text", "cli_argument": "--model-revision"}, + "framework": {"type": "framework", "cli_argument": "--repo-type", "required_for_plan_run": true}, + "workload": {"type": "text", "cli_argument": "--workload-id", "required_for_plan_run": true}, + "objective": {"type": "text", "required_for_plan_run": true}, + "plane": {"type": "choice", "choices": ["harness", "model", "infra"], "required_for_plan_run": true}, + "scenario": {"type": "text", "cli_argument": "--scenario"}, + "mode": {"type": "choice", "choices": ["config", "code", "semantic"], "cli_argument": "--mode"}, + "agent": {"type": "choice", "choices": ["manual", "codex", "external"], "cli_argument": "--agent"} + }, + "environment": { + "backend": {"type": "backend", "cli_argument": "--backend", "required_for_plan_run": true}, + "device_ids": {"type": "ids", "cli_argument": "--device-ids"}, + "device_model": {"type": "text", "cli_argument": "--device-model"}, + "python": {"type": "interpreter", "cli_argument": "--python"}, + "executor": {"type": "choice", "choices": ["local"], "cli_argument": "--executor"}, + "baseline_command": {"type": "argv", "cli_argument": "--baseline-command", "required_for_plan_run": true}, + "candidate_command": {"type": "argv", "cli_argument": "--candidate-command", "required_for_plan_run": true}, + "quality_command": {"type": "argv", "cli_argument": "--validation-command"}, + "repetitions": {"type": "positive_integer", "cli_argument": "--repetitions", "required_for_plan_run": true}, + "max_minutes": {"type": "minutes", "cli_argument": "--max-minutes", "required_for_plan_run": true}, + "output_directory": {"type": "path", "cli_argument": "--output-dir"} + }, + "acceptance": { + "reference": {"type": "reference", "required_for_plan_run": true}, + "baseline": {"type": "revision", "required_for_plan_run": true}, + "metric_key": {"type": "metric", "cli_argument": "--metric-key", "required_for_plan_run": true}, + "metric_unit": {"type": "text", "cli_argument": "--metric-unit", "required_for_plan_run": true}, + "direction": {"type": "choice", "choices": ["minimize", "maximize"], "cli_argument": "--direction", "required_for_plan_run": true}, + "min_improvement_pct": {"type": "threshold", "cli_argument": "--min-improvement-pct", "required_for_plan_run": true}, + "comparison_scope": {"type": "text", "cli_argument": "--comparison-scope"}, + "required_trial_checks": {"type": "checks", "cli_argument": "--require-trial-check"}, + "require_quality": {"type": "boolean"}, + "constraints": {"type": "text"} + } + } +} diff --git a/src/inferrsi/data/task_input/template-v1.yaml b/src/inferrsi/data/task_input/template-v1.yaml new file mode 100644 index 000000000..4b5948f2a --- /dev/null +++ b/src/inferrsi/data/task_input/template-v1.yaml @@ -0,0 +1,25 @@ +# Fill the required fields before plan/run. Null is a draft placeholder. +# Keep credentials in the existing runtime channels. +schema_version: 1 +optimization: + repository: . + model: null + framework: null + workload: null + objective: null + plane: null # harness, model, or infra +environment: + backend: null # cpu, cuda, rocm, ascend, or mlx + baseline_command: null # YAML argv list, including "{result}" where needed + candidate_command: null + quality_command: null + repetitions: null + max_minutes: null +acceptance: + reference: null # HTTP(S) acceptance issue URL or a local plan path + baseline: null # Full pinned Git revision as a quoted string + metric_key: null + metric_unit: null + direction: null # minimize or maximize + min_improvement_pct: null + constraints: null diff --git a/src/inferrsi/harness/runner.py b/src/inferrsi/harness/runner.py index a15c6b2a1..12d69be26 100644 --- a/src/inferrsi/harness/runner.py +++ b/src/inferrsi/harness/runner.py @@ -62,10 +62,16 @@ def record_status(phase: str, **fields: object) -> None: "candidate": Path(spec.candidate_repo or spec.repo).resolve(), } + def check_baseline_revision(identity: dict) -> None: + if spec.revision and identity.get("head") != spec.revision: + raise ValueError("Baseline Git checkout HEAD differs from the pinned RunSpec revision") + def execute_checked( command: tuple[str, ...], label: str, trial_dir: Path, metric_key: str | None ) -> dict: source_before = snapshot(repos[label], run_dir) + if label == "baseline": + check_baseline_revision(source_before) record = execute(command, repos[label], trial_dir, deadline, metric_key) if metric_key is not None and spec.required_trial_checks: record["trial_checks"] = collect_trial_checks(record, spec.required_trial_checks) @@ -161,6 +167,7 @@ def terminate() -> None: event("source_snapshot") before = {label: snapshot(repo, run_dir) for label, repo in repos.items()} _write_json(run_dir / "repositories.lock.json", before) + check_baseline_revision(before["baseline"]) commands = {"baseline": spec.baseline_command, "candidate": spec.candidate_command} for index in range(spec.repetitions): for label in ("baseline", "candidate") if index % 2 == 0 else ("candidate", "baseline"): diff --git a/src/inferrsi/infra/inspection.py b/src/inferrsi/infra/inspection.py index 597d3185a..f1c0acf3e 100644 --- a/src/inferrsi/infra/inspection.py +++ b/src/inferrsi/infra/inspection.py @@ -21,10 +21,13 @@ def inspect_environment( device_ids: tuple[str, ...] = (), device_model: str | None = None, python: str | None = None, + baseline_revision: str = "", command: Callable[[list[str]], dict] = probe_command, snapshot: Callable[[Path], dict] = repository_identity, ) -> dict: repository = snapshot(Path(repo).resolve()) + if baseline_revision and repository.get("head") != baseline_revision: + raise ValueError("acceptance.baseline: Git checkout HEAD differs from the pinned revision") python = python or sys.executable package_code = ( "import json,sys,importlib.metadata as m\n" diff --git a/src/inferrsi/inspection.py b/src/inferrsi/inspection.py index e7079b826..83a388d6c 100644 --- a/src/inferrsi/inspection.py +++ b/src/inferrsi/inspection.py @@ -12,6 +12,7 @@ def inspect_environment( device_ids: tuple[str, ...] = (), device_model: str | None = None, python: str | None = None, + baseline_revision: str = "", ) -> dict: from .execution import repository_identity @@ -22,6 +23,7 @@ def inspect_environment( device_ids=device_ids, device_model=device_model, python=python, + baseline_revision=baseline_revision, command=_command, snapshot=repository_identity, ) diff --git a/src/inferrsi/runspec.py b/src/inferrsi/runspec.py index 161408204..685ddeafb 100644 --- a/src/inferrsi/runspec.py +++ b/src/inferrsi/runspec.py @@ -4,6 +4,8 @@ from dataclasses import asdict, dataclass from pathlib import Path +MAX_EXPERIMENT_MINUTES = 7 * 24 * 60 + @dataclass(frozen=True) class RunSpec: @@ -109,7 +111,7 @@ def validate(self) -> None: raise ValueError(f"{label} must be finite") if self.max_minutes <= 0 or self.min_improvement_pct < 0: raise ValueError("max_minutes must be positive and min_improvement_pct nonnegative") - if self.max_minutes > 7 * 24 * 60: + if self.max_minutes > MAX_EXPERIMENT_MINUTES: raise ValueError("max_minutes cannot exceed seven days in this release") if not self.metric_key or any(not part for part in self.metric_key.split(".")): raise ValueError("metric_key must be a nonempty dotted JSON key") diff --git a/src/inferrsi/task_input.py b/src/inferrsi/task_input.py new file mode 100644 index 000000000..05424d67d --- /dev/null +++ b/src/inferrsi/task_input.py @@ -0,0 +1,326 @@ +"""Load and normalize the packaged v1 task contract without probing the target.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from urllib.parse import urlsplit + +from .arguments import EXPERIMENT_DEFAULTS, command_argv, make_spec +from .catalog import BACKENDS, REPO_TYPES +from .runspec import MAX_EXPERIMENT_MINUTES, RunSpec + +try: + import yaml + from yaml.events import AliasEvent, DocumentStartEvent + from yaml.nodes import MappingNode, ScalarNode, SequenceNode +except ImportError as exc: + raise ValueError( + "YAML task input requires PyYAML. Install the standard InferRSI package." + ) from exc + +_MAX_BYTES = 64 * 1024 +_INVALID_TEXT = re.compile(r"[\x00\uD800-\uDFFF]") +_SCALAR_TAG = "tag:yaml.org,2002:" +_SCALAR_TAGS = {_SCALAR_TAG + kind for kind in ("str", "null", "int", "float", "bool")} + + +class _TaskLoader(yaml.SafeLoader): + # Use a small set of unambiguous scalar spellings instead of YAML 1.1's + # dates, sexagesimal numbers and implicit yes/no/on/off booleans. + yaml_implicit_resolvers = {} + + def compose_document(self): + event = self.peek_event() + if isinstance(event, DocumentStartEvent) and (event.version or event.tags): + raise ValueError("YAML directives are unsupported in task files") + return super().compose_document() + + def compose_node(self, parent, index): + event = self.peek_event() + if isinstance(event, AliasEvent) or event.anchor or event.tag: + mark = event.start_mark + raise ValueError( + f"YAML line {mark.line + 1}: anchors, aliases and tags are unsupported" + ) + return super().compose_node(parent, index) + + +for _kind, _pattern in { + "null": r"^(?:null|~)?$", + "bool": r"^(?:true|false)$", + "int": r"^[+-]?(?:0|[1-9][0-9]*)$", + "float": r"^[+-]?(?:[0-9]+[.][0-9]*|[.][0-9]+)(?:[eE][+-]?[0-9]+)?$" + r"|^[+-]?[0-9]+[eE][+-]?[0-9]+$", +}.items(): + _TaskLoader.add_implicit_resolver(_SCALAR_TAG + _kind, re.compile(_pattern), None) + + +def _scalar(loader: _TaskLoader, node: ScalarNode, field: str): + if node.tag not in _SCALAR_TAGS: + raise ValueError(f"{field}: use a string or a plain numeric, boolean or null value") + try: + value = loader.construct_object(node) + except (ValueError, OverflowError): + raise ValueError(f"{field}: invalid scalar") from None + if isinstance(value, str) and _INVALID_TEXT.search(value): + raise ValueError(f"{field}: contains an invalid Unicode character") + return value + + +def _mapping(node, field: str) -> dict: + if not isinstance(node, MappingNode): + raise ValueError(f"{field}: expected a mapping") + result = {} + for key, value in node.value: + if ( + not isinstance(key, ScalarNode) + or key.tag != _SCALAR_TAG + "str" + or _INVALID_TEXT.search(key.value) + ): + raise ValueError(f"{field}: field names must be strings") + name = key.value + if name in result: + prefix = "" if field == "task" else f"{field}." + raise ValueError(f"{prefix}{name}: duplicate field") + result[name] = value + return result + + +def _read_task(path: Path, schema: dict) -> dict: + if not path.is_file(): + raise ValueError("task: expected a readable YAML file") + with path.open("rb") as stream: + raw = stream.read(_MAX_BYTES + 1) + if len(raw) > _MAX_BYTES: + raise ValueError("task: YAML exceeds 64 KiB") + loader = None + try: + loader = _TaskLoader(raw.decode("utf-8-sig")) + root = _mapping(loader.get_single_node(), "task") + version = root.pop("schema_version", None) + if not isinstance(version, ScalarNode): + raise ValueError("schema_version: expected integer 1") + number = _scalar(loader, version, "schema_version") + if type(number) is not int or number != schema["schema_version"]: + raise ValueError("schema_version: expected integer 1") + result = {} + for section, node in root.items(): + if section not in schema["sections"]: + raise ValueError(f"{section}: unknown section") + values = {} + for key, field_node in _mapping(node, section).items(): + field = f"{section}.{key}" + if key not in schema["sections"][section]: + raise ValueError(f"{field}: unknown field") + if isinstance(field_node, SequenceNode): + value = [ + _scalar(loader, item, f"{field}[{index}]") + if isinstance(item, ScalarNode) else None + for index, item in enumerate(field_node.value) + ] + elif isinstance(field_node, ScalarNode): + value = _scalar(loader, field_node, field) + else: + raise ValueError(f"{field}: expected a scalar or list") + if value is not None: + values[key] = value + result[section] = values + return result + except UnicodeError: + raise ValueError("task: expected UTF-8 YAML") from None + except yaml.YAMLError as exc: + mark = getattr(exc, "problem_mark", None) + location = f" at line {mark.line + 1}, column {mark.column + 1}" if mark else "" + raise ValueError(f"task: invalid YAML{location}") from None + except RecursionError: + raise ValueError("task: YAML is nested too deeply") from None + finally: + if loader is not None: + loader.dispose() + + +def _text(value, field: str) -> str: + if not isinstance(value, str) or not value.strip() or _INVALID_TEXT.search(value): + raise ValueError(f"{field}: expected a nonempty string") + return value.strip() + + +def _dotted_key(value, field: str) -> str: + key = _text(value, field) + if any(not part or part != part.strip() for part in key.split(".")): + raise ValueError(f"{field}: expected a dotted JSON key") + return key + + +def _path(value: str, field: str, base: Path) -> str: + try: + return str((base / value).resolve()) + except (OSError, RuntimeError, ValueError): + raise ValueError(f"{field}: invalid filesystem path") from None + + +def _normalize(value, rule: dict, field: str, base: Path, *, cli: bool = False): + kind = rule["type"] + if kind == "boolean": + if type(value) is not bool: + raise ValueError(f"{field}: expected a boolean") + return value + if kind == "positive_integer": + if type(value) is not int or value < 1: + raise ValueError(f"{field}: expected a positive integer") + return value + if kind in ("minutes", "threshold"): + # Convert only finite JSON-representable numbers. A YAML integer may be huge. + try: + number = float(value) if type(value) in (int, float) else math.nan + except OverflowError: + number = math.nan + if kind == "minutes" and not 0 < number <= MAX_EXPERIMENT_MINUTES: + raise ValueError(f"{field}: expected minutes greater than zero and at most seven days") + if kind == "threshold" and (not math.isfinite(number) or number < 0): + raise ValueError(f"{field}: expected a finite nonnegative percentage") + return number + if kind in ("argv", "ids", "checks"): + if cli and kind == "argv": + try: + value = list(command_argv(value)) + except ValueError: + raise ValueError(f"{field}: expected a nonempty quoted argv command") from None + elif cli and kind == "ids": + value = [item.strip() for item in value.split(",") if item.strip()] + if not isinstance(value, list) or any( + not isinstance(item, str) or _INVALID_TEXT.search(item) for item in value + ): + raise ValueError(f"{field}: expected a list of strings") + if kind == "argv": + if not value or not value[0].strip(): + raise ValueError(f"{field}: expected a nonempty executable as the first argv item") + return value + if kind == "checks": + value = [_dotted_key(item, field) for item in value] + else: + value = [_text(item, field) for item in value] + if any("," in item for item in value): + raise ValueError(f"{field}: individual device IDs cannot contain commas") + if len(value) != len(set(value)): + raise ValueError(f"{field}: values must be unique") + return value + + value = _text(value, field) + if kind in ("choice", "framework", "backend"): + choices = {"framework": REPO_TYPES, "backend": BACKENDS}.get(kind, rule.get("choices")) + if value not in choices: + raise ValueError(f"{field}: expected one of {', '.join(choices)}") + if kind == "revision": + if not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", value): + raise ValueError(f"{field}: expected a full Git object ID") + return value.lower() + if kind == "metric": + return _dotted_key(value, field) + if kind == "reference" and re.match(r"(?i)https?:", value): + try: + url = urlsplit(value) + valid = ( + url.hostname and url.username is None and url.password is None + and url.port != 0 and not url.query + ) + if valid and not any(ord(char) < 32 or char.isspace() for char in value): + return value + except ValueError: + pass + raise ValueError(f"{field}: expected an HTTP(S) issue URL or local plan path") + if kind == "reference" and "://" in value: + raise ValueError(f"{field}: only HTTP(S) issue URLs or local plan paths are supported") + if kind in ("path", "reference") or ( + kind == "interpreter" and (Path(value).drive or "/" in value or "\\" in value) + ): + return _path(value, field, base) + return value + + +@dataclass(frozen=True) +class TaskInput: + task: dict + missing: tuple[str, ...] + arguments: dict + + def preview(self) -> dict: + return {"task": self.task, "complete": not self.missing, "missing": self.missing} + + def execution_args(self, args: argparse.Namespace) -> argparse.Namespace: + if self.missing: + raise ValueError("task: required fields for plan/run: " + ", ".join(self.missing)) + reference = self.task["acceptance"]["reference"] + if not re.match(r"(?i)https?://", reference) and not Path(reference).is_file(): + raise ValueError("acceptance.reference: local plan must be an existing file") + return argparse.Namespace(**(vars(args) | self.arguments)) + + def make_spec(self, args: argparse.Namespace) -> RunSpec: + try: + spec = make_spec(args) + except ValueError as exc: + # Scalar and argv validation is complete at this point. RunSpec + # still checks filesystem-dependent invariants before any probe. + field = { + "repo": "optimization.repository", + "candidate_repo": "optimization.candidate_repository", + "output_dir": "environment.output_directory", + }.get(str(exc).partition(" ")[0], "task") + raise ValueError(f"{field}: {exc}") from None + output = Path(spec.output_dir) + if any(path.exists() and not path.is_dir() for path in (output, *output.parents)): + raise ValueError("environment.output_directory: expected a directory path") + return spec + + +def normalize_task(path: str | Path, overrides: dict | None = None) -> TaskInput: + """Normalize YAML and explicitly supplied CLI flags, without probing or writing.""" + schema_resource = files("inferrsi").joinpath("data", "task_input", "schema-v1.json") + schema = json.loads(schema_resource.read_text(encoding="utf-8")) + path = Path(path) + yaml_base = path.absolute().parent + raw = _read_task(path, schema) + overrides = overrides or {} + task = {"schema_version": schema["schema_version"]} + missing = [] + arguments = {} + for section, fields in schema["sections"].items(): + normalized = {} + task[section] = normalized + for key, rule in fields.items(): + field = f"{section}.{key}" + flag = rule.get("cli_argument") + arg = flag[2:].replace("-", "_") if flag else None + value = raw.get(section, {}).get(key) + if value is not None: + value = _normalize(value, rule, field, yaml_base) + if arg in overrides: + override = _normalize(overrides[arg], rule, field, Path.cwd(), cli=True) + if section == "acceptance" and override != value: + raise ValueError(f"{field}: {flag} conflicts with the task file") + value = override + if value is None: + if rule.get("required_for_plan_run"): + missing.append(field) + elif arg in EXPERIMENT_DEFAULTS: + default = EXPERIMENT_DEFAULTS[arg] + value = ( + _path(default, field, yaml_base) if rule["type"] == "path" and default + else [] if rule["type"] in ("ids", "checks") + else default + ) + if value is not None: + normalized[key] = value + if arg: + arguments[arg] = ",".join(value) if rule["type"] == "ids" else value + if task["acceptance"].get("require_quality") and "quality_command" not in task["environment"]: + missing.append("environment.quality_command") + if task["environment"].get("backend") == "mlx" and task["environment"].get("device_ids"): + raise ValueError("environment.device_ids: unsupported for the mlx backend") + return TaskInput(task, tuple(missing), arguments) diff --git a/tests/check_task_installed.py b/tests/check_task_installed.py new file mode 100644 index 000000000..2ab94144b --- /dev/null +++ b/tests/check_task_installed.py @@ -0,0 +1,36 @@ +"""Exercise the public task commands against an installed wheel without extras.""" + +import json +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + + +def command(*args, cwd): + return subprocess.check_output( + [sys.executable, "-I", "-m", "inferrsi", "task", *args], + cwd=cwd, + encoding="utf-8", + ) + + +fixture = Path(__file__).with_name("task-v1.yaml") +with TemporaryDirectory() as directory: + root = Path(directory).resolve() + path = root / fixture.name + path.write_bytes(fixture.read_bytes()) + assert json.loads(command("schema", cwd=root))["schema_version"] == 1 + template = root / "template.yaml" + template.write_text(command("template", cwd=root), encoding="utf-8") + assert json.loads(command("validate", template.name, "--json", cwd=root)) == {"valid": True} + assert json.loads(command("validate", path.name, "--json", cwd=root)) == {"valid": True} + preview = json.loads(command("preview", path.name, "--json", cwd=root)) + assert preview["complete"] and preview["missing"] == [] + assert preview["task"]["optimization"]["repository"] == str(root / "checkout") + assert preview["task"]["environment"]["backend"] == "cuda" + assert preview["task"]["environment"]["repetitions"] == 2 + assert preview["task"]["environment"]["max_minutes"] == 1.5 + assert preview["task"]["acceptance"]["min_improvement_pct"] == 7.5 + assert set(root.iterdir()) == {path, template} + assert path.read_bytes() == fixture.read_bytes() diff --git a/tests/fixtures/task-v1.yaml b/tests/fixtures/task-v1.yaml new file mode 100644 index 000000000..030a3c00c --- /dev/null +++ b/tests/fixtures/task-v1.yaml @@ -0,0 +1,26 @@ +# Synthetic input for read-only tests. Commands and revision are test placeholders. +schema_version: 1 +optimization: + repository: ./checkout + model: test-model + framework: vllm + workload: fixed-synthetic-input + objective: Verify task normalization + plane: harness +environment: + backend: cuda + device_ids: ["2"] + baseline_command: [fixture-benchmark, baseline, "{result}"] + candidate_command: [fixture-benchmark, candidate, "{result}"] + quality_command: [fixture-quality] + repetitions: 2 + max_minutes: 1.5 + output_directory: ./results +acceptance: + reference: ./frozen-plan.md + baseline: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + metric_key: metrics.latency_ms + metric_unit: ms + direction: minimize + min_improvement_pct: 7.5 + require_quality: true diff --git a/tests/test_knowledge_fresh_install.py b/tests/test_knowledge_fresh_install.py index 05b31aba9..69e30574a 100644 --- a/tests/test_knowledge_fresh_install.py +++ b/tests/test_knowledge_fresh_install.py @@ -1,4 +1,4 @@ -"""Fresh-venv install smoke: base install package-only, skills extra unlocks the checkout chain. +"""Fresh-venv install smoke for package and checkout knowledge reads. Opt-in because it builds a real venv and installs from this source tree (minutes of I/O, network access for the build-isolation backend, and an @@ -66,7 +66,7 @@ def _cli(venv_python: Path, *args: str) -> subprocess.CompletedProcess: ) -def test_base_install_package_only_then_skills_extra_chain(): +def test_base_install_supports_package_and_checkout_reads(): workspace = _short_workspace() try: venv = workspace / "venv" @@ -79,21 +79,12 @@ def test_base_install_package_only_then_skills_extra_chain(): assert created.returncode == 0, created.stderr python = _venv_python(venv) - _pip(python, "install", str(REPO)) # base only: no PyYAML, no pytest/ruff + _pip(python, "install", str(REPO)) package_only = _cli(python, "collect", "--limit", "2", "--json") assert package_only.returncode == 0, package_only.stderr assert json.loads(package_only.stdout)["items"] - blocked = _cli( - python, "collect", "--id", f"checkout:{SOURCE}", "--skill-root", str(SKILLS) - ) - assert blocked.returncode == 2 - assert "PyYAML" in blocked.stderr and "[skills]" in blocked.stderr - assert "Traceback" not in blocked.stderr - - _pip(python, "install", f"{REPO}[skills]") - collected = _cli( python, "collect", "--id", f"checkout:{SOURCE}", "--skill-root", str(SKILLS), "--json" ) diff --git a/tests/test_task_input.py b/tests/test_task_input.py new file mode 100644 index 000000000..b8e46bf9e --- /dev/null +++ b/tests/test_task_input.py @@ -0,0 +1,181 @@ +import json +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from inferrsi import cli, inspection +from inferrsi.task_input import normalize_task + +FIXTURE = Path(__file__).parent / "fixtures" / "task-v1.yaml" + + +@pytest.fixture +def task_file(tmp_path): + path = tmp_path / "inputs" / "task.yaml" + path.parent.mkdir() + path.write_bytes(FIXTURE.read_bytes()) + return path + + +def test_preview_and_validation_are_read_only(task_file, tmp_path, monkeypatch, capsys): + def unexpected(*args, **kwargs): + pytest.fail("read-only task command attempted to probe or execute") + + monkeypatch.setattr(cli, "inspect_environment", unexpected) + monkeypatch.setattr(subprocess, "Popen", unexpected) + monkeypatch.chdir(tmp_path) + before = sorted(tmp_path.rglob("*")) + assert cli.main(["task", "validate", str(task_file), "--json"]) == 0 + assert json.loads(capsys.readouterr().out) == {"valid": True} + assert cli.main(["task", "preview", str(task_file), "--json"]) == 0 + preview = json.loads(capsys.readouterr().out) + assert preview["task"] == normalize_task(task_file).task + assert preview["complete"] and not preview["missing"] + assert preview["task"]["optimization"]["repository"] == str(task_file.parent / "checkout") + assert preview["task"]["environment"]["backend"] == "cuda" + assert preview["task"]["environment"]["repetitions"] == 2 + assert preview["task"]["environment"]["max_minutes"] == 1.5 + assert preview["task"]["acceptance"]["min_improvement_pct"] == 7.5 + assert preview["task"]["acceptance"]["reference"] == str(task_file.parent / "frozen-plan.md") + assert sorted(tmp_path.rglob("*")) == before + + +def test_preview_uses_explicit_flags_and_respects_frozen_fields(task_file, tmp_path, monkeypatch, + capsys): + monkeypatch.chdir(tmp_path) + command = ["task", "preview", str(task_file), "--json"] + assert cli.main([ + *command, "--backend=cpu", "--device-ids=", "--repo", "cli-checkout", + "--repetitions", "5", "--min-improvement-pct", "7.5", + ]) == 0 + task = json.loads(capsys.readouterr().out)["task"] + assert task["environment"]["backend"] == "cpu" + assert task["environment"]["device_ids"] == [] + assert task["environment"]["repetitions"] == 5 + assert task["optimization"]["repository"] == str(tmp_path / "cli-checkout") + assert task["acceptance"]["min_improvement_pct"] == 7.5 + with pytest.raises(SystemExit) as exc: + cli.main([*command, "--min-improvement-pct", "0"]) + assert exc.value.code == 2 + assert "acceptance.min_improvement_pct" in capsys.readouterr().err + + +@pytest.mark.parametrize("command", ["plan", "run"]) +def test_incomplete_task_stops_before_probes(task_file, monkeypatch, capsys, command): + task_file.write_text("schema_version: 1\noptimization:\n model: null\n", encoding="utf-8") + assert cli.main(["task", "preview", str(task_file), "--json"]) == 0 + preview = json.loads(capsys.readouterr().out) + assert not preview["complete"] + assert {"optimization.model", "environment.backend", "acceptance.reference"} <= set( + preview["missing"] + ) + + def unexpected(*args, **kwargs): + pytest.fail("incomplete input reached an operation") + + monkeypatch.setattr(cli, "inspect_environment", unexpected) + monkeypatch.setattr(cli, "run_experiment", unexpected) + with pytest.raises(SystemExit) as exc: + cli.main([command, "--task", str(task_file)]) + assert exc.value.code == 2 + assert "optimization.model" in capsys.readouterr().err + assert list(task_file.parent.iterdir()) == [task_file] + + +@pytest.mark.parametrize(("source", "message"), [ + ("schema_version: true\n", "schema_version"), + ("schema_version: 2\n", "schema_version"), + ("schema_version: 1\nschema_version: 1\n", "schema_version: duplicate"), + ("schema_version: 1\nenvironment: null\n", "environment: expected a mapping"), + ("schema_version: 1\nenvironment: {backend: gpu}\n", "environment.backend"), + ("schema_version: 1\nenvironment: {backend: cpu, backend: cuda}\n", + "environment.backend: duplicate"), + ("schema_version: 1\nenvironment: {maxRounds: 2}\n", "environment.maxRounds"), + ("schema_version: 1\nenvironment: {repetitions: 2.0}\n", "environment.repetitions"), + ("schema_version: 1\nenvironment: {max_minutes: 1e999}\n", "environment.max_minutes"), + ("schema_version: 1\nenvironment: {device_ids: ['0', '0']}\n", "environment.device_ids"), + ("schema_version: 1\nenvironment: {baseline_command: [null]}\n", + "environment.baseline_command"), + ("schema_version: 1\nacceptance: {baseline: HEAD}\n", "acceptance.baseline"), + ("schema_version: 1\nacceptance: {reference: 'https://user:secret@example.invalid/issues/1'}\n", + "acceptance.reference"), + ("schema_version: 1\noptimization: {model: &alias model}\n", "anchors, aliases and tags"), + ("schema_version: 1\noptimization: {model: !!str model}\n", "anchors, aliases and tags"), + ('schema_version: 1\noptimization: {objective: "\\uD800"}\n', "optimization.objective"), + ('schema_version: 1\nacceptance: {reference: "https://@example.invalid/1"}\n', + "acceptance.reference"), + ('schema_version: 1\nacceptance: {reference: "https://example.invalid:abc/1"}\n', + "acceptance.reference"), + ("schema_version: 1\n# \0\n", "invalid YAML"), + ("schema_version: 1\n---\nschema_version: 1\n", "invalid YAML"), +]) +def test_invalid_yaml_has_a_field_or_location(task_file, source, message): + task_file.write_text(source, encoding="utf-8") + with pytest.raises(ValueError, match=message) as exc: + normalize_task(task_file, {"backend": "cpu"}) + assert "user:secret" not in str(exc.value) + + +def test_plan_and_explicit_run_share_task_and_keep_source_pin(task_file, capsys, monkeypatch): + repo = task_file.parent / "checkout" + repo.mkdir() + + def git(*args): + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() + + git("init", "-q") + git("-c", "commit.gpgsign=false", "-c", "user.name=Fixture", + "-c", "user.email=fixture@example.invalid", + "commit", "--allow-empty", "-qm", "synthetic") + head = git("rev-parse", "HEAD") + task = yaml.safe_load(task_file.read_text(encoding="utf-8")) + task["acceptance"]["baseline"] = head + (task_file.parent / "frozen-plan.md").write_text("Synthetic plan fixture", encoding="utf-8") + script = ( + "from pathlib import Path; import sys; " + "Path(sys.argv[1]).write_text('{\"metrics\":{\"latency_ms\":' + sys.argv[2] + '}}')" + ) + task["environment"].update( + baseline_command=[sys.executable, "-c", script, "{result}", "100"], + candidate_command=[sys.executable, "-c", script, "{result}", "90"], + quality_command=[sys.executable, "-c", "pass"], + device_ids=[], + ) + task_file.write_text(yaml.safe_dump(task), encoding="utf-8") + options = ["--task", str(task_file), "--backend", "cpu"] + assert cli.main(["plan", *options]) == 0 + plan = json.loads(capsys.readouterr().out) + spec = plan["run_spec"] + assert spec["revision"] == head + assert spec["backend"] == "cpu" + assert spec["repetitions"] == 2 and spec["max_minutes"] == 1.5 + assert spec["min_improvement_pct"] == 7.5 and spec["model"] == "test-model" + output = task_file.parent / "results" + assert not output.exists() + with pytest.raises(SystemExit) as exc: + cli.main(["run", *options]) + assert exc.value.code == 2 + assert "--confirm-acceptance" in capsys.readouterr().err + assert not output.exists() + assert cli.main(["run", *options, "--confirm-acceptance"]) == 0 + result = json.loads(capsys.readouterr().out) + run = Path(result["run_dir"]) + assert run.parent == output + assert json.loads((run / "run.lock.json").read_text()) == spec + assert json.loads((run / "environment.json").read_text())["task"] == plan["environment"]["task"] + + task["acceptance"]["baseline"] = "0" * 40 + task_file.write_text(yaml.safe_dump(task), encoding="utf-8") + + def unexpected(*args, **kwargs): + pytest.fail("revision mismatch reached a runtime probe") + + monkeypatch.setattr(inspection, "_command", unexpected) + with pytest.raises(SystemExit) as exc: + cli.main(["run", *options, "--confirm-acceptance"]) + assert exc.value.code == 2 + assert "acceptance.baseline" in capsys.readouterr().err + assert list(output.iterdir()) == [run] diff --git a/tools/plugin_bundle.py b/tools/plugin_bundle.py index f90c1ee91..0c87add48 100644 --- a/tools/plugin_bundle.py +++ b/tools/plugin_bundle.py @@ -56,6 +56,7 @@ def bundle_runtime(source: Path, runtime: Path, *, plugin_version: str, and relative.parts[-3:-1] == ("references", "licenses") ) or (path.suffix == ".html" and relative.parent.parts == ("data", "dashboard")) + or (path.suffix == ".yaml" and relative.parent.parts == ("data", "task_input")) or (path.name == "upstream-NOTICE" and relative.parent.name == "references") ) if path.suffix != ".py" and not package_data: