diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index fb9491e..556c07a 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -9,7 +9,6 @@ from __future__ import annotations -import glob import json import os import re @@ -17,7 +16,7 @@ import sys import uuid from pathlib import Path -from typing import Mapping, Sequence +from typing import Iterable, Mapping, Sequence from rich.console import Console @@ -218,6 +217,80 @@ def resolve_gateware_project( ) +# Highest build-summary schema this synapse understands. The SDK releases +# independently, so a newer schema must fail loudly rather than be misread. +_MAX_SUMMARY_SCHEMA = 2 + + +def _scan_bitstreams(bitstreams_dir: str) -> dict[str, tuple[int, int]]: + """Return ``{name: (mtime_ns, size)}`` for the regular files in *bitstreams_dir*. + + Size rides with mtime because some filesystems store whole seconds, so a + same-second rewrite would otherwise look untouched. + """ + if not os.path.isdir(bitstreams_dir): + return {} + files = {} + with os.scandir(bitstreams_dir) as entries: + for entry in entries: + if entry.is_file(): + st = entry.stat() + files[entry.name] = (st.st_mtime_ns, st.st_size) + return files + + +def _load_summary(path: str) -> dict | None: + """Return the build summary at *path* as a dict, or None if unreadable.""" + try: + with open(path, "r", encoding="utf-8") as fp: + summary = json.load(fp) + except (OSError, json.JSONDecodeError): + return None + return summary if isinstance(summary, dict) else None + + +def _summary_schema(path: str) -> tuple[int, dict] | None: + """Return ``(schema_version, summary)`` for a build summary, or None if unreadable.""" + summary = _load_summary(path) + if summary is None: + return None + version = summary.get("schema_version", 1) + if type(version) is not int: + raise ValueError( + f"Bitstream summary {path} has a non-integer schema_version {version!r}" + ) + return version, summary + + +def _schema2_bitstream( + bitstreams_dir: str, + summary_name: str, + summary: dict, + after: Mapping[str, tuple[int, int]], + emitted: set[str], +) -> str: + """Return the image a schema-2 summary names, checked to be from this build.""" + name = summary.get("bitstream") + # The readers find a summary by its image's stem, so the two must pair up. + expected = summary_name.removesuffix(".summary.json") + ".bit" + if name != expected: + raise ValueError( + f"Bitstream summary {summary_name} has 'bitstream' {name!r}; " + f"expected {expected!r}, the image next to it" + ) + if name not in after: + raise FileNotFoundError( + f"Bitstream summary {summary_name} names {name}, which does not " + f"exist in {bitstreams_dir}" + ) + if name not in emitted: + raise ValueError( + f"Bitstream summary {summary_name} names {name}, which this build " + "did not write (it predates the build)" + ) + return os.path.join(bitstreams_dir, name) + + def run_gateware_build( peripheral_dir: str, image_tag: str, @@ -231,24 +304,37 @@ def run_gateware_build( root either way, so a per-profile project can still reference shared sources above it (``../axon_test_source_peripheral.sv``). - Returns the absolute path to the newest ``sdk_*_extracted.bit`` emitted - under ``//build/bitstreams/``. This is the - *extracted* bitstream variant the SDK writes alongside the raw - ``sdk_*.bit``; the extracted form is what gets flashed to the probe, and - it carries its own same-stem ``.summary.json`` (same schema as the raw - one). Selecting it specifically — rather than the broader ``sdk_*.bit`` - glob, which would also match the extracted file and pick by mtime — - keeps the choice deterministic. + Returns the absolute path to the image this build emitted under + ``//build/bitstreams/``. Build directories + persist between builds, so only files the build created or changed (by + mtime and size) are considered; older files are ignored whatever their + name. Among those: + + - A schema-2 ``.summary.json`` names its image, ``.bit``, + in ``bitstream``; the newest such summary wins. + - Otherwise (schema 1, no ``schema_version``, or no summary) the newest + native ``sdk_*.bit`` wins, else the newest ``sdk_*_extracted.bit`` + (older SDKs emit only the extracted image). + + Unreadable summaries are skipped here. The image's same-stem + ``.summary.json`` describes it. Raises: LicenseUnsetError: if ``LM_LICENSE_FILE`` is unset (propagated from :func:`build_license_docker_args`). subprocess.CalledProcessError: if the container's build exits non-zero. - FileNotFoundError: if the build succeeds but no bitstream is emitted. + ValueError: if a new summary has a schema newer than this synapse + supports or a malformed ``schema_version``, or a schema-2 summary + names an invalid or pre-existing image. + FileNotFoundError: if the build emitted no bitstream, or a schema-2 + summary names a missing image. """ license_args = build_license_docker_args(env) abs_peripheral_dir = os.path.abspath(peripheral_dir) + bitstreams_dir = os.path.join( + abs_peripheral_dir, *project_subdir.split("/"), "build", "bitstreams" + ) argv = [ "docker", "run", @@ -265,24 +351,49 @@ def run_gateware_build( "-lc", f"axon-peripheral-sdk build --project {project_subdir}", ] + before = _scan_bitstreams(bitstreams_dir) subprocess.run(argv, check=True) + after = _scan_bitstreams(bitstreams_dir) + + emitted = {name for name, ident in after.items() if before.get(name) != ident} + + def newest(names: Iterable[str]) -> list[str]: + return sorted(names, key=lambda name: after[name][0], reverse=True) + + schema2 = {} + for name in emitted: + if not (name.startswith("sdk_") and name.endswith(".summary.json")): + continue + parsed = _summary_schema(os.path.join(bitstreams_dir, name)) + if parsed is None: + continue + version, summary = parsed + if version > _MAX_SUMMARY_SCHEMA: + raise ValueError( + f"Bitstream summary {name} uses schema_version {version}, newer " + f"than this synapse supports ({_MAX_SUMMARY_SCHEMA}). Upgrade " + "science-synapse to package this build." + ) + if version == 2: + schema2[name] = summary - bit_glob = os.path.join( - abs_peripheral_dir, - *project_subdir.split("/"), - "build", - "bitstreams", - "sdk_*_extracted.bit", - ) - matches = glob.glob(bit_glob) - if not matches: - raise FileNotFoundError( - "axon-peripheral-sdk build completed but no sdk_*_extracted.bit was " - f"emitted under {project_subdir}/build/bitstreams/" + if schema2: + matches = newest(schema2) + chosen = _schema2_bitstream( + bitstreams_dir, matches[0], schema2[matches[0]], after, emitted ) + else: + bits = [n for n in emitted if n.startswith("sdk_") and n.endswith(".bit")] + matches = [n for n in bits if not n.endswith("_extracted.bit")] or bits + if not matches: + raise FileNotFoundError( + "axon-peripheral-sdk build completed but no sdk_*.bit was " + f"emitted by this build under {project_subdir}/build/bitstreams/ " + "(files left from earlier builds are ignored)" + ) + matches = newest(matches) + chosen = os.path.join(bitstreams_dir, matches[0]) - matches.sort(key=os.path.getmtime, reverse=True) - chosen = matches[0] if len(matches) > 1: console.print( f"[yellow]Multiple bitstreams matched; selected newest: {chosen}[/yellow]" @@ -340,7 +451,7 @@ def read_usb_pid(bit_path: str) -> int: if not isinstance(summary, dict): raise ValueError( f"Bitstream summary {path} is not a JSON object; " - "expected a top-level usb_pid hex string (e.g. \"0x000B\")." + 'expected a top-level usb_pid hex string (e.g. "0x000B").' ) raw = summary.get("usb_pid") @@ -357,13 +468,13 @@ def read_usb_pid(bit_path: str) -> int: except ValueError: raise ValueError( f"Bitstream summary {path} usb_pid {raw!r} is not a valid hex " - "string (expected e.g. \"0x000B\")" + 'string (expected e.g. "0x000B")' ) if not (0 < usb_pid <= 0xFFFF): raise ValueError( f"Bitstream summary {path} usb_pid {raw!r} ({usb_pid}) is out of " - "range; expected a hex string in 1..0xFFFF (e.g. \"0x000B\")" + 'range; expected a hex string in 1..0xFFFF (e.g. "0x000B")' ) return usb_pid @@ -380,14 +491,8 @@ def read_identifier(bit_path: str) -> str | None: should stay within ``[A-Za-z0-9._-]`` — the identifier rides in ``scifi-probe-update-custom@:``. """ - path = summary_path_for(bit_path) - try: - with open(path, "r", encoding="utf-8") as fp: - summary = json.load(fp) - except (OSError, json.JSONDecodeError): - return None - - if not isinstance(summary, dict): + summary = _load_summary(summary_path_for(bit_path)) + if summary is None: return None target_profile = summary.get("target_profile") if not isinstance(target_profile, str) or not target_profile: @@ -407,14 +512,8 @@ def read_git_sha(bit_path: str) -> str | None: Best-effort: returns None when the summary or the field is missing/malformed. """ - path = summary_path_for(bit_path) - try: - with open(path, "r", encoding="utf-8") as fp: - summary = json.load(fp) - except (OSError, json.JSONDecodeError): - return None - - if not isinstance(summary, dict): + summary = _load_summary(summary_path_for(bit_path)) + if summary is None: return None project = summary.get("project") if not isinstance(project, dict): diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index ea36322..aef7091 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -295,9 +295,7 @@ def build_peripheral_so( # of profile doesn't depend on which one a given repo happens to take. # shlex.quote keeps a hostile profile name from breaking out of the # `bash -c` string; the value reaches us from --profile or the environment. - profile_flag = ( - f" -DAXON_TARGET_PROFILE={shlex.quote(profile)}" if profile else "" - ) + profile_flag = f" -DAXON_TARGET_PROFILE={shlex.quote(profile)}" if profile else "" build_cmd_str = ( "cd /home/workspace && " "if [ -f CMakePresets.json ]; then " @@ -456,9 +454,7 @@ def build_peripheral_deb( # 1. Stage the plugin .so at /usr/lib/scifi/plugins/.so if not os.path.exists(so_path): - console.print( - f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}" - ) + console.print(f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}") return False plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins") os.makedirs(plugin_dst, exist_ok=True) @@ -505,9 +501,7 @@ def build_peripheral_deb( ) return False arch_suffix = detect_arch() - platform_opt = ( - "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" - ) + platform_opt = "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" console.print( f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]" @@ -534,9 +528,7 @@ def build_peripheral_deb( return False sdk_files = [ - f - for f in os.listdir(sdk_dst) - if f.startswith("libscifi-peripheral-sdk.so") + f for f in os.listdir(sdk_dst) if f.startswith("libscifi-peripheral-sdk.so") ] if not sdk_files: console.print( @@ -788,7 +780,7 @@ def _run_gateware_half(peripheral_dir: str, project_subdir: str) -> Optional[str except LicenseUnsetError as exc: console.print(f"[bold red]Error:[/bold red] {exc}") return None - except (subprocess.CalledProcessError, FileNotFoundError) as exc: + except (subprocess.CalledProcessError, FileNotFoundError, ValueError) as exc: console.print(f"[bold red]Error:[/bold red] Gateware build failed: {exc}") return None @@ -840,7 +832,9 @@ def _build_debs( else None ) if resolved_profile: - console.print(f"[bold]Target profile:[/bold] [yellow]{resolved_profile}[/yellow]") + console.print( + f"[bold]Target profile:[/bold] [yellow]{resolved_profile}[/yellow]" + ) if do_gateware and clean: try: @@ -892,7 +886,9 @@ def _build_debs( git_hash=gateware.read_git_sha(bit_path), ): return None - deb = find_deb_package(dist_dir, f"{_gateware_package_name(identifier)}_{version}") + deb = find_deb_package( + dist_dir, f"{_gateware_package_name(identifier)}_{version}" + ) if deb is None: return None debs.append(deb) diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index 3dec7e5..3fb727e 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -15,9 +15,13 @@ import importlib import json import os +from pathlib import Path import pytest +from synapse.tests.cli.conftest import fake_fpm_run +from synapse.tests.cli.test_gateware_runner import _new_ns, _summary, _write + @pytest.fixture() def gateware(): @@ -244,7 +248,9 @@ def test_read_identifier_missing_target_profile_returns_none(gateware, tmp_path) def test_read_identifier_missing_project_name_returns_none(gateware, tmp_path): bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"usb_pid": "0x000B", "target_profile": "via-devkit", "project": {}}) + _write_summary( + bit, {"usb_pid": "0x000B", "target_profile": "via-devkit", "project": {}} + ) assert gateware.read_identifier(str(bit)) is None @@ -291,7 +297,9 @@ def test_read_git_sha_missing_git_sha_returns_none(gateware, tmp_path): def test_read_git_sha_empty_string_returns_none(gateware, tmp_path): bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"usb_pid": "0x000B", "project": {"name": "gateware", "git_sha": ""}}) + _write_summary( + bit, {"usb_pid": "0x000B", "project": {"name": "gateware", "git_sha": ""}} + ) assert gateware.read_git_sha(str(bit)) is None @@ -336,10 +344,6 @@ def test_find_deb_package_version_anchored_prefix_skips_stale(buildmod, tmp_path # peripherals.build_gateware_deb # --------------------------------------------------------------------------- -# synapse/tests/cli/ is a package (has __init__.py), so import the helper -# package-qualified rather than relying on pytest's rootdir sys.path insert. -from synapse.tests.cli.conftest import fake_fpm_run - def _spy_mkdtemp(peripherals, monkeypatch, holder: list): real_mkdtemp = peripherals.tempfile.mkdtemp @@ -369,9 +373,13 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( monkeypatch.setattr(peripherals.subprocess, "run", fake_fpm_run(dist_dir, calls)) ok = peripherals.build_gateware_deb( - str(pd), manifest, bit_path=str(bit), usb_pid=4, - bitstream_name="via-devkit_gateware", git_hash="e6890a3", - version="0.2.0" + str(pd), + manifest, + bit_path=str(bit), + usb_pid=4, + bitstream_name="via-devkit_gateware", + git_hash="e6890a3", + version="0.2.0", ) assert ok is True assert len(staging) == 1 @@ -380,7 +388,11 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( staging[0], "opt", "scifi", "bitstreams", "custom", "via-devkit_gateware.bit" ) frag_dst = os.path.join( - staging[0], "opt", "scifi", "bitstreams", "custom", + staging[0], + "opt", + "scifi", + "bitstreams", + "custom", "via-devkit_gateware.manifest.json", ) assert os.path.exists(bit_dst), "bitstream staged under custom/ as .bit" @@ -427,14 +439,22 @@ def test_build_gateware_deb_omit_bitstream_name_falls_back_to_plugin_name( staging[0], "opt", "scifi", "bitstreams", "custom", "scifi-my-chip.bit" ) frag_dst = os.path.join( - staging[0], "opt", "scifi", "bitstreams", "custom", + staging[0], + "opt", + "scifi", + "bitstreams", + "custom", "scifi-my-chip.manifest.json", ) assert os.path.exists(bit_dst), "fallback: bit staged as .bit" with open(frag_dst, "r", encoding="utf-8") as fh: frag = json.load(fh) # 3-key fragment, NO git_hash key - assert frag == {"name": "scifi-my-chip", "usb_pid": 4, "artifact": "custom/scifi-my-chip.bit"} + assert frag == { + "name": "scifi-my-chip", + "usb_pid": 4, + "artifact": "custom/scifi-my-chip.bit", + } fpm_call = next(c for c in calls if "fpm" in c) assert fpm_call[fpm_call.index("-n") + 1] == "axon-gateware-scifi-my-chip" @@ -448,3 +468,70 @@ def test_build_gateware_deb_missing_bit_errors(peripherals, tmp_path, capsys): ) assert ok is False assert "not found" in capsys.readouterr().out.lower() + + +def test_selected_native_bytes_and_summary_become_custom_package( + gateware, peripherals, tmp_path, monkeypatch +): + pd = tmp_path / "plugin" + bitstreams = pd / "src" / "gateware" / "build" / "bitstreams" + native = bitstreams / "sdk_profile-alpha_gateware.bit" + native_payload = b"\x96\x01\x80\x7f\x00\xa5" + + def emit_sdk_outputs(): + _write(native, native_payload, _new_ns(1)) + _summary( + native, + _new_ns(1), + schema_version=1, + usb_pid="0x000B", + target_profile="profile-alpha", + project={"name": "gateware", "git_sha": "123abcd"}, + ) + extracted = bitstreams / "sdk_profile-alpha_gateware_extracted.bit" + _write(extracted, b"\x69\x80\x01\xfe\x00\xa5", _new_ns(2)) + _summary( + extracted, + _new_ns(2), + schema_version=1, + usb_pid="0x0004", + target_profile="wrong-profile", + project={"name": "decoy", "git_sha": "badbeef"}, + ) + + staging = [] + _spy_mkdtemp(peripherals, monkeypatch, staging) + fpm_run = fake_fpm_run(str(pd / "dist"), []) + + def fake_run(argv, *args, **kwargs): + # The SDK build writes its images; every other call goes to the fpm stub. + if any("axon-peripheral-sdk build" in str(element) for element in argv): + emit_sdk_outputs() + return fpm_run(argv, *args, **kwargs) + + monkeypatch.setattr(peripherals.subprocess, "run", fake_run) + + selected = gateware.run_gateware_build( + str(pd), "fake-gw:latest", env={"LM_LICENSE_FILE": "7788@licenseserver"} + ) + ok = peripherals.build_gateware_deb( + str(pd), + {"name": "scifi-plugin", "version": "0.2.0"}, + bit_path=selected, + usb_pid=gateware.read_usb_pid(selected), + bitstream_name=gateware.read_identifier(selected), + git_hash=gateware.read_git_sha(selected), + version="0.2.0", + ) + + assert ok is True + custom = Path(staging[0]) / "opt" / "scifi" / "bitstreams" / "custom" + assert (custom / "profile-alpha_gateware.bit").read_bytes() == native_payload + assert json.loads( + (custom / "profile-alpha_gateware.manifest.json").read_text() + ) == { + "name": "profile-alpha_gateware", + "usb_pid": 11, + "artifact": "custom/profile-alpha_gateware.bit", + "git_hash": "123abcd", + } diff --git a/synapse/tests/cli/test_gateware_profiles.py b/synapse/tests/cli/test_gateware_profiles.py index dfc2b64..805e835 100644 --- a/synapse/tests/cli/test_gateware_profiles.py +++ b/synapse/tests/cli/test_gateware_profiles.py @@ -19,9 +19,18 @@ from __future__ import annotations import importlib +import subprocess +from pathlib import Path import pytest +from synapse.tests.cli.test_gateware_runner import ( + _new_ns, + _project_args, + _summary, + _write, +) + @pytest.fixture() def gw(): @@ -163,38 +172,68 @@ def test_bare_gateware_dir_without_manifest_raises(gw, tmp_path): # --------------------------------------------------------------------------- -def test_build_command_and_bit_glob_follow_the_project_subdir( +def test_build_targets_each_profile_project_and_selects_its_native_image( gw, tmp_path, monkeypatch ): - """The SDK ``--project`` argument and the bitstream glob must agree; a - mismatch would build one profile and package the other's stale .bit.""" - project = "src/gateware/via-devkit" - bitstreams = tmp_path / project / "build" / "bitstreams" - bitstreams.mkdir(parents=True) - expected_bit = bitstreams / "sdk_via-devkit_gateware_extracted.bit" - expected_bit.write_bytes(b"\x00") - - # A decoy under the *other* profile proves the glob is scoped. - other = tmp_path / "src/gateware/nerv512u-devkit/build/bitstreams" - other.mkdir(parents=True) - (other / "sdk_nerv512u-devkit_gateware_extracted.bit").write_bytes(b"\x00") - - recorded = {} + """The SDK stub writes only into the project the command names, so a build + that ignores the selected profile finds nothing (or the wrong image).""" + profiles = { + "profile-alpha": {"usb_pid": "0x0004", "git_sha": "123abcd"}, + "profile-beta": {"usb_pid": "0x000B", "git_sha": "456def0"}, + } + _make_repo(tmp_path, profiles=profiles) + expected = {} + + def emit(project): + profile = Path(project).name + metadata = profiles[profile] + bitstreams = tmp_path / project / "build" / "bitstreams" + native = bitstreams / f"sdk_{profile}_gateware.bit" + payload = bytes([len(expected) + 1, 0x96, 0x80, 0x7F]) + _write(native, payload, _new_ns(1)) + _summary( + native, + _new_ns(1), + schema_version=1, + target_profile=profile, + usb_pid=metadata["usb_pid"], + project={"name": "gateware", "git_sha": metadata["git_sha"]}, + ) + extracted = bitstreams / f"sdk_{profile}_gateware_extracted.bit" + _write(extracted, b"wrong extracted bytes", _new_ns(9)) + _summary( + extracted, + _new_ns(9), + usb_pid="0xFFFF", + target_profile="wrong-profile", + project={"name": "decoy", "git_sha": "badbeef"}, + ) + expected[profile] = (native, payload) - def fake_run(argv, **kwargs): - recorded["argv"] = argv - import subprocess as sp + requested = [] - return sp.CompletedProcess(argv, 0) + def fake_run(argv, *args, **kwargs): + projects = _project_args(argv) + requested.append(projects) + if len(projects) == 1: + emit(projects[0]) + return subprocess.CompletedProcess(argv, 0) monkeypatch.setattr(gw.subprocess, "run", fake_run) - got = gw.run_gateware_build( - str(tmp_path), - "fake-gw:latest", - env={"LM_LICENSE_FILE": "7788@licenseserver"}, - project_subdir=project, - ) - - assert got == str(expected_bit) - assert f"axon-peripheral-sdk build --project {project}" in recorded["argv"][-1] + for profile, metadata in profiles.items(): + project = gw.resolve_gateware_project(str(tmp_path), profile, {}) + selected = gw.run_gateware_build( + str(tmp_path), + "fake-gw:latest", + env={"LM_LICENSE_FILE": "7788@licenseserver"}, + project_subdir=project, + ) + assert requested[-1] == [f"src/gateware/{profile}"] + native, payload = expected[profile] + assert Path(selected) == native + assert Path(selected).read_bytes() == payload + assert gw.summary_path_for(selected) == str(native.with_suffix(".summary.json")) + assert gw.read_usb_pid(selected) == int(metadata["usb_pid"], 16) + assert gw.read_identifier(selected) == f"{profile}_gateware" + assert gw.read_git_sha(selected) == metadata["git_sha"] diff --git a/synapse/tests/cli/test_gateware_runner.py b/synapse/tests/cli/test_gateware_runner.py index df08af7..7bfdbff 100644 --- a/synapse/tests/cli/test_gateware_runner.py +++ b/synapse/tests/cli/test_gateware_runner.py @@ -1,53 +1,64 @@ -"""AC-10 / AC-6: unit tests for run_gateware_build(). - -The runner lives in `synapse.cli.gateware` per the plan's File Structure -section. Signature: - - run_gateware_build(peripheral_dir: str, image_tag: str, - env: Mapping[str, str] = os.environ) -> str - -Behavior (per AC-6): - 1. Calls build_license_docker_args(env); LicenseUnsetError propagates. - 2. Issues `docker run --rm --user dev -v :/home/workspace - -w /home/workspace /bin/bash -lc - 'axon-peripheral-sdk build --project src/gateware'`. - 3. Non-zero exit -> raises subprocess.CalledProcessError. - 4. After success, globs - /src/gateware/build/bitstreams/sdk_*_extracted.bit - and returns the newest by mtime (warns on multi-match). The *extracted* - variant is what gets flashed; the raw sdk_*.bit is ignored. - 5. Empty glob -> FileNotFoundError with message mentioning - "sdk_*_extracted.bit". - -Sub-phase 4.4 (Tester): the xfail marker is removed — these tests now run -as live AC-6 acceptance gates and must fail until the Implementer lands -``run_gateware_build``. +"""Public SDK runner contract: return the image this build produced. + +Only files the build wrote or changed are candidates. A schema-2 summary names +the image outright; otherwise the newest native ``sdk_*.bit`` wins, falling +back to the newest ``sdk_*_extracted.bit``. """ from __future__ import annotations import importlib +import json import os +import shlex import subprocess import time +from pathlib import Path import pytest +IMAGE_TAG = "myplugin-gateware:latest-arm64" + + +# Earlier builds' files sit an hour in the past; this build's land whole +# seconds after "now", so ordering never depends on the wall clock moving. +def _stale_ns() -> int: + return time.time_ns() - 3600 * 10**9 + + +def _new_ns(order: int) -> int: + return time.time_ns() + order * 10**9 + + +def _write(path: Path, data: bytes, mtime_ns: int) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + os.utime(path, ns=(mtime_ns, mtime_ns)) + return path + + +def _summary(bit_path: Path, mtime_ns: int, **fields) -> Path: + """Write *bit_path*'s same-stem ``.summary.json``.""" + path = bit_path.with_name(bit_path.name.removesuffix(".bit") + ".summary.json") + return _write(path, json.dumps(fields).encode(), mtime_ns) + + +def _project_args(argv) -> list[str]: + """Every ``--project`` value in *argv*, including inside ``bash -lc``.""" + values = [] + for element in argv: + tokens = shlex.split(str(element)) + values += [b for a, b in zip(tokens, tokens[1:]) if a == "--project"] + return values + @pytest.fixture() def gateware(): - """Lazy import — avoids module-collection failure before AC-5/AC-6 land.""" return importlib.import_module("synapse.cli.gateware") -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - @pytest.fixture() def peripheral_dir(tmp_path): - """Create a minimal peripheral dir with src/gateware/ + a license file.""" pd = tmp_path / "myplugin" (pd / "src" / "gateware").mkdir(parents=True) license_file = tmp_path / "license.dat" @@ -55,240 +66,415 @@ def peripheral_dir(tmp_path): return pd, license_file -def _bitstreams_dir(peripheral_dir): - bs = os.path.join(str(peripheral_dir), "src", "gateware", "build", "bitstreams") - os.makedirs(bs, exist_ok=True) - return bs +@pytest.fixture() +def bitstreams(peripheral_dir): + path = peripheral_dir[0] / "src" / "gateware" / "build" / "bitstreams" + path.mkdir(parents=True) + return path -# --------------------------------------------------------------------------- -# AC-10 case 10 / 13: docker-run argv shape -# --------------------------------------------------------------------------- +@pytest.fixture() +def build(gateware, peripheral_dir, bitstreams, monkeypatch): + """Run the build once; *emit* plays the SDK, writing into ``bitstreams``.""" + pd, license_file = peripheral_dir + + def run(emit) -> Path: + before = len(run.argvs) + def fake_run(argv, *args, **kwargs): + assert len(run.argvs) == before, "build invoked the SDK twice" + run.argvs.append(argv) + emit(bitstreams) + return subprocess.CompletedProcess(argv, 0, b"", b"") + + monkeypatch.setattr(gateware.subprocess, "run", fake_run) + selected = gateware.run_gateware_build( + str(pd), IMAGE_TAG, env={"LM_LICENSE_FILE": str(license_file)} + ) + assert len(run.argvs) == before + 1 + assert Path(selected).is_absolute() + return Path(selected) -def test_runner_builds_docker_run_argv_with_project_flag( + run.argvs = [] + return run + + +def test_runner_raises_when_license_unset_and_does_not_invoke_docker( gateware, peripheral_dir, monkeypatch ): - """Case 10/13: captured docker-run argv has the correct shape and ends - with the exact axon-peripheral-sdk invocation (`--project src/gateware`; - the SDK no longer accepts `--pdc`/`--impl`). - """ - pd, license_file = peripheral_dir - recorded: list[list[str]] = [] + pd, _ = peripheral_dir + called = [] def fake_run(argv, *args, **kwargs): - recorded.append(list(argv)) - # Drop a fake extracted .bit so the post-run glob succeeds. - bs = _bitstreams_dir(pd) - bit = os.path.join(bs, "sdk_topbuild_extracted.bit") - with open(bit, "w") as fp: - fp.write("bitstream") + called.append(argv) return subprocess.CompletedProcess(argv, 0, b"", b"") monkeypatch.setattr(gateware.subprocess, "run", fake_run) - result = gateware.run_gateware_build( - str(pd), - "myplugin-gateware:latest-arm64", - env={"LM_LICENSE_FILE": str(license_file)}, - ) + with pytest.raises(gateware.LicenseUnsetError): + gateware.run_gateware_build(str(pd), IMAGE_TAG, env={}) - assert len(recorded) == 1, "runner should issue exactly one docker run" - argv = recorded[0] + assert called == [] - # Sanity: docker run --rm - assert argv[0:3] == ["docker", "run", "--rm"] - # The image tag is the second-to-last block before the entrypoint. - assert "myplugin-gateware:latest-arm64" in argv +def test_runner_passes_default_project_to_sdk(build): + build(lambda bs: _write(bs / "sdk_gateware.bit", b"native", _new_ns(1))) - # Workdir is /home/workspace (case 13) - assert "-w" in argv - assert argv[argv.index("-w") + 1] == "/home/workspace" + assert [_project_args(argv) for argv in build.argvs] == [["src/gateware"]] - # Bind-mount the peripheral_dir abspath to /home/workspace. - abs_pd = os.path.abspath(str(pd)) - assert "-v" in argv - # There may be multiple -v (license + workspace); check that the - # workspace bind-mount is present. - v_indices = [i for i, tok in enumerate(argv) if tok == "-v"] - bind_targets = {argv[i + 1] for i in v_indices} - assert f"{abs_pd}:/home/workspace" in bind_targets - # The SDK command is the final shell -lc payload. - assert "/bin/bash" in argv - bash_idx = argv.index("/bin/bash") - assert argv[bash_idx + 1] == "-lc" - sdk_cmd = argv[bash_idx + 2] - assert sdk_cmd == "axon-peripheral-sdk build --project src/gateware" +# --------------------------------------------------------------------------- +# Legacy rule (summary schema_version missing or 1) +# --------------------------------------------------------------------------- - # And the returned path is the extracted .bit we dropped. - assert result.endswith("sdk_topbuild_extracted.bit") +legacy_schema = pytest.mark.parametrize( + "schema_field", + [{}, {"schema_version": 1}], + ids=["schema_version_missing", "schema_version_1"], +) -# --------------------------------------------------------------------------- -# AC-10 case 14: LM_LICENSE_FILE forwarded -# --------------------------------------------------------------------------- +def test_runner_returns_native_only_output(build, bitstreams): + selected = build( + lambda bs: _write(bs / "sdk_gateware.bit", b"\x96\x01\x80\x7f", _new_ns(1)) + ) + assert selected == bitstreams / "sdk_gateware.bit" + assert selected.read_bytes() == b"\x96\x01\x80\x7f" -def test_runner_forwards_floating_license_arg(gateware, peripheral_dir, monkeypatch): - """Case 14: port@host floating license -> -e LM_LICENSE_FILE= in argv.""" - pd, _ = peripheral_dir - recorded: list[list[str]] = [] - def fake_run(argv, *args, **kwargs): - recorded.append(list(argv)) - bs = _bitstreams_dir(pd) - with open(os.path.join(bs, "sdk_topbuild_extracted.bit"), "w") as fp: - fp.write("x") - return subprocess.CompletedProcess(argv, 0, b"", b"") +def test_runner_prefers_native_over_newer_extracted(build, bitstreams): + def emit(bs): + _write(bs / "sdk_gateware.bit", b"native", _new_ns(1)) + _write(bs / "sdk_gateware_extracted.bit", b"extracted", _new_ns(3)) - monkeypatch.setattr(gateware.subprocess, "run", fake_run) + selected = build(emit) + + assert selected == bitstreams / "sdk_gateware.bit" + assert selected.read_bytes() == b"native" + + +def test_runner_returns_newest_native_candidate(build, bitstreams): + def emit(bs): + _write(bs / "sdk_old.bit", b"old native", _new_ns(1)) + _write(bs / "sdk_new.bit", b"new native", _new_ns(2)) + _write(bs / "sdk_new_extracted.bit", b"extracted decoy", _new_ns(3)) + + selected = build(emit) - gateware.run_gateware_build( - str(pd), - "myplugin-gateware:latest-arm64", - env={"LM_LICENSE_FILE": "27000@licenseserver"}, + assert selected == bitstreams / "sdk_new.bit" + assert selected.read_bytes() == b"new native" + + +def test_runner_falls_back_to_extracted_when_no_native_image(build, bitstreams): + selected = build( + lambda bs: _write(bs / "sdk_gateware_extracted.bit", b"extracted", _new_ns(1)) ) - argv = recorded[0] - e_indices = [i for i, tok in enumerate(argv) if tok == "-e"] - e_pairs = [argv[i + 1] for i in e_indices] - assert "LM_LICENSE_FILE=27000@licenseserver" in e_pairs + assert selected == bitstreams / "sdk_gateware_extracted.bit" + assert selected.read_bytes() == b"extracted" - # And no bind-mount for the license (port@host mode is mount-free). - v_indices = [i for i, tok in enumerate(argv) if tok == "-v"] - bind_targets = [argv[i + 1] for i in v_indices] - assert not any("/opt/lattice/license.dat" in t for t in bind_targets) +def test_runner_falls_back_to_newest_extracted_candidate(build, bitstreams): + def emit(bs): + _write(bs / "sdk_new_extracted.bit", b"new extracted", _new_ns(2)) + _write(bs / "sdk_old_extracted.bit", b"old extracted", _new_ns(1)) -# --------------------------------------------------------------------------- -# AC-10 case 15: unset env -> error, no subprocess.run -# --------------------------------------------------------------------------- + selected = build(emit) + assert selected == bitstreams / "sdk_new_extracted.bit" + assert selected.read_bytes() == b"new extracted" -def test_runner_raises_when_license_unset_and_does_not_invoke_docker( - gateware, peripheral_dir, monkeypatch -): - """Case 15: LM_LICENSE_FILE unset -> LicenseUnsetError, subprocess.run unused.""" - pd, _ = peripheral_dir - called = [] - def fake_run(argv, *args, **kwargs): # pragma: no cover - must NOT be called - called.append(argv) - return subprocess.CompletedProcess(argv, 0, b"", b"") +def test_runner_ignores_newer_non_sdk_and_non_bit_files(build, bitstreams): + def emit(bs): + _write(bs / "sdk_gateware.bit", b"native", _new_ns(1)) + _write(bs / "gateware.bit", b"decoy", _new_ns(3)) + _write(bs / "sdk_gateware.mcs", b"decoy", _new_ns(3)) - monkeypatch.setattr(gateware.subprocess, "run", fake_run) + assert build(emit) == bitstreams / "sdk_gateware.bit" - with pytest.raises(gateware.LicenseUnsetError): - gateware.run_gateware_build( - str(pd), - "myplugin-gateware:latest-arm64", - env={}, + +@legacy_schema +def test_legacy_summary_ignores_bitstream_field_and_prefers_native( + build, bitstreams, schema_field +): + """Below schema 2 a ``bitstream`` key carries no meaning.""" + + def emit(bs): + _write(bs / "sdk_gateware.bit", b"native", _new_ns(1)) + _write(bs / "sdk_gateware_extracted.bit", b"extracted", _new_ns(2)) + _summary( + bs / "sdk_gateware.bit", + _new_ns(1), + bitstream="sdk_gateware_extracted.bit", + **schema_field, ) - assert called == [] + assert build(emit) == bitstreams / "sdk_gateware.bit" + + +@legacy_schema +def test_legacy_summary_falls_back_to_extracted(build, bitstreams, schema_field): + def emit(bs): + _write(bs / "sdk_gateware_extracted.bit", b"extracted", _new_ns(1)) + _summary(bs / "sdk_gateware_extracted.bit", _new_ns(1), **schema_field) + + assert build(emit) == bitstreams / "sdk_gateware_extracted.bit" # --------------------------------------------------------------------------- -# AC-10 case 11: bitstream glob returns newest of many +# Only this build's output counts # --------------------------------------------------------------------------- -def test_runner_returns_newest_bit_when_multiple_emitted( - gateware, peripheral_dir, monkeypatch +def test_runner_returns_extracted_image_when_stale_native_predates_build( + build, bitstreams ): - """Case 11: glob with two extracted .bit files of different mtimes -> newest wins.""" - pd, license_file = peripheral_dir - bs = _bitstreams_dir(pd) - older = os.path.join(bs, "sdk_old_extracted.bit") - newer = os.path.join(bs, "sdk_new_extracted.bit") - with open(older, "w") as fp: - fp.write("old") - time.sleep(0.05) - with open(newer, "w") as fp: - fp.write("new") - # Belt-and-suspenders: force mtimes so the test isn't flaky on - # coarse-granularity filesystems. - os.utime(older, (1_000_000, 1_000_000)) - os.utime(newer, (2_000_000, 2_000_000)) + """A native image left by an earlier build must not mask this build's + extracted-only output.""" + _write(bitstreams / "sdk_x.bit", b"earlier native", _stale_ns()) - def fake_run(argv, *args, **kwargs): - return subprocess.CompletedProcess(argv, 0, b"", b"") + selected = build( + lambda bs: _write(bs / "sdk_y_extracted.bit", b"this build", _new_ns(1)) + ) - monkeypatch.setattr(gateware.subprocess, "run", fake_run) + assert selected == bitstreams / "sdk_y_extracted.bit" + assert selected.read_bytes() == b"this build" + + +def test_runner_ignores_unchanged_preexisting_file_with_future_mtime(build, bitstreams): + """Unchanged means unchanged, even when the old file's mtime is ahead.""" + _write(bitstreams / "sdk_x.bit", b"earlier native", _new_ns(100)) - result = gateware.run_gateware_build( - str(pd), - "myplugin-gateware:latest-arm64", - env={"LM_LICENSE_FILE": str(license_file)}, + selected = build( + lambda bs: _write(bs / "sdk_y_extracted.bit", b"this build", _new_ns(1)) ) - assert os.path.abspath(result) == os.path.abspath(newer) + assert selected == bitstreams / "sdk_y_extracted.bit" + + +def test_runner_returns_preexisting_image_the_build_rewrote(build, bitstreams): + _write(bitstreams / "sdk_gateware.bit", b"earlier bytes", _stale_ns()) + + selected = build( + lambda bs: _write(bs / "sdk_gateware.bit", b"rebuilt bytes!", _new_ns(1)) + ) + + assert selected == bitstreams / "sdk_gateware.bit" + assert selected.read_bytes() == b"rebuilt bytes!" + + +def test_sequential_builds_each_return_their_own_native_image(build, bitstreams): + first = build(lambda bs: _write(bs / "sdk_a.bit", b"build A", _new_ns(5))) + # B's image is older on disk than A's: newness to *this* build is what + # matters, not mtime across builds. + second = build(lambda bs: _write(bs / "sdk_b.bit", b"build B", _new_ns(2))) + + assert first == bitstreams / "sdk_a.bit" + assert second == bitstreams / "sdk_b.bit" + assert second.read_bytes() == b"build B" + + +def test_runner_raises_when_only_non_candidate_files_emitted(build): + def emit(bs): + _write(bs / "gateware.bit", b"decoy", _new_ns(1)) + _write(bs / "sdk_gateware.mcs", b"decoy", _new_ns(1)) + + with pytest.raises(FileNotFoundError): + build(emit) + + +def test_runner_raises_when_build_emits_nothing(build): + with pytest.raises(FileNotFoundError): + build(lambda bs: None) + + +def test_runner_raises_when_build_emits_nothing_but_old_images_exist(build, bitstreams): + _write(bitstreams / "sdk_x.bit", b"earlier native", _stale_ns()) + _write(bitstreams / "sdk_x_extracted.bit", b"earlier extracted", _stale_ns()) + _summary(bitstreams / "sdk_x.bit", _stale_ns(), schema_version=1) + + with pytest.raises(FileNotFoundError): + build(lambda bs: None) # --------------------------------------------------------------------------- -# AC-10 case 12: no .bit -> clear FileNotFoundError naming the glob +# Schema 2: the summary names the image # --------------------------------------------------------------------------- -def test_runner_raises_with_glob_in_message_when_no_bit_emitted( - gateware, peripheral_dir, monkeypatch -): - """Case 12: docker run succeeds but no .bit lands -> FileNotFoundError - whose message names the expected glob pattern. - """ - pd, license_file = peripheral_dir - _bitstreams_dir(pd) # exists but empty +def test_schema2_summary_selects_named_image_over_newer_native(build, bitstreams): + def emit(bs): + _write(bs / "sdk_gw_extracted.bit", b"named image", _new_ns(1)) + _summary( + bs / "sdk_gw_extracted.bit", + _new_ns(1), + schema_version=2, + bitstream="sdk_gw_extracted.bit", + ) + _write(bs / "sdk_other.bit", b"newer native decoy", _new_ns(5)) - def fake_run(argv, *args, **kwargs): - return subprocess.CompletedProcess(argv, 0, b"", b"") + selected = build(emit) + + assert selected == bitstreams / "sdk_gw_extracted.bit" + assert selected.read_bytes() == b"named image" - monkeypatch.setattr(gateware.subprocess, "run", fake_run) - with pytest.raises(FileNotFoundError) as excinfo: - gateware.run_gateware_build( - str(pd), - "myplugin-gateware:latest-arm64", - env={"LM_LICENSE_FILE": str(license_file)}, +def _schema2_emitter(bitstream, extra=lambda bs: None, schema_version=2): + """Emit ``sdk_gw.bit`` plus its summary, whose ``bitstream`` is *bitstream*.""" + + def emit(bs): + _write(bs / "sdk_gw.bit", b"native", _new_ns(1)) + extra(bs) + _summary( + bs / "sdk_gw.bit", + _new_ns(1), + schema_version=schema_version, + bitstream=bitstream, ) - assert "sdk_*_extracted.bit" in str(excinfo.value) + return emit + + +def test_schema2_summary_selects_its_own_sibling_image(build, bitstreams): + def emit(bs): + _write(bs / "sdk_other.bit", b"newer native decoy", _new_ns(5)) + + selected = build(_schema2_emitter("sdk_gw.bit", extra=emit)) + + assert selected == bitstreams / "sdk_gw.bit" + assert selected.read_bytes() == b"native" + + +def test_schema2_rejects_summary_naming_a_different_new_image(build): + """Each summary may only name its own ``.bit``, even when the other + image is a valid file this build wrote.""" + emit = _schema2_emitter( + "sdk_other.bit", + extra=lambda bs: _write(bs / "sdk_other.bit", b"other", _new_ns(2)), + ) + + with pytest.raises(ValueError): + build(emit) + + +@pytest.mark.parametrize( + "bitstream", + # The backslash form is a legal POSIX file name, so only the rule rejects it. + ["sub/sdk_gw.bit", "sub\\sdk_gw.bit", "../sdk_gw.bit"], + ids=["forward_slash", "backslash", "parent_relative"], +) +def test_schema2_rejects_bitstream_that_is_not_a_basename(build, bitstream): + """Rejected even though a fresh image exists where the path points.""" + emit = _schema2_emitter( + bitstream, extra=lambda bs: _write(bs / bitstream, b"escape", _new_ns(2)) + ) + + with pytest.raises(ValueError): + build(emit) + + +def test_schema2_rejects_absolute_bitstream(build, bitstreams): + with pytest.raises(ValueError): + build(_schema2_emitter(str(bitstreams / "sdk_gw.bit"))) + + +def test_schema2_rejects_bitstream_without_bit_suffix(build): + emit = _schema2_emitter( + "sdk_gw.mcs", extra=lambda bs: _write(bs / "sdk_gw.mcs", b"mcs", _new_ns(1)) + ) + + with pytest.raises(ValueError): + build(emit) + + +def test_schema2_rejects_missing_sibling_image(build): + """A fresh fallback image must not paper over the summary's missing one.""" + + def emit(bs): + _write(bs / "sdk_other.bit", b"fallback native", _new_ns(1)) + _summary( + bs / "sdk_gw.bit", _new_ns(1), schema_version=2, bitstream="sdk_gw.bit" + ) + + with pytest.raises(FileNotFoundError): + build(emit) + + +def test_schema2_rejects_sibling_image_from_an_earlier_build(build, bitstreams): + _write(bitstreams / "sdk_old.bit", b"earlier native", _stale_ns()) + + def emit(bs): + _write(bs / "sdk_other.bit", b"fallback native", _new_ns(1)) + _summary( + bs / "sdk_old.bit", _new_ns(1), schema_version=2, bitstream="sdk_old.bit" + ) + + with pytest.raises((ValueError, FileNotFoundError)): + build(emit) + + +@pytest.mark.parametrize("schema_version", [3, 17]) +def test_newer_summary_schema_asks_for_upgrade(build, schema_version): + with pytest.raises(ValueError) as exc: + build(_schema2_emitter("sdk_gw.bit", schema_version=schema_version)) + + message = str(exc.value) + assert str(schema_version) in message + assert "upgrade" in message.lower() # --------------------------------------------------------------------------- -# Extracted-bitstream selection: the raw sdk_*.bit is ignored +# Summaries left by earlier builds carry no weight # --------------------------------------------------------------------------- -def test_runner_ignores_raw_bit_and_picks_extracted( - gateware, peripheral_dir, monkeypatch -): - """When both the raw sdk_*.bit and its sdk_*_extracted.bit sit side by - side, the runner must select the extracted variant — even when the raw - .bit is newer by mtime (the raw glob would otherwise win on mtime). - """ - pd, license_file = peripheral_dir - bs = _bitstreams_dir(pd) - raw = os.path.join(bs, "sdk_topbuild.bit") - extracted = os.path.join(bs, "sdk_topbuild_extracted.bit") - with open(raw, "w") as fp: - fp.write("raw") - with open(extracted, "w") as fp: - fp.write("extracted") - # Make the raw .bit strictly newer so a mtime-only selection would pick it. - os.utime(extracted, (1_000_000, 1_000_000)) - os.utime(raw, (2_000_000, 2_000_000)) +def test_stale_newer_schema_summary_is_ignored_next_to_fresh_image(build, bitstreams): + """A schema-3 summary from an earlier build must not block this build.""" + _summary( + bitstreams / "sdk_gw.bit", + _stale_ns(), + schema_version=3, + bitstream="sdk_gw.bit", + ) - def fake_run(argv, *args, **kwargs): - return subprocess.CompletedProcess(argv, 0, b"", b"") + selected = build(lambda bs: _write(bs / "sdk_gw.bit", b"fresh", _new_ns(1))) + + assert selected == bitstreams / "sdk_gw.bit" + assert selected.read_bytes() == b"fresh" - monkeypatch.setattr(gateware.subprocess, "run", fake_run) - result = gateware.run_gateware_build( - str(pd), - "myplugin-gateware:latest-arm64", - env={"LM_LICENSE_FILE": str(license_file)}, +def test_stale_schema2_summary_does_not_redirect_selection(build, bitstreams): + _write(bitstreams / "sdk_old.bit", b"earlier native", _stale_ns()) + _summary( + bitstreams / "sdk_old.bit", + _stale_ns(), + schema_version=2, + bitstream="sdk_old.bit", ) - assert os.path.abspath(result) == os.path.abspath(extracted) + selected = build( + lambda bs: _write(bs / "sdk_new_extracted.bit", b"fresh", _new_ns(1)) + ) + + assert selected == bitstreams / "sdk_new_extracted.bit" + + +# --------------------------------------------------------------------------- +# Schema 2 without a usable ``bitstream`` field +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "bitstream_field", + [{}, {"bitstream": None}, {"bitstream": 7}, {"bitstream": ["sdk_gw.bit"]}], + ids=["missing", "null", "integer", "list"], +) +def test_schema2_summary_without_string_bitstream_raises(build, bitstream_field): + def emit(bs): + _write(bs / "sdk_gw.bit", b"native", _new_ns(1)) + _summary(bs / "sdk_gw.bit", _new_ns(1), schema_version=2, **bitstream_field) + + with pytest.raises(ValueError): + build(emit)