Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 143 additions & 44 deletions synapse/cli/gateware.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,14 @@

from __future__ import annotations

import glob
import json
import os
import re
import subprocess
import sys
import uuid
from pathlib import Path
from typing import Mapping, Sequence
from typing import Iterable, Mapping, Sequence

from rich.console import Console

Expand Down Expand Up @@ -218,6 +217,80 @@ def resolve_gateware_project(
)


# Highest build-summary schema this synapse understands. The SDK releases
# independently, so a newer schema must fail loudly rather than be misread.
_MAX_SUMMARY_SCHEMA = 2


def _scan_bitstreams(bitstreams_dir: str) -> dict[str, tuple[int, int]]:
"""Return ``{name: (mtime_ns, size)}`` for the regular files in *bitstreams_dir*.

Size rides with mtime because some filesystems store whole seconds, so a
same-second rewrite would otherwise look untouched.
"""
if not os.path.isdir(bitstreams_dir):
return {}
files = {}
with os.scandir(bitstreams_dir) as entries:
for entry in entries:
if entry.is_file():
st = entry.stat()
files[entry.name] = (st.st_mtime_ns, st.st_size)
return files


def _load_summary(path: str) -> dict | None:
"""Return the build summary at *path* as a dict, or None if unreadable."""
try:
with open(path, "r", encoding="utf-8") as fp:
summary = json.load(fp)
except (OSError, json.JSONDecodeError):
return None
return summary if isinstance(summary, dict) else None


def _summary_schema(path: str) -> tuple[int, dict] | None:
"""Return ``(schema_version, summary)`` for a build summary, or None if unreadable."""
summary = _load_summary(path)
if summary is None:
return None
version = summary.get("schema_version", 1)
if type(version) is not int:
raise ValueError(
f"Bitstream summary {path} has a non-integer schema_version {version!r}"
)
return version, summary


def _schema2_bitstream(
bitstreams_dir: str,
summary_name: str,
summary: dict,
after: Mapping[str, tuple[int, int]],
emitted: set[str],
) -> str:
"""Return the image a schema-2 summary names, checked to be from this build."""
name = summary.get("bitstream")
# The readers find a summary by its image's stem, so the two must pair up.
expected = summary_name.removesuffix(".summary.json") + ".bit"
if name != expected:
raise ValueError(
f"Bitstream summary {summary_name} has 'bitstream' {name!r}; "
f"expected {expected!r}, the image next to it"
)
if name not in after:
raise FileNotFoundError(
f"Bitstream summary {summary_name} names {name}, which does not "
f"exist in {bitstreams_dir}"
)
if name not in emitted:
raise ValueError(
f"Bitstream summary {summary_name} names {name}, which this build "
"did not write (it predates the build)"
)
return os.path.join(bitstreams_dir, name)


def run_gateware_build(
peripheral_dir: str,
image_tag: str,
Expand All @@ -231,24 +304,37 @@ def run_gateware_build(
root either way, so a per-profile project can still reference shared
sources above it (``../axon_test_source_peripheral.sv``).

Returns the absolute path to the newest ``sdk_*_extracted.bit`` emitted
under ``<peripheral_dir>/<project_subdir>/build/bitstreams/``. This is the
*extracted* bitstream variant the SDK writes alongside the raw
``sdk_*.bit``; the extracted form is what gets flashed to the probe, and
it carries its own same-stem ``.summary.json`` (same schema as the raw
one). Selecting it specifically — rather than the broader ``sdk_*.bit``
glob, which would also match the extracted file and pick by mtime —
keeps the choice deterministic.
Returns the absolute path to the image this build emitted under
``<peripheral_dir>/<project_subdir>/build/bitstreams/``. Build directories
persist between builds, so only files the build created or changed (by
mtime and size) are considered; older files are ignored whatever their
name. Among those:

- A schema-2 ``<stem>.summary.json`` names its image, ``<stem>.bit``,
in ``bitstream``; the newest such summary wins.
- Otherwise (schema 1, no ``schema_version``, or no summary) the newest
native ``sdk_*.bit`` wins, else the newest ``sdk_*_extracted.bit``
(older SDKs emit only the extracted image).

Unreadable summaries are skipped here. The image's same-stem
``.summary.json`` describes it.

Raises:
LicenseUnsetError: if ``LM_LICENSE_FILE`` is unset (propagated from
:func:`build_license_docker_args`).
subprocess.CalledProcessError: if the container's build exits non-zero.
FileNotFoundError: if the build succeeds but no bitstream is emitted.
ValueError: if a new summary has a schema newer than this synapse
supports or a malformed ``schema_version``, or a schema-2 summary
names an invalid or pre-existing image.
FileNotFoundError: if the build emitted no bitstream, or a schema-2
summary names a missing image.
"""
license_args = build_license_docker_args(env)

abs_peripheral_dir = os.path.abspath(peripheral_dir)
bitstreams_dir = os.path.join(
abs_peripheral_dir, *project_subdir.split("/"), "build", "bitstreams"
)
argv = [
"docker",
"run",
Expand All @@ -265,24 +351,49 @@ def run_gateware_build(
"-lc",
f"axon-peripheral-sdk build --project {project_subdir}",
]
before = _scan_bitstreams(bitstreams_dir)
subprocess.run(argv, check=True)
after = _scan_bitstreams(bitstreams_dir)

emitted = {name for name, ident in after.items() if before.get(name) != ident}

def newest(names: Iterable[str]) -> list[str]:
return sorted(names, key=lambda name: after[name][0], reverse=True)

schema2 = {}
for name in emitted:
if not (name.startswith("sdk_") and name.endswith(".summary.json")):
continue
parsed = _summary_schema(os.path.join(bitstreams_dir, name))
if parsed is None:
continue
version, summary = parsed
if version > _MAX_SUMMARY_SCHEMA:
raise ValueError(
f"Bitstream summary {name} uses schema_version {version}, newer "
f"than this synapse supports ({_MAX_SUMMARY_SCHEMA}). Upgrade "
"science-synapse to package this build."
)
if version == 2:
schema2[name] = summary

bit_glob = os.path.join(
abs_peripheral_dir,
*project_subdir.split("/"),
"build",
"bitstreams",
"sdk_*_extracted.bit",
)
matches = glob.glob(bit_glob)
if not matches:
raise FileNotFoundError(
"axon-peripheral-sdk build completed but no sdk_*_extracted.bit was "
f"emitted under {project_subdir}/build/bitstreams/"
if schema2:
matches = newest(schema2)
chosen = _schema2_bitstream(
bitstreams_dir, matches[0], schema2[matches[0]], after, emitted
)
else:
bits = [n for n in emitted if n.startswith("sdk_") and n.endswith(".bit")]
matches = [n for n in bits if not n.endswith("_extracted.bit")] or bits
if not matches:
raise FileNotFoundError(
"axon-peripheral-sdk build completed but no sdk_*.bit was "
f"emitted by this build under {project_subdir}/build/bitstreams/ "
"(files left from earlier builds are ignored)"
)
matches = newest(matches)
chosen = os.path.join(bitstreams_dir, matches[0])

matches.sort(key=os.path.getmtime, reverse=True)
chosen = matches[0]
if len(matches) > 1:
console.print(
f"[yellow]Multiple bitstreams matched; selected newest: {chosen}[/yellow]"
Expand Down Expand Up @@ -340,7 +451,7 @@ def read_usb_pid(bit_path: str) -> int:
if not isinstance(summary, dict):
raise ValueError(
f"Bitstream summary {path} is not a JSON object; "
"expected a top-level usb_pid hex string (e.g. \"0x000B\")."
'expected a top-level usb_pid hex string (e.g. "0x000B").'
)

raw = summary.get("usb_pid")
Expand All @@ -357,13 +468,13 @@ def read_usb_pid(bit_path: str) -> int:
except ValueError:
raise ValueError(
f"Bitstream summary {path} usb_pid {raw!r} is not a valid hex "
"string (expected e.g. \"0x000B\")"
'string (expected e.g. "0x000B")'
)

if not (0 < usb_pid <= 0xFFFF):
raise ValueError(
f"Bitstream summary {path} usb_pid {raw!r} ({usb_pid}) is out of "
"range; expected a hex string in 1..0xFFFF (e.g. \"0x000B\")"
'range; expected a hex string in 1..0xFFFF (e.g. "0x000B")'
)
return usb_pid

Expand All @@ -380,14 +491,8 @@ def read_identifier(bit_path: str) -> str | None:
should stay within ``[A-Za-z0-9._-]`` — the identifier rides in
``scifi-probe-update-custom@<serial>:<name>``.
"""
path = summary_path_for(bit_path)
try:
with open(path, "r", encoding="utf-8") as fp:
summary = json.load(fp)
except (OSError, json.JSONDecodeError):
return None

if not isinstance(summary, dict):
summary = _load_summary(summary_path_for(bit_path))
if summary is None:
return None
target_profile = summary.get("target_profile")
if not isinstance(target_profile, str) or not target_profile:
Expand All @@ -407,14 +512,8 @@ def read_git_sha(bit_path: str) -> str | None:
Best-effort: returns None when the summary or the field is
missing/malformed.
"""
path = summary_path_for(bit_path)
try:
with open(path, "r", encoding="utf-8") as fp:
summary = json.load(fp)
except (OSError, json.JSONDecodeError):
return None

if not isinstance(summary, dict):
summary = _load_summary(summary_path_for(bit_path))
if summary is None:
return None
project = summary.get("project")
if not isinstance(project, dict):
Expand Down
26 changes: 11 additions & 15 deletions synapse/cli/peripherals.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,9 +295,7 @@ def build_peripheral_so(
# of profile doesn't depend on which one a given repo happens to take.
# shlex.quote keeps a hostile profile name from breaking out of the
# `bash -c` string; the value reaches us from --profile or the environment.
profile_flag = (
f" -DAXON_TARGET_PROFILE={shlex.quote(profile)}" if profile else ""
)
profile_flag = f" -DAXON_TARGET_PROFILE={shlex.quote(profile)}" if profile else ""
build_cmd_str = (
"cd /home/workspace && "
"if [ -f CMakePresets.json ]; then "
Expand Down Expand Up @@ -456,9 +454,7 @@ def build_peripheral_deb(

# 1. Stage the plugin .so at /usr/lib/scifi/plugins/<name>.so
if not os.path.exists(so_path):
console.print(
f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}"
)
console.print(f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}")
return False
plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins")
os.makedirs(plugin_dst, exist_ok=True)
Expand Down Expand Up @@ -505,9 +501,7 @@ def build_peripheral_deb(
)
return False
arch_suffix = detect_arch()
platform_opt = (
"linux/arm64" if arch_suffix == "arm64" else "linux/amd64"
)
platform_opt = "linux/arm64" if arch_suffix == "arm64" else "linux/amd64"

console.print(
f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]"
Expand All @@ -534,9 +528,7 @@ def build_peripheral_deb(
return False

sdk_files = [
f
for f in os.listdir(sdk_dst)
if f.startswith("libscifi-peripheral-sdk.so")
f for f in os.listdir(sdk_dst) if f.startswith("libscifi-peripheral-sdk.so")
]
if not sdk_files:
console.print(
Expand Down Expand Up @@ -788,7 +780,7 @@ def _run_gateware_half(peripheral_dir: str, project_subdir: str) -> Optional[str
except LicenseUnsetError as exc:
console.print(f"[bold red]Error:[/bold red] {exc}")
return None
except (subprocess.CalledProcessError, FileNotFoundError) as exc:
except (subprocess.CalledProcessError, FileNotFoundError, ValueError) as exc:
console.print(f"[bold red]Error:[/bold red] Gateware build failed: {exc}")
return None

Expand Down Expand Up @@ -840,7 +832,9 @@ def _build_debs(
else None
)
if resolved_profile:
console.print(f"[bold]Target profile:[/bold] [yellow]{resolved_profile}[/yellow]")
console.print(
f"[bold]Target profile:[/bold] [yellow]{resolved_profile}[/yellow]"
)

if do_gateware and clean:
try:
Expand Down Expand Up @@ -892,7 +886,9 @@ def _build_debs(
git_hash=gateware.read_git_sha(bit_path),
):
return None
deb = find_deb_package(dist_dir, f"{_gateware_package_name(identifier)}_{version}")
deb = find_deb_package(
dist_dir, f"{_gateware_package_name(identifier)}_{version}"
)
if deb is None:
return None
debs.append(deb)
Expand Down
Loading
Loading