diff --git a/CLAUDE.md b/CLAUDE.md index 9bd6a9cf..7e0658c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -319,6 +319,28 @@ Second device: C64 Ultimate "Starlight", `U64_HOST=10.53.21.158`, fw 1.1.0, Scratch DMA addresses come from `build_policy_and_arbiter()` (`_memory_policy.py`, parses `build/labels.txt`) — never hardcode them. +**The acquire budget is `C64_DEVICE_LOCK_TIMEOUT`, default 1800 s**, and +it is one number: every rig here and `tests/rig_ip65_rrnet_hw.py` takes +the lock through `acquire_device_lock()` in `_device_lock_helper.py`, and +`tools/test_device_lock_timeout.py` fails if one grows its own back (it +also pins the malformed-value and progress-window behaviour). The budget +is **not** how long you may queue behind a running rig: the harness's +`acquire` re-arms its deadline indefinitely behind a live, progressing +holder, so the budget only bounds waits it refuses to extend — a wedged +holder, or a *handoff chain*. **The chain boundary is the FOURTH change +of holder identity, not the third**: `_MAX_HOLDER_HANDOFFS` is 3, but the +harness extends while `handoffs <= 3` and its own docstring says +otherwise (a harness doc bug — its WARNING text agrees with its code). +Pinned by `test_the_handoff_boundary_is_four_changes_not_three`; do not +re-derive it from their prose. That case is what several lanes cycling +one U64E *would* produce — the mechanism is lab-measured, no such +timeout has been captured from a real run here — and it is why the old +hardcoded 120 s could fail against a device that was merely busy. A +malformed value is fatal before the device is touched; a wait prints +progress to stderr every 30 s (elapsed, budget, holder PID, lockfile age +with a STALE flag past the 60 s progress window, queue depth), never to +stdout. + boot_check.py / phase2_check.py / phase3_tcp_echo.py — boot, DHCP, TCP rig_http_local.py / rig_http_live.py — plaintext HTTP rig_https_local.py — HTTPS vs local TLS 1.3 listener diff --git a/README.md b/README.md index 475118cb..73e386e4 100644 --- a/README.md +++ b/README.md @@ -739,6 +739,37 @@ The device address comes from `U64_HOST`. Every rig script defaults it to device-queue helper — defaults to a different address, so **set `U64_HOST` explicitly** rather than relying on a default agreeing with itself. +The device is shared — across sessions and across the `c64-*` repos — so every +script here queues for it, and the wait has **one** budget: +`C64_DEVICE_LOCK_TIMEOUT`, default **1800 s (30 min)**. Every rig takes the +lock through `acquire_device_lock()` in `tools/uci/_device_lock_helper.py`, and +`tools/test_device_lock_timeout.py` fails if any script grows a budget of its +own back. A malformed value (`30m`, `2 min`, `0`) is a hard error before the +device is touched rather than a silent fall back to the default; an empty value +falls back with a notice. While waiting, a progress line goes to **stderr** +every 30 s — elapsed, budget, holder PID, lockfile age (flagged stale past the +60 s progress window) and queue depth — so a half-hour block is distinguishable +from a hang. stdout stays parseable. + +That budget is **not** the longest run you may queue behind. The harness +re-arms its deadline on every poll while the holder is alive and its lockfile +is fresh, so a waiter sits behind one long healthy holder indefinitely; the +budget bounds only the waits it refuses to extend — a dead or wedged holder, +and a **handoff chain**. Several lanes cycling one device is the case that +bites: after the **fourth** change of holder identity the harness stops +extending for the rest of that acquire (four, not three — `_MAX_HOLDER_HANDOFFS` +is 3 but extension survives `handoffs <= 3`), and from there the budget is +everything. Take that boundary from `tools/test_device_lock_timeout.py`, which +asserts it against the harness's real behaviour: the upstream docstring that +said three was fixed in c64-test-harness#232 and is now deliberately +literal-free, so there is no number there to check yours against. That is issue #212, and it +is why the old hardcoded 120 s could fail against a device that was merely +busy. Raise it when you know you are behind a long run: + +```bash +C64_DEVICE_LOCK_TIMEOUT=7200 python3 tools/uci/rig_https_live.py # queue for 2 h +``` + **Prerequisite — `c64-test-harness` (same as the VICE suites above).** It is a separate public package, not vendored here; `requirements.txt` carries only `cryptography`. Without it every script in this directory dies at import with `ModuleNotFoundError: No module named 'c64_test_harness'`: ```bash @@ -816,6 +847,7 @@ control. `tools/test_finished_verify.py` is the VICE-only equivalent. Environment variables honored by `rig_https_local.py`: - `U64_HOST` (default `192.168.1.81`) — U64E address +- `C64_DEVICE_LOCK_TIMEOUT` (default `1800`) — seconds to queue for the shared device; see the paragraph above - `TURBO_MHZ` (default `48`) — C64 CPU speed. `TURBO_MHZ=1` runs the test at stock 1 MHz with every wall-clock budget auto-scaled, and is validated end-to-end on real U64E hardware; the handshake + GET itself measured 1,157.7 s (~19 min) there, not the full budget. - `HTTPS_PORT` (default `443`, falls back to `4433` if the bind fails) - `SENTINEL_POLL_TIMEOUT`, `ACCEPT_TIMEOUT` — per-test overrides in seconds; default to `600 * max(1, 48 / TURBO_MHZ)`. diff --git a/pytest.ini b/pytest.ini index 10c36693..bdfdd441 100644 --- a/pytest.ini +++ b/pytest.ini @@ -46,6 +46,7 @@ [pytest] testpaths = tools/test_build_flags_stamp.py + tools/test_device_lock_timeout.py tools/test_flags_stamp_skip_is_loud.py tools/test_ip65_hw_checks_unit.py tools/test_net_test_env.py diff --git a/tests/rig_ip65_rrnet_hw.py b/tests/rig_ip65_rrnet_hw.py index 2e12a20e..887046d7 100755 --- a/tests/rig_ip65_rrnet_hw.py +++ b/tests/rig_ip65_rrnet_hw.py @@ -183,12 +183,16 @@ PROJECT_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(PROJECT_ROOT / "tools")) +sys.path.insert(0, str(PROJECT_ROOT / "tools" / "uci")) for _p in ("/Users/someone/Documents/c64-test-harness/src",): if os.path.isdir(_p) and _p not in sys.path: sys.path.insert(0, _p) import ip65_hw_checks as hw # noqa: E402 from _skip_policy import cannot_run, not_applicable # noqa: E402 +from _device_lock_helper import ( # noqa: E402 + LockTimeoutConfigError, acquire_device_lock, +) from https_e2e import ( # noqa: E402 press_key, get_screen_text, wait_for_screen_text, start_https_listener, stop_https_listener, @@ -228,7 +232,9 @@ U64_HOST = os.environ.get("U64_HOST", "10.43.23.81") HTTPS_PORT = int(os.environ.get("HTTPS_PORT", "4433")) FETCH_BUDGET_S = float(os.environ.get("E2E_TIMEOUT", "4800")) -LOCK_TIMEOUT_S = 120.0 +# The device-lock budget is no longer a constant here: it is the one +# shared, C64_DEVICE_LOCK_TIMEOUT-overridable number owned by +# tools/uci/_device_lock_helper.py. TURBO_MHZ = int(os.environ.get("TURBO_MHZ", "1")) #: Phase-timeline resolution. At 1 MHz a phase lasts minutes and a 5 s #: cadence is free; at turbo the whole handshake is under a minute, so 5 s @@ -813,7 +819,10 @@ def main() -> int: lock = DeviceLock(U64_HOST) try: - lock.acquire_or_raise(timeout=LOCK_TIMEOUT_S) + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + return cannot_run(str(exc), executed=0, total=1, certifies=CERTIFIES, + opt_out_env=None) except DeviceLockTimeout as exc: return cannot_run(f"another lane holds {U64_HOST}: {exc}", executed=0, total=1, certifies=CERTIFIES, diff --git a/tools/test_device_lock_timeout.py b/tools/test_device_lock_timeout.py new file mode 100644 index 00000000..69f288b0 --- /dev/null +++ b/tools/test_device_lock_timeout.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""Guard the shared device-lock acquire budget (tools/uci/_device_lock_helper.py). + +Nothing here touches hardware, VICE, the network or a build. The +``DeviceLock`` is faked, so this is pure logic and runs in milliseconds — +the same shape as ``tools/test_reu_preflight.py``, and for the same +reason: the thing being guarded protects a *hardware* run, and its only +failure mode is to stop guarding, which no hardware run can observe. + +Why this exists +--------------- +Every rig used to hardcode its own acquire timeout (120 s in four HTTPS +rigs and the RR-Net rig, 60 s in five more, 300 s in one) with no way to +raise it from the environment. That is survivable only because the +harness's ``acquire`` is queue-aware: behind one live, progressing holder +the deadline is re-armed indefinitely and ``timeout`` is never consulted. +It stops being survivable the moment the harness *stops* extending — +after the **fourth** change of holder identity, or against a holder whose +heartbeat has died — at which point the hardcoded number is the entire +budget and a rig queued behind a live device fails while the device is +merely busy. (Four, not the three the harness's docstring implies: +``_MAX_HOLDER_HANDOFFS`` is 3 but extension survives ``handoffs <= 3``. +Measured with a scripted identity sequence: 0/1/2/3 changes still waited +indefinitely, 4 and 5 returned False at ``timeout``.) + +Both halves are pinned here: + +1. **The budget is one number, read from the environment.** + ``C64_DEVICE_LOCK_TIMEOUT`` reaches ``DeviceLock.acquire``; unset + yields the documented default; a malformed value is fatal, never a + silent fallback to a budget nobody asked for. + +2. **Nothing keeps its own.** An AST sweep over ``tools/uci/*.py`` and + ``tests/rig_*.py``: a module that constructs a ``DeviceLock`` must go + through ``acquire_device_lock``, and no module outside the helper may + call ``acquire``/``acquire_or_raise`` on it. A third sweep, over a + *wider* glob, forbids a literal ``lock_timeout=`` anywhere — that is + how ``UnifiedManager(backend="u64", lock_timeout=120.0)`` in + ``tools/test_ecdsa_p384_kat.py`` kept a thirteenth budget that the + first two checks could not see, since it is neither in a rig + directory nor a ``DeviceLock`` call. This is the half that keeps the + fix from decaying — a new rig copied from an old one reintroduces the + literal, and only a structural check catches that. + +Invariant 1 also pins what must NOT change: the helper passes +``progress_window`` through, because ``progress_window=None`` would +restore the legacy hard timeout and destroy the indefinite wait behind a +healthy long holder — the one part of the old behaviour that worked. + +Runs under pytest, and standalone for anyone without pytest installed +(the repo declares no pytest dependency):: + + python3 tools/test_device_lock_timeout.py +""" + +import ast +import importlib.util +import io +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +HELPER_PATH = REPO / "tools" / "uci" / "_device_lock_helper.py" + +# tools/uci/ is in pytest.ini's norecursedirs (it is a rig directory), so +# the module under test is loaded by path rather than imported by name. +_spec = importlib.util.spec_from_file_location("_device_lock_helper_ut", HELPER_PATH) +dlh = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(dlh) + +from c64_test_harness.backends.device_lock import DeviceLockTimeout # noqa: E402 + +ENV = dlh.LOCK_TIMEOUT_ENV +DEFAULT = dlh.DEFAULT_LOCK_TIMEOUT_S + + +class FakeLock: + """Stands in for ``DeviceLock``: records the acquire call, never locks. + + ``_lock_dir`` is a real empty temp directory so the helper's + best-effort holder/queue-depth probes look at nothing instead of at + this machine's live lock directory — a unit test must not read, let + alone prune, another lane's queue entries. + """ + + def __init__(self, *, raises: BaseException | None = None, host="fake.invalid"): + self.device_host = host + self._lock_dir = Path(tempfile.mkdtemp(prefix="dl-ut-")) + self.calls: list[dict] = [] + self._raises = raises + + def acquire_or_raise(self, timeout=None, *, progress_window=None): + self.calls.append({"timeout": timeout, "progress_window": progress_window}) + if self._raises is not None: + raise self._raises + + +def _timeout_exc(host="fake.invalid"): + """A real ``DeviceLockTimeout`` with the diagnostics a rig prints.""" + return DeviceLockTimeout( + device_host=host, + holder_pid=4242, + pid_alive=True, + lockfile_age_seconds=3.0, + device_reachable_rest=True, + timeout=1800.0, + progress_window=60.0, + ) + + +# -------------------------------------------------------------------------- +# 1. Reading the budget +# -------------------------------------------------------------------------- + + +def test_default_when_unset() -> None: + assert dlh.lock_timeout_s({}) == DEFAULT + assert DEFAULT == 1800.0, "the documented default moved; update the docs too" + + +def test_env_override_is_honoured() -> None: + assert dlh.lock_timeout_s({ENV: "7200"}) == 7200.0 + assert dlh.lock_timeout_s({ENV: " 90.5 "}) == 90.5 + + +def test_empty_env_falls_back_loudly() -> None: + """``VAR=`` is the shell's "unset", so it falls back — but it says so. + + Silent would be defensible for a value that is merely absent; this + one was *written* and expanded to nothing, which is usually a bug in + the caller's own quoting. + """ + buf = io.StringIO() + assert dlh.lock_timeout_s({ENV: ""}, stream=buf) == DEFAULT + assert dlh.lock_timeout_s({ENV: " "}, stream=buf) == DEFAULT + text = buf.getvalue() + assert text.count(ENV) == 2, f"expected a notice per empty read, got {text!r}" + + +def test_malformed_env_is_fatal() -> None: + """A typo must not silently reinstate the default. + + The override exists so a lane can say "I know I am behind an + 80-minute run". ``C64_DEVICE_LOCK_TIMEOUT=30m`` quietly meaning 1800 + would fail that lane half an hour later with a message describing a + budget nobody asked for — the project's standing rule is that a + guard which silently passes when it cannot read its input is worse + than no guard. + """ + for bad in ("30m", "2 min", "abc", "1,800", "'900'", "1800s"): + try: + value = dlh.lock_timeout_s({ENV: bad}) + except dlh.LockTimeoutConfigError as exc: + assert ENV in str(exc) and bad in str(exc), f"unhelpful message: {exc}" + else: + raise AssertionError(f"{bad!r} accepted, gave {value}") + + +def test_non_positive_and_non_finite_are_fatal() -> None: + for bad in ("0", "-1", "-0.5", "nan", "inf", "-inf"): + try: + value = dlh.lock_timeout_s({ENV: bad}) + except dlh.LockTimeoutConfigError: + pass + else: + raise AssertionError(f"{bad!r} accepted, gave {value}") + + +# -------------------------------------------------------------------------- +# 2. The budget reaching DeviceLock.acquire +# -------------------------------------------------------------------------- + + +def test_default_reaches_acquire() -> None: + lock = FakeLock() + dlh.acquire_device_lock(lock, env={}, stream=io.StringIO(), progress_interval=0) + assert lock.calls == [{"timeout": DEFAULT, "progress_window": dlh.PROGRESS_WINDOW_S}] + + +def test_env_override_reaches_acquire() -> None: + lock = FakeLock() + dlh.acquire_device_lock( + lock, env={ENV: "5400"}, stream=io.StringIO(), progress_interval=0 + ) + assert lock.calls[0]["timeout"] == 5400.0 + + +def test_explicit_timeout_beats_the_environment() -> None: + """A caller that names a budget owns it; the env does not override it.""" + lock = FakeLock() + dlh.acquire_device_lock( + lock, timeout=12.0, env={ENV: "5400"}, stream=io.StringIO(), + progress_interval=0, + ) + assert lock.calls[0]["timeout"] == 12.0 + + +def test_progress_window_is_never_disabled() -> None: + """Passing ``progress_window=None`` would be a real regression. + + It restores the legacy hard timeout, which is what makes an + indefinite wait behind one healthy long holder possible. Measured on + the real harness with no hardware: ``acquire(timeout=2.0)`` behind an + 8 s hold returned True after 8.0 s with the default window, and False + after 2.0 s with ``progress_window=None``. + """ + lock = FakeLock() + dlh.acquire_device_lock(lock, env={}, stream=io.StringIO(), progress_interval=0) + window = lock.calls[0]["progress_window"] + assert window is not None and window > 0, f"progress_window defeated: {window!r}" + + +def test_malformed_env_stops_the_acquire() -> None: + lock = FakeLock() + try: + dlh.acquire_device_lock(lock, env={ENV: "30m"}, stream=io.StringIO()) + except dlh.LockTimeoutConfigError: + pass + else: + raise AssertionError("acquire proceeded on a malformed budget") + assert lock.calls == [], "the device was touched despite a bad budget" + + +# -------------------------------------------------------------------------- +# 3. Diagnostics and progress output +# -------------------------------------------------------------------------- + + +def test_timeout_propagates_with_the_harness_diagnostics() -> None: + """The rigs' ``except DeviceLockTimeout`` arms must still fire, unchanged. + + The harness message is the part that tells "queued behind a healthy + holder" from "wedged" from "device unreachable"; wrapping it in a + helper-specific exception would cost every rig that signal. + """ + exc = _timeout_exc() + lock = FakeLock(raises=exc) + buf = io.StringIO() + try: + dlh.acquire_device_lock(lock, env={}, stream=buf, progress_interval=0) + except DeviceLockTimeout as caught: + assert caught is exc + assert "4242" in str(caught), "holder PID lost" + else: + raise AssertionError("timeout swallowed") + note = buf.getvalue() + assert "gave up" in note and ENV in note, f"no budget note: {note!r}" + + +def test_progress_and_notices_never_touch_stdout() -> None: + """Rig stdout is parsed; progress output must not appear in it. + + Covers the slow path too — a wait long enough to tick — because that + is the only path that emits the periodic line at all. + """ + import contextlib + import time + + class SlowLock(FakeLock): + def acquire_or_raise(self, timeout=None, *, progress_window=None): + super().acquire_or_raise(timeout=timeout, progress_window=progress_window) + time.sleep(0.25) + + lock = SlowLock() + err = io.StringIO() + out = io.StringIO() + with contextlib.redirect_stdout(out): + dlh.acquire_device_lock(lock, env={ENV: ""}, stream=err, progress_interval=0.05) + assert out.getvalue() == "", f"progress leaked to stdout: {out.getvalue()!r}" + text = err.getvalue() + assert "still waiting" in text, f"no progress line emitted: {text!r}" + assert ENV in text, "the progress line does not say how to wait longer" + + +# -------------------------------------------------------------------------- +# 4. No rig keeps its own budget (the half that stops this decaying) +# -------------------------------------------------------------------------- + +#: Every module that may take the device lock. Directories, not a list of +#: filenames, so a rig added tomorrow is covered the day it lands. +RIG_GLOBS = (("tools/uci", "*.py"), ("tests", "rig_*.py")) + +HELPER_NAME = "_device_lock_helper.py" + + +def _rig_sources(): + for rel, pattern in RIG_GLOBS: + for path in sorted((REPO / rel).glob(pattern)): + yield path, ast.parse(path.read_text()) + + +def _calls_to(tree, names): + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + if node.func.attr in names: + yield node + + +def _constructs_device_lock(tree): + return any( + isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) + and n.func.id == "DeviceLock" + for n in ast.walk(tree) + ) + + +def _imported_names(tree): + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + names.update(a.asname or a.name for a in node.names) + return names + + +def test_no_rig_calls_acquire_itself() -> None: + """Only the helper may drive ``DeviceLock.acquire*``. + + A direct call is how a rig gets its own budget back, and a literal + timeout in a rig is invisible from the outside — nothing in a passing + hardware run reveals that this one script still cannot be told to + wait. + """ + offenders = [] + for path, tree in _rig_sources(): + if path.name == HELPER_NAME: + continue + for call in _calls_to(tree, {"acquire", "acquire_or_raise"}): + offenders.append(f"{path.relative_to(REPO)}:{call.lineno}") + assert not offenders, ( + "these call DeviceLock.acquire directly instead of going through " + f"acquire_device_lock(): {offenders}" + ) + + +def test_every_locking_rig_uses_the_helper() -> None: + offenders = [] + for path, tree in _rig_sources(): + if path.name == HELPER_NAME: + continue + if _constructs_device_lock(tree) and "acquire_device_lock" not in _imported_names(tree): + offenders.append(str(path.relative_to(REPO))) + assert not offenders, ( + "these construct a DeviceLock without importing acquire_device_lock: " + f"{offenders}" + ) + + +def test_the_helper_passes_progress_window_through() -> None: + """Structural counterpart to ``test_progress_window_is_never_disabled``. + + The behavioural test uses a fake lock, so it would still pass if the + helper's single real call site were edited to pass ``None`` for a + different argument name. This reads the source. + """ + tree = ast.parse(HELPER_PATH.read_text()) + calls = [c for c in _calls_to(tree, {"acquire_or_raise"})] + assert len(calls) == 1, f"expected one acquire_or_raise call, found {len(calls)}" + kwargs = {k.arg: k.value for k in calls[0].keywords} + assert set(kwargs) == {"timeout", "progress_window"}, sorted(kwargs) + assert not isinstance(kwargs["progress_window"], ast.Constant) or ( + kwargs["progress_window"].value is not None + ), "progress_window=None restores the legacy hard timeout" + + +#: Wider than RIG_GLOBS on purpose: a literal budget can hide anywhere a +#: device is driven, and the one that did was a UnifiedManager keyword in +#: a suite that is neither a rig nor a DeviceLock caller. +LOCK_TIMEOUT_GLOBS = (("tools", "*.py"), ("tools/uci", "*.py"), + ("tools/package", "*.py"), ("tests", "*.py")) + + +def test_no_module_passes_a_literal_lock_timeout() -> None: + """``lock_timeout=`` is a budget the environment cannot raise. + + ``UnifiedManager(backend="u64", lock_timeout=...)`` takes the same + DeviceLock by the same ``acquire_or_raise``, so a literal there is + the identical defect one layer up — and invisible to the two checks + above, which look for ``DeviceLock`` calls in rig directories. + """ + offenders = [] + for rel, pattern in LOCK_TIMEOUT_GLOBS: + for path in sorted((REPO / rel).glob(pattern)): + if path.name in (HELPER_NAME, Path(__file__).name): + continue + for node in ast.walk(ast.parse(path.read_text())): + if not isinstance(node, ast.Call): + continue + for kw in node.keywords: + if kw.arg == "lock_timeout" and isinstance(kw.value, ast.Constant): + offenders.append( + f"{path.relative_to(REPO)}:{kw.value.lineno}" + ) + assert not offenders, ( + "these hardcode a device-lock budget the environment cannot " + f"raise; pass lock_timeout_s() instead: {offenders}" + ) + + +def test_progress_line_carries_the_lockfile_age() -> None: + """Age is what separates "healthy long run" from "wedged". + + Without it that verdict arrives only in the final DeviceLockTimeout, + up to a whole budget later — the time-to-diagnosis cost of a larger + default. One stat buys it back at the first tick. + """ + import json + import os + import time as _time + from c64_test_harness.backends.device_lock import device_lock_path + + class SlowLock(FakeLock): + def acquire_or_raise(self, timeout=None, *, progress_window=None): + super().acquire_or_raise(timeout=timeout, progress_window=progress_window) + _time.sleep(0.15) + + lock = SlowLock() + path = device_lock_path(lock.device_host, lock_dir=lock._lock_dir) + path.write_text(json.dumps({"pid": os.getpid(), "ts": _time.time(), + "device_host": lock.device_host})) + + buf = io.StringIO() + dlh.acquire_device_lock(lock, env={}, stream=buf, progress_interval=0.05) + assert "lockfile age=" in buf.getvalue(), buf.getvalue() + assert "STALE" not in buf.getvalue(), "a fresh lockfile read as wedged" + + old = _time.time() - (dlh.PROGRESS_WINDOW_S * 5) + os.utime(path, (old, old)) + buf = io.StringIO() + dlh.acquire_device_lock(lock, env={}, stream=buf, progress_interval=0.05) + assert "STALE" in buf.getvalue(), ( + "a lockfile older than the progress window must read as wedged: " + f"{buf.getvalue()!r}" + ) + + +def test_the_handoff_boundary_is_four_changes_not_three() -> None: + """Pin the number four files of prose depend on. + + It had no coverage in either direction, in this repo or upstream, + which is exactly how a wrong value (three, read off the harness's own + docstring) sat in four files under a green suite. + + Real ``DeviceLock``, no hardware and no flock race: ``_holder_progress`` + is overridden to return a scripted sequence of live holder identities + and ``_try_acquire_once`` to never grant, which is precisely what a + waiter behind a handoff chain observes. Three changes must still + extend past the caller's timeout; four must not. + """ + import threading + import time as _time + from c64_test_harness.backends.device_lock import DeviceLock + + for attr in ("_holder_progress", "_try_acquire_once"): + assert hasattr(DeviceLock, attr), ( + f"c64-test-harness no longer has DeviceLock.{attr}; this test " + "and the handoff prose in _device_lock_helper.py, CLAUDE.md and " + "tools/uci/README.md all rest on that shape — re-measure, do " + "not delete" + ) + + def stops_extending(changes: int, timeout: float = 0.15, + observe: float = 0.9) -> bool: + # heartbeat_interval=None: granting the lock at the end of the + # observation window would otherwise start a heartbeat thread + # that sleeps a full interval before noticing the lockfile is + # not there. No holder here is real, so none needs one. + lock = DeviceLock("fake.invalid", lock_dir=Path(tempfile.mkdtemp()), + heartbeat_interval=None) + identities = [1000 + i for i in range(changes + 1)] + seen = {"i": 0} + + def scripted(_window): + i = min(seen["i"], len(identities) - 1) + seen["i"] += 1 + return True, identities[i] + + # Never grant while we are observing; grant afterwards so the + # still-extending arm's thread ends instead of polling at 10 Hz + # for the life of the process. A test that leaks a daemon thread + # per call is cheap here and expensive in a suite that grows. + give_up_at = _time.monotonic() + observe + 0.2 + lock._holder_progress = scripted + lock._try_acquire_once = lambda: _time.monotonic() >= give_up_at + done = threading.Event() + thread = threading.Thread( + target=lambda: (lock.acquire(timeout=timeout), done.set()), + daemon=True, + ) + thread.start() + stopped = done.wait(observe) + thread.join(timeout=2.0) + assert not thread.is_alive(), ( + "the probe thread outlived its acquire; the overrides no longer " + "control DeviceLock.acquire and this result means nothing" + ) + return stopped + + # A rename is caught by the hasattr checks above; a *signature* change + # is not -- the override would simply never be called the way acquire + # calls it, and the arm would fail through the boundary assertions with + # a message blaming the boundary. Call it once here so that failure + # names the real cause. + try: + stops_extending(0, observe=0.2) + except TypeError as exc: + raise AssertionError( + "DeviceLock._holder_progress / _try_acquire_once changed shape " + f"({exc}); this probe drives them directly, so re-measure the " + "handoff boundary rather than trusting either assertion below" + ) from exc + + assert not stops_extending(3), ( + "three identity changes stopped extending; the boundary moved to " + "three and every 'fourth' in the docs is now wrong" + ) + assert stops_extending(4), ( + "four identity changes still extended; the boundary moved past " + "four and the docs understate how long a starved waiter survives" + ) + + +# -------------------------------------------------------------------------- + + +def _main(tests=None) -> int: + """Run the checks without pytest, one line per case. + + Catches more than ``AssertionError`` deliberately: the code under + test raises ``LockTimeoutConfigError`` by design, and a regression + that let one escape would otherwise abort the run partway and leave + later cases silently unrun. An unrun test is not a passing one. + """ + if tests is None: + tests = { + n: f + for n, f in globals().items() + if n.startswith("test_") and callable(f) + } + failures = 0 + for name, fn in sorted(tests.items()): + try: + fn() + except AssertionError as exc: + failures += 1 + print(f"FAIL {name}: {exc}") + except Exception as exc: # noqa: BLE001 — a raise is a result + failures += 1 + summary = str(exc).strip().splitlines() + print(f"ERROR {name}: {type(exc).__name__}: " + f"{summary[0] if summary else ''}") + else: + print(f"ok {name}") + print("FAILED" if failures else "PASS") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/tools/test_ecdsa_p384_kat.py b/tools/test_ecdsa_p384_kat.py index ce8b3c25..aa179d34 100644 --- a/tools/test_ecdsa_p384_kat.py +++ b/tools/test_ecdsa_p384_kat.py @@ -96,6 +96,13 @@ from _skip_policy import cannot_run # noqa: E402 +# The device-lock budget is one number for the whole repo, and it lives +# with the rigs that take the lock most often. This suite is the only +# module outside tools/uci/ and tests/ that drives a real device. +if os.path.join(_HERE, "uci") not in sys.path: + sys.path.insert(0, os.path.join(_HERE, "uci")) +from _device_lock_helper import lock_timeout_s # noqa: E402 + PROJECT_ROOT = Path(__file__).resolve().parents[1] PRG_PATH = PROJECT_ROOT / "build" / "c64-https.prg" LABELS_PATH = PROJECT_ROOT / "build" / "labels.txt" @@ -703,7 +710,10 @@ def _run_backend(*, backend: str, vectors: list[dict], ) mgr = UnifiedManager(backend="vice", vice_config=config) else: - mgr = UnifiedManager(backend="u64", lock_timeout=120.0) + # C64_DEVICE_LOCK_TIMEOUT, else 30 min. UnifiedManager takes the + # lock itself, so the budget is passed rather than the helper + # called; the number is the same one every rig uses. + mgr = UnifiedManager(backend="u64", lock_timeout=lock_timeout_s()) passed = failed = 0 details: list[dict] = [] diff --git a/tools/uci/README.md b/tools/uci/README.md index 69c9278a..8a89861b 100644 --- a/tools/uci/README.md +++ b/tools/uci/README.md @@ -71,13 +71,57 @@ and never `pkill` a run you did not start, because the device is shared across the `c64-*` repos and the lock queue is the only thing keeping concurrent sessions from clobbering each other. +### Queueing for the device — `C64_DEVICE_LOCK_TIMEOUT` + +Every script here, plus `tests/rig_ip65_rrnet_hw.py` and +`tools/test_ecdsa_p384_kat.py`'s U64 arm, takes the lock through +`acquire_device_lock()` in `_device_lock_helper.py` (or passes +`lock_timeout_s()` where the harness takes the lock for you), which owns +the one acquire budget: **`C64_DEVICE_LOCK_TIMEOUT` seconds, default +1800 (30 min)**. Nothing keeps its own any more, and +`tools/test_device_lock_timeout.py` fails if anything grows one back — +including a literal `lock_timeout=` passed to `UnifiedManager`, which is +how the thirteenth budget hid from the first version of that guard. + +```sh +C64_DEVICE_LOCK_TIMEOUT=7200 python3 tools/uci/rig_https_live.py # queue for 2 h +``` + +That budget is **not** the longest run you may queue behind. The +harness's `acquire` is queue aware: while the holder's PID is alive and +its lockfile mtime is fresh, the deadline is re-armed on every poll, so a +waiter sits behind one long healthy holder indefinitely. The budget +bounds the waits it refuses to extend — a dead or wedged holder, and a +**handoff chain**: after the **fourth** change of holder identity the +harness stops extending for the rest of that acquire, and then the +budget is everything. Four, not three: `_MAX_HOLDER_HANDOFFS` is 3 but +extension survives `handoffs <= 3`, so the harness's docstring is one +out and its WARNING text is right. Several lanes cycling one U64E would +hit that within seconds, which is why 120 s could fail against a device +that was merely busy — the mechanism is lab-measured; no timeout from a +real run here has been captured. + +A malformed value (`30m`, `2 min`, `0`) is a hard error before the device +is touched, not a silent fall back to the default: the override exists so +a lane can say "I know I am behind an 80-minute run", and a typo that +quietly reinstated 1800 s would fail it later against a budget nobody +asked for. `C64_DEVICE_LOCK_TIMEOUT=` (empty) falls back, with a notice. + +While waiting, a line goes to **stderr** every 30 s naming the elapsed +wait, the budget, the holder PID, the **lockfile age** (flagged `STALE, +holder may be wedged` past the 60 s progress window) and the queue depth +— a silent half-hour block is indistinguishable from a hang, and the age +is the field that tells a healthy long run from a wedged one without +waiting out the whole budget for the final diagnostic. Nothing is added +to stdout, which stays parseable. + Other environment variables read by scripts here: `TURBO_MHZ`, `TURBO_SETTLE`, `HTTPS_PORT`, `BOOT_TIMEOUT`, `ACCEPT_TIMEOUT`, `SENTINEL_POLL_TIMEOUT`, `C64_INIT_WAIT`, `DEBUG_CAPTURE`, `UCI_DEBUG_DIR`, `KEEP_DEBUG_ON_PASS`, `EXTERNAL_LISTENER`, `EXTERNAL_HOST`, `EXTERNAL_PORT`, `FINISHED_MODE`, `BACKEND`, `ECDSA_MHZ_LIST`, `ECDSA_REPEATS`, `ECDSA_POLL_S`, `ECDSA_DEBUG_CAPTURE`, -`ECDSA_DEBUG_DIR`. See each script's docstring for which ones it honors +`ECDSA_DEBUG_DIR`, `C64_DEVICE_LOCK_TIMEOUT` (above). See each script's docstring for which ones it honors and what the defaults are; `CLAUDE.md`'s "UCI test scripts" section has the prose. diff --git a/tools/uci/_device_lock_helper.py b/tools/uci/_device_lock_helper.py index ebd964e0..9122eb3f 100644 --- a/tools/uci/_device_lock_helper.py +++ b/tools/uci/_device_lock_helper.py @@ -1,40 +1,314 @@ -"""tools/uci/_device_lock_helper.py - U64E shared-queue conventions for CI. - -CI-bot use only. Existing interactive ``tools/uci/*.py`` scripts continue -using ``c64_test_harness.DeviceLock`` directly without going through this -helper; the helper exists so an automated runner can yield cleanly -instead of camping on the U64E queue indefinitely. - -See user memory ``u64e_shared_queue`` for the canonical motivation: the -U64E (default 10.43.23.81) is shared across the c64-* projects, so a -poorly-timed CI run that waits forever blocks both itself and any -interactive user. The 30-min hard timeout plus the (future) -queue-depth gate lets the CI bot drop out cleanly and retry on the next -cron tick. - -Public surface: - -* :class:`QueueSaturatedError` - raised when the lock could not be - acquired inside the configured budget. -* :func:`acquire_with_queue_budget` - thin wrapper around - ``DeviceLock.acquire`` with bounded wait and queue-position logging. - -Known limitation - ``c64_test_harness.DeviceLock`` does not currently -expose the number of agents waiting behind us in the kernel-level -``flock()`` queue, so today the helper can only gate on -``lock_timeout_sec``. The ``max_queue_depth`` argument is accepted (and -its semantics documented) so the CI bot's call sites do not need to -change once the harness grows a queue-introspection API. +"""tools/uci/_device_lock_helper.py - shared U64E device-lock conventions. + +Every rig in this repo that drives a real device takes +``c64_test_harness.DeviceLock`` first. This module owns the two things +they all need and used to spell out one at a time: **how long to wait** +and **what to say while waiting**. + +The wait budget +--------------- +``DeviceLock.acquire(timeout=..., progress_window=60.0)`` is *queue +aware*: while the current holder's PID is alive and the lockfile mtime is +fresh, the deadline is re-armed on every poll, so a waiter sits behind one +long healthy holder indefinitely and ``timeout`` is never consulted. +Measured, no hardware: ``acquire(timeout=2.0)`` behind an 8 s hold +returned True after 8.0 s. + +``timeout`` is therefore **not** the length of the longest run you may +queue behind. It is the bound on waits the harness refuses to extend: + +* a holder whose PID is dead, or whose mtime has gone stale past + ``progress_window`` (a wedged holder, or one whose heartbeat gave up); +* a **handoff chain** - after the **fourth** change of holder identity + the harness stops extending for the rest of that acquire, permanently, + and the caller's ``timeout`` runs out normally. + + Four, not three. ``_MAX_HOLDER_HANDOFFS`` is 3 and the harness's own + docstring says "after ``_MAX_HOLDER_HANDOFFS`` identity changes", but + the code extends while ``handoffs <= _MAX_HOLDER_HANDOFFS`` and + increments *on* a change, so three changes still extend for ever. The + harness's own WARNING agrees with its code ("the holder changed 4 + times"), not with its prose. Measured here by driving a scripted + identity sequence, no hardware: 0/1/2/3 changes then a stable holder + still waited past the observation window; 4 and 5 returned False at + ``timeout``. A doc issue for the harness is drafted; do not "correct" + this to three from reading their docstring. + +The second case is the one that would bite this fleet: several lanes +cycling the same U64E overtake a queued rig within seconds, after which +its hardcoded 120 s was the whole budget - and there was no way to raise +it without editing the rig. No such timeout has been captured from a +real run here; the mechanism above is lab-measured, the field frequency +is not. Hence :data:`LOCK_TIMEOUT_ENV`. + +Default: :data:`DEFAULT_LOCK_TIMEOUT_S` = 1800 s (30 min). Justified +against both failure modes. Too short and a rig fails while the device +is merely busy - 120 s could not survive one overtake by a lane doing a +45 s comb boot, and 30 min outlasts every UCI rig's own hold (the long +one, the RR-Net e2e at 33-80 min, runs alone). Too long and a rig +blocking on a genuinely wedged device becomes its own outage: at 30 min a +lane fails the same session with diagnostics naming the holder, rather +than being indistinguishable from a hang - which is the same reason +CLAUDE.md's "Design note - bounded timeouts" refuses unbounded waits on +the C64 side. It also matches the budget the CI path in this file +already used, so the repo has one number instead of two. + +Residual limitation, stated rather than hidden: a waiter that has been +overtaken a fourth time and is *then* queued behind a fresh 80-minute +holder still fails at the budget, because the harness never resumes +extending within one acquire. ``C64_DEVICE_LOCK_TIMEOUT=7200`` is the answer, and the +timeout diagnostic says which case you are in. + +Progress output +--------------- +A silent 30-minute block is indistinguishable from a hang - that +ambiguity is what prompted this module. :func:`acquire_device_lock` +therefore prints a line every :data:`PROGRESS_INTERVAL_S` seconds naming +the elapsed wait, the budget, the holder PID and the queue depth. It +goes to **stderr**, never stdout: rig stdout is parsed (by supervisors, +and by ``tools/package/verify_release.py``-style greps), and a +progress line on it would be a new record in someone's log format. + +Public surface +-------------- +* :data:`LOCK_TIMEOUT_ENV` / :func:`lock_timeout_s` - the budget and how + it is read. +* :class:`LockTimeoutConfigError` - raised for a malformed override. +* :func:`acquire_device_lock` - acquire with that budget, progress + output, and the harness's ``DeviceLockTimeout`` diagnostics preserved. +* :class:`QueueSaturatedError` / :func:`acquire_with_queue_budget` - the + CI-bot path: same budget, plus a queue-depth gate and a yield-cleanly + contract for a cron runner. """ + from __future__ import annotations +import math import os +import sys +import threading import time from contextlib import contextmanager -from typing import Iterator +from typing import IO, Iterator, Mapping + +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, device_lock_path, +) + + +#: Environment variable that overrides the device-lock acquire budget, +#: in seconds. Unset (or empty) means :data:`DEFAULT_LOCK_TIMEOUT_S`; +#: anything that is not a positive finite number is a hard error, never +#: a silent fallback - see :func:`lock_timeout_s`. +LOCK_TIMEOUT_ENV = "C64_DEVICE_LOCK_TIMEOUT" + +#: Default acquire budget in seconds. See the module docstring for the +#: justification against both failure modes; do not change it here +#: without changing that paragraph. +DEFAULT_LOCK_TIMEOUT_S = 1800.0 -from c64_test_harness.backends.device_lock import DeviceLock +#: How often :func:`acquire_device_lock` says it is still waiting. +PROGRESS_INTERVAL_S = 30.0 + +#: How fresh the holder's lockfile mtime must be for the harness to keep +#: extending the deadline. Restated here (it is the harness's own +#: default) so that passing it explicitly is a decision this repo owns: +#: ``progress_window=None`` would restore the legacy hard timeout and +#: destroy the indefinite wait behind a healthy long holder, which is the +#: one part of this that already worked. +PROGRESS_WINDOW_S = 60.0 + + +class LockTimeoutConfigError(RuntimeError): + """Raised when ``C64_DEVICE_LOCK_TIMEOUT`` cannot be read as a budget. + + Deliberately fatal rather than a fall back to the default. The + override exists precisely so a lane can say "I know I am behind an + 80-minute run"; a typo (``30m``, ``2 min``, a stray quote) that + silently reinstated 1800 s would fail two hours later with a message + describing a budget nobody asked for. This repo's standing rule -- a + guard that silently passes when it cannot read its input is worse + than no guard -- applies to the budget as much as to the REU + preflight it is modelled on (``tools/uci/_reu_preflight.py``). + """ + + +def lock_timeout_s( + env: Mapping[str, str] | None = None, + *, + default: float = DEFAULT_LOCK_TIMEOUT_S, + stream: IO[str] | None = None, +) -> float: + """The device-lock acquire budget in seconds. + + Read at call time, not import time, so a test (and a long-lived + supervisor process) can flip it. + + * unset -> *default*, silently: that is the ordinary case. + * set but empty -> *default*, with a note on *stream*. ``VAR=`` is + the shell's own spelling of "not set" and refusing it would break + ``env -u``-style call sites, but it is worth one line, because a + variable that expanded to nothing usually meant to expand to + something. + * anything else -> parsed as a float; a value that is not finite and + strictly positive raises :class:`LockTimeoutConfigError`. + + :raises LockTimeoutConfigError: on any unparseable or non-positive + value. + """ + source = os.environ if env is None else env + raw = source.get(LOCK_TIMEOUT_ENV) + if raw is None: + return float(default) + text = raw.strip() + if not text: + print( + f"[device-lock] {LOCK_TIMEOUT_ENV} is set but empty; " + f"using the default {default:g}s", + file=stream if stream is not None else sys.stderr, + flush=True, + ) + return float(default) + try: + value = float(text) + except ValueError: + raise LockTimeoutConfigError( + f"{LOCK_TIMEOUT_ENV}={raw!r} is not a number of seconds. " + f"Give a positive number, e.g. {LOCK_TIMEOUT_ENV}=7200 for a " + f"two-hour queue; unset it for the default {default:g}s." + ) from None + if not math.isfinite(value) or value <= 0: + raise LockTimeoutConfigError( + f"{LOCK_TIMEOUT_ENV}={raw!r} is not a positive, finite number " + f"of seconds. A zero or negative budget would turn every " + f"queued run into an instant failure, and an infinite one " + f"would make a wedged device indistinguishable from a busy " + f"one; unset it for the default {default:g}s." + ) + return value + + +def _holder_note(lock: DeviceLock) -> str: + """Best-effort ``holder pid=..., queue depth=...`` tag for a progress line. + + Filesystem-only and never raises: this decorates a message, it is not + a check. ``foreign_holder`` (not ``read_info``) is the "who holds it + right now" query -- ``read_info`` names whoever held it *last*, which + after any completed run is a dead PID. + """ + bits = [] + try: + holder = DeviceLock.foreign_holder( + lock.device_host, lock_dir=getattr(lock, "_lock_dir", None) + ) + except Exception: # noqa: BLE001 + holder = None + if holder is not None: + bits.append(f"holder pid={holder.get('pid')}") + # Lockfile age is the one field that separates "healthy long run" + # from "wedged", and without it that verdict arrives only in the + # final DeviceLockTimeout -- up to a full budget later. One stat + # buys it back at the first tick. Same rule as the harness's + # DeviceLockTimeout message: older than the progress window is the + # wedged reading, and it is why a long wait is not, by itself, a + # reason to touch the device. + try: + age = time.time() - os.stat( + device_lock_path( + lock.device_host, lock_dir=getattr(lock, "_lock_dir", None) + ) + ).st_mtime + except Exception: # noqa: BLE001 + age = None + if age is not None: + stale = " STALE, holder may be wedged" if age > PROGRESS_WINDOW_S else "" + bits.append(f"lockfile age={max(0.0, age):.0f}s{stale}") + try: + depth = DeviceLock.peek_queue_depth( + lock.device_host, lock_dir=getattr(lock, "_lock_dir", None) + ) + except Exception: # noqa: BLE001 + depth = None + if depth is not None: + bits.append(f"queue depth={depth}") + return ("; " + ", ".join(bits)) if bits else "" + + +def acquire_device_lock( + lock: DeviceLock, + *, + timeout: float | None = None, + env: Mapping[str, str] | None = None, + stream: IO[str] | None = None, + progress_interval: float = PROGRESS_INTERVAL_S, +) -> float: + """Take *lock* within the shared budget, saying so while it waits. + + The one entry point every rig in this repo uses, so that the budget + is a single env-overridable number instead of a literal per script. + + Returns the wall-clock seconds spent waiting (0.0 for an uncontended + acquire). On failure raises the harness's + :class:`DeviceLockTimeout`, unchanged, so existing ``except + DeviceLockTimeout`` arms and their holder/liveness/reachability + diagnostics keep working; a final stderr line adds the elapsed wait + and the budget that was in force, which the harness cannot know. + + :param timeout: explicit budget; ``None`` reads + :data:`LOCK_TIMEOUT_ENV` via :func:`lock_timeout_s`. + :param stream: where progress goes; ``None`` means ``sys.stderr``. + Never stdout -- see the module docstring. + :raises LockTimeoutConfigError: malformed :data:`LOCK_TIMEOUT_ENV`. + :raises DeviceLockTimeout: budget exhausted. + """ + out = stream if stream is not None else sys.stderr + budget = lock_timeout_s(env, stream=out) if timeout is None else float(timeout) + + started = time.monotonic() + stop = threading.Event() + + def _tick() -> None: + while not stop.wait(progress_interval): + waited = time.monotonic() - started + print( + f"[device-lock] still waiting {waited:.0f}s of {budget:.0f}s " + f"for {lock.device_host}{_holder_note(lock)} " + f"(raise {LOCK_TIMEOUT_ENV} to wait longer)", + file=out, + flush=True, + ) + + ticker: threading.Thread | None = None + if progress_interval and progress_interval > 0: + ticker = threading.Thread( + target=_tick, name="device-lock-progress", daemon=True + ) + ticker.start() + try: + lock.acquire_or_raise(timeout=budget, progress_window=PROGRESS_WINDOW_S) + except DeviceLockTimeout: + elapsed = time.monotonic() - started + print( + f"[device-lock] gave up on {lock.device_host} after {elapsed:.0f}s " + f"(budget {budget:.0f}s from " + f"{LOCK_TIMEOUT_ENV if timeout is None else 'the caller'})", + file=out, + flush=True, + ) + raise + finally: + stop.set() + if ticker is not None: + ticker.join(timeout=1.0) + elapsed = time.monotonic() - started + if elapsed >= progress_interval > 0: + print( + f"[device-lock] acquired {lock.device_host} after {elapsed:.0f}s", + file=out, + flush=True, + ) + return elapsed class QueueSaturatedError(RuntimeError): @@ -50,7 +324,7 @@ def __init__( *, device_host: str, elapsed_sec: float, - lock_timeout_sec: int, + lock_timeout_sec: float, max_queue_depth: int, reason: str, ) -> None: @@ -62,7 +336,7 @@ def __init__( super().__init__( f"QueueSaturatedError: device={device_host!r} reason={reason!r} " f"elapsed={elapsed_sec:.1f}s " - f"(lock_timeout_sec={lock_timeout_sec}, " + f"(lock_timeout_sec={lock_timeout_sec:g}, " f"max_queue_depth={max_queue_depth})" ) @@ -70,16 +344,17 @@ def __init__( def _peek_queue_depth(lock: DeviceLock) -> int | None: """Best-effort inspection of the queue depth behind ``lock``. - Returns the observed waiter count, or ``None`` if the underlying - ``c64_test_harness`` build does not yet surface that information. + Returns the observed waiter count, or ``None`` if it is unobservable + (an unreadable sidecar directory, or a harness too old to have the + accessor). - TODO(c64-test-harness): ``DeviceLock`` does not currently expose the - number of agents waiting on its flock. ``read_info()`` only returns - metadata for the **current holder**, not the queue tail. Until the - upstream harness grows a ``queue_depth()`` (or equivalent) accessor, - this function returns ``None`` and the helper gates only on - ``lock_timeout_sec``. When the API arrives, route it through here so - no call site has to change. + The harness grew this after the helper was written: waiters register + an intent file in ``.queue/`` and ``DeviceLock.queue_depth`` + counts the live ones, so the probe below now returns a real number + rather than ``None``. It still probes by attribute name so an older + ``c64-test-harness`` (editable install from a sibling working tree, + so its version is not ours to pin) degrades to ``None`` instead of + raising. """ for attr in ("queue_depth", "waiters", "wait_count"): probe = getattr(lock, attr, None) @@ -99,7 +374,7 @@ def acquire_with_queue_budget( device_host: str | None = None, *, max_queue_depth: int = 3, - lock_timeout_sec: int = 1800, + lock_timeout_sec: float | None = None, ) -> Iterator[DeviceLock]: """Acquire ``DeviceLock(device_host)`` with bounded wait + queue logging. @@ -112,18 +387,26 @@ def acquire_with_queue_budget( distinguish "queued" from "wedged" from "in flight" without parsing the harness's internal state. - 2. **Hard timeout** - if the lock is not granted inside - ``lock_timeout_sec`` seconds (default 30 min = 1800 s), - :class:`QueueSaturatedError` is raised. The CI bot decides the - retry policy (typically: log and exit, retry on the next cron - tick). + 2. **Bounded wait** - the acquire goes through + :func:`acquire_device_lock`, so it uses the one shared budget + (``lock_timeout_sec``, else ``C64_DEVICE_LOCK_TIMEOUT``, else + :data:`DEFAULT_LOCK_TIMEOUT_S`) and prints the same progress + lines. If the budget is exhausted, :class:`QueueSaturatedError` + is raised carrying the harness's own holder/liveness/reachability + diagnostics in ``reason``. The CI bot decides the retry policy + (typically: log and exit, retry on the next cron tick). + + Note what the budget is not: a live, progressing holder extends + the harness's deadline indefinitely, so this does not cap the + time spent behind one healthy long run. See the module + docstring. 3. **Queue-depth threshold** - if the observed queue depth at acquire time exceeds ``max_queue_depth``, :class:`QueueSaturatedError` is raised immediately rather than - getting in line. Currently a no-op until the upstream harness - grows queue introspection; see the TODO in - :func:`_peek_queue_depth`. + getting in line. Live now that the harness exposes + ``queue_depth``; still a no-op against a harness that does not, + where the depth reads ``None``. The default device host is ``"10.43.23.81"`` (the U64E in this lab), overridable via the ``U64_HOST`` environment variable. @@ -132,12 +415,33 @@ def acquire_with_queue_budget( ``$U64_HOST`` then ``"10.43.23.81"``. :param max_queue_depth: bail out with :class:`QueueSaturatedError` if the queue depth exceeds this value at acquire time. - :param lock_timeout_sec: hard wall-clock limit on the acquire wait. - Defaults to 30 minutes. + :param lock_timeout_sec: budget for the acquire wait. ``None`` + (the default) reads ``C64_DEVICE_LOCK_TIMEOUT``, falling back to + :data:`DEFAULT_LOCK_TIMEOUT_S`. + + Two things a cron runner adopting this must know, since nothing in + this repo calls it and so nothing here would notice: + + * **Keep the budget under your tick.** The default is 1800 s. A bot + that ticks more often than its budget stacks a new waiter on every + tick until ``max_queue_depth`` self-trips, which reads as queue + saturation caused by the bot itself. Pass ``lock_timeout_sec`` + explicitly rather than inheriting the interactive default. + * :class:`LockTimeoutConfigError` escapes this contextmanager + un-wrapped, deliberately: a malformed ``C64_DEVICE_LOCK_TIMEOUT`` + is a misconfiguration to fix, not a busy queue to retry on the + next tick, and wrapping it as :class:`QueueSaturatedError` would + make a typo look like contention for ever. :raises QueueSaturatedError: when the acquire fails inside the configured budget(s). """ host = device_host or os.environ.get("U64_HOST", "10.43.23.81") + # Resolved before anything else: a malformed C64_DEVICE_LOCK_TIMEOUT + # must fail before the run starts, not after the queue gate has + # already reported a budget it never used. + budget = ( + lock_timeout_s() if lock_timeout_sec is None else float(lock_timeout_sec) + ) lock = DeviceLock(host) depth = _peek_queue_depth(lock) depth_label = "unknown" if depth is None else str(depth) @@ -147,21 +451,27 @@ def acquire_with_queue_budget( raise QueueSaturatedError( device_host=host, elapsed_sec=0.0, - lock_timeout_sec=lock_timeout_sec, + lock_timeout_sec=budget, max_queue_depth=max_queue_depth, reason=f"queue depth {depth} > max_queue_depth {max_queue_depth}", ) start = time.monotonic() - if not lock.acquire(timeout=float(lock_timeout_sec)): - elapsed = time.monotonic() - start + try: + acquire_device_lock(lock, timeout=budget) + except DeviceLockTimeout as exc: + # The harness message names the holder, its liveness, the + # lockfile age and whether the device answers REST -- everything + # needed to tell "queued" from "wedged". Carrying it in `reason` + # keeps that in the CI log; discarding it for a bare + # "timeout exhausted" is what this rewrite is fixing. raise QueueSaturatedError( device_host=host, - elapsed_sec=elapsed, - lock_timeout_sec=lock_timeout_sec, + elapsed_sec=time.monotonic() - start, + lock_timeout_sec=budget, max_queue_depth=max_queue_depth, - reason=f"lock_timeout_sec={lock_timeout_sec} exhausted", - ) + reason=str(exc), + ) from exc try: yield lock finally: diff --git a/tools/uci/bench_ecdsa_u64e.py b/tools/uci/bench_ecdsa_u64e.py index b0e38b08..80ae1c91 100644 --- a/tools/uci/bench_ecdsa_u64e.py +++ b/tools/uci/bench_ecdsa_u64e.py @@ -49,7 +49,9 @@ import traceback from pathlib import Path -from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, +) from c64_test_harness.labels import Labels from c64_test_harness.backends.ultimate64 import Ultimate64Transport from c64_test_harness.backends.ultimate64_client import Ultimate64Client @@ -64,6 +66,9 @@ ) from c64_test_harness.uci_network import enable_uci, disable_uci from c64_test_harness.keyboard import send_text +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import build_policy_and_arbiter from _reu_preflight import ReuPreflightError, preflight_reu @@ -419,8 +424,16 @@ def main() -> int: print(f"Debug dir: {run_dir}") lock = DeviceLock(HOST) - if not lock.acquire(timeout=60.0): - print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + try: + # Budget: C64_DEVICE_LOCK_TIMEOUT, else 30 min. The old + # bare-bool acquire() discarded every diagnostic the + # harness had gathered and reported only False. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 3 print(f"Acquired DeviceLock({HOST})") diff --git a/tools/uci/boot_check.py b/tools/uci/boot_check.py index 865832b9..ed4c9d10 100644 --- a/tools/uci/boot_check.py +++ b/tools/uci/boot_check.py @@ -50,9 +50,14 @@ import time from pathlib import Path -from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, +) from c64_test_harness.backends.ultimate64_client import Ultimate64Client from c64_test_harness.uci_network import disable_uci, enable_uci +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) HOST = os.environ.get("U64_HOST", "192.168.1.81") BACKEND = os.environ.get("BACKEND", "uci").strip().lower() @@ -269,8 +274,16 @@ def main() -> int: ) lock = DeviceLock(HOST) - if not lock.acquire(timeout=60.0): - print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + try: + # Budget: C64_DEVICE_LOCK_TIMEOUT, else 30 min. The old + # bare-bool acquire() discarded every diagnostic the + # harness had gathered and reported only False. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 3 print(f"\nAcquired DeviceLock({HOST})") diff --git a/tools/uci/phase2_check.py b/tools/uci/phase2_check.py index c21f49e4..c38540a2 100644 --- a/tools/uci/phase2_check.py +++ b/tools/uci/phase2_check.py @@ -32,10 +32,15 @@ from pathlib import Path from c64_test_harness import Labels -from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, +) from c64_test_harness.backends.ultimate64 import Ultimate64Transport from c64_test_harness.backends.ultimate64_client import Ultimate64Client from c64_test_harness.uci_network import enable_uci, disable_uci +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) HOST = os.environ.get("U64_HOST", "192.168.1.81") REPO_ROOT = Path(__file__).resolve().parents[2] @@ -116,8 +121,16 @@ def main() -> int: print(f"Loaded {len(prg)} bytes from {PRG_PATH}") lock = DeviceLock(HOST) - if not lock.acquire(timeout=60.0): - print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + try: + # Budget: C64_DEVICE_LOCK_TIMEOUT, else 30 min. The old + # bare-bool acquire() discarded every diagnostic the + # harness had gathered and reported only False. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 3 print(f"Acquired DeviceLock({HOST})") diff --git a/tools/uci/phase3_tcp_echo.py b/tools/uci/phase3_tcp_echo.py index 964e0dbd..05c3ccf8 100644 --- a/tools/uci/phase3_tcp_echo.py +++ b/tools/uci/phase3_tcp_echo.py @@ -36,11 +36,16 @@ from pathlib import Path from c64_test_harness import Labels -from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, +) from c64_test_harness.backends.ultimate64 import Ultimate64Transport from c64_test_harness.backends.ultimate64_client import Ultimate64Client from c64_test_harness.uci_network import enable_uci, disable_uci from c64_test_harness.keyboard import send_text +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import build_policy_and_arbiter @@ -384,8 +389,16 @@ def main() -> int: prg = PRG_PATH.read_bytes() lock = DeviceLock(HOST) - if not lock.acquire(timeout=60.0): - print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + try: + # Budget: C64_DEVICE_LOCK_TIMEOUT, else 30 min. The old + # bare-bool acquire() discarded every diagnostic the + # harness had gathered and reported only False. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 3 print(f"Acquired DeviceLock({HOST})") diff --git a/tools/uci/rig_http_live.py b/tools/uci/rig_http_live.py index ca8d7649..e7ea1789 100644 --- a/tools/uci/rig_http_live.py +++ b/tools/uci/rig_http_live.py @@ -19,11 +19,16 @@ from pathlib import Path from c64_test_harness import Labels -from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, +) from c64_test_harness.backends.ultimate64 import Ultimate64Transport from c64_test_harness.backends.ultimate64_client import Ultimate64Client from c64_test_harness.uci_network import enable_uci, disable_uci from c64_test_harness.keyboard import send_text +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import build_policy_and_arbiter @@ -253,8 +258,16 @@ def main() -> int: prg = PRG_PATH.read_bytes() lock = DeviceLock(HOST) - if not lock.acquire(timeout=60.0): - print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + try: + # Budget: C64_DEVICE_LOCK_TIMEOUT, else 30 min. The old + # bare-bool acquire() discarded every diagnostic the + # harness had gathered and reported only False. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 3 print(f"Acquired DeviceLock({HOST})") diff --git a/tools/uci/rig_http_local.py b/tools/uci/rig_http_local.py index 043575a6..a2ad6b0b 100644 --- a/tools/uci/rig_http_local.py +++ b/tools/uci/rig_http_local.py @@ -27,11 +27,16 @@ from pathlib import Path from c64_test_harness import Labels -from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, +) from c64_test_harness.backends.ultimate64 import Ultimate64Transport from c64_test_harness.backends.ultimate64_client import Ultimate64Client from c64_test_harness.uci_network import enable_uci, disable_uci from c64_test_harness.keyboard import send_text +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import build_policy_and_arbiter @@ -374,8 +379,16 @@ def main() -> int: prg = PRG_PATH.read_bytes() lock = DeviceLock(HOST) - if not lock.acquire(timeout=60.0): - print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + try: + # Budget: C64_DEVICE_LOCK_TIMEOUT, else 30 min. The old + # bare-bool acquire() discarded every diagnostic the + # harness had gathered and reported only False. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 3 print(f"Acquired DeviceLock({HOST})") diff --git a/tools/uci/rig_https_bad_finished.py b/tools/uci/rig_https_bad_finished.py index bf9fea7e..d73346c0 100755 --- a/tools/uci/rig_https_bad_finished.py +++ b/tools/uci/rig_https_bad_finished.py @@ -114,6 +114,9 @@ from c64_test_harness.labels import Labels sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _device_lock_helper import ( # noqa: E402 + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import ( # noqa: E402 build_policy_and_arbiter_with_overlay_carveout, ) @@ -529,7 +532,12 @@ def main() -> int: lock = DeviceLock(HOST) try: - lock.acquire_or_raise(timeout=300.0) + # Shared, env-overridable budget: C64_DEVICE_LOCK_TIMEOUT, else + # 30 min (tools/uci/_device_lock_helper.py). + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 except DeviceLockTimeout as exc: print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 2 diff --git a/tools/uci/rig_https_banner.py b/tools/uci/rig_https_banner.py index 054208ea..b3c909e4 100755 --- a/tools/uci/rig_https_banner.py +++ b/tools/uci/rig_https_banner.py @@ -26,11 +26,16 @@ import time from pathlib import Path -from c64_test_harness.backends.device_lock import DeviceLock +from c64_test_harness.backends.device_lock import ( + DeviceLock, DeviceLockTimeout, +) from c64_test_harness.backends.ultimate64_client import Ultimate64Client from c64_test_harness.uci_network import disable_uci, enable_uci sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _device_lock_helper import ( # noqa: E402 + LockTimeoutConfigError, acquire_device_lock, +) from boot_check import decode_screen, screen_text # noqa: E402 HOST = os.environ.get("U64_HOST", "192.168.1.81") @@ -86,8 +91,16 @@ def main() -> int: print(f"Expecting the banner to name: {EXPECT_HOST}") lock = DeviceLock(HOST) - if not lock.acquire(timeout=120.0): - print(f"ERROR: could not acquire DeviceLock({HOST})", file=sys.stderr) + try: + # Budget: C64_DEVICE_LOCK_TIMEOUT, else 30 min. The old + # bare-bool acquire() discarded every diagnostic the + # harness had gathered and reported only False. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 3 print(f"Acquired DeviceLock({HOST})") diff --git a/tools/uci/rig_https_live.py b/tools/uci/rig_https_live.py index 05296fff..273b44aa 100644 --- a/tools/uci/rig_https_live.py +++ b/tools/uci/rig_https_live.py @@ -95,6 +95,9 @@ from c64_test_harness.keyboard import send_text from c64_test_harness.labels import Labels +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import build_policy_and_arbiter_with_overlay_carveout from _reu_preflight import ReuPreflightError, preflight_reu from _temp_gc import gc_temp @@ -395,7 +398,12 @@ def main() -> int: lock = DeviceLock(HOST) try: - lock.acquire_or_raise(timeout=120.0) + # Shared, env-overridable budget: C64_DEVICE_LOCK_TIMEOUT, else + # 30 min (tools/uci/_device_lock_helper.py). + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 except DeviceLockTimeout as exc: print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 2 diff --git a/tools/uci/rig_https_local.py b/tools/uci/rig_https_local.py index 96b8165a..3c8497aa 100644 --- a/tools/uci/rig_https_local.py +++ b/tools/uci/rig_https_local.py @@ -110,6 +110,9 @@ from c64_test_harness.keyboard import send_text from c64_test_harness.labels import Labels +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import ( build_policy_and_arbiter, build_policy_and_arbiter_with_overlay_carveout, @@ -1487,16 +1490,22 @@ def main() -> int: lock = DeviceLock(HOST) try: - # acquire_or_raise (c64-test-harness PR #88) replaces the legacy - # bare-bool acquire+if pattern. On timeout it gathers holder - # PID/liveness, lockfile age, and a quick REST reachability probe - # and raises DeviceLockTimeout with a diagnostic message that - # disambiguates "queued behind healthy holder" from - # "wedged / stale / unreachable" -- supervisors and humans need - # this signal to know whether to wait, kill the holder, or call - # for a recover() (the last requires explicit user authorization, - # never automated here). - lock.acquire_or_raise(timeout=120.0) + # acquire_device_lock wraps the harness's acquire_or_raise + # (c64-test-harness PR #88), which replaced the legacy bare-bool + # acquire+if pattern: on timeout it gathers holder PID/liveness, + # lockfile age, and a quick REST reachability probe and raises + # DeviceLockTimeout with a message that disambiguates "queued + # behind healthy holder" from "wedged / stale / unreachable" -- + # supervisors and humans need this signal to know whether to + # wait, kill the holder, or call for a recover() (the last + # requires explicit user authorization, never automated here). + # The helper adds the shared, env-overridable budget + # (C64_DEVICE_LOCK_TIMEOUT, else 30 min) and a stderr progress + # line every 30 s, so a long queue is not mistaken for a hang. + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 except DeviceLockTimeout as exc: print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 2 diff --git a/tools/uci/rig_https_wiki.py b/tools/uci/rig_https_wiki.py index 6514fdea..ab272551 100644 --- a/tools/uci/rig_https_wiki.py +++ b/tools/uci/rig_https_wiki.py @@ -112,6 +112,9 @@ from c64_test_harness.labels import Labels from c64_test_harness.uci_network import disable_uci, enable_uci +from _device_lock_helper import ( + LockTimeoutConfigError, acquire_device_lock, +) from _memory_policy import (build_policy, build_policy_and_arbiter_with_overlay_carveout) from _reu_preflight import ReuPreflightError, preflight_reu @@ -596,7 +599,12 @@ def main() -> int: lock = DeviceLock(HOST) try: - lock.acquire_or_raise(timeout=120.0) + # Shared, env-overridable budget: C64_DEVICE_LOCK_TIMEOUT, else + # 30 min (tools/uci/_device_lock_helper.py). + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 except DeviceLockTimeout as exc: print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) return 2