diff --git a/Makefile b/Makefile index 4a97baa..6dabf4e 100644 --- a/Makefile +++ b/Makefile @@ -616,7 +616,7 @@ LINK_MAP := build/c64-https.map LINK_DBG := build/c64-https.dbg LINK_OUTPUTS := $(PRG) $(LABELS) $(LINK_MAP) $(LINK_DBG) -.PHONY: all link run clean ip65-libs ip65-blob package package-verify +.PHONY: all link run clean ip65-libs ip65-blob package package-verify package-m3-demo all: $(PRG) @@ -1305,5 +1305,12 @@ package-verify: $(PACKAGE_PYTHON) tools/test_package_verify.py $(PACKAGE_PYTHON) tools/package/verify_release.py +# The BACKEND=uci-m3 "just run it" demo (TLS on the Ultimate's ESP32): one +# PRG + .d64 + README in dist/m3-demo/. Not part of `package`: the M3 +# variant is secondary and stays out of PACKAGE_VARIANTS. The script builds +# from clean twice (reproducibility) and reads the PRG back out of the .d64. +package-m3-demo: + bash tools/package/build_m3_demo.sh + # The per-object dependency files written by the build/%.o rule above. -include $(ALL_OBJS:.o=.d) diff --git a/tools/package/build_m3_demo.sh b/tools/package/build_m3_demo.sh new file mode 100755 index 0000000..30d3fc0 --- /dev/null +++ b/tools/package/build_m3_demo.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +# ============================================================================= +# tools/package/build_m3_demo.sh — the "just run it" BACKEND=uci-m3 demo. +# +# Produces dist/m3-demo/ from a clean tree: +# +# c64-https-uci-m3-demo.prg the M3 client, default target en.wikipedia.org +# c64-https-uci-m3-demo.d64 that PRG alone, bootable with LOAD"*",8,1 +# README.txt what it is, what it needs, how to run it, and +# the provenance (commit, flags, sha256s) +# SHA256SUMS sha256 of the PRG, the .d64 and README.txt, +# in `shasum -a 256 -c` form +# +# These four are the GitHub pre-release's assets; nothing else is written. +# +# Separate from `make package` on purpose. The M3 variant is SECONDARY (the +# 6510 crypto is the product), so it is not in PACKAGE_VARIANTS, and nothing +# here reads or writes that matrix or the shipped products in dist/. +# +# It refuses a working tree with any change, untracked files included: the +# README names a commit, and the PRG must be what that commit builds. +# +# make runs in a scrubbed environment (PATH, HOME, TMPDIR only), so neither +# an exported knob (HTTPS_SNI=..., VIC_BLANK=0) nor one given on the +# `make package-m3-demo` command line (it reaches here through MAKEFLAGS) +# can leak into the demo build. build/flags.stamp is then checked to hold +# the uci-m3 cfg and no -D beyond the backend's own. +# +# It proves its own output before writing the README: +# - a second clean build reproduces the PRG byte for byte; +# - c1541 reads the PRG back out of the .d64, byte for byte. +# Any failure exits non-zero and leaves no README, so a half-made demo never +# looks finished. +# +# Usage: make package-m3-demo [C1541=... to override the tool] +# ============================================================================= +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$PROJECT_ROOT" + +C1541="${C1541:-c1541}" +OUT="dist/m3-demo" +NAME="c64-https-uci-m3-demo" +DISK_FILE="c64-https-m3" # the 1541 file name (<= 16 chars) +DEMO_HOST="en.wikipedia.org" +DEMO_PATH="/wiki/Commodore_64" +MAKE_ARGS=(BACKEND=uci-m3 "HTTPS_HOST=$DEMO_HOST" "HTTPS_PATH=$DEMO_PATH") + +# What the firmware side published; quoted in the README, checked nowhere +# here (the device is not involved in a build). +# The requirement is the interface; the build named is only where it was +# validated. The upstream PR numbers are not known yet; the README carries a +# marked slot for them (UPSTREAM_PRS overrides it once they exist). +FIRMWARE="M3-SPEC v1 + errata v1.1/v1.2 firmware (validated on esp-tls/m3-master 3a1ff9ff: fw 3.15, FPGA 126, ESP 1.309)" +UPSTREAM_PRS="${UPSTREAM_PRS:-[TO BE FILLED IN: 1541ultimate upstream PR number(s)]}" +SPEC="M3-SPEC v1 (= r7.7), sha256 c265fdfcfdd08dee989a96eee25fae66fb6650ee4be8bb24f8dd791d9ee3db9f, with errata v1.1 (f9a39ff334b34cb0e1a63d5d508675de552ac8eacb23ccd6095a7f4c72788f9a) and v1.2 (a6802f40db3229d53591ef6e0773641d723186e779dd8726abff5915cbe551a4)" + +die() { echo "[m3-demo] ERROR: $*" >&2; exit 1; } +sha() { shasum -a 256 "$1" | cut -d' ' -f1; } + +command -v "$C1541" >/dev/null 2>&1 \ + || die "c1541 not found in PATH (it ships with VICE). Set C1541=... to override." + +[ -z "$(git status --porcelain --untracked-files=all)" ] \ + || { git status --short --untracked-files=all >&2 + die "the working tree is not clean (untracked files count): commit or remove the changes above"; } + +rm -rf "$OUT" +mkdir -p "$OUT" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +scrubbed_make() { + env -i PATH="$PATH" HOME="$HOME" ${TMPDIR:+TMPDIR="$TMPDIR"} make "$@" +} + +# The demo is the bare backend: its -D set, its cfg, the default target. +check_stamp() { + local stamp=build/flags.stamp defines + grep -qx 'BACKEND=uci-m3' "$stamp" || die "$stamp: BACKEND is not uci-m3" + grep -q '^LD65FLAGS=-C cfg/c64-https-uci-m3.cfg ' "$stamp" \ + || die "$stamp: not linked with cfg/c64-https-uci-m3.cfg" + defines="$(grep '^CA65FLAGS=' "$stamp" | grep -o -- '-D [^ ]*' | tr '\n' ' ')" + [ "$defines" = "-D BACKEND_UCI=1 -D BACKEND_UCI_M3=1 " ] \ + || die "$stamp: unexpected defines: $defines" + grep -qx ".define HTTPS_HOST_STR \"$DEMO_HOST\"" build/https_host.inc \ + && grep -qx ".define HTTPS_PATH_STR \"$DEMO_PATH\"" build/https_host.inc \ + && grep -qx '.define HTTPS_SNI_STR ""' build/https_host.inc \ + || die "build/https_host.inc does not hold the demo target" +} + +build_once() { + scrubbed_make clean >/dev/null + scrubbed_make "${MAKE_ARGS[@]}" >"$tmp/build.log" 2>&1 \ + || { tail -20 "$tmp/build.log" >&2; die "build failed: make ${MAKE_ARGS[*]}"; } + check_stamp +} + +echo "[m3-demo] building: make ${MAKE_ARGS[*]}" +build_once +cp build/c64-https.prg "$OUT/$NAME.prg" +flags_line="$(grep '^CA65FLAGS=' build/flags.stamp | sed 's/--bin-include-dir [^ ]*//g; s/ */ /g')" +# Path and sha256 of each tool make linked, from the stamp: a ca65 that is +# not cc65's shows here, since nothing else in this script can tell. +toolchain="$(grep '^TOOLCHAIN=' build/flags.stamp | sed 's/^TOOLCHAIN=//' \ + | awk '{ for (i = 1; i + 2 <= NF; i += 3) print " " $i, $(i+1), $(i+2) }')" + +echo "[m3-demo] rebuilding from clean to prove the PRG is reproducible" +build_once +[ "$(sha build/c64-https.prg)" = "$(sha "$OUT/$NAME.prg")" ] \ + || die "the second clean build differs from the first" + +echo "[m3-demo] writing $NAME.d64" +"$C1541" -format "c64-https-m3,m3" d64 "$OUT/$NAME.d64" >/dev/null +"$C1541" -attach "$OUT/$NAME.d64" -write "$OUT/$NAME.prg" "$DISK_FILE,p" >/dev/null +"$C1541" -attach "$OUT/$NAME.d64" -read "$DISK_FILE" "$tmp/readback.prg" >/dev/null +cmp -s "$tmp/readback.prg" "$OUT/$NAME.prg" \ + || die "the PRG read back out of the .d64 differs from the one written" +listing="$("$C1541" -attach "$OUT/$NAME.d64" -list \ + | grep -Ev '^(OPENCBM:|D64 disk image |Unit [0-9]+ drive )')" + +commit="$(git rev-parse HEAD)" +prg_sha="$(sha "$OUT/$NAME.prg")" +d64_sha="$(sha "$OUT/$NAME.d64")" +prg_bytes="$(wc -c < "$OUT/$NAME.prg" | tr -d ' ')" + +cat > "$OUT/README.txt" <.nip.io name for a site whose certificate does + not name it.) + +Measured with THIS image (PRG sha256 above), validated on an Ultimate 64 +Elite (U64E), firmware 3.15 git 3a1ff9ff, 48 MHz, REU disabled, the .d64 +mounted on drive 8 and loaded with LOAD"*",8,1 +(tools/uci/rig_https_m3_demo.py), 2026-10-06: + 1. HTTP 200, 751,848 B, complete by its chunked framing, 207 s from G to + CONNECTION CLOSED (the page size changes as the article is edited). + 2. HTTP 200, 16,739 B consumed == Content-Length, 9 s. + 3. Refused: 94,CERTIFICATE NAME MISMATCH: 0x00000004. + Other Ultimate models, and other clock speeds, have not been tested. + +What an unsupported device shows +-------------------------------- +(Per the M3 spec and our model tests, not yet observed on hardware with +this image.) + - Firmware without the M3 commands: I prints NETWORK INIT FAILED and the + firmware's answer, 21,UNKNOWN COMMAND. + - No Command Interface (a plain C64, or the interface disabled): NETWORK + INIT FAILED. + - M3 firmware but no usable TLS module (WiFi off, no ESP32, a build without + TLS): G waits up to 30 s for the module, then prints TLS HANDSHAKE FAILED + and 90,TLS NOT AVAILABLE (or 87 no entropy / 92 no trusted time). + - Any refusal prints the firmware's own status line, e.g. 93 (certificate + not trusted), 94 (name mismatch), 95 (expired). + - NETWORK INTERFACE NOT RESPONDING - PRESS RESET means the interface + stopped answering; the program halts on purpose. Press the reset button + (if it repeats, power-cycle the Ultimate). + +Source: https://github.com/JC-000/c64-https (src/net/uci-m3/). +EOF + +# SHA256SUMS last, over everything a release would attach (shasum -c format). +( cd "$OUT" && shasum -a 256 "$NAME.prg" "$NAME.d64" README.txt > SHA256SUMS ) + +echo "[m3-demo] done: $OUT" +echo "[m3-demo] $NAME.prg $prg_sha" +echo "[m3-demo] $NAME.d64 $d64_sha" diff --git a/tools/test_device_prep.py b/tools/test_device_prep.py index aa6648b..4354ef7 100644 --- a/tools/test_device_prep.py +++ b/tools/test_device_prep.py @@ -67,6 +67,7 @@ # on-chip row generator is the REU profile; the equate at 0 is on-chip. LABELS_REU = "al 006000 .ecdsa_verify_256\nal 00A000 .tls_rec_buf\n" LABELS_ONCHIP = "al 000000 .LIB_NISTCURVES_REU_BANKS_USED\nal 006000 .gen_mul_row\n" +LABELS_M3 = "al 00AF9F .m3_owned\nal 00BC00 .sqtab_lo\n" #: A device at the factory default: 1 MHz, turbo off, no REU. This is the #: ordinary state of a freshly power-cycled U64E, not a broken one — which is @@ -362,6 +363,36 @@ def test_onchip_build_never_writes_the_reu() -> None: assert "no REU configuration needed" in out +def test_m3_build_switches_the_reu_off() -> None: + """uci-m3 needs no REU, and its runs prove it: the REU is turned OFF.""" + report, writers, out = _run(FakeClient([READY_STATE, DEFAULT_STATE]), + labels=LABELS_M3) + assert report["profile"] == "none", out + assert writers.reu == [(False, None)], f"REU not switched off:\n{out}" + assert report["wanted"]["reu"] == "Disabled" + assert report["after"][f"{CAT_CART}/RAM Expansion Unit"] == "Disabled" + + +def test_m3_build_with_the_reu_off_writes_nothing() -> None: + report, writers, out = _run(FakeClient([DEFAULT_STATE]), labels=LABELS_M3) + assert writers.reu == [], f"redundant REU write:\n{out}" + assert "reu" in report["skipped_write"] + + +def test_m3_build_unreadable_reu_is_switched_off_anyway() -> None: + """Disabling is the safe direction, so an unreadable state writes.""" + blind = dict(READY_STATE) + blind[f"{CAT_CART}/RAM Expansion Unit"] = _HarnessError("read refused") + report, writers, out = _run(FakeClient([blind]), labels=LABELS_M3) + assert writers.reu == [(False, None)], out + + +def test_m3_build_refused_disable_is_exit_4() -> None: + exc = _assert_raises_prep(FakeClient([READY_STATE]), "refused disable", + labels=LABELS_M3, reu_raises=_HarnessError("400")) + assert "Disabled" in str(exc) + + # --------------------------------------------- the REU write can itself fail def test_a_refused_reu_write_is_exit_4_with_a_ladder_not_a_traceback() -> None: @@ -558,6 +589,16 @@ def test_skipping_prep_warns_that_the_clock_is_unmanaged() -> None: #: changes. KNOWN_UNPREPPED: dict[str, str] = {} +#: Rigs the discovery rule below cannot see that still depend on the clock +#: and REU state the prep sets: the demo rig boots from the drive +#: (LOAD"*",8,1), not by DMA. Registered by name. +REGISTERED_RIGS = ("rig_https_m3_demo.py",) + + +def _covered_rigs(): + """What the contract covers: the discovered rigs plus the registered.""" + return sorted(set(_crypto_path_rigs()) | {UCI / n for n in REGISTERED_RIGS}) + def _crypto_path_rigs(): """Discover the rigs this contract covers, rather than listing them. @@ -609,7 +650,7 @@ def test_the_discovery_rule_finds_the_known_rigs() -> None: "rig_https_live.py", "rig_https_local.py", "rig_https_wiki.py", "rig_https_banner.py", "rig_close_retry.py", "rig_https_refetch.py", - "rig_trust_policy.py"} + "rig_trust_policy.py", "rig_https_m3.py"} assert names == expected, ( f"the discovery rule now selects {sorted(names)}, not " f"{sorted(expected)}. If a rig was added or renamed that is fine — " @@ -618,6 +659,14 @@ def test_the_discovery_rule_finds_the_known_rigs() -> None: ) +def test_registered_rigs_exist_and_are_not_discovered() -> None: + """A registration is for a rig the rule misses; one it finds is noise.""" + for name in REGISTERED_RIGS: + assert (UCI / name).is_file(), f"REGISTERED_RIGS names a missing file: {name}" + found = {p.name for p in _crypto_path_rigs()} & set(REGISTERED_RIGS) + assert not found, f"{sorted(found)} are discovered now: drop the registration" + + def _call_lines(tree, name): """Line numbers of every ``name(...)`` call in *tree*.""" return sorted( @@ -635,7 +684,7 @@ def test_every_crypto_rig_prepares_before_it_checks() -> None: a convention, not a fix, and #197's defect was exactly four rigs holding a guard with no setup behind it. """ - for path in _crypto_path_rigs(): + for path in _covered_rigs(): tree = ast.parse(path.read_text(), filename=str(path)) prep = _call_lines(tree, "prepare_device") flight = _call_lines(tree, "preflight_reu") @@ -680,7 +729,7 @@ def test_every_prepping_rig_hands_over_its_run_artifact_dir() -> None: supplies its own Python-level default) it would write nothing at all. So every call site passes `artifact_dir` explicitly. """ - for path in _crypto_path_rigs(): + for path in _covered_rigs(): if path.name in KNOWN_UNPREPPED: continue tree = ast.parse(path.read_text(), filename=str(path)) diff --git a/tools/test_reu_preflight.py b/tools/test_reu_preflight.py index b5ffe99..c797377 100644 --- a/tools/test_reu_preflight.py +++ b/tools/test_reu_preflight.py @@ -74,6 +74,8 @@ LABELS_REU = "al 006000 .ecdsa_verify_256\nal 00A000 .tls_rec_buf\n" # LIB_NISTCURVES_REU_BANKS_USED = 0 — the manifest equate saying "no REU". LABELS_ONCHIP = "al 000000 .LIB_NISTCURVES_REU_BANKS_USED\nal 006000 .gen_mul_row\n" +#: BACKEND=uci-m3: the M3 adapter, no nistcurves equate, no row generator. +LABELS_M3 = "al 00AF9F .m3_owned\nal 00BC00 .sqtab_lo\n" class _HarnessError(Exception): @@ -292,6 +294,32 @@ def test_onchip_build_makes_no_device_call() -> None: assert "no REU required" in out +def test_m3_build_needs_no_reu_and_makes_no_device_call() -> None: + """uci-m3 links no 6510 crypto: it is not the fail-closed REU answer.""" + client = FakeClient(item=AssertionError("no device call for a uci-m3 build"), + value=AssertionError("no device call for a uci-m3 build")) + result, out = _run(client, LABELS_M3) + assert result == "none", f"expected none, got {result!r}\n{out}" + assert client.calls == [], f"uci-m3 build touched the device: {client.calls!r}" + + +def test_an_m3_build_with_the_reu_body_sink_needs_the_reu() -> None: + """HTTPS_BODY_TO_REU=1 links the viewer: the REU is needed after all.""" + with tempfile.TemporaryDirectory() as tmp: + labels = Path(tmp) / "labels.txt" + labels.write_text(LABELS_M3 + "al 00243A .viewer_enter\n") + assert pf.detect_crypto_profile(labels)[0] == "reu" + + +def test_the_m3_marker_does_not_outvote_the_manifest_equate() -> None: + """The equate stays authoritative: a REU-claiming build is still REU.""" + with tempfile.TemporaryDirectory() as tmp: + labels = Path(tmp) / "labels.txt" + labels.write_text("al 000004 .LIB_NISTCURVES_REU_BANKS_USED\n" + "al 00AF9F .m3_owned\n") + assert pf.detect_crypto_profile(labels)[0] == "reu" + + def test_empty_current_is_unreadable_not_disabled() -> None: """An empty ``current`` is an inconclusive read, not a disabled REU. diff --git a/tools/uci/_device_prep.py b/tools/uci/_device_prep.py index dcf296c..f11a7f6 100644 --- a/tools/uci/_device_prep.py +++ b/tools/uci/_device_prep.py @@ -398,7 +398,7 @@ def prepare_device( "profile_reason": reason, "wanted": { "turbo_mhz": turbo_mhz, - "reu": None if profile == "onchip" else reu_size, + "reu": {"onchip": None, "none": "Disabled"}.get(profile, reu_size), }, "wrote": [], "skipped_write": [], @@ -425,9 +425,32 @@ def prepare_device( # On-chip builds need no REU, so nothing is read, written or reported # about one. Anything else gets the configuration it needs, and an # unreadable probe degrades toward WRITING it (#197's asymmetry). + # A build with no 6510 crypto (uci-m3) is run with the REU switched + # OFF, so a "needs no REU" claim is measured, not inherited from the + # previous lane. Disabling is safe, so an unreadable state writes too. if profile == "onchip": print(f"device prep: on-chip profile ({reason}) — no REU " "configuration needed", file=out, flush=True) + elif profile == "none": + if before.get(f"{CAT_CART}/{ITEM_REU_ENABLED}") == "Disabled": + print(f"device prep: {reason} — REU already Disabled", + file=out, flush=True) + report["skipped_write"].append("reu") + else: + print(f"device prep: {reason} — setting REU Disabled — " + "runtime-only, REVERTS ON POWER CYCLE", file=out, flush=True) + writer = set_reu if set_reu is not None else _harness_set_reu() + try: + writer(client, False) + except Exception as exc: # noqa: BLE001 — becomes exit 4 + raise DevicePrepError( + f"\nDEVICE PREP FAILED — the device refused 'RAM " + f"Expansion Unit: Disabled' ({exc.__class__.__name__}: " + f"{exc}). This build is run with the REU off; not " + f"starting it with the REU in an unknown state.\n" + ) from exc + report["wrote"].append("reu") + time.sleep(float(os.environ.get("REU_SETTLE", "3.0"))) else: match = _reu_matches(before, reu_size) if match is True: diff --git a/tools/uci/_reu_preflight.py b/tools/uci/_reu_preflight.py index 3733a63..b942a09 100644 --- a/tools/uci/_reu_preflight.py +++ b/tools/uci/_reu_preflight.py @@ -111,6 +111,17 @@ #: Manifest equate: REU banks the nistcurves archive claims. 0 == onchip. _BANKS_EQUATE = "LIB_NISTCURVES_REU_BANKS_USED" +#: The BACKEND=uci-m3 adapter's handle flag. That build links no 6510 crypto +#: (the Ultimate's ESP32 does the TLS), so it has neither marker above. +_M3_SYMBOL = "m3_owned" + +#: The REU viewer, linked by HTTPS_BODY_TO_REU=1: an M3 build carrying it +#: streams the body into the REU, so it is NOT the no-REU case. +_REU_SINK_SYMBOL = "viewer_enter" + +#: Profiles that need no REU: the preflight has nothing to check for them. +NO_REU_PROFILES = ("onchip", "none") + SKIP_ENV = "C64_SKIP_REU_PREFLIGHT" #: Values that read as "off" for the three flags that share this parser: @@ -173,12 +184,13 @@ def _parse_labels(labels_path: Path) -> dict[str, int]: def detect_crypto_profile(labels_path: Path | str) -> tuple[str, str]: - """Classify the linked build as ``"onchip"`` or ``"reu"``. + """Classify the linked build as ``"onchip"``, ``"none"`` or ``"reu"``. :param labels_path: path to ``build/labels.txt`` from the same link as the PRG about to be run. - :returns: ``(profile, reason)`` where *profile* is ``"onchip"`` or - ``"reu"`` and *reason* is a short human-readable justification + :returns: ``(profile, reason)`` where *profile* is ``"onchip"``, + ``"none"`` (BACKEND=uci-m3: no 6510 crypto at all) or ``"reu"``, + and *reason* is a short human-readable justification suitable for printing. """ labels = _parse_labels(Path(labels_path)) @@ -210,6 +222,13 @@ def detect_crypto_profile(labels_path: Path | str) -> tuple[str, str]: return ("onchip", f"{_BANKS_EQUATE}=0") return ("reu", f"{_BANKS_EQUATE}=${banks:02X} — claims REU bank(s)") + # No manifest equate and the M3 adapter: no 6510 crypto to need a REU, + # unless the body sink is linked. Recognised by a positive marker, so + # anything unrecognised still falls through to the fail-closed REU + # answer below. + if _M3_SYMBOL in labels and _REU_SINK_SYMBOL not in labels: + return ("none", f"{_M3_SYMBOL}: BACKEND=uci-m3, no 6510 crypto") + # No manifest equate: fall back to the symbol union. Union rather than # conjunction so renaming any single symbol upstream cannot silently # reclassify an onchip build as REU-profile and block the configuration @@ -302,7 +321,7 @@ def preflight_reu( :param client: connected ``Ultimate64Client``. :param labels_path: ``build/labels.txt`` from the current link. :param stream: where to print progress (default ``sys.stdout``). - :returns: the detected profile, ``"onchip"`` or ``"reu"``. + :returns: the detected profile (see :func:`detect_crypto_profile`). :raises ReuPreflightError: on a REU-profile build, for any of three outcomes — the device reported the REU **Disabled**; the read **raised**; or the read returned nothing usable (an unrecognised @@ -318,11 +337,11 @@ def preflight_reu( return "skipped" profile, reason = detect_crypto_profile(labels_path) - if profile == "onchip": - # No REST call: the onchip image needs no REU, so there is + if profile in NO_REU_PROFILES: + # No REST call: the image needs no REU, so there is # nothing to check and nothing to slow down. print( - f"REU preflight: build is the on-chip profile ({reason}) — " + f"REU preflight: build is the {profile} profile ({reason}) — " "no REU required, skipping device check", file=out, flush=True, diff --git a/tools/uci/rig_https_m3.py b/tools/uci/rig_https_m3.py index 0c9e6d3..1d21ea9 100644 --- a/tools/uci/rig_https_m3.py +++ b/tools/uci/rig_https_m3.py @@ -33,6 +33,7 @@ from __future__ import annotations import os +import re import sys import time import zlib @@ -47,10 +48,12 @@ from _device_lock_helper import LockTimeoutConfigError, acquire_device_lock # noqa: E402 from _device_prep import DevicePrepError, prepare_device # noqa: E402 from _prg_load import PrgLoadError, load_verified_and_run # noqa: E402 -from _reu_preflight import ReuPreflightError, preflight_reu # noqa: E402 +from _reu_preflight import ( # noqa: E402 + ReuPreflightError, detect_crypto_profile, preflight_reu) from boot_check import decode_screen, screen_text # noqa: E402 from _petscii_keys import push_keys, target_keys # noqa: E402 from _rig_lifecycle import guard_socket_teardown # noqa: E402 +from rig_https_local import _create_run_dir # noqa: E402 HOST = os.environ.get("U64_HOST", "192.168.1.81") REPO = Path(__file__).resolve().parents[2] @@ -96,6 +99,56 @@ def wait_for(client, markers, budget: float): time.sleep(1.0) +def wait_after(client, anchor, markers, budget: float): + """Like wait_for, but only counts text after the LAST `anchor` on the + screen: an earlier step's lines (CONNECTION CLOSED, DHCP OK) are still + there. Nothing counts until `anchor` is on screen, so the caller must + know it will be (or wait_for it first). `markers` are regular + expressions; the one that matched is returned.""" + end = time.monotonic() + budget + while True: + lines, text = screen(client) + tail = text[text.rfind(anchor):] if anchor in text else "" + for m in markers: + if re.search(m, tail): + return m, lines + if time.monotonic() > end: + return None, lines + time.sleep(1.0) + + +#: The end of do_net_init: the whole address is printed (then CR, RTS). +DHCP_DONE = r"DHCP OK - IP: \d+\.\d+\.\d+\.\d+" + + +def press_init(client) -> None: + """'I', then wait for THAT init's verdict. + + The boot runs the same init before it prints the menu, so "DHCP OK" is + already on screen when 'I' is pressed; waiting for it returned at once + and let 'G' land while the re-init was still running. Only text after + the menu line belongs to this 'I'. The menu is on screen when 'I' is + pressed (the caller waited for it), with a blank row under it, so at + row 22 at the lowest; a successful re-init prints four rows, so the + menu is still there when DHCP OK is. (A failure long enough to scroll + it off ends in a timeout: still a FAIL.) The address is matched while + it may still be printing, so the screen must then read the same twice. + """ + client.send_text("I", finish_with_return=False) + m, lines = wait_after(client, "Q=QUIT", [DHCP_DONE, "FAILED"], 120) + if m != DHCP_DONE: + dump(lines, "init") + raise Fail("network init: %s" % m) + for _ in range(10): + time.sleep(1.0) + again, _text = screen(client) + if again == lines: + return + lines = again + dump(lines, "init") + raise Fail("the screen kept changing after DHCP OK") + + def dump(lines, title): print(f"--- {title} ---") for i, line in enumerate(lines): @@ -120,11 +173,7 @@ def run_fetch(client, prg_path: Path, host: str, path: str): if not m: dump(lines, "boot") raise Fail("menu never appeared") - client.send_text("I", finish_with_return=False) - m, lines = wait_for(client, ["DHCP OK", "FAILED"], 120) - if m != "DHCP OK": - dump(lines, "init") - raise Fail("network init: %s" % m) + press_init(client) client.send_text("G", finish_with_return=False) push_keys(client, target_keys(f"{host}\r{path}\r")) m, lines = wait_for(client, ENDS, FETCH_TIMEOUT) @@ -200,6 +249,14 @@ def main(argv) -> int: host, path = argv[2], argv[3] runs = [("A (uci-m3)", PRG_A, LABELS_A), ("B (6510 TLS)", Path(argv[4]), Path(argv[5]))] + # The device is prepared for A, which needs no REU (prep turns it + # off), so B must need none either: a REU-profile or comb B would + # spin. Refused rather than prepared twice. + b_profile, why = detect_crypto_profile(argv[5]) + if b_profile != "onchip": + print(f"[fatal] B must be an on-chip build (no REU); {argv[5]} " + f"is {b_profile!r} ({why})", file=sys.stderr) + return 2 lock = DeviceLock(HOST) try: acquire_device_lock(lock) @@ -218,10 +275,11 @@ def main(argv) -> int: print(f"device: {info.get('product')} fw {info.get('firmware_version')} " f"git {info.get('git_commit_hash')} core {info.get('core_version')}") enable_uci(client) - RUN_DIR.mkdir(parents=True, exist_ok=True) + run_dir = _create_run_dir(RUN_DIR) + print(f"run dir: {run_dir} (device_state.json)") try: prepare_device(client, LABELS_A, turbo_mhz=TURBO_MHZ, - artifact_dir=RUN_DIR) + artifact_dir=run_dir) except DevicePrepError as exc: print(str(exc), file=sys.stderr) return 4 diff --git a/tools/uci/rig_https_m3_demo.py b/tools/uci/rig_https_m3_demo.py new file mode 100644 index 0000000..222f09c --- /dev/null +++ b/tools/uci/rig_https_m3_demo.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""The M3 demo disk, the way a user runs it: from the drive, not by DMA. + +`make package-m3-demo` writes dist/m3-demo/c64-https-uci-m3-demo.d64. This +rig mounts THAT image on the Ultimate's drive A (device 8), types +LOAD"*",8,1 and RUN, and then drives the menu: + + 1. 'I', then 'G' with RETURN, RETURN: the built-in default target + (en.wikipedia.org /wiki/Commodore_64). It must end HTTP 200 with the + body complete by its own framing (tools/http_body_checks.py). + 1b. 'G' with github.com /robots.txt: HTTP 200, complete (timed too). + 2. 'G' with a name the certificate does not carry (NEGATIVE_HOST, a + nip.io name for a Wikipedia address): it must end "TLS HANDSHAKE + FAILED" with the firmware's "94,..." line on screen, no handle held, + net_last_error $8D UCI_ERR_OPEN_REFUSED. + +The PRG on the disk must be the one build/labels.txt describes: the rig +reads the PRG back out of the .d64 with c1541 and refuses to run if it +differs from build/c64-https.prg (run it right after `make +package-m3-demo`, which leaves that build in build/). + +Drive A's state is read first and restored at the end: our image removed, +the old image re-mounted by path when it had one, the drive switched back +off when it was off, and the result compared with what was read. The +upload (/Temp/cache/upload/image[_N].d64: the U64E numbers a repeat rather +than overwrite it) is deleted over FTP once unmounted. +Device prep, the REU preflight, the DeviceLock, the init wait and the +refusal verdict are rig_https_m3.py's. + + U64_HOST=10.43.23.81 tools/uci/rig_https_m3_demo.py [path/to.d64] + +C1541 overrides the c1541 binary, as in `make package-m3-demo`. + +Exit: 0 pass, 1 fail, 2 fatal, 3 DeviceLock timeout, 4 device prep. +""" +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import time +import ftplib +from ftplib import FTP +from pathlib import Path + +from c64_test_harness.backends.device_lock import DeviceLock, DeviceLockTimeout +from c64_test_harness.backends.ultimate64_client import Ultimate64Client +from c64_test_harness.uci_network import disable_uci, enable_uci + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _device_lock_helper import LockTimeoutConfigError, acquire_device_lock # noqa: E402 +from _device_prep import DevicePrepError, prepare_device # noqa: E402 +from _reu_preflight import ReuPreflightError, preflight_reu # noqa: E402 +from _rig_lifecycle import guard_socket_teardown # noqa: E402 +from _petscii_keys import push_keys, target_keys # noqa: E402 +from http_body_checks import ( # noqa: E402 + SYMBOLS, check_body_complete, check_http_status, decode_body_state) +from rig_https_local import _create_run_dir # noqa: E402 +from rig_https_m3 import ( # noqa: E402 + SETTLED, Fail, dump, labels, press_init, screen, verdict_refuse, + wait_after, wait_for) + +HOST = os.environ.get("U64_HOST", "192.168.1.81") +REPO = Path(__file__).resolve().parents[2] +D64 = REPO / "dist" / "m3-demo" / "c64-https-uci-m3-demo.d64" +DISK_FILE = "c64-https-m3" +PRG = REPO / "build" / "c64-https.prg" +LABELS = REPO / "build" / "labels.txt" +SECOND_HOST, SECOND_PATH = "github.com", "/robots.txt" +NEGATIVE_HOST = os.environ.get("NEGATIVE_HOST", "208-80-153-224.nip.io") +TURBO_MHZ = int(os.environ.get("TURBO_MHZ", "48")) +FETCH_TIMEOUT = float(os.environ.get("FETCH_TIMEOUT", "900")) +RUN_DIR = Path(os.environ.get("UCI_DEBUG_DIR", "/tmp/uci_https_debug")) +C1541 = os.environ.get("C1541", "c1541") +DRIVE_FIELDS = ("enabled", "image_path", "image_file") + + +def disk_prg(d64: Path) -> bytes: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "prg" + subprocess.run([C1541, "-attach", str(d64), "-read", DISK_FILE, + str(out)], check=True, capture_output=True) + return out.read_bytes() + + +def since(lines, anchor): + """The rows from the last one holding `anchor` on (all rows if none).""" + rows = [i for i, ln in enumerate(lines) if anchor in ln.upper()] + return lines[rows[-1]:] if rows else lines + + +def body_state(client, lab): + raw = {n: bytes(client.read_mem(lab[n], w)) for n, w in SYMBOLS.items()} + return decode_body_state(raw) + + +def drive_a(client) -> dict: + info = client.list_drives() + for d in info.get("drives", []): + if "a" in d: + return d["a"] + return {} + + +def delete_upload(ours: dict, before: dict) -> None: + """FTP-delete the /Temp file our mount created (never one found mounted). + Never raises: it is the last step of the restore, and best-effort.""" + name = ours.get("image_file", "") + if not name: + print("WARNING: our mount reported no image_file; its /Temp upload " + "(if any) is not deleted") + return + if not name.startswith("/Temp/") or name == before.get("image_file"): + return + try: + with FTP(HOST, timeout=10.0) as ftp: + ftp.login() + ftp.delete(name) + print(f" deleted {name}") + except ftplib.all_errors as exc: + print(f"WARNING: {name} not deleted: {exc}") + + +def restore_drive_a(client, before: dict, ours: dict, mounted: bool) -> None: + """Put drive A back. `mounted` is False when our mount itself failed: + whatever the drive held is still there, so only the power is undone. + The upload is deleted last, so nothing it does can skip the rest.""" + # fw 3.15 reports the whole path in image_file and image_path empty. + path, file = before.get("image_path"), before.get("image_file") + if mounted: + client.unmount_disk("a") + if file: + client.mount_disk_path( + "a", f"{path.rstrip('/')}/{file}" if path else file) + else: + print("WARNING: our mount failed; drive A's power is put back, " + "its disk was not touched") + if not before.get("enabled", True): + client.drive_off("a") + after = drive_a(client) + diff = {k: (before.get(k), after.get(k)) for k in DRIVE_FIELDS + if before.get(k) != after.get(k)} + if diff: + print(f"WARNING: drive A not as found (before, after): {diff}") + else: + print(f"drive A restored: {after}") + if mounted: + delete_upload(ours, before) + + +def main() -> int: + d64 = Path(sys.argv[1]) if len(sys.argv) > 1 else D64 + if not d64.is_file(): + print(f"[fatal] no {d64}: run `make package-m3-demo` first", file=sys.stderr) + return 2 + try: + on_disk = disk_prg(d64) + except (OSError, subprocess.CalledProcessError) as exc: + print(f"[fatal] c1541 could not read {DISK_FILE} from {d64}: {exc}", + file=sys.stderr) + return 2 + if not PRG.is_file() or on_disk != PRG.read_bytes(): + print("[fatal] the PRG on the disk is not build/c64-https.prg, so " + "build/labels.txt does not describe it", file=sys.stderr) + return 2 + lab = labels(LABELS) + if "m3_wedged" not in lab: + print("[fatal] build/ is not a BACKEND=uci-m3 build", file=sys.stderr) + return 2 + + lock = DeviceLock(HOST) + try: + acquire_device_lock(lock) + except LockTimeoutConfigError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + except DeviceLockTimeout as exc: + print(f"[fatal] DeviceLock({HOST}): {exc}", file=sys.stderr) + return 3 + client = None + fetch_in_flight = False + before = {} + ours = {} + touched = mounted = False + try: + client = Ultimate64Client(host=HOST, timeout=30.0) + info = client.get_info() + print(f"device: {info.get('product')} fw {info.get('firmware_version')} " + f"git {info.get('git_commit_hash')} core {info.get('core_version')}") + enable_uci(client) + run_dir = _create_run_dir(RUN_DIR) + print(f"run dir: {run_dir} (device_state.json)") + try: + prepare_device(client, LABELS, turbo_mhz=TURBO_MHZ, + artifact_dir=run_dir) + except DevicePrepError as exc: + print(str(exc), file=sys.stderr) + return 4 + try: + preflight_reu(client, LABELS) + except ReuPreflightError as exc: + print(str(exc), file=sys.stderr) + return 4 + try: + from _temp_gc import gc_temp + print(f"/Temp GC: removed {gc_temp(HOST)} stale file(s)") + except Exception as exc: # noqa: BLE001 + print(f"WARNING: /Temp GC skipped: {exc}") + + before = drive_a(client) + print(f"drive A before: {before}") + touched = True # from here drive A needs restoring + if not before.get("enabled", True): + client.drive_on("a") + client.mount_disk("a", d64.read_bytes(), "d64", mode="readonly") + mounted = True + ours = drive_a(client) + print(f"drive A now: {ours}") + + try: + client.reset() + time.sleep(3.0) + m, lines = wait_for(client, ["READY."], 30) + if not m: + dump(lines, "after reset") + raise Fail("no READY. after reset") + client.send_text('LOAD"*",8,1') + m, lines = wait_for(client, ["LOADING"], 60) + if not m: + dump(lines, "load") + raise Fail("the drive never started LOADING") + time.sleep(1.0) + end = time.monotonic() + 180 + while time.monotonic() < end: + lines, text = screen(client) + if text.count("READY.") >= 2: + break + time.sleep(1.0) + else: + dump(lines, "load") + raise Fail("LOAD never finished") + print(" loaded from drive 8; typing RUN") + client.send_text("RUN") + m, lines = wait_for(client, ["Q=QUIT"], 120) + if not m: + dump(lines, "run") + raise Fail("the menu never appeared after RUN") + press_init(client) + + # --- 1. the default target ---------------------------------- + t0 = time.monotonic() + fetch_in_flight = True # #234: a socket may be live + client.send_text("G", finish_with_return=False) + push_keys(client, target_keys("\r\r")) + m, lines = wait_for(client, ["CONNECTION CLOSED", "TLS HANDSHAKE FAILED", + "NOT RESPONDING"], FETCH_TIMEOUT) + took = time.monotonic() - t0 + if m in SETTLED: + fetch_in_flight = False + dump(lines, "default fetch") + if m != "CONNECTION CLOSED": + raise Fail(f"default fetch ended with {m!r}") + state = body_state(client, lab) + status, body = check_http_status(state), check_body_complete(state) + print(f" {state.summary()}") + print(f" HTTP status : {status.reason}") + print(f" body complete : {body.reason}") + print(f" default fetch : {took:.0f} s from 'G' to CONNECTION CLOSED" + " (1 s screen polling)") + if not (status.ok and body.ok): + raise Fail("the default fetch is not HTTP 200 + complete") + + # --- 1b. a second positive, the README's second URL ---------- + t0 = time.monotonic() + fetch_in_flight = True + client.send_text("G", finish_with_return=False) + push_keys(client, target_keys(f"{SECOND_HOST}\r{SECOND_PATH}\r")) + # The default fetch's CONNECTION CLOSED is still on screen: judge + # only what follows this fetch's own banner, once it is printed. + banner = "HTTPS GET " + SECOND_HOST.upper() + if wait_for(client, [banner], 60)[0] is None: + raise Fail(f"no {banner!r} banner") + m, lines = wait_after(client, banner, + ["CONNECTION CLOSED", "TLS HANDSHAKE FAILED", + "NOT RESPONDING"], FETCH_TIMEOUT) + took = time.monotonic() - t0 + if m in SETTLED: + fetch_in_flight = False + if m != "CONNECTION CLOSED": + dump(lines, "second fetch") + raise Fail(f"{SECOND_HOST}{SECOND_PATH} ended with {m!r}") + state = body_state(client, lab) + status, body = check_http_status(state), check_body_complete(state) + print(f" {SECOND_HOST}{SECOND_PATH}: {status.reason}; {body.reason}; " + f"{took:.0f} s") + if not (status.ok and body.ok): + raise Fail(f"{SECOND_HOST}{SECOND_PATH} is not HTTP 200 + complete") + + # --- 2. the negative ---------------------------------------- + fetch_in_flight = True + client.send_text("G", finish_with_return=False) + push_keys(client, target_keys(f"{NEGATIVE_HOST}\r/\r")) + banner = "HTTPS GET " + NEGATIVE_HOST.upper() + if wait_for(client, [banner], 60)[0] is None: + raise Fail(f"no {banner!r} banner") + m, lines = wait_after(client, banner, + ["TLS HANDSHAKE FAILED", "CONNECTION CLOSED", + "NOT RESPONDING"], 300) + if m in SETTLED: + fetch_in_flight = False + time.sleep(2.0) # let the status line print + lines, _ = screen(client) + # Only this fetch's rows: the default fetch's REQUEST SENT is + # still on screen, and verdict_refuse rejects one. + verdict_refuse(client, lab, NEGATIVE_HOST, "94", m, + since(lines, banner)) + client.send_text("Q", finish_with_return=False) + print("PASS: the demo disk LOADs from drive 8, fetches the default " + "target complete, and shows the name-mismatch refusal") + return 0 + except Fail as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + finally: + if fetch_in_flight and client is not None: + guard_socket_teardown(client.read_mem, lab.get("net_tcp_state")) + if client is not None: + if touched: + try: + restore_drive_a(client, before, ours, mounted) + except Exception as exc: # noqa: BLE001 + print(f"WARNING: drive A not restored: {exc}") + try: + disable_uci(client) + except Exception as exc: # noqa: BLE001 + print(f"WARNING: disable_uci failed: {exc}") + lock.release() + print(f"Released DeviceLock({HOST})") + + +if __name__ == "__main__": + sys.exit(main())