diff --git a/CLAUDE.md b/CLAUDE.md index b9a32a7..1fd16fe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,7 +117,10 @@ Variables: (M3 firmware; M3-SPEC v1 + errata v1.1/v1.2), `src/net/uci-m3/` + `cfg/c64-https-uci-m3.cfg`, no 6510 TLS/crypto linked, every crypto knob `$(error)`s, not in `PACKAGE_VARIANTS`. `HTTPS_PIN_SPKI_SHA256` = trust - PIN, `M3_ALLOW_TLS12=1` offers hardened 1.2. Model + mutants: + PIN, `M3_ALLOW_TLS12=1` offers hardened 1.2. UCI reads are unfenced + (writes keep `uci_fence`); `M3_READ_FENCE=1` restores the old client + byte for byte, `M3_TRACE=1` = per-READ timing records (measurement only; + `rig_https_m3.py timed`). Model + mutants: `tools/check_m3_client.py`, `tools/mutate_m3_client.py` (standalone, need an m3 build); hardware: `rig_https_banner.py`, `rig_https_m3.py`. - `USE_NISTCURVES_ONCHIP=1` — libs/nistcurves FP_ONCHIP_MUL profile: no diff --git a/Makefile b/Makefile index a18185f..9499c27 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,12 @@ $(foreach k,$(M3_REFUSED_KNOBS),$(if $(strip $($(k))),$(error $(k)=$($(k)) does ifneq ($(filter-out 0 1,$(M3_ALLOW_TLS12)),) $(error M3_ALLOW_TLS12 must be 1 (offer TLS 1.3 and hardened 1.2) or 0/unset (TLS 1.3 only)) endif +# M3_READ_FENCE=1: a uci_fence after every UCI register READ too, as before +# (src/net/uci-m3/m3.inc, "Read pacing"): the A/B arm and the fallback. +# M3_TRACE=1: per-net_poll timing records, a measurement build only. +$(foreach k,M3_READ_FENCE M3_TRACE,$(if $(filter-out 0 1,$($(k))),$(error $(k) must be 1 or 0/unset))) +else +$(foreach k,M3_READ_FENCE M3_TRACE,$(if $(filter-out 0,$($(k))),$(error $(k)=$($(k)) applies only to BACKEND=uci-m3))) endif # X25519 is always the libs/x25519 sibling (issue #245 retired the in-tree @@ -116,6 +122,12 @@ CA65FLAGS += -D BACKEND_UCI=1 -D BACKEND_UCI_M3=1 -I src/net/uci ifeq ($(M3_ALLOW_TLS12),1) CA65FLAGS += -D M3_ALLOW_TLS12=1 endif +ifeq ($(M3_READ_FENCE),1) +CA65FLAGS += -D M3_READ_FENCE=1 +endif +ifeq ($(M3_TRACE),1) +CA65FLAGS += -D M3_TRACE=1 +endif else CA65FLAGS += -D BACKEND_IP65=1 endif diff --git a/src/net/uci-m3/m3.inc b/src/net/uci-m3/m3.inc index ce7af21..186ed6e 100644 --- a/src/net/uci-m3/m3.inc +++ b/src/net/uci-m3/m3.inc @@ -61,6 +61,34 @@ M3_B_READY = 3000 ; ER-10 ready-bit polling budget: 30 s M3_B_READY_POLL = 100 ; ...polled once a second M3_B_HTTP_IDLE = 6000 ; http_recv_body: 60 s with no byte arrived +; --- Read pacing (M3_READ_FENCE) ------------------------------------------ +; uci_fence's gap stays after every WRITE to $DF1C/$DF1D. READS of $DF1C-$DF1F +; are unfenced: the 1541ultimate firmware side says back-to-back reads need +; no gap, only correct state polling (DATA_AV is still tested before every +; byte, ER-12), and the U64E A/B that came with this change is the evidence. +; Two fences per body byte were most of a large fetch's time. +; `make BACKEND=uci-m3 M3_READ_FENCE=1` puts a fence back after every read +; and assembles the pre-change client byte for byte: the A/B arm, and the +; fallback if some device turns out to need it. + +; --- Timing trace (M3_TRACE, a measurement build only) --------------------- +; net_poll appends one M3_TR_REC-byte record per call made while CONNECTED +; to M3_TR_BASE (the MAIN tail, link-asserted free), up to M3_TR_CAP; +; m3_tr_count keeps counting past it. Each delta is (dTB, dTA): CIA2 timer B units and timer +; A cycles, ticks = dTB * M3_UNIT_CYCLES + (signed) dTA. Record layout: +; +0 rtt PUSH of the READ to REPLY_VALID seen (0/0: no READ was run) +; +4 total the whole net_poll call +; +8 bytes put into the ring (16-bit) +; +10 m3_poll_result +; +11 $DF1C read right after that READ's PUSH and its fence +M3_TR_BASE = $6000 +M3_TR_REC = 12 +M3_TR_CAP = ($A000 - M3_TR_BASE) / M3_TR_REC +M3_TS_PUSH = 0 ; m3_tr_t slots +M3_TS_REPLY = 4 +M3_TS_START = 8 +M3_TS_END = 12 + ; Deadline slots (offsets into m3_dl). M3_DL_CMD = 0 M3_DL_APP = 4 diff --git a/src/net/uci-m3/m3_cmd.s b/src/net/uci-m3/m3_cmd.s index 798e273..91d13a6 100644 --- a/src/net/uci-m3/m3_cmd.s +++ b/src/net/uci-m3/m3_cmd.s @@ -54,6 +54,17 @@ .export m3_code .export m3_wedged .export m3_dl +.ifdef M3_TRACE +.export m3_tr_sample +.export m3_tr_delta +.export m3_tr_t +.export m3_tr_d +.export m3_tr_got +.export m3_tr_post +.export m3_tr_pushes +.export m3_tr_bit3 +.export m3_tr_or +.endif CIA2_TA_LO = $DD04 CIA2_TA_HI = $DD05 @@ -139,10 +150,14 @@ m3_settle: uci_fence rts -; m3_get_status — A = $DF1C, read and settled. +; m3_get_status — A = $DF1C. Unfenced unless M3_READ_FENCE (m3.inc). m3_get_status: lda UCI_STATUS +.ifdef M3_READ_FENCE jmp m3_settle +.else + rts +.endif ; ============================================================================= ; m3_wait_clear / m3_wait_set — A = mask. Spin until ($DF1C & mask) is zero @@ -282,7 +297,14 @@ m3_exec: jsr m3_settle lda #UCI_CTRL_PUSH_CMD sta UCI_CONTROL +.ifdef M3_TRACE + ldy #M3_TS_PUSH + jsr m3_tr_sample +.endif jsr m3_settle +.ifdef M3_TRACE + jsr m3_tr_post_push +.endif lda m3_tmp2 ldx m3_tmp2+1 ldy #M3_DL_CMD @@ -290,6 +312,12 @@ m3_exec: lda #(UCI_STAT_REPLY_VALID | UCI_STAT_ERROR) jsr m3_wait_set bcs @x_timeout +.ifdef M3_TRACE + ldy #M3_TS_REPLY + jsr m3_tr_sample + lda #1 + sta m3_tr_got +.endif jsr m3_get_status and #UCI_STAT_ERROR beq @x_reply @@ -337,7 +365,9 @@ m3_read_data: and #UCI_STAT_DATA_AV beq @rd_done lda UCI_RESP_DATA +.ifdef M3_READ_FENCE jsr m3_settle +.endif @rd_store: sta $FFFF,y ; SMC: m3_rd_dst iny @@ -361,7 +391,9 @@ m3_discard_data: and #UCI_STAT_DATA_AV beq @dd_done lda UCI_RESP_DATA +.ifdef M3_READ_FENCE jsr m3_settle +.endif lda #1 sta m3_discarded dey @@ -387,7 +419,9 @@ m3_read_status: and #UCI_STAT_STAT_AV beq @rs_done lda UCI_STATUS_DATA +.ifdef M3_READ_FENCE jsr m3_settle +.endif cpy #M3_STATUS_MAX bcs @rs_skip sta m3_status,y @@ -438,6 +472,76 @@ m3_finish: jsr m3_read_status jmp m3_accept +.ifdef M3_TRACE +; ============================================================================= +; M3_TRACE only (m3.inc, "Timing trace"): never in a default build. +; m3_tr_sample — Y = slot (M3_TS_*): CIA2 timer B (2) then timer A (2) into +; m3_tr_t+Y, a consistent set (read again until neither timer moved between +; its bytes). Clobbers: A. Preserves X, Y. +; ============================================================================= +m3_tr_sample: + lda CIA2_TB_HI + sta m3_tr_t+1,y + lda CIA2_TB_LO + sta m3_tr_t+0,y + lda CIA2_TA_HI + sta m3_tr_t+3,y + lda CIA2_TA_LO + sta m3_tr_t+2,y + lda CIA2_TA_HI + cmp m3_tr_t+3,y + bne m3_tr_sample + lda CIA2_TB_LO + cmp m3_tr_t+0,y + bne m3_tr_sample + lda CIA2_TB_HI + cmp m3_tr_t+1,y + bne m3_tr_sample + rts + +; m3_tr_delta — X = earlier slot, Y = later slot: m3_tr_d = (dTB, dTA), each +; 16-bit little-endian; both timers count down, so earlier - later. +; Clobbers: A. +m3_tr_delta: + sec + lda m3_tr_t+0,x + sbc m3_tr_t+0,y + sta m3_tr_d+0 + lda m3_tr_t+1,x + sbc m3_tr_t+1,y + sta m3_tr_d+1 + sec + lda m3_tr_t+2,x + sbc m3_tr_t+2,y + sta m3_tr_d+2 + lda m3_tr_t+3,x + sbc m3_tr_t+3,y + sta m3_tr_d+3 + rts + +; m3_tr_post_push — $DF1C as the PUSH left it (after its fence): kept for +; the READ record, OR-ed into m3_tr_or, and bit 3 (the FPGA's error_busy, +; a refused push) counted. Clobbers: A. +m3_tr_post_push: + lda UCI_STATUS +.ifdef M3_READ_FENCE + jsr m3_settle +.endif + sta m3_tr_post + ora m3_tr_or + sta m3_tr_or + inc m3_tr_pushes + bne :+ + inc m3_tr_pushes+1 +: lda m3_tr_post + and #UCI_STAT_ERROR + beq :+ + inc m3_tr_bit3 + bne :+ + inc m3_tr_bit3+1 +: rts +.endif + .segment "UCI_BSS" m3_dl: .res 8 ; two deadline slots: start(2), length(2) @@ -453,3 +557,12 @@ m3_status: .res M3_STATUS_MAX m3_status_len: .res 1 m3_status_seen: .res 1 m3_code: .res 1 +.ifdef M3_TRACE +m3_tr_t: .res 16 ; M3_TS_* slots: TB (2), TA (2) +m3_tr_d: .res 4 +m3_tr_got: .res 1 ; 1: the last m3_exec saw its reply (rtt valid) +m3_tr_post: .res 1 +m3_tr_pushes: .res 2 +m3_tr_bit3: .res 2 +m3_tr_or: .res 1 +.endif diff --git a/src/net/uci-m3/net.s b/src/net/uci-m3/net.s index e95dcc3..5432770 100644 --- a/src/net/uci-m3/net.s +++ b/src/net/uci-m3/net.s @@ -82,6 +82,21 @@ .import m3_status_seen .import m3_code .import m3_wedged +.ifdef M3_TRACE +.import m3_tr_sample +.import m3_tr_delta +.import m3_tr_d +.import m3_tr_got +.import m3_tr_post +.import m3_tr_pushes +.import m3_tr_bit3 +.import m3_tr_or +.export m3_tr_count +.export m3_tr_n +; The records live in MAIN's unused tail: the image must end below them. +.import __MAIN_LAST__ +.assert __MAIN_LAST__ <= M3_TR_BASE, lderror, "M3_TRACE: the image reaches M3_TR_BASE" +.endif .import tcp_recv_head .import tcp_recv_tail @@ -129,6 +144,22 @@ net_init: sta m3_status_len sta net_tcp_state ; NET_TCP_CLOSED sta net_last_error +.ifdef M3_TRACE + ldx #(m3_tr_zero_end - m3_tr_zero - 1) +: sta m3_tr_zero,x + dex + bpl :- + sta m3_tr_pushes + sta m3_tr_pushes+1 + sta m3_tr_bit3 + sta m3_tr_bit3+1 + sta m3_tr_or + lda #M3_TR_BASE + sta m3_tr_ptr+1 + lda #0 +.endif ldx #3 @zero_ip: sta net_local_ip,x @@ -140,7 +171,9 @@ net_init: ; so do not ABORT and do not wait (ER-2). $49 = $C9 with the UCI ; IRQ active. lda UCI_ID +.ifdef M3_READ_FENCE jsr m3_settle +.endif and #$7F cmp #(UCI_ID_VALUE & $7F) beq @present @@ -697,7 +730,26 @@ m3_status_number: ; handle is CLOSEd by the caller (ERROR, owned). ; Clobbers: A, X, Y ; ============================================================================= +.ifdef M3_TRACE net_poll: + lda net_tcp_state ; the menu loop polls with no session: + cmp #NET_TCP_CONNECTED ; untraced, or it fills the records + bne m3_poll + lda #0 + sta m3_tr_got + sta m3_tr_post + ldy #M3_TS_START + jsr m3_tr_sample + lda tcp_recv_tail+0 + sta m3_tr_tail0 + lda tcp_recv_tail+1 + sta m3_tr_tail0+1 + jsr m3_poll + jmp m3_tr_end +m3_poll: +.else +net_poll: +.endif lda net_tcp_state cmp #NET_TCP_CONNECTED beq @p_go @@ -758,7 +810,9 @@ net_poll: sta m3_bad ; 0, or the code a bad block earns ; ER-12: test bit 7 before the header. No data = an empty reply. lda UCI_STATUS +.ifdef M3_READ_FENCE jsr m3_settle +.endif and #UCI_STAT_DATA_AV bne :+ jsr m3_finish @@ -802,6 +856,7 @@ net_poll: lda m3_req+1 sta m3_hdr+1 @p_copy_init: +.ifdef M3_READ_FENCE @p_copy: lda m3_hdr+0 ora m3_hdr+1 @@ -836,6 +891,56 @@ net_poll: dec m3_hdr+1 : dec m3_hdr+0 jmp @p_copy +.else + ; n > 0 here: $0000 went to @p_0000, and a capped n is the request, + ; which is >= 1. X = n's low byte, Y = its pages, one more when the + ; low byte is not 0: dex/bne then dey/bne runs exactly n times. + ldx m3_hdr+0 + ldy m3_hdr+1 + txa + beq :+ + iny +: + clc + lda tcp_recv_tail+0 + adc #tcp_recv_buf + sta @p_store+2 +@p_copy: + lda UCI_STATUS ; ER-12: DATA_AV before EVERY byte, or a + bpl @p_short ; short block reads as $00s (bit 7) + lda UCI_RESP_DATA +@p_store: + sta $FFFF ; SMC: tcp_recv_buf + tail, advanced here + inc @p_store+1 + bne @p_next + inc @p_store+2 ; a new page: wrap at the ring's end + lda @p_store+2 + cmp #>(tcp_recv_buf + TCP_RECV_MASK + 1) + bne @p_next + lda #>tcp_recv_buf + sta @p_store+2 +@p_next: + dex + bne @p_copy + dey + bne @p_copy + beq @p_tail ; always +@p_short: + lda #UCI_ERR_SHORT_READ ; short of its header: a hole (ER-5) + sta m3_bad +@p_tail: + lda @p_store+1 ; the tail is where the store points + sta tcp_recv_tail+0 + lda @p_store+2 + sec + sbc #>tcp_recv_buf + sta tcp_recv_tail+1 +.assert UCI_STAT_DATA_AV = $80, error, "@p_copy tests DATA_AV with bpl" +.assert M3_TR_CAP + bcc :+ + jmp @te_out ; full: counted, not kept +: ldy #3 + lda #0 +: sta m3_tr_rec,y ; rtt 0/0: no READ reply in this call + dey + bpl :- + lda m3_tr_got + beq @te_total + ldx #M3_TS_PUSH + ldy #M3_TS_REPLY + jsr m3_tr_delta + ldy #3 +: lda m3_tr_d,y + sta m3_tr_rec,y + dey + bpl :- +@te_total: + ldx #M3_TS_START + ldy #M3_TS_END + jsr m3_tr_delta + ldy #3 +: lda m3_tr_d,y + sta m3_tr_rec+4,y + dey + bpl :- + lda tcp_recv_tail+0 + sec + sbc m3_tr_tail0 + sta m3_tr_rec+8 + lda tcp_recv_tail+1 + sbc m3_tr_tail0+1 + and #>TCP_RECV_MASK + sta m3_tr_rec+9 + lda m3_poll_result + sta m3_tr_rec+10 + lda m3_tr_post ; the post-PUSH $DF1C of this call's READ + sta m3_tr_rec+11 ; (0: no PUSH in this call) + lda m3_tr_ptr + sta @te_st+1 + lda m3_tr_ptr+1 + sta @te_st+2 + ldy #M3_TR_REC - 1 +: lda m3_tr_rec,y +@te_st: sta $FFFF,y ; SMC: m3_tr_ptr + dey + bpl :- + lda m3_tr_ptr + clc + adc #M3_TR_REC + sta m3_tr_ptr + bcc :+ + inc m3_tr_ptr+1 +: inc m3_tr_n + bne @te_out + inc m3_tr_n+1 +@te_out: + rts +.endif + ; ============================================================================= ; net_tcp_send — A/X = data, net_send_len = length. ; @@ -1183,3 +1364,12 @@ m3_rem: .res 2 m3_piece: .res 2 m3_cnt: .res 2 m3_wresp: .res 2 +.ifdef M3_TRACE +m3_tr_zero: ; net_init zeroes these +m3_tr_count: .res 2 ; CONNECTED net_poll calls since net_init +m3_tr_n: .res 2 ; records kept (<= M3_TR_CAP) +m3_tr_zero_end: +m3_tr_ptr: .res 2 +m3_tr_tail0: .res 2 +m3_tr_rec: .res M3_TR_REC +.endif diff --git a/tools/check_m3_client.py b/tools/check_m3_client.py index cbf75f3..3d5199d 100644 --- a/tools/check_m3_client.py +++ b/tools/check_m3_client.py @@ -455,12 +455,33 @@ def __init__(self, image, load_addr): self.ram[0xFFE4:0xFFE8] = bytes([0xAD, 0x01, 0xFE, 0x60]) self.screen = bytearray() self.keys = bytearray() + # Fence trace (test_read_pacing): None, or one [steps, "r"/"w", + # addr, SBC #1 executions before the next UCI access] per access. + self.uci_trace = None + self.sbc1 = 0 + + def _uci_access(self, kind, addr): + if self.uci_trace is not None and 0xDF1C <= addr <= 0xDF1F: + if self.uci_trace: + self.uci_trace[-1][3] = self.sbc1 + self.uci_trace.append([self.cpu.steps, kind, addr, None]) + self.sbc1 = 0 + + def uci_trace_close(self): + if self.uci_trace: + self.uci_trace[-1][3] = self.sbc1 def units(self): return (self.cpu.steps // STEPS_PER_UNIT) if self.cpu else 0 def read(self, addr): addr &= 0xFFFF + if self.uci_trace is not None: + # An opcode fetch of SBC #$01: uci_fence's inner loop. + if (addr == self.cpu.pc and self.ram[addr] == 0xE9 + and self.ram[(addr + 1) & 0xFFFF] == 0x01): + self.sbc1 += 1 + self._uci_access("r", addr) if 0xDF00 <= addr <= 0xDFFF: return self.dev.read(addr) if addr == 0xFE01: @@ -475,6 +496,8 @@ def read(self, addr): def write(self, addr, value): addr &= 0xFFFF + if self.uci_trace is not None: + self._uci_access("w", addr) if 0xDF00 <= addr <= 0xDFFF: self.dev.write(addr, value & 0xFF) return @@ -520,6 +543,42 @@ def _body_to_reu(labels=None): raise Unavailable("%s has no CA65FLAGS line" % stamp) +def _read_fence(labels=None): + """True for an M3_READ_FENCE=1 image (read off flags.stamp, as + _body_to_reu does): every UCI READ fenced, the pre-change client.""" + stamp = Path(labels or LABELS).parent / "flags.stamp" + if not stamp.is_file(): + raise Unavailable("%s is missing: cannot tell whether the image " + "fences its UCI reads" % stamp) + for line in stamp.read_text().splitlines(): + if line.startswith("CA65FLAGS="): + return "-D M3_READ_FENCE" in line + raise Unavailable("%s has no CA65FLAGS line" % stamp) + + +def _refuse_trace(labels=None): + """An M3_TRACE=1 image runs m3_tr_end after net_poll's last register + access, which test_read_pacing's READ_GAP_BUDGET would count as a delay + after a read. The model rules are the plain image's: refuse, like the + mutator does.""" + stamp = Path(labels or LABELS).parent / "flags.stamp" + if stamp.is_file() and "-D M3_TRACE" in stamp.read_text(): + raise Unavailable("%s is an M3_TRACE=1 image (a measurement build): " + "the model checks are written against images built " + "without M3_TRACE" % stamp) + + +def _fence_sbc(): + """SBC #1 executions in one uci_fence (OUTER x INNER, uci_regs.inc).""" + text = (REPO / "src" / "net" / "uci" / "uci_regs.inc").read_text() + val = {} + for name in ("UCI_FENCE_OUTER", "UCI_FENCE_INNER"): + for line in text.splitlines(): + if line.startswith(name): + val[name] = int(line.split("=")[1].split(";")[0]) + return val["UCI_FENCE_OUTER"] * val["UCI_FENCE_INNER"] + + def _labels(path=None): path = Path(path or LABELS) if not path.is_file(): @@ -1460,6 +1519,221 @@ def test_held_session_message(prg=None, labels=None): _code(m, ERR_CONNECT_FAIL, "an Open refused over a held session") +def _pacing_flows(prg, labels): + """Every UCI access of four sessions, with the SBC #1 count after it: + a whole fetch (init, Open, the GET's drain READ and WRITE, a 2,000 B + body over full and partial blocks, an idle READ, 01, CLOSE), a CLOSE + whose PUSH is refused once (the rejected path's CLR_ERR write), a block + longer than its header (the discard path), and a Data More block (the + ABORT path).""" + traces = [] + m = Machine(prg, labels) + m.mem.uci_trace = [] + _connected(m) + body = bytes((i * 7) & 0x7F | 0x20 for i in range(2000)) + resp = b"HTTP/1.1 200 OK\r\nContent-Length: 2000\r\n\r\n" + body + carry, status, total = _http_fetch( + m, [("data", resp[:1500]), ("idle",), ("data", resp[1500:]), + ("end", CLOSED_BY_HOST)]) + _check(carry is False and status == 200 and total == 2000, + "the traced fetch failed: C=%s, HTTP %d, %d B" % (carry, status, total)) + m.call("net_tcp_close") + m.mem.uci_trace_close() + m.no_violations() + traces.append(("fetch", m.mem.uci_trace)) + # A CLOSE whose PUSH the FPGA refuses once: m3_exec's CLR_ERR write on + # the rejected path, then the retry. + m = Machine(prg, labels) + m.mem.uci_trace = [] + _connected(m) + _reject_ops(m, 0x09, 1) + m.call("net_tcp_close") + m.mem.uci_trace_close() + _check(m.dev.pushes_rejected == 1 and not m.dev.sessions, + "rejected: the refused CLOSE was not retried to the end") + m.no_violations() + traces.append(("rejected", m.mem.uci_trace)) + for name, blocks in (("tail", [b"\x03\x00abcdefgh"]), + ("more", [b"\x03\x00abc", b"\x02\x00de"])): + m = Machine(prg, labels) + m.mem.uci_trace = [] + _connected(m) + m.dev.forced_blocks = (blocks, OK) + m.call("net_poll") + m.call("net_tcp_close") + m.mem.uci_trace_close() + _check(m.peek("net_tcp_state") != NET_TCP_CONNECTED, + "%s: the bad block was accepted" % name) + m.no_violations() + traces.append((name, m.mem.uci_trace)) + return traces + + +#: Default image: the most instructions that may run between a READ of +#: this register and the next UCI access, on the traced flows. Measured +#: maxima at 4464886: $DF1C 49, $DF1D 10, $DF1E 35, $DF1F 7. A delay of +#: any shape (a fence, a NOP sled, a loop) past these is caught; one that +#: fits under them is not -- that is the detector's resolution. +READ_GAP_BUDGET = {0xDF1C: 64, 0xDF1D: 16, 0xDF1E: 40, 0xDF1F: 16} + + +def test_read_pacing(prg=None, labels=None): + """Read pacing (m3.inc): a uci_fence after EVERY write to $DF1C/$DF1D, + and no delay after any read of $DF1C-$DF1F -- unless the image is + M3_READ_FENCE=1, where every read is fenced too (the pre-change + client). The body copy must cost a handful of instructions per byte. + + What it detects, on the four traced flows only (_pacing_flows): + - a write with no uci_fence after it, seen by execution: fewer SBC #1 + opcode fetches than 0.9 x OUTER x INNER before the next UCI access; + - default image: any SBC #1 after a read, AND more instructions after + a read than READ_GAP_BUDGET allows, whatever the delay's shape -- + except a delay made of UCI register accesses (e.g. a loop of + status reads), since each access restarts the gap; + - M3_READ_FENCE=1 image: a read with no uci_fence after it.""" + fenced = _read_fence(labels) + need = _fence_sbc() * 9 // 10 # SBC #1 with C=0 takes 2 first + for name, trace in _pacing_flows(prg, labels): + writes = [e for e in trace if e[1] == "w"] + reads = [e for e in trace if e[1] == "r"] + _check(len(writes) > 10 and len(reads) > 100, + "%s: only %d writes / %d reads traced" % (name, len(writes), len(reads))) + bare = [e for e in writes if e[3] < need] + _check(not bare, "%s: %d write(s) with no fence before the next UCI " + "access, first $%04X at step %d (%d SBC #1, need %d): the FPGA " + "needs the gap after a WRITE" % (name, len(bare), bare[0][2], + bare[0][0], bare[0][3], need) + if bare else "") + if fenced: + bad = [e for e in reads[:-1] if e[3] < need] + what = "unfenced read(s) in an M3_READ_FENCE=1 image" + else: + bad = [e for e in reads[:-1] if e[3] != 0] + what = "fenced read(s) in the default image (M3_READ_FENCE is off)" + _check(not bad, "%s: %d %s, first $%04X at step %d (%d SBC #1)" + % (name, len(bad), what, bad[0][2], bad[0][0], bad[0][3]) + if bad else "") + if not fenced: + slow = [(trace[i], trace[i + 1][0] - trace[i][0]) + for i in range(len(trace) - 1) if trace[i][1] == "r" + and trace[i + 1][0] - trace[i][0] > READ_GAP_BUDGET[trace[i][2]]] + _check(not slow, "%s: %d read(s) followed by a delay before the " + "next UCI access, first $%04X at step %d: %d instructions, " + "budget %d" % (name, len(slow), slow[0][0][2], slow[0][0][0], + slow[0][1], READ_GAP_BUDGET[slow[0][0][2]]) + if slow else "") + if name != "fetch": + continue + # The copy loop: consecutive $DF1E reads with one $DF1C between. + gaps = [trace[i + 2][0] - trace[i][0] for i in range(len(trace) - 2) + if trace[i][2] == 0xDF1E and trace[i + 1][2] == 0xDF1C + and trace[i + 2][2] == 0xDF1E] + _check(len(gaps) > 1900, "fetch: only %d copy-loop byte pairs seen" + % len(gaps)) + gaps.sort() + median = gaps[len(gaps) // 2] + if fenced: + _check(median >= 2 * need, "fetch: %d instructions per body byte " + "in an M3_READ_FENCE=1 image: its two read fences are gone" + % median) + else: + _check(median <= 10, "fetch: %d instructions per body byte, " + "expected <= 10 (the unfenced copy loop is 8)" % median) + + +def _ring_case(m, tail, head, blocks): + for name, v in (("tcp_recv_head", head), ("tcp_recv_tail", tail)): + m.poke(name, v & 0xFF) + m.poke(name, v >> 8, 1) + m.dev.forced_blocks = (blocks, OK) + m.call("net_poll") + return m.peek("tcp_recv_tail") | (m.peek("tcp_recv_tail", 1) << 8) + + +def test_copy_counts_at_page_edges(prg=None, labels=None): + """The copy's X/Y page count (@p_copy_init) and ring wrap, at the byte + counts where a page counter goes wrong -- 1, 255, 256, 257, 512, 768, + 893 -- and at ring tails $0000, $0E7D, $0F00 and $0FFF: every byte in + order, the tail where it belongs, still CONNECTED, nothing stored past + the copy or at $D000+. Then, per count, a short block ($8F), a header + that is exactly the request, and an over-claim ($8B), at the edges.""" + pattern = bytes((i * 13 + 7) & 0xFF for i in range(893)) + m = _connected(Machine(prg, labels)) + for tail in (0x0000, 0x0E7D, 0x0F00, 0x0FFF): + for n in (1, 255, 256, 257, 512, 768, 893): + data = pattern[:n] + for a in range(0xC000, 0xD000): + m.mem.ram[a] = 0xA5 + got = _ring_case(m, tail, tail, [bytes([n & 0xFF, n >> 8]) + data]) + note = "n=%d from tail $%04X" % (n, tail) + _check(m.peek("net_tcp_state") == NET_TCP_CONNECTED, + "%s: the session died (net_last_error $%02X)" + % (note, m.peek("net_last_error"))) + _check(got == (tail + n) & 0x0FFF, "%s: tail $%04X, expected $%04X" + % (note, got, (tail + n) & 0x0FFF)) + _check(bytes(m.mem.ram[0xC000 + ((tail + i) & 0x0FFF)] + for i in range(n)) == data, "%s: ring bytes differ" % note) + _check(m.mem.ram[0xC000 + got] == 0xA5, "%s: a byte stored past " + "the copy" % note) + _check(not any(m.mem.ram[0xD000:0xD400]), "%s: a byte stored at " + "$D000+" % note) + m.no_violations() + for n in (1, 255, 256, 257, 512, 768): + for tail in (0x0F00, 0x0FFF): + note = "n=%d from tail $%04X" % (n, tail) + # short: the header says n + 1, the block holds n + m = _connected(Machine(prg, labels)) + _ring_case(m, tail, tail, [bytes([(n + 1) & 0xFF, (n + 1) >> 8]) + + pattern[:n]]) + _check(m.peek("net_tcp_state") == NET_TCP_ERROR, "%s, short: still " + "CONNECTED" % note) + _code(m, ERR_SHORT_READ, note + ", short") + # exactly the request: free - 1 == n + m = _connected(Machine(prg, labels)) + head = (tail + n + 1) & 0x0FFF + got = _ring_case(m, tail, head, [bytes([n & 0xFF, n >> 8]) + + pattern[:n]]) + reads = [c for c in m.dev.commands() if c[1] == 0x10] + _check((reads[-1][3] | reads[-1][4] << 8) == n, "%s: asked for %d, " + "expected the ring's free - 1" % (note, reads[-1][3] + | reads[-1][4] << 8)) + _check(m.peek("net_tcp_state") == NET_TCP_CONNECTED + and got == (tail + n) & 0x0FFF, "%s, exact request: state " + "$%02X tail $%04X" % (note, m.peek("net_tcp_state"), got)) + # over-claim: header n + 1 over a request of n; n are kept + m = _connected(Machine(prg, labels)) + got = _ring_case(m, tail, head, [bytes([(n + 1) & 0xFF, (n + 1) >> 8]) + + pattern[:n] + b"\x77"]) + _check(m.peek("net_tcp_state") == NET_TCP_ERROR + and got == (tail + n) & 0x0FFF, "%s, over-claim: state $%02X " + "tail $%04X" % (note, m.peek("net_tcp_state"), got)) + _code(m, ERR_BAD_READ_HDR, note + ", over-claim") + + +def test_ring_wraps_in_one_block(prg=None, labels=None): + """One 893 B block that crosses the end of the 4 KB ring: every byte + lands in order across $CFFF -> $C000, the tail wraps, and nothing is + stored at $D000 (VIC-II, on hardware) or past it.""" + m = _connected(Machine(prg, labels)) + for name in ("tcp_recv_head", "tcp_recv_tail"): + m.poke(name, 0x80) + m.poke(name, 0x0F, 1) + data = bytes((i * 13 + 5) & 0xFF for i in range(893)) + m.dev.sessions[5]["rx"] = [("data", data)] + m.call("net_poll") + _check(m.peek("net_tcp_state") == NET_TCP_CONNECTED, + "a wrapping block ended the connection (net_last_error $%02X)" + % m.peek("net_last_error")) + tail = m.peek("tcp_recv_tail") | (m.peek("tcp_recv_tail", 1) << 8) + _check(tail == (0x0F80 + 893) & 0x0FFF, "tail $%04X after 893 B from " + "$0F80, expected $%04X" % (tail, (0x0F80 + 893) & 0x0FFF)) + _check(m.ring() == data, "the ring does not hold the block in order " + "across the wrap") + _check(not any(m.mem.ram[0xD000:0xD400]), "bytes stored at $D000+: the " + "copy ran off the end of the ring") + m.no_violations() + + TESTS = ( test_startup_sequence, test_no_uci_writes_nothing, @@ -1511,6 +1785,9 @@ def test_held_session_message(prg=None, labels=None): test_data_more_wedge_keeps_89, test_held_session_message, test_boot_reu_dma_only_for_the_sink, + test_read_pacing, + test_ring_wraps_in_one_block, + test_copy_counts_at_page_edges, ) # Need `make BACKEND=uci-m3 HTTPS_BODY_TO_REU=1` (flags.stamp says so); main @@ -1523,6 +1800,7 @@ def test_held_session_message(prg=None, labels=None): def run(prg=None, labels=None, only=None, quiet=False): """Run TESTS (or `only`, names) on one image. Returns {name: error|None}.""" results = {} + _refuse_trace(labels) tests = TESTS + (VIEWER_TESTS if _body_to_reu(labels) else ()) for fn in tests: if only and fn.__name__ not in only: @@ -1544,6 +1822,7 @@ def main(): _labels() if not PRG.is_file(): raise Unavailable("no PRG") + _refuse_trace() except Unavailable as exc: return cannot_run(str(exc), executed=0, total=len(TESTS), certifies=CERTIFIES, opt_out_env=OPT_OUT_ENV) diff --git a/tools/check_m3_oracle.py b/tools/check_m3_oracle.py new file mode 100644 index 0000000..e41215b --- /dev/null +++ b/tools/check_m3_oracle.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Red cases for rig_https_m3.py's large-page body oracle (`timed` mode). + +The oracle excuses MediaWiki's per-render metadata between the body the +C64 fetched and a host reference fetched just before. These cases pin +that a corruption cannot widen its own excuse (adv-296 rounds 3-4): a +block that breaks a report comment's `` or the +wgPageParseReport object's closing `}` must be DIFFERENT, as must an +insertion or deletion just outside a region, a payload hidden inside one +(REGION_SLACK), and the controls around them. + +Synthetic pages with the same structure as en.wikipedia.org's, so no +fetched content is committed. No hardware, no build. + + python3 tools/check_m3_oracle.py [path/to/rig_https_m3.py] + python3 tools/check_m3_oracle.py --mutants + +The optional path checks another copy of the rig (an older oracle) with +the same cases. --mutants breaks each rule of the oracle in turn (MUTANTS) +and requires some case to go red on every one. Exit 0 all pass / all +killed, 1 otherwise. +""" +from __future__ import annotations + +import importlib.util +import sys +import tempfile +from pathlib import Path + +RIG = Path(__file__).resolve().parent / "uci" / "rig_https_m3.py" + +ARTICLE = b" ".join(b"Commodore 64 paragraph %d: the C64 is an 8-bit home " + b"computer introduced in 1982." % i for i in range(40)) + + +def page(rid, ms, host, cpu, wall, ts, pct, gen): + return (b"\n\n\n" + b"\n

" + ARTICLE + b"

\n

" + ARTICLE + + b"

\n\n" + b"\n\n\n" + b"\n" + b"

Commodore 64

\n" + b"") + + +CATEGORIES = b",".join(b"\"Home computer category %d\"" % i for i in range(30)) + +REF = page(b"4428ca7e-172e-424e-b439-7cf1d498867f", b"155", b"mw-web.codfw.a", + b"3.374", b"3.647", b"20261014002700", b"100.00% 2710.452", + b"20261014002701") +BODY = page(b"9f1c22aa-0b3d-4e11-8c0d-2a7e55e1c9b0", b"4302", b"mw-web.eqiad.b", + b"3.101", b"3.388", b"20261013120001", b"100.00% 2533.019", + b"20261013120002") + + +def flip(b, lo, hi): + """XOR 0x20 over [lo, hi), newlines kept (the line structure survives).""" + return b[:lo] + bytes(c if c == 10 else c ^ 0x20 for c in b[lo:hi]) + b[hi:] + + +def cases(): + obj_end = BODY.index(b"}},\"wgCategories") + 2 # the report object's } + c3 = BODY.index(b"-->", BODY.index(b"Transclusion")) + 3 + gap = BODY.index(b" into the gap", flip(BODY, c3 - 12, gap - 1), "DIFFERENT" + yield "3: one bit, article text far past a key", BODY[:art] + bytes([BODY[art] ^ 1]) + BODY[art + 1:], "DIFFERENT" + yield "4: one bit on a key's line, 400 B past it", BODY[:cat] + bytes([BODY[cat] ^ 1]) + BODY[cat + 1:], "DIFFERENT" + yield "5: truncated before ", BODY[:-40], "DIFFERENT" + drop = BODY.index(b"

", BODY.index(b"

")) + yield "6: an article line dropped", BODY[:drop] + BODY[BODY.index(b"\n", drop) + 1:], "DIFFERENT" + # adv-296 round 4: each pins one rule (MUTANTS names which) + yield "7: the object's closing } replaced by 200 B", BODY[:obj_end - 1] + b"q" * 200 + BODY[obj_end:], "DIFFERENT" + yield "7b: the object's last 60 B, } included, overwritten", BODY[:obj_end - 60] + b"q" * 60 + BODY[obj_end:], "DIFFERENT" + yield "8: 50 B inserted right after the object's }", BODY[:obj_end] + b"Z" * 50 + BODY[obj_end:], "DIFFERENT" + yield "9: 6 B deleted right after the object's }", BODY[:obj_end] + BODY[obj_end + 6:], "DIFFERENT" + val = BODY.index(b",", BODY.index(b"wgBackendResponseTime")) + yield "10: 'x,yy' inserted at a value's end", BODY[:val] + b"x,yy" + BODY[val:], "DIFFERENT" + newpp = BODY.index(b"CPU time usage:") + yield "11: 50 KB inside the NewPP report comment", BODY[:newpp] + b"A" * 50000 + BODY[newpp:], "DIFFERENT" + rid_end = BODY.index(b" -->") + yield "12: the --> after a Render ID value changed", BODY[:rid_end + 3] + b"!" + BODY[rid_end + 4:], "DIFFERENT" + opener = BODY.index(b" ends a key's window", + 'VALUE_END = re.compile(rb"[,}\\]<\\n]|-->")', + 'VALUE_END = re.compile(rb"[,}\\]<\\n]")'), + ("no-slack-cap", "a region grows or shrinks by at most REGION_SLACK", + " if abs((b1 - b0) - (r1 - r0)) > REGION_SLACK:\n", " if False:\n"), +] + + +def load(path: Path): + sys.path.insert(0, str(RIG.parent)) + spec = importlib.util.spec_from_file_location("rig_under_test", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def run_cases(mod, quiet=False): + red = [] + for name, body, want in cases(): + got = mod.body_oracle(body, REF)[0] + if got != want: + red.append(name) + if not quiet: + print("%s %-55s %s%s" % ("PASS" if got == want else "FAIL", name, + got, "" if got == want else " (want %s)" % want)) + return red + + +def mutants() -> int: + src = RIG.read_text() + survivors = 0 + with tempfile.TemporaryDirectory() as tmp: + for name, rule, old, new in MUTANTS: + if src.count(old) != 1: + print("BROKEN %-20s mutant text matches %d times" % (name, src.count(old))) + survivors += 1 + continue + path = Path(tmp) / ("rig_%s.py" % name.replace("-", "_")) + path.write_text(src.replace(old, new)) + red = run_cases(load(path), quiet=True) + if red: + print("KILLED %-20s %-48s by %s" % (name, rule, ", ".join(c.split(":")[0] for c in red))) + else: + survivors += 1 + print("SURVIVED %-20s %s" % (name, rule)) + print("\n%d mutants, %d killed" % (len(MUTANTS), len(MUTANTS) - survivors)) + return 1 if survivors else 0 + + +def main(argv) -> int: + if len(argv) > 1 and argv[1] == "--mutants": + return mutants() + rig = Path(argv[1]) if len(argv) > 1 else RIG + n = len(list(cases())) + red = run_cases(load(rig)) + print("\n%d/%d passed (%s)" % (n - len(red), n, rig)) + return 1 if red else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools/mutate_m3_client.py b/tools/mutate_m3_client.py index dccbd1e..9ffc660 100644 --- a/tools/mutate_m3_client.py +++ b/tools/mutate_m3_client.py @@ -234,8 +234,8 @@ " bne @p_dead_code ; the reason the block was refused\n lda #0", ["test_empty_read_reply_is_not_eof"]), ("short-not-8f", "finding 7: $8F on a block short of its header", NET, - " lda #UCI_ERR_SHORT_READ ; the block ended short of its header", - " lda #UCI_ERR_BAD_READ_HDR ; the block ended short of its header", + " lda #UCI_ERR_SHORT_READ ; short of its header: a hole (ER-5)", + " lda #UCI_ERR_BAD_READ_HDR ; short of its header: a hole (ER-5)", ["test_short_block_is_dead"]), ("tail-no-code", "finding 7: $8B on bytes past the header", NET, " lda #UCI_ERR_BAD_READ_HDR ; bytes past the header: dropped\n sta m3_bad", @@ -276,6 +276,52 @@ ["test_held_session_message"]), ("tls12-offered", "S 1.1/1.7 flags: TLS 1.3 only by default", NET, "M3_FLAGS = M3_FLAG_TLS13_ONLY", "M3_FLAGS = 0", ["test_open_layout"]), + # read pacing (m3.inc): fence every write, no read; the fast copy loop + ("copy-read-fenced", "no fence after a READ (copy loop)", NET, + " lda UCI_RESP_DATA\n@p_store:\n sta $FFFF ; SMC: tcp_recv_buf + tail, advanced here", + " lda UCI_RESP_DATA\n uci_fence\n@p_store:\n sta $FFFF", + ["test_read_pacing"]), + ("status-read-fenced", "no fence after a READ (status polls)", CMD, + ".ifdef M3_READ_FENCE\n jmp m3_settle\n.else\n rts\n.endif", + ".ifdef M3_READ_FENCE\n jmp m3_settle\n.else\n jmp m3_settle\n.endif", + ["test_read_pacing"]), + ("put-unfenced", "a fence after every command-byte WRITE", CMD, + "m3_put:\n sta UCI_CMD_DATA\n jmp m3_settle", + "m3_put:\n sta UCI_CMD_DATA\n rts", ["test_read_pacing"]), + ("push-unfenced", "a fence after the PUSH WRITE", CMD, + " jsr m3_tr_sample\n.endif\n jsr m3_settle\n", + " jsr m3_tr_sample\n.endif\n", ["test_read_pacing"]), + ("accept-unfenced", "a fence after the data-accept WRITE", CMD, + " lda #UCI_CTRL_DATA_ACC\n sta UCI_CONTROL\n jmp m3_settle", + " lda #UCI_CTRL_DATA_ACC\n sta UCI_CONTROL\n rts", + ["test_read_pacing"]), + ("copy-blind", "ER-12: DATA_AV before every body byte", NET, + " lda UCI_STATUS ; ER-12: DATA_AV before EVERY byte, or a\n" + " bpl @p_short ; short block reads as $00s (bit 7)\n", "", + ["test_short_block_is_dead"]), + ("copy-no-wrap", "the copy wraps at the ring's end", NET, + " cmp #>(tcp_recv_buf + TCP_RECV_MASK + 1)", " cmp #$D1", + ["test_ring_wraps_in_one_block"]), + ("copy-count-pages", "the copy runs exactly n times", NET, + " txa\n beq :+\n iny\n:\n", "", + ["test_http_content_length_end_to_end"]), + ("copy-tail-stale", "the ring tail follows the copy", NET, + " lda @p_store+1 ; the tail is where the store points\n" + " sta tcp_recv_tail+0\n", "", + ["test_http_content_length_end_to_end", "test_ring_wraps_in_one_block"]), + # adv-296 + ("count-always-iny", "the copy runs exactly n times at n = k*256", NET, + " txa\n beq :+\n iny\n:\n clc\n lda tcp_recv_tail+0", + " iny\n clc\n lda tcp_recv_tail+0", + ["test_copy_counts_at_page_edges"]), + ("rejected-clr-unfenced", "a fence after the rejected path's CLR_ERR write", CMD, + " lda #UCI_CTRL_CLR_ERR\n sta UCI_CONTROL\n jsr m3_settle\n lda #M3_EXEC_REJECTED", + " lda #UCI_CTRL_CLR_ERR\n sta UCI_CONTROL\n lda #M3_EXEC_REJECTED", + ["test_read_pacing"]), + ("statline-nopsled", "no delay of any shape after a READ", CMD, + " lda UCI_STATUS_DATA\n.ifdef M3_READ_FENCE\n jsr m3_settle\n.endif\n", + " lda UCI_STATUS_DATA\n.repeat 40\n nop\n.endrep\n", + ["test_read_pacing"]), ] @@ -295,6 +341,13 @@ def _stamp(): # and the rules here are the plain image's. raise SystemExit("build/ is an HTTPS_BODY_TO_REU image: the mutants " "are written against plain `make BACKEND=uci-m3`") + for knob in ("M3_READ_FENCE", "M3_TRACE"): + # The pacing mutants target the default image's code; the fenced + # copy loop is not assembled there, and the trace moves timings. + if "-D " + knob in vals.get("CA65FLAGS", ""): + raise SystemExit("build/ is a %s=1 image: the mutants are " + "written against plain `make BACKEND=uci-m3`" + % knob) return vals diff --git a/tools/package/build_m3_demo.sh b/tools/package/build_m3_demo.sh index 30d3fc0..1ad22d1 100755 --- a/tools/package/build_m3_demo.sh +++ b/tools/package/build_m3_demo.sh @@ -163,9 +163,10 @@ What it needs - Its WiFi module up (the ESP32 does the TLS), the network connected, and the clock set by SNTP (certificate dates are checked). - The Ultimate's Command Interface enabled. - - No REU. Turbo is optional but recommended: the client paces every UCI - register access for the FPGA, so large pages are slow at 1 MHz (not - measured there; see the measured 48 MHz times below). + - No REU. Turbo is optional. The same client source, in its REU-sink + build, fetched en.wikipedia.org /robots.txt (28,275 B) in 16 s from G + at 1 MHz (turbo off) and in 6 s at 48 MHz (U64E, 2026-10-13); large + pages were not timed at 1 MHz. Interface implemented: $SPEC. How to run it @@ -197,10 +198,10 @@ Three URLs to try Measured with THIS image (PRG sha256 above), validated on an Ultimate 64 Elite (U64E), firmware 3.15 git 3a1ff9ff, 48 MHz, REU disabled, the .d64 mounted on drive 8 and loaded with LOAD"*",8,1 -(tools/uci/rig_https_m3_demo.py), 2026-10-06: - 1. HTTP 200, 751,848 B, complete by its chunked framing, 207 s from G to +(tools/uci/rig_https_m3_demo.py), 2026-10-13: + 1. HTTP 200, 751,823 B consumed == Content-Length, 18 s from G to CONNECTION CLOSED (the page size changes as the article is edited). - 2. HTTP 200, 16,739 B consumed == Content-Length, 9 s. + 2. HTTP 200, 16,739 B consumed == Content-Length, 4 s. 3. Refused: 94,CERTIFICATE NAME MISMATCH: 0x00000004. Other Ultimate models, and other clock speeds, have not been tested. diff --git a/tools/uci/rig_https_m3.py b/tools/uci/rig_https_m3.py index 1d21ea9..a034c08 100644 --- a/tools/uci/rig_https_m3.py +++ b/tools/uci/rig_https_m3.py @@ -18,6 +18,24 @@ CRC-32 of the bytes both keep in http_resp_buf (the whole body when it is <= 512 B, else its first 512 B) must match. + timed SCHEDULE NAME=DIR [NAME=DIR ...] + Timed A/B of M3 builds (each DIR holds c64-https.prg, + labels.txt and flags.stamp of a `make BACKEND=uci-m3 + HTTPS_BODY_TO_REU=1` link), e.g. M3_READ_FENCE=1 vs the + default. SCHEDULE is PAGE:NAMES[,PAGE:NAMES], e.g. + small:FSSFFS,large:FSSFFS (ABBA order); PAGE is small + (en.wikipedia.org /robots.txt) or large (/wiki/ + Commodore_64). Per attempt: reset, verified load, 'I', + then the clock starts at 'G' and stops when the REU + viewer opens (the whole body sunk); 'Q' closes; the + body is read back out of REU $10:0000 and compared with + a host fetch made just before (curl, the client's UA). + A dynamic page that differs only in per-render metadata + lines is reported as such, never as equal. An M3_TRACE + build's per-net_poll records are read before 'Q' and + summarised. TURBO_OFF=1 runs with turbo off (1 MHz). + One JSON line per attempt to $UCI_DEBUG_DIR// + timed.jsonl. Exit 1 if any attempt is not complete. Both drive the MENU the way an operator does ('I', 'G', the typed target), take the harness DeviceLock, set 48 MHz and the REU through @@ -27,13 +45,19 @@ U64_HOST=10.43.23.81 tools/uci/rig_https_m3.py refuse wrong.host.badssl.com 94 U64_HOST=10.43.23.81 tools/uci/rig_https_m3.py ab en.wikipedia.org / B.prg B.labels + U64_HOST=10.43.23.81 tools/uci/rig_https_m3.py timed small:FSSF F=bf S=bs Exit: 0 pass, 1 fail, 2 fatal, 3 DeviceLock timeout, 4 device prep / load. """ from __future__ import annotations +import bisect +import difflib +import hashlib +import json import os import re +import subprocess import sys import time import zlib @@ -60,6 +84,7 @@ PRG_A = REPO / "build" / "c64-https.prg" LABELS_A = REPO / "build" / "labels.txt" TURBO_MHZ = int(os.environ.get("TURBO_MHZ", "48")) +TURBO_OFF = os.environ.get("TURBO_OFF") == "1" # timed: prep, then turbo off INIT_WAIT = float(os.environ.get("C64_INIT_WAIT", "120")) FETCH_TIMEOUT = float(os.environ.get("FETCH_TIMEOUT", "900")) RUN_DIR = Path(os.environ.get("UCI_DEBUG_DIR", "/tmp/uci_https_debug")) @@ -234,15 +259,380 @@ def verdict_ab(rows) -> None: print(f"PASS: same status, same length ({ta:,} B), same CRC-32 over {whole}") +# --- timed ------------------------------------------------------------------ + +PAGES = {"small": ("en.wikipedia.org", "/robots.txt"), + "large": ("en.wikipedia.org", "/wiki/Commodore_64")} +PAGE_BUDGET = {"small": 900.0, "large": 3600.0} +UA = "c64-https/0.1" # src/http.s's User-Agent +#: The REU viewer's status line: OOOOOO/TTTTTT (decode_screen lowercases). +VIEWER = re.compile(r"^\s*[0-9a-f]{6}/[0-9a-f]{6}", re.I) +REU_BODY = 0x100000 # HTTPS_BODY_TO_REU's region +#: After a reset (BASIC idle, no program), REU -> C64 staging for readback. +STAGE = 0x0800 +STAGE_LEN = 0x8000 +#: MediaWiki per-render values: the value after one of these keys is an +#: excusable region (report_spans, key_window_end). +RENDER_KEYS = re.compile( + rb"wgRequestId|wgBackendResponseTime|wgHostname|wgPageParseReport|" + rb"cputime|walltime|timestamp|\"ttl\"|transientcontent|cachereport|" + rb"limitreport|timingprofile|ppvisitednodes|postexpandincludesize|" + rb"Parsed by|Cached time|Cache expiry|CPU time usage|Real time usage|" + rb"Saved in parser cache|Render ID|Transclusion expansion time report|" + rb"Rendering was triggered") +TOKEN = re.compile(rb"\w+|\W") + + +def host_reference(host: str, path: str) -> bytes | None: + r = subprocess.run(["curl", "-s", "--http1.1", "-A", UA, "--max-time", + "60", f"https://{host}{path}"], capture_output=True) + return r.stdout if r.returncode == 0 else None + + +def reu_read(client, off: int, n: int) -> bytes: + """REU [off, off+n) through STAGE, a chunk at a time (C64 reset first).""" + out = bytearray() + for o in range(0, n, STAGE_LEN): + k = min(STAGE_LEN, n - o) + a = off + o + client.write_mem(0xDF02, bytes([STAGE & 0xFF, STAGE >> 8, a & 0xFF, + (a >> 8) & 0xFF, (a >> 16) & 0xFF, + k & 0xFF, (k >> 8) & 0xFF, 0, 0])) + client.write_mem(0xDF01, bytes([0x91])) # execute, REU -> C64 + time.sleep(0.05) + out += bytes(client.read_mem(STAGE, k)) + return bytes(out) + + +#: MediaWiki's parser reports: HTML comments re-written on every render. +REPORT_COMMENT = re.compile(rb").)*?(?:NewPP limit report|" + rb"Transclusion expansion time report|" + rb"Saved in parser cache|cache key).*?-->", re.S) +PARSE_REPORT = re.compile(rb'"wgPageParseReport":\{') +KEY_WINDOW = 120 +VALUE_END = re.compile(rb"[,}\]<\n]|-->") +#: A region's body side may differ in length from its reference side by at +#: most this. Measured over the 33 complete saved large bodies against their +#: references (2026-10-13): every region was 0 or +1 B longer in the body. +#: Re-render values are numbers, timestamps, ids and host names whose +#: lengths move by a few bytes each; 64 B is generous for that and still +#: stops a payload hidden inside a delimited region. +REGION_SLACK = 64 + + +def key_window_end(page: bytes, at: int) -> int: + """End of the value that starts at `at`: the first , } ] < newline or + -->, and never more than KEY_WINDOW bytes on.""" + m = VALUE_END.search(page, at, at + KEY_WINDOW) + return m.start() if m else min(at + KEY_WINDOW, len(page)) + + +def report_spans(page: bytes) -> list[tuple[int, int, int, int]]: + """Excusable regions of `page`: (start, end, opening delimiter length, + closing delimiter length). The REPORT_COMMENT comments (), + the "wgPageParseReport":{...} object (braces matched), and the + value after each RENDER_KEYS key, up to the first , } ] < newline or --> + and at most KEY_WINDOW bytes (the key is the opening delimiter; a window + has no closing one).""" + out = [(m.start(), m.end(), 4, 3) for m in REPORT_COMMENT.finditer(page)] + for m in PARSE_REPORT.finditer(page): + depth, k = 0, m.end() - 1 + while k < len(page): + depth += {123: 1, 125: -1}.get(page[k], 0) + k += 1 + if depth == 0: + break + out.append((m.start(), k, m.end() - m.start(), 1)) + for m in RENDER_KEYS.finditer(page): + out.append((m.start(), key_window_end(page, m.end()), + m.end() - m.start(), 0)) + return out + + +def body_oracle(body: bytes, ref: bytes | None) -> tuple[str, dict]: + """("equal" | "metadata-only" | "DIFFERENT" | "no-reference", facts). + + A line diff, then a token diff inside each changed line pair, gives the + changed runs and an alignment of the bytes the two share. The excusable + regions are taken from the REFERENCE (report_spans(ref)), never from + the body under test, and a region counts only if its delimiters are + unchanged bytes of that alignment, which also places it in the body. + "metadata-only" = every changed run lies wholly inside one such region + on BOTH sides (ref extent inside the ref region, body extent inside the + same region as mapped into the body), the region's body side is within + REGION_SLACK bytes of its reference length, and no line was added or + dropped on its own. tools/check_m3_oracle.py pins each of these rules + (its --mutants mode breaks them one at a time). So a corruption cannot widen its own excuse: breaking a + `-->` or the report's closing `}` voids that region. What stays + accepted is a change inside a region whose delimiters survived. + The body's own length is pinned separately, by the response framing.""" + facts = {"sha256": hashlib.sha256(body).hexdigest(), + "crc32": f"{zlib.crc32(body) & 0xFFFFFFFF:08X}", "len": len(body)} + if ref is None: + return "no-reference", facts + facts.update(ref_len=len(ref), ref_sha256=hashlib.sha256(ref).hexdigest()) + if body == ref: + return "equal", facts + a, b = body.split(b"\n"), ref.split(b"\n") + sa, sb = [0], [0] + for ln in a: + sa.append(sa[-1] + len(ln) + 1) + for ln in b: + sb.append(sb[-1] + len(ln) + 1) + same = [] # (ref_lo, ref_hi, body_lo): bytes the two share + changed = [] # (ref_lo, ref_hi, body_lo, body_hi, text) + unexplained = [] + for tag, i1, i2, j1, j2 in difflib.SequenceMatcher( + None, a, b, autojunk=False).get_opcodes(): + if tag == "equal": + same.append((sb[j1], sb[j2], sa[i1])) + continue + if tag != "replace" or i2 - i1 != j2 - j1: + unexplained.append(f"lines {i1}-{i2} vs {j1}-{j2}: {tag}") + continue + for li, lj in zip(range(i1, i2), range(j1, j2)): + ta, tb = TOKEN.findall(a[li]), TOKEN.findall(b[lj]) + pa, pb = [sa[li]], [sb[lj]] + for t in ta: + pa.append(pa[-1] + len(t)) + for t in tb: + pb.append(pb[-1] + len(t)) + for t2, k1, k2, m1, m2 in difflib.SequenceMatcher( + None, ta, tb, autojunk=False).get_opcodes(): + if t2 == "equal": + same.append((pb[m1], pb[m2], pa[k1])) + else: + changed.append((pb[m1], pb[m2], pa[k1], pa[k2], + a[li][pa[k1] - sa[li]:pa[k1] - sa[li] + 60])) + same.append((sb[lj + 1] - 1, sb[lj + 1], sa[li + 1] - 1)) # "\n" + same.sort() + starts = [x[0] for x in same] + + def to_body(lo, hi): + """Body offset of ref [lo, hi) if it is one shared stretch, else None.""" + k = bisect.bisect_right(starts, lo) - 1 + if k >= 0 and same[k][0] <= lo and hi <= same[k][1]: + return same[k][2] + (lo - same[k][0]) + return None + + regions = [] # (ref_lo, ref_hi, body_lo, body_hi) + for r0, r1, d_open, d_close in report_spans(ref): + b0 = to_body(r0, r0 + d_open) + if b0 is None: + continue + if d_close: + bc = to_body(r1 - d_close, r1) + if bc is None: + continue + b1 = bc + d_close + else: # a key window, in the body too + b1 = key_window_end(body, b0 + d_open) + if abs((b1 - b0) - (r1 - r0)) > REGION_SLACK: + continue # grew or shrank past a re-render + regions.append((r0, r1, b0, b1)) + for r0, r1, b0, b1, text in changed: + if not any(g0 <= r0 and r1 <= g1 and h0 <= b0 and b1 <= h1 + for g0, g1, h0, h1 in regions): + unexplained.append(text.decode("latin-1")) + facts.update(diff_runs=len(changed), unexplained=len(unexplained), + first_unexplained=unexplained[:3]) + return ("metadata-only" if changed and not unexplained else "DIFFERENT"), facts + + +def read_trace(client, lab) -> dict | None: + """An M3_TRACE build's records (src/net/uci-m3/m3.inc, "Timing trace").""" + if "m3_tr_count" not in lab: + return None + unit = 10227 # M3_UNIT_CYCLES + n = le(client, lab["m3_tr_n"], 2) + raw = bytes(client.read_mem(0x6000, n * 12)) if n else b"" + + def ticks(b, o): + dtb = b[o] | b[o + 1] << 8 + dta = b[o + 2] | b[o + 3] << 8 + return dtb * unit + (dta - 0x10000 if dta & 0x8000 else dta) + recs = [] + for i in range(n): + r = raw[12 * i:12 * i + 12] + recs.append({"rtt": ticks(r, 0) if any(r[0:4]) else None, + "total": ticks(r, 4), "bytes": r[8] | r[9] << 8, + "result": r[10], "post": r[11]}) + return {"count": le(client, lab["m3_tr_count"], 2), "kept": n, + "pushes": le(client, lab["m3_tr_pushes"], 2), + "bit3": le(client, lab["m3_tr_bit3"], 2), + "or": bytes(client.read_mem(lab["m3_tr_or"], 1))[0], + "records": recs} + + +def _dist(xs): + xs = sorted(xs) + if not xs: + return None + return {"n": len(xs), "min": xs[0], "median": xs[len(xs) // 2], + "max": xs[-1]} + + +def summarise_trace(tr: dict) -> dict: + """CIA ticks are PHI2 cycles at the real bus clock (~1.0227 MHz NTSC): + ms = ticks / 1022.7. Distributions over READs that returned data, over + idle READs, and per byte.""" + ms = lambda t: round(t / 1022.7, 3) # noqa: E731 + data = [r for r in tr["records"] if r["result"] == 0 and r["rtt"] is not None] + idle = [r for r in tr["records"] if r["result"] == 1 and r["rtt"] is not None] + return { + "net_poll_calls": tr["count"], "records_kept": tr["kept"], + "pushes": tr["pushes"], "pushes_with_bit3": tr["bit3"], + "post_push_status_or": f"${tr['or']:02X}", + "post_push_status_values": sorted({f"${r['post']:02X}" for r in tr["records"] if r["rtt"] is not None}), + "data_reads": len(data), "idle_reads": len(idle), + "data_rtt_ms": _dist([ms(r["rtt"]) for r in data]), + "data_total_ms": _dist([ms(r["total"]) for r in data]), + "idle_rtt_ms": _dist([ms(r["rtt"]) for r in idle]), + "bytes_per_data_read": _dist([r["bytes"] for r in data]), + "full_893_reads": sum(1 for r in data if r["bytes"] == 893), + } + + +def timed_attempt(client, name, bdir: Path, page: str, run_dir: Path) -> dict: + lab = labels(bdir / "labels.txt") + host, path = PAGES[page] + rec = {"arm": name, "dir": str(bdir), "page": page, "host": host, + "path": path, "start": time.strftime("%Y-%m-%dT%H:%M:%S"), + "prg_sha256": hashlib.sha256((bdir / "c64-https.prg").read_bytes()).hexdigest()} + ref = host_reference(host, path) + client.reset() + time.sleep(2.5) + load_verified_and_run(client, (bdir / "c64-https.prg").read_bytes()) + m, lines = wait_for(client, ["Q=QUIT"], INIT_WAIT) + if not m: + dump(lines, "boot") + raise Fail(f"{name}: menu never appeared") + press_init(client) + t0 = time.monotonic() + client.send_text("G", finish_with_return=False) + push_keys(client, target_keys(f"{host}\r{path}\r")) + t_hs = None + end = t0 + PAGE_BUDGET[page] + while True: + lines, text = screen(client) + now = time.monotonic() + if t_hs is None and "TLS HANDSHAKE OK" in text: + t_hs = now + if VIEWER.match(lines[0]): + outcome = "VIEWER" + break + hit = [e for e in ENDS + ["BODY INCOMPLETE"] if e in text] + if hit: + outcome = hit[0] + break + if now > end: + outcome = "TIMEOUT" + break + time.sleep(0.5) + rec.update(outcome=outcome, t_body=round(time.monotonic() - t0, 2), + t_handshake=round(t_hs - t0, 2) if t_hs else None) + rec["http_status"] = le(client, lab["http_status"], 2) + rec["body_total"] = le(client, lab["http_body_total"], 3) + rec["net_last_error"] = f"${bytes(client.read_mem(lab['net_last_error'], 1))[0]:02X}" + tr = read_trace(client, lab) + if tr is not None: + (run_dir / f"trace-{name}-{page}-{rec['start']}.json").write_text(json.dumps(tr)) + rec["trace"] = summarise_trace(tr) + rec["screen_at_end"] = [x for x in lines if x.strip()] + if outcome != "VIEWER": + dump(lines, f"{name} final screen") + rec["verdict"] = f"INCOMPLETE: {outcome}" + if outcome not in SETTLED: + guard_socket_teardown(client.read_mem, lab["net_tcp_state"], + nudge=lambda: client.send_text("Q", finish_with_return=False)) + return rec + client.send_text("Q", finish_with_return=False) + m, lines = wait_for(client, ["CONNECTION CLOSED"], 60) + if not m: + dump(lines, f"{name} after Q") + guard_socket_teardown(client.read_mem, lab["net_tcp_state"]) + raise Fail(f"{name}: no CONNECTION CLOSED after Q") + client.reset() # the socket is closed: safe + time.sleep(3.0) + body = reu_read(client, REU_BODY, rec["body_total"]) if rec["body_total"] else b"" + (run_dir / f"body-{name}-{page}-{rec['start']}.bin").write_bytes(body) + if ref is not None: + (run_dir / f"ref-{page}-{rec['start']}.bin").write_bytes(ref) + oracle, facts = body_oracle(body, ref) + rec.update(oracle=oracle, body=facts) + ok = rec["http_status"] == 200 and oracle in ("equal", "metadata-only") + rec["verdict"] = "PASS" if ok else "FAIL" + return rec + + +def run_timed(client, sched, arms, run_dir: Path) -> int: + out = run_dir / "timed.jsonl" + bad = 0 + for page, name in sched: + print(f"=== {name} {page} {time.strftime('%H:%M:%S')}", flush=True) + try: + rec = timed_attempt(client, name, arms[name], page, run_dir) + except BaseException: + # #234: never leave this attempt's socket live behind the lock. + lab = labels(arms[name] / "labels.txt") + guard_socket_teardown(client.read_mem, lab["net_tcp_state"]) + raise + rec["turbo_mhz"] = 1 if TURBO_OFF else TURBO_MHZ + with out.open("a") as f: + f.write(json.dumps(rec) + "\n") + short = {k: rec.get(k) for k in ("arm", "page", "verdict", "oracle", + "t_handshake", "t_body", "http_status", "body_total", + "net_last_error")} + print(" " + json.dumps(short), flush=True) + if "trace" in rec: + print(" trace: " + json.dumps(rec["trace"]), flush=True) + if rec["verdict"] != "PASS": + bad += 1 + if rec["verdict"].startswith("INCOMPLETE"): + print("stopping: an incomplete fetch leaves the device state unknown") + break + print(f"results: {out}") + return 1 if bad else 0 + + +def parse_timed(argv): + sched = [(page, n) for item in argv[2].split(",") + for page, names in [item.split(":")] for n in names] + arms = {} + for a in argv[3:]: + n, d = a.split("=", 1) + arms[n] = Path(d) + for page, n in sched: + if page not in PAGES or n not in arms: + raise ValueError(f"schedule names {page}:{n}, not a page/arm") + for n, d in arms.items(): + lab = labels(d / "labels.txt") + if "m3_wedged" not in lab or "http_sink_full" not in lab: + raise ValueError(f"{d} is not a BACKEND=uci-m3 HTTPS_BODY_TO_REU=1 build") + return sched, arms + + def main(argv) -> int: - if (len(argv) < 2 or argv[1] not in ("refuse", "ab") + timed = len(argv) >= 4 and argv[1] == "timed" + if timed: + try: + sched, arms = parse_timed(argv) + except (ValueError, OSError) as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 2 + prep_labels = next(iter(arms.values())) / "labels.txt" + elif (len(argv) < 2 or argv[1] not in ("refuse", "ab") or len(argv) != {"refuse": 4, "ab": 6}[argv[1]]): print(__doc__) return 2 - if not PRG_A.is_file() or "m3_wedged" not in labels(LABELS_A): + else: + prep_labels = LABELS_A + if not timed and (not PRG_A.is_file() or "m3_wedged" not in labels(LABELS_A)): print("[fatal] build/ is not a BACKEND=uci-m3 build", file=sys.stderr) return 2 - if argv[1] == "refuse": + if timed: + runs = [] + elif argv[1] == "refuse": host, path = argv[2], "/" runs = [("A (uci-m3)", PRG_A, LABELS_A)] else: @@ -273,21 +663,44 @@ def main(argv) -> int: client = Ultimate64Client(host=HOST, timeout=20.0) info = client.get_info() print(f"device: {info.get('product')} fw {info.get('firmware_version')} " - f"git {info.get('git_commit_hash')} core {info.get('core_version')}") + f"git {info.get('git_commit_hash')} fpga {info.get('fpga_version')} " + f"core {info.get('core_version')}") + want_git = os.environ.get("M3_EXPECT_GIT") + if want_git and info.get("git_commit_hash") != want_git: + print(f"[fatal] firmware git {info.get('git_commit_hash')}, " + f"expected {want_git} (M3_EXPECT_GIT): not benching", + file=sys.stderr) + return 2 enable_uci(client) run_dir = _create_run_dir(RUN_DIR) print(f"run dir: {run_dir} (device_state.json)") try: - prepare_device(client, LABELS_A, turbo_mhz=TURBO_MHZ, + prepare_device(client, prep_labels, turbo_mhz=TURBO_MHZ, artifact_dir=run_dir) except DevicePrepError as exc: print(str(exc), file=sys.stderr) return 4 + if timed and TURBO_OFF: # stock 1 MHz: turbo off + from c64_test_harness.backends.ultimate64_helpers import set_turbo_mhz + set_turbo_mhz(client, None) + time.sleep(3.0) + print("turbo: " + str(client.get_config_value( + "U64 Specific Settings", "Turbo Control"))) try: - preflight_reu(client, LABELS_A) # the backstop behind the prep + preflight_reu(client, prep_labels) # the backstop behind the prep except ReuPreflightError as exc: print(str(exc), file=sys.stderr) return 4 + if timed: + (run_dir / "device_info.json").write_text(json.dumps(info)) + try: + return run_timed(client, sched, arms, run_dir) + except PrgLoadError as exc: + print(f"[fatal] {exc}", file=sys.stderr) + return 4 + except Fail as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 rows = [] try: for name, prg, lab_path in runs: