From af328a907e9c21d8b9762c1221f4df860123f83e Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 02:34:33 +0200 Subject: [PATCH] ci(testnet): round_elapse_time 20 s, fail a run stuck in a bootstrap, a bootstrap-livelock reproduction contracts-net-loss stalled at block 831 for over an hour (run 39866725719). Fable's analysis: a round slowed by the loss lasted more than round_elapse_time (10 s here), a bootstrap of the full 7-confidant table started, and it can never complete: each node sends its stage-1 hash as soon as it takes the table, a hash that reaches a node before that node took the table is dropped (same round number), the last node never gets 3 hashes, and the consensus's fallback for a silent node (about 12.5 s) is longer than the 10 s after which the table is sent again. Official behaviour, not caused by #70/#71; with mainnet's 60 s default the round ends after the fallback instead. - run.sh: ROUND_ELAPSE_MS (default 20000, was a fixed 10000) - a run fails if n1's last three round headers are bootstrap sub-rounds (a stall after the target used to pass: run 39846198921 ended stuck at R-1'192.1) - robustness: bootstrap-livelock (loss on n3 and n5, 10 s, target 3000) keeps the reproduction; it runs only when named in `only` (manual scenarios), until the node keeps those hashes Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 9 ++++++++- ci/testnet/run.sh | 13 ++++++++++++- docs/CHANGES.md | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 3034092a9..5609b6f22 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -286,6 +286,12 @@ jobs: - name: contracts-net-loss env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1 contracts: true + # reproduction of an official stage-1 hash race: with round_elapse_time 10 s a slow round + # under loss triggers a bootstrap of the full table, which never completes; fails until the + # node keeps the hashes of a same-round bootstrap (manual: run by name) + - name: bootstrap-livelock + env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 + manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000 @@ -302,7 +308,8 @@ jobs: # a manual run can pick scenarios; the others stop at once env: # the scenarios with a NODE_STORAGE list are laid out for 7 nodes - SELECTED: ${{ (inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }} + # manual scenarios (known failures kept as reproductions) run only when named in `only` + SELECTED: ${{ ((inputs.only == '' && matrix.manual != true) || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }} steps: - name: Skip unselected scenario if: env.SELECTED != 'true' diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index ba5d4c1e8..37ebb15d5 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -9,6 +9,7 @@ # WORK (default ./testnet-run) # TRUSTED_NODES (default NODES): only the first TRUSTED_NODES nodes are initial confidants # (odd, 5 to 25); the others, up to 61 nodes in all, are ordinary nodes +# ROUND_ELAPSE_MS (default 20000): round_elapse_time of every node # SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU # of the nodes and executors in resources.tsv, and prints a summary at the end # FUND (default 0; needs a -DCREDITS_TESTNET=ON binary): genesis funds go to a generated master @@ -138,6 +139,11 @@ CONTRACT_LOAD_SECONDS=${CONTRACT_LOAD_SECONDS:-0} CONTRACT_WRITE_RATE=${CONTRACT_WRITE_RATE:-1} CONTRACT_READ_RATE=${CONTRACT_READ_RATE:-5} SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} +# a node starts a bootstrap when its round lasts this long. The consensus's own fallback for a +# silent trusted node takes about 12.5 s, so below that a slow round is pre-empted by a bootstrap, +# and a bootstrap of a full table re-sent every 10 s never completes (official stage-1 hash race): +# 20 s by default (mainnet default 60 s); bootstrap-livelock keeps 10 s to reproduce the race +ROUND_ELAPSE_MS=${ROUND_ELAPSE_MS:-20000} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} RESTART_AT=${RESTART_AT:-300} @@ -358,7 +364,7 @@ ipv6=false traverse_nat=false min_neighbours=1 max_neighbours=16 -round_elapse_time=10000 +round_elapse_time=$ROUND_ELAPSE_MS [host_input] ip=127.0.0.1 @@ -1209,6 +1215,11 @@ if funded and 1000 in seen and max(w for _, w in seen[1000].values()) <= 2: ok = False sys.exit(0 if ok else 1) PY +# a network still restarting at the end is stuck, even if it reached the target before (a bootstrap +# round, R-x.1 and up, that never gives way to R-(x+1).0): the last three round headers of n1 +stuck=$(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -3 | grep -vc '\.0 ' || true) +[ "$stuck" -lt 3 ] || { echo "FAIL: the network ended stuck in a bootstrap round ($(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -1 | sed -E 's/.*(R-[^ ]*).*/\1/'))"; exit 1; } + # who took part in consensus: trusted per round (from n1's round headers) and how many nodes were # trusted at least once (a node logs TRUSTED [index] in the rounds it is trusted in) per_round=$(grep -ah 'Trusted count:' "$WORK/n1/node.log" | sed -E 's/.*Trusted count: ([0-9]+).*/\1/' | sort -n | uniq -c \ diff --git a/docs/CHANGES.md b/docs/CHANGES.md index e6730bd8c..81a3c9968 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -42,6 +42,7 @@ lines under the new release. | #72 | ci: contract reads on every node and a contract load phase; networks of up to 61 nodes with 5 to 25 initial confidants, stake-based election and nodes below the minimum stake; a scale workflow | | #73 | api: the executor version check asks every second up to `executor_check_version_delay` instead of once after it (the node starts as soon as its executor answers); with `executor_multi_instance` a stopping node no longer kills the other nodes' executors | | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | +| #76 | ci(testnet): round_elapse_time 20 s by default (10 s pre-empted slow rounds with a bootstrap); a run that ends stuck in a bootstrap round fails; bootstrap-livelock reproduces an official stage-1 hash race (run by name) | ## community.9 — 2026-10-22