diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 3034092a9..baed3f6b4 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -286,6 +286,12 @@ jobs: - name: contracts-net-loss env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1 contracts: true + # an official stage-1 hash race: with round_elapse_time 10 s a slow round under loss can + # trigger a bootstrap of the full table, which then never completes. Intermittent (it + # stalled 1 run of 3 at 10 s, none at 3000 blocks in run 39878938046): run by name, repeated + - name: bootstrap-livelock + env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 + manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000 @@ -302,7 +308,8 @@ jobs: # a manual run can pick scenarios; the others stop at once env: # the scenarios with a NODE_STORAGE list are laid out for 7 nodes - SELECTED: ${{ (inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }} + # manual scenarios (known failures kept as reproductions) run only when named in `only` + SELECTED: ${{ ((inputs.only == '' && matrix.manual != true) || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }} steps: - name: Skip unselected scenario if: env.SELECTED != 'true' diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index ba5d4c1e8..37ebb15d5 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -9,6 +9,7 @@ # WORK (default ./testnet-run) # TRUSTED_NODES (default NODES): only the first TRUSTED_NODES nodes are initial confidants # (odd, 5 to 25); the others, up to 61 nodes in all, are ordinary nodes +# ROUND_ELAPSE_MS (default 20000): round_elapse_time of every node # SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU # of the nodes and executors in resources.tsv, and prints a summary at the end # FUND (default 0; needs a -DCREDITS_TESTNET=ON binary): genesis funds go to a generated master @@ -138,6 +139,11 @@ CONTRACT_LOAD_SECONDS=${CONTRACT_LOAD_SECONDS:-0} CONTRACT_WRITE_RATE=${CONTRACT_WRITE_RATE:-1} CONTRACT_READ_RATE=${CONTRACT_READ_RATE:-5} SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} +# a node starts a bootstrap when its round lasts this long. The consensus's own fallback for a +# silent trusted node takes about 12.5 s, so below that a slow round is pre-empted by a bootstrap, +# and a bootstrap of a full table re-sent every 10 s never completes (official stage-1 hash race): +# 20 s by default (mainnet default 60 s); bootstrap-livelock keeps 10 s to reproduce the race +ROUND_ELAPSE_MS=${ROUND_ELAPSE_MS:-20000} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} RESTART_AT=${RESTART_AT:-300} @@ -358,7 +364,7 @@ ipv6=false traverse_nat=false min_neighbours=1 max_neighbours=16 -round_elapse_time=10000 +round_elapse_time=$ROUND_ELAPSE_MS [host_input] ip=127.0.0.1 @@ -1209,6 +1215,11 @@ if funded and 1000 in seen and max(w for _, w in seen[1000].values()) <= 2: ok = False sys.exit(0 if ok else 1) PY +# a network still restarting at the end is stuck, even if it reached the target before (a bootstrap +# round, R-x.1 and up, that never gives way to R-(x+1).0): the last three round headers of n1 +stuck=$(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -3 | grep -vc '\.0 ' || true) +[ "$stuck" -lt 3 ] || { echo "FAIL: the network ended stuck in a bootstrap round ($(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -1 | sed -E 's/.*(R-[^ ]*).*/\1/'))"; exit 1; } + # who took part in consensus: trusted per round (from n1's round headers) and how many nodes were # trusted at least once (a node logs TRUSTED [index] in the rounds it is trusted in) per_round=$(grep -ah 'Trusted count:' "$WORK/n1/node.log" | sed -E 's/.*Trusted count: ([0-9]+).*/\1/' | sort -n | uniq -c \ diff --git a/docs/CHANGES.md b/docs/CHANGES.md index e0203040a..8e886809a 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -43,6 +43,7 @@ lines under the new release. | #73 | api: the executor version check asks every second up to `executor_check_version_delay` instead of once after it (the node starts as soon as its executor answers); with `executor_multi_instance` a stopping node no longer kills the other nodes' executors | | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | | #75 | ci(testnet): the contract load window starts after funding its writers, block rates from the first block (found in the slowdown analysis of the scale runs) | +| #76 | ci(testnet): round_elapse_time 20 s by default (10 s pre-empted slow rounds with a bootstrap); a run that ends stuck in a bootstrap round fails; bootstrap-livelock exercises an official stage-1 hash race (intermittent; run by name) | ## community.9 — 2026-10-22