From 306836d532c263af20c85fd3ad2159e1447b0422 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 05:44:58 +0200 Subject: [PATCH 1/3] ci(testnet): pause a trusted node to reproduce the bootstrap livelock; debug logs on chosen nodes Fable's analysis of the bootstrap-livelock stall with #77 (run 39886642406): the cause is not the stage-1 hash race but timing. The starter re-sends its bootstrap table every round_elapse_time and every same-round table restarts the consensus on each node; a round with one late stage message needs the fallbacks (12.5-16.5 s), so with 10 s it never completes. Official behaviour (master showed the same stall, run 39866725719). - PAUSE_AT / PAUSE_SECONDS: a trusted node (not n1) is stopped with SIGSTOP and continued, so a bootstrap with a silent member happens every run - LOG_DEBUG_NODES: those nodes log at debug level (stage and timer messages) - robustness: bootstrap-pause (10 s, 15 s pause, debug on n1-n3), run by name, fails until the node stops restarting the consensus on a re-sent table; comments name the real cause Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 13 ++++++++++--- ci/testnet/run.sh | 33 ++++++++++++++++++++++++++++++-- docs/CHANGES.md | 1 + 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index baed3f6b4..e2f038212 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -286,12 +286,19 @@ jobs: - name: contracts-net-loss env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1 contracts: true - # an official stage-1 hash race: with round_elapse_time 10 s a slow round under loss can - # trigger a bootstrap of the full table, which then never completes. Intermittent (it - # stalled 1 run of 3 at 10 s, none at 3000 blocks in run 39878938046): run by name, repeated + # an official bootstrap livelock: with round_elapse_time 10 s a slow round under loss can + # trigger a bootstrap; the starter re-sends the table every 10 s and each re-sent table + # restarts the consensus, which needs up to 12.5-16.5 s when a stage message is late. + # Intermittent and long (3000 blocks): run by name, repeated - name: bootstrap-livelock env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 manual: true + # the same livelock made deterministic: a trusted node frozen for 15 s (> round_elapse_time) + # is in the bootstrap table but silent; debug logs on three nodes. Fails until the node + # stops restarting the consensus on a re-sent table (run by name) + - name: bootstrap-pause + env: PAUSE_AT=600 PAUSE_SECONDS=15 ROUND_ELAPSE_MS=10000 LOG_DEBUG_NODES=1,2,3 TARGET_SEQ=1500 + manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 37ebb15d5..eb0124278 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -142,7 +142,9 @@ SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} # a node starts a bootstrap when its round lasts this long. The consensus's own fallback for a # silent trusted node takes about 12.5 s, so below that a slow round is pre-empted by a bootstrap, # and a bootstrap of a full table re-sent every 10 s never completes (official stage-1 hash race): -# 20 s by default (mainnet default 60 s); bootstrap-livelock keeps 10 s to reproduce the race +# 20 s by default (mainnet default 60 s). The starter re-sends its table every round_elapse_time +# and each same-round table restarts the consensus on every node: with 10 s a round that needs the +# fallbacks never completes (bootstrap-livelock, bootstrap-pause reproduce it) ROUND_ELAPSE_MS=${ROUND_ELAPSE_MS:-20000} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} @@ -229,6 +231,14 @@ NET_FAULT_PCT=${NET_FAULT_PCT:-10} SLOW_NODES=${SLOW_NODES:-} SLOW_CPU=${SLOW_CPU:-20} GARBAGE_AT=${GARBAGE_AT:-0} +# LOG_DEBUG_NODES (comma separated, default none): those nodes log at debug level (stage and timer +# messages of the consensus; node.log grows a lot) +LOG_DEBUG_NODES=${LOG_DEBUG_NODES:-} +# PAUSE_AT (default 0 = off): at that sequence a node that is trusted in the current round (not n1) +# is stopped with SIGSTOP for PAUSE_SECONDS (default 15) and continued: the round stalls, the others +# bootstrap with the paused node in the table, and the network must make progress again after it +PAUSE_AT=${PAUSE_AT:-0} +PAUSE_SECONDS=${PAUSE_SECONDS:-15} GARBAGE_SECONDS=${GARBAGE_SECONDS:-300} GARBAGE_RATE=${GARBAGE_RATE:-200} for v in "$NET_FAULT_NODES" "$SLOW_NODES"; do @@ -377,7 +387,7 @@ diag_port=0 $(storage_section "$i") [Core] -Filter="%Severity% >= info" +Filter="%Severity% >= $( [[ ",$LOG_DEBUG_NODES," == *",$i,"* ]] && echo debug || echo info )" [Sinks.file] Destination=TextFile @@ -618,6 +628,25 @@ while :; do # the whole network killed at once, as after a power cut # with contracts, only once the first calls are done: a cut during the deploys would end contracts.py + # a trusted node frozen longer than round_elapse_time: a bootstrap with a silent member + if [ "$PAUSE_AT" -gt 0 ] && [ "${paused:-0}" -eq 0 ] && [ "$min" -ge "$PAUSE_AT" ]; then + paused=1 + pause_node="" + for i in $(seq 2 "$NODES"); do + if grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n$i/node.log" | tail -1 | grep -q TRUSTED; then pause_node=$i; break; fi + done + if [ -n "$pause_node" ]; then + echo "pause test: SIGSTOP n$pause_node (trusted) at sequence $min for $PAUSE_SECONDS s" + kill -STOP "${PIDS[$pause_node]}" 2>/dev/null || true + sleep "$PAUSE_SECONDS" + kill -CONT "${PIDS[$pause_node]}" 2>/dev/null || true + pause_seq=$(seq_of 1); pause_seq=${pause_seq:-0} + echo "pause test: n$pause_node continued at sequence $pause_seq" + else + echo "pause test: no trusted node other than n1 at sequence $min, nothing paused" + fi + fi + if [ "$CRASH_ALL_AT" -gt 0 ] && [ "$crashed_all" -eq 0 ] && [ "$min" -ge "$CRASH_ALL_AT" ] \ && { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/contracts-ready" ]; }; then echo "crash test: SIGKILL every node at sequence $min" diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 8e886809a..13010d407 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,6 +44,7 @@ lines under the new release. | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | | #75 | ci(testnet): the contract load window starts after funding its writers, block rates from the first block (found in the slowdown analysis of the scale runs) | | #76 | ci(testnet): round_elapse_time 20 s by default (10 s pre-empted slow rounds with a bootstrap); a run that ends stuck in a bootstrap round fails; bootstrap-livelock exercises an official stage-1 hash race (intermittent; run by name) | +| #78 | ci(testnet): a deterministic reproduction of the official bootstrap livelock (a trusted node paused longer than round_elapse_time), debug logs on chosen nodes, the livelock's cause corrected in the comments | ## community.9 — 2026-10-22 From 8f78b900d3867c6822c595d2f839579625a65686 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 06:11:23 +0200 Subject: [PATCH 2/3] ci: bootstrap-pause recovers in both runs: a diagnosis scenario, not a reproduction Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index e2f038212..f4f37a06e 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -293,9 +293,9 @@ jobs: - name: bootstrap-livelock env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 manual: true - # the same livelock made deterministic: a trusted node frozen for 15 s (> round_elapse_time) - # is in the bootstrap table but silent; debug logs on three nodes. Fails until the node - # stops restarting the consensus on a re-sent table (run by name) + # a trusted node frozen for 15 s (> round_elapse_time) with debug logs on three nodes: a + # bootstrap with a silent member. The network recovered in both runs so far, so this alone + # does not reproduce the livelock seen under loss (run by name, for diagnosis) - name: bootstrap-pause env: PAUSE_AT=600 PAUSE_SECONDS=15 ROUND_ELAPSE_MS=10000 LOG_DEBUG_NODES=1,2,3 TARGET_SEQ=1500 manual: true From 20b19388777a80cb946151444b359545e4a654ce Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 06:55:16 +0200 Subject: [PATCH 3/3] ci: bootstrap-livelock-debug, the loss scenario with debug logs on every node Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index f4f37a06e..4c7bcf76f 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -293,6 +293,11 @@ jobs: - name: bootstrap-livelock env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 manual: true + # bootstrap-livelock with debug logs on every node and a shorter target, to see which stage + # message is late when it stalls (run by name, several times: it is intermittent) + - name: bootstrap-livelock-debug + env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 LOG_DEBUG_NODES=1,2,3,4,5,6,7 TARGET_SEQ=2000 + manual: true # a trusted node frozen for 15 s (> round_elapse_time) with debug logs on three nodes: a # bootstrap with a silent member. The network recovered in both runs so far, so this alone # does not reproduce the livelock seen under loss (run by name, for diagnosis)