From fc6f25f8658cd2a99753ee104c8acf74124a74e3 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 10 Oct 2026 15:55:24 +0200 Subject: [PATCH 1/7] ci(testnet): a node with pruned storage With PRUNE_NODE (workflow input prune_node) that node runs with a checkpoint every 1'000 blocks, checkpoint_keep=2 and prune_keep_blocks=300. The run fails if it never prunes, and reports the pruning log lines and every node's block DB size; its state digest is checked as for any node. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testnet.yml | 4 ++++ ci/testnet/run.sh | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index fa8bbbda4..cd220cbfb 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -16,6 +16,9 @@ on: disk_full_node: description: 'Node whose block DB disk fills up for a while (0 = off)' default: '0' + prune_node: + description: 'Node running with pruned storage (0 = off)' + default: '0' restart_at: description: 'Sequence at which the last node is restarted (0 = off)' default: '300' @@ -127,6 +130,7 @@ jobs: TIMEOUT_MIN: ${{ inputs.timeout_min || '45' }} DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }} RESTART_AT: ${{ inputs.restart_at || '300' }} + PRUNE_NODE: ${{ inputs.prune_node || '0' }} DPOS_AT: ${{ inputs.dpos_at || '0' }} REWARD_ROUND: ${{ inputs.reward_round || '0' }} DELEGATE: ${{ inputs.delegate || '0' }} diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 32db116ba..c4410dd42 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -15,6 +15,9 @@ # restart test, the restarted node must apply it again after its quick start # DELEGATE (default 0, needs FUND=1): nodes delegate to each other, with and without a time # limit (DELEGATION_SECONDS, default 240), and some delegations are withdrawn later +# PRUNE_NODE (default 0 = off): that node runs with pruned storage (a checkpoint every 1'000 +# blocks, prune_keep_blocks=300); it must remove old blocks, survive the restart test if it is +# the restarted node, and end with the same state digest # DISK_FULL_NODE (default 0 = off): put that node's block DB on a DISK_TMPFS_MB (default 64) tmpfs, # fill it when the chain reaches DISK_FULL_AT (default 400) and free it DISK_FULL_SECONDS # (default 60) later, restarting the node if it stopped; it must catch up and end with the same @@ -38,6 +41,7 @@ DPOS_AT=${DPOS_AT:-0} REWARD_ROUND=${REWARD_ROUND:-0} DELEGATE=${DELEGATE:-0} MIN_STAKE=${MIN_STAKE:-0} +PRUNE_NODE=${PRUNE_NODE:-0} DELEGATION_SECONDS=${DELEGATION_SECONDS:-240} DISK_FULL_AT=${DISK_FULL_AT:-400} DISK_FULL_SECONDS=${DISK_FULL_SECONDS:-60} @@ -83,6 +87,7 @@ port=$([ "$FUND" = "1" ] && [ "$i" -eq 1 ] && echo 9090 || echo 0) apiexec_port=0 ajax_port=0 diag_port=0 +$([ "$i" -eq "$PRUNE_NODE" ] && printf '\n[storage]\ncheckpoint_every=1000\ncheckpoint_keep=2\nprune_keep_blocks=300\n') [Core] Filter="%Severity% >= info" @@ -215,6 +220,18 @@ if [ "$MIN_STAKE" -gt 0 ] && [ "$RESTART_AT" -gt 0 ]; then fi fi +if [ "$PRUNE_NODE" -gt 0 ]; then + echo "pruned storage: n$PRUNE_NODE logged:" + grep -ah "pruned storage" "$WORK/n$PRUNE_NODE"/node.log | sed 's/^/ /' | tail -n 8 + for i in $(seq 1 "$NODES"); do + echo " n$i block DB: $(du -sh "$WORK/n$i/db" 2>/dev/null | cut -f1)" + done + if ! grep -aq "pruned storage: blocks before .* removed" "$WORK/n$PRUNE_NODE"/node.log; then + echo "FAIL: n$PRUNE_NODE never pruned its storage" + exit 1 + fi +fi + # every node logs "STATE DIGEST # wallets "; at a sequence logged by several nodes the # digests must be identical python3 - "$WORK" "$NODES" "$FUND" <<'PY' From 4e152e8697a47995374ab7540d19c3babfad7db7 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 10 Oct 2026 18:50:54 +0200 Subject: [PATCH 2/7] ci(testnet): pruned-node report file by file Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index c4410dd42..5d2517d9b 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -226,6 +226,10 @@ if [ "$PRUNE_NODE" -gt 0 ]; then for i in $(seq 1 "$NODES"); do echo " n$i block DB: $(du -sh "$WORK/n$i/db" 2>/dev/null | cut -f1)" done + # file by file, for the pruned node and a full one: block records, hash index, logs, environment + for i in 1 "$PRUNE_NODE"; do + echo " n$i files: $(cd "$WORK/n$i/db" && du -k * 2>/dev/null | sort -k2 | awk '{printf "%s=%sK ", $2, $1}')" + done if ! grep -aq "pruned storage: blocks before .* removed" "$WORK/n$PRUNE_NODE"/node.log; then echo "FAIL: n$PRUNE_NODE never pruned its storage" exit 1 From 51785792677a767ad7a9229ff2e97901d159d24f Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 10 Oct 2026 19:53:02 +0200 Subject: [PATCH 3/7] ci(testnet): a node joining from block 0 with pruned storage With JOIN_PRUNED_AT (workflow input join_pruned_at) one more node, in nobody's hosts or trusted list and without funds or checkpoints, starts once the network reaches that sequence. It syncs every block from block 0 through the normal block requests, validates and applies them, saves a checkpoint every 1'000 blocks and prunes behind itself (prune_keep_blocks=300). The run waits for it to reach the target too, requires it to have pruned, and checks its state digests with the others'. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testnet.yml | 4 +++ ci/testnet/run.sh | 53 ++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index cd220cbfb..cc5bb8bd4 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -19,6 +19,9 @@ on: prune_node: description: 'Node running with pruned storage (0 = off)' default: '0' + join_pruned_at: + description: 'Sequence at which one more node joins from block 0 with pruned storage (0 = off)' + default: '0' restart_at: description: 'Sequence at which the last node is restarted (0 = off)' default: '300' @@ -131,6 +134,7 @@ jobs: DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }} RESTART_AT: ${{ inputs.restart_at || '300' }} PRUNE_NODE: ${{ inputs.prune_node || '0' }} + JOIN_PRUNED_AT: ${{ inputs.join_pruned_at || '0' }} DPOS_AT: ${{ inputs.dpos_at || '0' }} REWARD_ROUND: ${{ inputs.reward_round || '0' }} DELEGATE: ${{ inputs.delegate || '0' }} diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 5d2517d9b..c7bfaef3d 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -18,6 +18,9 @@ # PRUNE_NODE (default 0 = off): that node runs with pruned storage (a checkpoint every 1'000 # blocks, prune_keep_blocks=300); it must remove old blocks, survive the restart test if it is # the restarted node, and end with the same state digest +# JOIN_PRUNED_AT (default 0 = off): one more node, outside the initial trusted set and without +# funds or checkpoints, starts when the network reaches that sequence; it syncs from block 0 +# with pruned storage like PRUNE_NODE and must end with the same state digest # DISK_FULL_NODE (default 0 = off): put that node's block DB on a DISK_TMPFS_MB (default 64) tmpfs, # fill it when the chain reaches DISK_FULL_AT (default 400) and free it DISK_FULL_SECONDS # (default 60) later, restarting the node if it stopped; it must catch up and end with the same @@ -42,6 +45,10 @@ REWARD_ROUND=${REWARD_ROUND:-0} DELEGATE=${DELEGATE:-0} MIN_STAKE=${MIN_STAKE:-0} PRUNE_NODE=${PRUNE_NODE:-0} +JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} +# the joining node is node NODES + 1; it is in nobody's hosts or trusted list +JOIN_NODE=$([ "$JOIN_PRUNED_AT" -gt 0 ] && echo $((NODES + 1)) || echo 0) +ALL_NODES=$([ "$JOIN_NODE" -gt 0 ] && echo "$JOIN_NODE" || echo "$NODES") DELEGATION_SECONDS=${DELEGATION_SECONDS:-240} DISK_FULL_AT=${DISK_FULL_AT:-400} DISK_FULL_SECONDS=${DISK_FULL_SECONDS:-60} @@ -51,7 +58,7 @@ BASE_PORT=6000 declare -a PIDS KEYS -mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$NODES" "$WORK") +mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$ALL_NODES" "$WORK") if [ "$FUND" = "1" ]; then read -r MASTER_SEED MASTER_PUB < <(python3 -c "import base58, nacl.signing; k = nacl.signing.SigningKey.generate(); print(base58.b58encode(bytes(k)).decode(), base58.b58encode(bytes(k.verify_key)).decode())") @@ -60,7 +67,7 @@ if [ "$FUND" = "1" ]; then echo "testnet master key $MASTER_PUB" fi -for i in $(seq 1 "$NODES"); do +for i in $(seq 1 "$ALL_NODES"); do dir="$WORK/n$i" : > "$dir/hosts.txt" : > "$dir/trusted.txt" @@ -87,7 +94,7 @@ port=$([ "$FUND" = "1" ] && [ "$i" -eq 1 ] && echo 9090 || echo 0) apiexec_port=0 ajax_port=0 diag_port=0 -$([ "$i" -eq "$PRUNE_NODE" ] && printf '\n[storage]\ncheckpoint_every=1000\ncheckpoint_keep=2\nprune_keep_blocks=300\n') +$({ [ "$i" -eq "$PRUNE_NODE" ] || [ "$i" -eq "$JOIN_NODE" ]; } && printf '\n[storage]\ncheckpoint_every=1000\ncheckpoint_keep=2\nprune_keep_blocks=300\n') [Core] Filter="%Severity% >= info" @@ -108,11 +115,11 @@ start_node() { } stop_all() { - for i in $(seq 1 "$NODES"); do + for i in $(seq 1 "$ALL_NODES"); do [ -n "${PIDS[$i]:-}" ] && kill -TERM "${PIDS[$i]}" 2>/dev/null || true done sleep 15 - for i in $(seq 1 "$NODES"); do + for i in $(seq 1 "$ALL_NODES"); do [ -n "${PIDS[$i]:-}" ] && kill -KILL "${PIDS[$i]}" 2>/dev/null || true done if [ "$DISK_FULL_NODE" -gt 0 ]; then @@ -152,8 +159,18 @@ while :; do if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then line+="(down)"; fi if [ "$min" -lt 0 ] || [ "$s" -lt "$min" ]; then min=$s; fi done + join_seq=-1 + if [ "$JOIN_NODE" -gt 0 ] && [ -n "${PIDS[$JOIN_NODE]:-}" ]; then + join_seq=$(seq_of "$JOIN_NODE"); join_seq=${join_seq:-0} + line+=" n$JOIN_NODE=$join_seq(joined)" + fi echo "$line" + if [ "$JOIN_NODE" -gt 0 ] && [ -z "${PIDS[$JOIN_NODE]:-}" ] && [ "$min" -ge "$JOIN_PRUNED_AT" ]; then + echo "join test: n$JOIN_NODE starts from block 0 with pruned storage at sequence $min" + start_node "$JOIN_NODE" + fi + if [ "$RESTART_AT" -gt 0 ] && [ "$restarted" -eq 0 ] && [ "$min" -ge "$RESTART_AT" ]; then echo "restart test: stopping n$NODES at sequence $min" kill -TERM "${PIDS[$NODES]}" @@ -189,7 +206,7 @@ while :; do FUND_PID=$! fi - if [ "$min" -ge "$TARGET_SEQ" ]; then break; fi + if [ "$min" -ge "$TARGET_SEQ" ] && { [ "$JOIN_NODE" -eq 0 ] || [ "$join_seq" -ge "$TARGET_SEQ" ]; }; then break; fi if [ "$(date +%s)" -ge "$deadline" ]; then echo "FAIL: not every node reached sequence $TARGET_SEQ within $TIMEOUT_MIN min" exit 1 @@ -220,25 +237,23 @@ if [ "$MIN_STAKE" -gt 0 ] && [ "$RESTART_AT" -gt 0 ]; then fi fi -if [ "$PRUNE_NODE" -gt 0 ]; then - echo "pruned storage: n$PRUNE_NODE logged:" - grep -ah "pruned storage" "$WORK/n$PRUNE_NODE"/node.log | sed 's/^/ /' | tail -n 8 - for i in $(seq 1 "$NODES"); do - echo " n$i block DB: $(du -sh "$WORK/n$i/db" 2>/dev/null | cut -f1)" - done +for p in "$PRUNE_NODE" "$JOIN_NODE"; do + [ "$p" -gt 0 ] || continue + echo "pruned storage: n$p logged:" + grep -ah "pruned storage" "$WORK/n$p"/node.log | sed 's/^/ /' | tail -n 8 # file by file, for the pruned node and a full one: block records, hash index, logs, environment - for i in 1 "$PRUNE_NODE"; do - echo " n$i files: $(cd "$WORK/n$i/db" && du -k * 2>/dev/null | sort -k2 | awk '{printf "%s=%sK ", $2, $1}')" + for i in 1 "$p"; do + echo " n$i block DB $(du -sh "$WORK/n$i/db" 2>/dev/null | cut -f1): $(cd "$WORK/n$i/db" && du -k * 2>/dev/null | sort -k2 | awk '{printf "%s=%sK ", $2, $1}')" done - if ! grep -aq "pruned storage: blocks before .* removed" "$WORK/n$PRUNE_NODE"/node.log; then - echo "FAIL: n$PRUNE_NODE never pruned its storage" + if ! grep -aq "pruned storage: blocks before .* removed" "$WORK/n$p"/node.log; then + echo "FAIL: n$p never pruned its storage" exit 1 fi -fi +done # every node logs "STATE DIGEST # wallets "; at a sequence logged by several nodes the # digests must be identical -python3 - "$WORK" "$NODES" "$FUND" <<'PY' +python3 - "$WORK" "$ALL_NODES" "$FUND" <<'PY' import collections, glob, re, sys work, nodes, funded = sys.argv[1], int(sys.argv[2]), sys.argv[3] == "1" seen = collections.defaultdict(dict) @@ -261,4 +276,4 @@ if funded and 1000 in seen and max(w for _, w in seen[1000].values()) <= 2: ok = False sys.exit(0 if ok else 1) PY -echo "PASS: $NODES nodes reached sequence $TARGET_SEQ with matching state digests" +echo "PASS: $ALL_NODES nodes reached sequence $TARGET_SEQ with matching state digests" From 7a325a6131874cb94d21374f5e200df483aca273 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 10 Oct 2026 21:50:46 +0200 Subject: [PATCH 4/7] ci(testnet): report the LMDB caches of pruned nodes Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index c7bfaef3d..d374b6197 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -244,6 +244,8 @@ for p in "$PRUNE_NODE" "$JOIN_NODE"; do # file by file, for the pruned node and a full one: block records, hash index, logs, environment for i in 1 "$p"; do echo " n$i block DB $(du -sh "$WORK/n$i/db" 2>/dev/null | cut -f1): $(cd "$WORK/n$i/db" && du -k * 2>/dev/null | sort -k2 | awk '{printf "%s=%sK ", $2, $1}')" + # LMDB caches: block hashes (seqdb, hashdb) and the transaction index (indexdb) + echo " n$i caches $(du -sh "$WORK/n$i/caches" 2>/dev/null | cut -f1): $(cd "$WORK/n$i/caches" 2>/dev/null && du -sk * 2>/dev/null | sort -k2 | awk '{printf "%s=%sK ", $2, $1}')" done if ! grep -aq "pruned storage: blocks before .* removed" "$WORK/n$p"/node.log; then echo "FAIL: n$p never pruned its storage" From ea6d4313f4a71697011ca381a7d6edd291d95f57 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 11 Oct 2026 00:11:08 +0200 Subject: [PATCH 5/7] ci(testnet): report node exit status and crash backtraces A node that dies is now reported with its exit status (and signal), and with core dumps on (kernel.core_pattern into the run directory, ulimit -c unlimited) the run prints the backtrace of every core file with gdb. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testnet.yml | 5 +++++ ci/testnet/run.sh | 21 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index cc5bb8bd4..b9fa623cb 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -125,6 +125,11 @@ jobs: ccache --max-size=2G cmake --build build -j"$(nproc)" --target node + - name: Enable core dumps + run: | + sudo apt-get install -y -qq gdb + sudo sysctl -w kernel.core_pattern="${{ github.workspace }}/testnet-run/core.%e.%p" + - name: Run private network timeout-minutes: 300 env: diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index d374b6197..99e12c59a 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -56,7 +56,7 @@ DISK_TMPFS_MB=${DISK_TMPFS_MB:-64} WORK=$(mkdir -p "${WORK:-testnet-run}" && cd "${WORK:-testnet-run}" && pwd) BASE_PORT=6000 -declare -a PIDS KEYS +declare -a PIDS KEYS REPORTED_DOWN mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$ALL_NODES" "$WORK") @@ -110,7 +110,8 @@ done start_node() { local i=$1 - (cd "$WORK/n$i" && exec "$BIN" --db-path db --config-file config.ini > stdout.log 2>&1) & + # core dumps (see core_pattern in the workflow) let a crash be traced afterwards + (cd "$WORK/n$i" && ulimit -c unlimited && exec "$BIN" --db-path db --config-file config.ini > stdout.log 2>&1) & PIDS[$i]=$! } @@ -156,7 +157,14 @@ while :; do for i in $(seq 1 "$NODES"); do s=$(seq_of "$i"); s=${s:-0} line+=" n$i=$s" - if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then line+="(down)"; fi + if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then + line+="(down)" + if [ -z "${REPORTED_DOWN[$i]:-}" ]; then + REPORTED_DOWN[$i]=1 + status=0; wait "${PIDS[$i]}" 2>/dev/null || status=$? + echo "n$i exited with status $status$([ "$status" -gt 128 ] && echo " (signal $((status - 128)))")" + fi + fi if [ "$min" -lt 0 ] || [ "$s" -lt "$min" ]; then min=$s; fi done join_seq=-1 @@ -237,6 +245,13 @@ if [ "$MIN_STAKE" -gt 0 ] && [ "$RESTART_AT" -gt 0 ]; then fi fi +# a crashed node left a core file: print where it stopped +for core in "$WORK"/core.* "$WORK"/n*/core*; do + [ -f "$core" ] || continue + echo "core dump $core:" + gdb -q -batch -ex "bt 40" "$BIN" "$core" 2>/dev/null | grep -E '^#' | head -n 40 +done + for p in "$PRUNE_NODE" "$JOIN_NODE"; do [ "$p" -gt 0 ] || continue echo "pruned storage: n$p logged:" From 61a4814efc36a4f2370a9936aaa052b342c851df Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 11 Oct 2026 01:48:42 +0200 Subject: [PATCH 6/7] ci(testnet): trace a crashed node as soon as it is found down Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 99e12c59a..da680c8f7 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -163,6 +163,13 @@ while :; do REPORTED_DOWN[$i]=1 status=0; wait "${PIDS[$i]}" 2>/dev/null || status=$? echo "n$i exited with status $status$([ "$status" -gt 128 ] && echo " (signal $((status - 128)))")" + # the run may end on its timeout later, so trace a crash right away + for core in "$WORK"/core.*."${PIDS[$i]}"; do + [ -f "$core" ] || continue + echo "n$i core dump $core:" + gdb -q -batch -ex "thread apply all bt 25" "$BIN" "$core" 2>/dev/null | grep -E '^(#|Thread)' | head -n 120 + mv "$core" "$core.traced" + done fi fi if [ "$min" -lt 0 ] || [ "$s" -lt "$min" ]; then min=$s; fi From 218434d557481391c068be82178d651389f56bc3 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 11 Oct 2026 02:56:26 +0200 Subject: [PATCH 7/7] ci(testnet): print the crashing thread's full backtrace first Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index da680c8f7..429b4af75 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -167,7 +167,7 @@ while :; do for core in "$WORK"/core.*."${PIDS[$i]}"; do [ -f "$core" ] || continue echo "n$i core dump $core:" - gdb -q -batch -ex "thread apply all bt 25" "$BIN" "$core" 2>/dev/null | grep -E '^(#|Thread)' | head -n 120 + gdb -q -batch -ex "bt 60" -ex "thread apply all bt 6" "$BIN" "$core" 2>/dev/null | grep -E '^(#|Thread)' | head -n 200 mv "$core" "$core.traced" done fi