From 9f7484e2eca8941dc12f01ae212819da42d4d3d8 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 11:49:33 +0200 Subject: [PATCH 1/3] ci(testnet): sample host and per-process resources; nodes input for testnet and robustness - ci/testnet/sampler.py: every SAMPLE_SECONDS (default 60) host memory, swap, CPU busy/steal, load, disk, and RSS/CPU/threads of each node, executor JVM and harness script, into resources.tsv; on SIGTERM a summary (peaks, sums, block rate, spread, down samples) to resources-summary.txt and the log. It never fails a run. - run.sh: NODES must be odd, 5 to 25 (warning above 15, max_neighbours=16); the main loop writes t/min/max to progress; stop_all stops the sampler first, so its last sample sees the nodes, and prints the summary last. - testnet.yml: `nodes` input (5 to 15, checked before the build), job name with the node count, the summary in the step summary, resources files in the artifact. - robustness.yml: `nodes` and `timeout_min` inputs; the NODE_STORAGE scenarios (laid out for 7 nodes) run only at 7; longer job timeouts; contracts.log, node log files and resources in the artifact; the summary in the step summary. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 46 +++++- .github/workflows/testnet.yml | 20 ++- ci/testnet/run.sh | 26 +++ ci/testnet/sampler.py | 272 +++++++++++++++++++++++++++++++ 4 files changed, 355 insertions(+), 9 deletions(-) create mode 100644 ci/testnet/sampler.py diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 5463f9c26..5bfba5120 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -1,7 +1,8 @@ name: Robustness -# Fork hunting: the private 7-node network of testnet.yml under conditions that could make honest -# nodes disagree, each scenario on its own runner. Every node reports a state digest every block, +# Fork hunting: the private network of testnet.yml (7 nodes, or the `nodes` input) under +# conditions that could make honest nodes disagree, each scenario on its own runner. Every node +# reports a state digest every block, # and a scenario passes only if all nodes report the same digests and reach the target sequence: # - mixed block databases (BerkeleyDB, LMDB, RocksDB in the same network); # - a node killed with SIGKILL six times, half of them right after it saved a checkpoint; @@ -21,6 +22,12 @@ on: only: description: 'Comma-separated scenario names to run (empty = all)' default: '' + nodes: + description: 'Nodes in the network (odd, 5 to 15); the NODE_STORAGE scenarios run only at 7' + default: '7' + timeout_min: + description: 'Minutes each scenario has to reach its target sequence' + default: '75' permissions: contents: read @@ -36,6 +43,14 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 180 steps: + - name: Check inputs + env: + NODES: ${{ inputs.nodes || '7' }} + TIMEOUT: ${{ inputs.timeout_min || '75' }} + run: | + case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac + [ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 120 ] || { echo "timeout_min must be 10 to 120, got '$TIMEOUT'"; exit 1; } + - name: Checkout uses: actions/checkout@v4 @@ -105,7 +120,7 @@ jobs: scenario: needs: build runs-on: ubuntu-24.04 - timeout-minutes: 120 + timeout-minutes: 150 strategy: fail-fast: false # a few runners at a time: the suite is long and heavy use got Actions disabled once @@ -139,6 +154,7 @@ jobs: # peers must answer NoSuchBlocks, not AwaitAnswer and then nothing - name: pruned-peers-join env: NODE_STORAGE=lmdb,lmdb,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000 + fixed: true # kills that land while the node prunes, or right after a checkpoint it prunes from - name: rocksdb-pruned-crash env: DB_BACKEND=rocksdb PRUNE_NODE=6 CRASH_NODE=6 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=2800 @@ -196,10 +212,13 @@ jobs: # with a power cut and packet corruption - name: all-storage env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=5 SNAPSHOT_AT=2300 TARGET_SEQ=3300 + fixed: true - name: all-storage-crash env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_NODE=4 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=3300 + fixed: true - name: all-storage-power-cut env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_ALL_AT=2500 NET_FAULT=corrupt NET_FAULT_PCT=3 NET_FAULT_NODES=2,6 TARGET_SEQ=3300 + fixed: true # a node joining from a snapshot taken after the first pruning, and from a truncated one - name: snapshot-join env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3100 @@ -213,7 +232,8 @@ jobs: name: ${{ matrix.name }} # a manual run can pick scenarios; the others stop at once env: - SELECTED: ${{ inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name)) }} + # the scenarios with a NODE_STORAGE list are laid out for 7 nodes + SELECTED: ${{ (inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }} steps: - name: Skip unselected scenario if: env.SELECTED != 'true' @@ -247,12 +267,12 @@ jobs: - name: Run private network if: env.SELECTED == 'true' - timeout-minutes: 100 + timeout-minutes: 130 env: SCENARIO: ${{ matrix.env }} - NODES: '7' + NODES: ${{ inputs.nodes || '7' }} TARGET_SEQ: '1100' # a scenario can raise it - TIMEOUT_MIN: '75' + TIMEOUT_MIN: ${{ inputs.timeout_min || '75' }} RESTART_AT: '300' FUND: '1' LOAD_SECONDS: '300' @@ -267,6 +287,13 @@ jobs: echo "scenario: $SCENARIO" ci/testnet/run.sh build/node + - name: Resource summary + if: always() && env.SELECTED == 'true' + run: | + if [ -f testnet-run/resources-summary.txt ]; then + { echo "### Resources (${{ matrix.name }})"; echo '```'; cat testnet-run/resources-summary.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload logs if: always() && env.SELECTED == 'true' uses: actions/upload-artifact@v4 @@ -275,5 +302,10 @@ jobs: path: | testnet-run/n*/*.log testnet-run/fund.log + testnet-run/contracts.log + testnet-run/n*/log/*.txt testnet-run/n*/config.ini + testnet-run/resources.tsv + testnet-run/resources-summary.txt + testnet-run/sampler.log retention-days: 14 diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index 91a396ed2..4901f7c1b 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -49,6 +49,9 @@ on: restart_nodes: description: 'Nodes stopped and restarted together in the restart test, e.g. "6 7" (empty = the last node)' default: '' + nodes: + description: 'Nodes in the network (odd, 5 to 15; 15 is the full-mesh limit of max_neighbours=16)' + default: '7' schedule: - cron: '17 2 * * *' pull_request: @@ -71,11 +74,17 @@ env: jobs: testnet: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'testnet') - name: Private network (7 nodes) + name: Private network (${{ inputs.nodes || '7' }} nodes) runs-on: ubuntu-24.04 timeout-minutes: 340 steps: + - name: Check inputs + env: + NODES: ${{ inputs.nodes || '7' }} + run: | + case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac + - name: Checkout uses: actions/checkout@v4 @@ -259,7 +268,7 @@ jobs: - name: Run private network timeout-minutes: 300 env: - NODES: '7' + NODES: ${{ inputs.nodes || '7' }} TARGET_SEQ: ${{ inputs.target_seq || '1100' }} TIMEOUT_MIN: ${{ inputs.timeout_min || '45' }} DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }} @@ -283,6 +292,9 @@ jobs: free -m df -h "${GITHUB_WORKSPACE}" du -sh testnet-run 2>/dev/null || true + if [ -f testnet-run/resources-summary.txt ]; then + { echo '### Resources'; echo '```'; cat testnet-run/resources-summary.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload node logs if: always() @@ -295,4 +307,8 @@ jobs: testnet-run/contracts.log testnet-run/n*/log/*.txt testnet-run/n*/config.ini + testnet-run/resources.tsv + testnet-run/resources-summary.txt + testnet-run/sampler.log + testnet-run/progress retention-days: 14 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 1ae58daab..5bfc97c85 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -6,6 +6,8 @@ # TARGET_SEQ (default 1100: past the first STATE DIGEST at block 1000) # TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test) # WORK (default ./testnet-run) +# SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU +# of the nodes and executors in resources.tsv, and prints a summary at the end # FUND (default 0; needs a -DCREDITS_TESTNET=ON binary): genesis funds go to a generated master # key, which funds every node through node 1's API (port 9090) once the chain reaches FUND_AT # (default 30) and then keeps random transfers flowing for LOAD_SECONDS (default 300) @@ -104,6 +106,13 @@ BIN_ALT=${BIN_ALT:+$(realpath "$BIN_ALT")} HERE=$(cd "$(dirname "$0")" && pwd) REPO=$(cd "$HERE/../.." && pwd) NODES=${NODES:-7} +# MinTrustedNodes is 4; an even set of ready nodes drops one at bootstrap +case "$NODES" in + 5|7|9|11|13|15|17|19|21|23|25) ;; + *) echo "NODES must be odd, 5 to 25, got '$NODES'"; exit 1 ;; +esac +[ "$NODES" -le 15 ] || echo "warning: $NODES nodes, max_neighbours is 16: not a full mesh" +SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} RESTART_AT=${RESTART_AT:-300} @@ -376,6 +385,13 @@ net_fault_stats() { } stop_all() { + # the sampler's last sample still sees the nodes running + if [ -n "${SAMPLER_PID:-}" ]; then + kill -TERM "$SAMPLER_PID" 2>/dev/null || true + for _ in $(seq 1 20); do kill -0 "$SAMPLER_PID" 2>/dev/null || break; sleep 1; done + kill -KILL "$SAMPLER_PID" 2>/dev/null || true + wait "$SAMPLER_PID" 2>/dev/null || true + fi [ -z "${API_LOAD_PID:-}" ] || kill -TERM "$API_LOAD_PID" 2>/dev/null || true for i in $(seq 1 "$ALL_NODES"); do [ -n "${PIDS[$i]:-}" ] && kill -TERM "${PIDS[$i]}" 2>/dev/null || true @@ -396,6 +412,7 @@ stop_all() { fi [ -z "$NET_FAULT" ] || net_fault_off [ -z "${GARBAGE_PID:-}" ] || kill "$GARBAGE_PID" 2>/dev/null || true + [ ! -f "$WORK/resources-summary.txt" ] || cat "$WORK/resources-summary.txt" } trap stop_all EXIT @@ -451,6 +468,12 @@ fi for i in $(seq 1 "$NODES"); do start_node "$i"; done echo "started $NODES nodes in $WORK" +SAMPLER_PID= +if [ "$SAMPLE_SECONDS" -gt 0 ]; then + python3 "$HERE/sampler.py" "$WORK" "$SAMPLE_SECONDS" "$WORK/resources.tsv" --nodes "$NODES" \ + --summary-file "$WORK/resources-summary.txt" > "$WORK/sampler.log" 2>&1 & + SAMPLER_PID=$! +fi deadline=$(( $(date +%s) + TIMEOUT_MIN * 60 )) restarted=0 @@ -478,6 +501,7 @@ while :; do sleep 20 line="t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) ))s" min=-1 + max=0 for i in $(seq 1 "$NODES"); do s=$(seq_of "$i"); s=${s:-0} line+=" n$i=$s" @@ -497,6 +521,7 @@ while :; do fi fi if [ "$min" -lt 0 ] || [ "$s" -lt "$min" ]; then min=$s; fi + if [ "$s" -gt "$max" ]; then max=$s; fi done join_seq=-1 if [ "$JOIN_NODE" -gt 0 ] && [ -n "${PIDS[$JOIN_NODE]:-}" ]; then @@ -504,6 +529,7 @@ while :; do line+=" n$JOIN_NODE=$join_seq(joined)" fi echo "$line" + echo "t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) )) min=$min max=$max" > "$WORK/progress" if [ "$JOIN_PRUNED_AT" -gt 0 ] && [ -z "${PIDS[$JOIN_NODE]:-}" ] && [ "$min" -ge "$JOIN_PRUNED_AT" ]; then echo "join test: n$JOIN_NODE starts from block 0 with pruned storage at sequence $min" diff --git a/ci/testnet/sampler.py b/ci/testnet/sampler.py new file mode 100644 index 000000000..c9862ec5f --- /dev/null +++ b/ci/testnet/sampler.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Sample the resources a private test network uses, to size the machine that runs it. + +Usage: sampler.py --nodes N --summary-file PATH + +Every interval it writes one host row and one row per process of the network to out.tsv: +the nodes (cwd /n, named n), their contract executors (cs.testnet.node=n, named +x) and the harness scripts (py: