diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 5385b2957..bdea659f6 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -1,8 +1,9 @@ name: Robustness -# Fork hunting: the private 7-node network of testnet.yml under conditions that could make honest -# nodes disagree, each scenario on its own runner. Every node reports a state digest every block, -# and a scenario passes only if all nodes report the same digests and reach the target sequence: +# Fork hunting: the private network of testnet.yml (7 nodes, or the `nodes` input) under +# conditions that could make honest nodes disagree, each scenario on its own runner. Every node +# reports a state digest every block, and a scenario passes only if all nodes report the same +# digests and reach the target sequence: # - mixed block databases (BerkeleyDB, LMDB, RocksDB in the same network); # - a node killed with SIGKILL six times, half of them right after it saved a checkpoint; # - the whole network killed at once and restarted, as after a power cut; @@ -21,6 +22,12 @@ on: only: description: 'Comma-separated scenario names to run (empty = all)' default: '' + nodes: + description: 'Nodes in the network (odd, 7 to 15); the NODE_STORAGE scenarios run only at 7' + default: '7' + timeout_min: + description: 'Minutes each scenario has to reach its target sequence' + default: '75' node_alt_ref: description: 'Branch or commit of this repository built and run by the even nodes (empty = all the same)' default: '' @@ -39,6 +46,15 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 180 steps: + - name: Check inputs + env: + NODES: ${{ inputs.nodes || '7' }} + TIMEOUT: ${{ inputs.timeout_min || '75' }} + run: | + # the scenarios name nodes up to n6 (crashes, pruning, faults), so at least 7 + case "$NODES" in 7|9|11|13|15) ;; *) echo "nodes must be odd, 7 to 15, got '$NODES'"; exit 1 ;; esac + [ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 120 ] || { echo "timeout_min must be 10 to 120, got '$TIMEOUT'"; exit 1; } + - name: Checkout uses: actions/checkout@v4 @@ -147,7 +163,7 @@ jobs: scenario: needs: build runs-on: ubuntu-24.04 - timeout-minutes: 120 + timeout-minutes: 150 strategy: fail-fast: false # a few runners at a time: the suite is long and heavy use got Actions disabled once @@ -181,6 +197,7 @@ jobs: # peers must answer NoSuchBlocks, not AwaitAnswer and then nothing - name: pruned-peers-join env: NODE_STORAGE=lmdb,lmdb,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000 + fixed: true # kills that land while the node prunes, or right after a checkpoint it prunes from - name: rocksdb-pruned-crash env: DB_BACKEND=rocksdb PRUNE_NODE=6 CRASH_NODE=6 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=2800 @@ -238,10 +255,13 @@ jobs: # with a power cut and packet corruption - name: all-storage env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=5 SNAPSHOT_AT=2300 TARGET_SEQ=3300 + fixed: true - name: all-storage-crash env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_NODE=4 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=3300 + fixed: true - name: all-storage-power-cut env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_ALL_AT=2500 NET_FAULT=corrupt NET_FAULT_PCT=3 NET_FAULT_NODES=2,6 TARGET_SEQ=3300 + fixed: true # a node joining from a snapshot taken after the first pruning, and from a truncated one - name: snapshot-join env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3100 @@ -255,7 +275,8 @@ jobs: name: ${{ matrix.name }} # a manual run can pick scenarios; the others stop at once env: - SELECTED: ${{ inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name)) }} + # the scenarios with a NODE_STORAGE list are laid out for 7 nodes + SELECTED: ${{ (inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }} steps: - name: Skip unselected scenario if: env.SELECTED != 'true' @@ -296,12 +317,12 @@ jobs: - name: Run private network if: env.SELECTED == 'true' - timeout-minutes: 100 + timeout-minutes: 130 env: SCENARIO: ${{ matrix.env }} - NODES: '7' + NODES: ${{ inputs.nodes || '7' }} TARGET_SEQ: '1100' # a scenario can raise it - TIMEOUT_MIN: '75' + TIMEOUT_MIN: ${{ inputs.timeout_min || '75' }} RESTART_AT: '300' FUND: '1' LOAD_SECONDS: '300' @@ -320,6 +341,13 @@ jobs: echo "scenario: $SCENARIO" ci/testnet/run.sh build/node + - name: Resource summary + if: always() && env.SELECTED == 'true' + run: | + if [ -f testnet-run/resources-summary.txt ]; then + { echo "### Resources (${{ matrix.name }})"; echo '```'; cat testnet-run/resources-summary.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload logs if: always() && env.SELECTED == 'true' uses: actions/upload-artifact@v4 @@ -328,5 +356,10 @@ jobs: path: | testnet-run/n*/*.log testnet-run/fund.log + testnet-run/contracts.log + testnet-run/n*/log/*.txt testnet-run/n*/config.ini + testnet-run/resources.tsv + testnet-run/resources-summary.txt + testnet-run/sampler.log retention-days: 14 diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index b467e9a97..0b99cdbf6 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -49,6 +49,9 @@ on: restart_nodes: description: 'Nodes stopped and restarted together in the restart test, e.g. "6 7" (empty = the last node)' default: '' + nodes: + description: 'Nodes in the network (odd, 5 to 15; 15 is the full-mesh limit of max_neighbours=16)' + default: '7' schedule: - cron: '17 2 * * *' pull_request: @@ -71,11 +74,17 @@ env: jobs: testnet: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'testnet') - name: Private network (7 nodes) + name: Private network (${{ inputs.nodes || '7' }} nodes) runs-on: ubuntu-24.04 timeout-minutes: 340 steps: + - name: Check inputs + env: + NODES: ${{ inputs.nodes || '7' }} + run: | + case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac + - name: Checkout uses: actions/checkout@v4 @@ -259,7 +268,7 @@ jobs: - name: Run private network timeout-minutes: 300 env: - NODES: '7' + NODES: ${{ inputs.nodes || '7' }} TARGET_SEQ: ${{ inputs.target_seq || '1100' }} TIMEOUT_MIN: ${{ inputs.timeout_min || '45' }} DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }} @@ -283,6 +292,9 @@ jobs: free -m df -h "${GITHUB_WORKSPACE}" du -sh testnet-run 2>/dev/null || true + if [ -f testnet-run/resources-summary.txt ]; then + { echo '### Resources'; echo '```'; cat testnet-run/resources-summary.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload node logs if: always() @@ -295,4 +307,8 @@ jobs: testnet-run/contracts.log testnet-run/n*/log/*.txt testnet-run/n*/config.ini + testnet-run/resources.tsv + testnet-run/resources-summary.txt + testnet-run/sampler.log + testnet-run/progress retention-days: 14 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index f83305d7e..76b4bcaf1 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -6,6 +6,8 @@ # TARGET_SEQ (default 1100: past the first STATE DIGEST at block 1000) # TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test) # WORK (default ./testnet-run) +# SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU +# of the nodes and executors in resources.tsv, and prints a summary at the end # FUND (default 0; needs a -DCREDITS_TESTNET=ON binary): genesis funds go to a generated master # key, which funds every node through node 1's API (port 9090) once the chain reaches FUND_AT # (default 30) and then keeps random transfers flowing for LOAD_SECONDS (default 300) @@ -105,6 +107,13 @@ BIN_ALT=${BIN_ALT:+$(realpath "$BIN_ALT")} HERE=$(cd "$(dirname "$0")" && pwd) REPO=$(cd "$HERE/../.." && pwd) NODES=${NODES:-7} +# MinTrustedNodes is 4; an even set of ready nodes drops one at bootstrap +case "$NODES" in + 5|7|9|11|13|15|17|19|21|23|25) ;; + *) echo "NODES must be odd, 5 to 25, got '$NODES'"; exit 1 ;; +esac +[ "$NODES" -le 15 ] || echo "warning: $NODES nodes, max_neighbours is 16: not a full mesh" +SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} RESTART_AT=${RESTART_AT:-300} @@ -377,6 +386,13 @@ net_fault_stats() { } stop_all() { + # the sampler's last sample still sees the nodes running + if [ -n "${SAMPLER_PID:-}" ]; then + kill -TERM "$SAMPLER_PID" 2>/dev/null || true + for _ in $(seq 1 20); do kill -0 "$SAMPLER_PID" 2>/dev/null || break; sleep 1; done + kill -KILL "$SAMPLER_PID" 2>/dev/null || true + wait "$SAMPLER_PID" 2>/dev/null || true + fi [ -z "${API_LOAD_PID:-}" ] || kill -TERM "$API_LOAD_PID" 2>/dev/null || true for i in $(seq 1 "$ALL_NODES"); do [ -n "${PIDS[$i]:-}" ] && kill -TERM "${PIDS[$i]}" 2>/dev/null || true @@ -397,6 +413,7 @@ stop_all() { fi [ -z "$NET_FAULT" ] || net_fault_off [ -z "${GARBAGE_PID:-}" ] || kill "$GARBAGE_PID" 2>/dev/null || true + [ ! -f "$WORK/resources-summary.txt" ] || cat "$WORK/resources-summary.txt" } trap stop_all EXIT @@ -452,6 +469,12 @@ fi for i in $(seq 1 "$NODES"); do start_node "$i"; done echo "started $NODES nodes in $WORK" +SAMPLER_PID= +if [ "$SAMPLE_SECONDS" -gt 0 ]; then + python3 "$HERE/sampler.py" "$WORK" "$SAMPLE_SECONDS" "$WORK/resources.tsv" --nodes "$NODES" \ + --summary-file "$WORK/resources-summary.txt" > "$WORK/sampler.log" 2>&1 & + SAMPLER_PID=$! +fi deadline=$(( $(date +%s) + TIMEOUT_MIN * 60 )) restarted=0 @@ -479,6 +502,7 @@ while :; do sleep 20 line="t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) ))s" min=-1 + max=0 for i in $(seq 1 "$NODES"); do s=$(seq_of "$i"); s=${s:-0} line+=" n$i=$s" @@ -498,6 +522,7 @@ while :; do fi fi if [ "$min" -lt 0 ] || [ "$s" -lt "$min" ]; then min=$s; fi + if [ "$s" -gt "$max" ]; then max=$s; fi done join_seq=-1 if [ "$JOIN_NODE" -gt 0 ] && [ -n "${PIDS[$JOIN_NODE]:-}" ]; then @@ -505,6 +530,7 @@ while :; do line+=" n$JOIN_NODE=$join_seq(joined)" fi echo "$line" + echo "t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) )) min=$min max=$max" > "$WORK/progress" if [ "$JOIN_PRUNED_AT" -gt 0 ] && [ -z "${PIDS[$JOIN_NODE]:-}" ] && [ "$min" -ge "$JOIN_PRUNED_AT" ]; then echo "join test: n$JOIN_NODE starts from block 0 with pruned storage at sequence $min" diff --git a/ci/testnet/sampler.py b/ci/testnet/sampler.py new file mode 100644 index 000000000..c66ed6fe6 --- /dev/null +++ b/ci/testnet/sampler.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Sample the resources a private test network uses, to size the machine that runs it. + +Usage: sampler.py --nodes N --summary-file PATH + +Every interval it writes one host row and one row per process of the network to out.tsv: +the nodes (cwd /n, named n), their contract executors (cs.testnet.node=n, named +x) and the harness scripts (py: