Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 41 additions & 8 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
name: Robustness

# Fork hunting: the private 7-node network of testnet.yml under conditions that could make honest
# nodes disagree, each scenario on its own runner. Every node reports a state digest every block,
# and a scenario passes only if all nodes report the same digests and reach the target sequence:
# Fork hunting: the private network of testnet.yml (7 nodes, or the `nodes` input) under
# conditions that could make honest nodes disagree, each scenario on its own runner. Every node
# reports a state digest every block, and a scenario passes only if all nodes report the same
# digests and reach the target sequence:
# - mixed block databases (BerkeleyDB, LMDB, RocksDB in the same network);
# - a node killed with SIGKILL six times, half of them right after it saved a checkpoint;
# - the whole network killed at once and restarted, as after a power cut;
Expand All @@ -21,6 +22,12 @@ on:
only:
description: 'Comma-separated scenario names to run (empty = all)'
default: ''
nodes:
description: 'Nodes in the network (odd, 7 to 15); the NODE_STORAGE scenarios run only at 7'
default: '7'
timeout_min:
description: 'Minutes each scenario has to reach its target sequence'
default: '75'
node_alt_ref:
description: 'Branch or commit of this repository built and run by the even nodes (empty = all the same)'
default: ''
Expand All @@ -39,6 +46,15 @@ jobs:
runs-on: ubuntu-24.04
timeout-minutes: 180
steps:
- name: Check inputs
env:
NODES: ${{ inputs.nodes || '7' }}
TIMEOUT: ${{ inputs.timeout_min || '75' }}
run: |
# the scenarios name nodes up to n6 (crashes, pruning, faults), so at least 7
case "$NODES" in 7|9|11|13|15) ;; *) echo "nodes must be odd, 7 to 15, got '$NODES'"; exit 1 ;; esac
[ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 120 ] || { echo "timeout_min must be 10 to 120, got '$TIMEOUT'"; exit 1; }

- name: Checkout
uses: actions/checkout@v4

Expand Down Expand Up @@ -147,7 +163,7 @@ jobs:
scenario:
needs: build
runs-on: ubuntu-24.04
timeout-minutes: 120
timeout-minutes: 150
strategy:
fail-fast: false
# a few runners at a time: the suite is long and heavy use got Actions disabled once
Expand Down Expand Up @@ -181,6 +197,7 @@ jobs:
# peers must answer NoSuchBlocks, not AwaitAnswer and then nothing
- name: pruned-peers-join
env: NODE_STORAGE=lmdb,lmdb,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000
fixed: true
# kills that land while the node prunes, or right after a checkpoint it prunes from
- name: rocksdb-pruned-crash
env: DB_BACKEND=rocksdb PRUNE_NODE=6 CRASH_NODE=6 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=2800
Expand Down Expand Up @@ -238,10 +255,13 @@ jobs:
# with a power cut and packet corruption
- name: all-storage
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=5 SNAPSHOT_AT=2300 TARGET_SEQ=3300
fixed: true
- name: all-storage-crash
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_NODE=4 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=3300
fixed: true
- name: all-storage-power-cut
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_ALL_AT=2500 NET_FAULT=corrupt NET_FAULT_PCT=3 NET_FAULT_NODES=2,6 TARGET_SEQ=3300
fixed: true
# a node joining from a snapshot taken after the first pruning, and from a truncated one
- name: snapshot-join
env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3100
Expand All @@ -255,7 +275,8 @@ jobs:
name: ${{ matrix.name }}
# a manual run can pick scenarios; the others stop at once
env:
SELECTED: ${{ inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name)) }}
# the scenarios with a NODE_STORAGE list are laid out for 7 nodes
SELECTED: ${{ (inputs.only == '' || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }}
steps:
- name: Skip unselected scenario
if: env.SELECTED != 'true'
Expand Down Expand Up @@ -296,12 +317,12 @@ jobs:

- name: Run private network
if: env.SELECTED == 'true'
timeout-minutes: 100
timeout-minutes: 130
env:
SCENARIO: ${{ matrix.env }}
NODES: '7'
NODES: ${{ inputs.nodes || '7' }}
TARGET_SEQ: '1100' # a scenario can raise it
TIMEOUT_MIN: '75'
TIMEOUT_MIN: ${{ inputs.timeout_min || '75' }}
RESTART_AT: '300'
FUND: '1'
LOAD_SECONDS: '300'
Expand All @@ -320,6 +341,13 @@ jobs:
echo "scenario: $SCENARIO"
ci/testnet/run.sh build/node

- name: Resource summary
if: always() && env.SELECTED == 'true'
run: |
if [ -f testnet-run/resources-summary.txt ]; then
{ echo "### Resources (${{ matrix.name }})"; echo '```'; cat testnet-run/resources-summary.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
fi

- name: Upload logs
if: always() && env.SELECTED == 'true'
uses: actions/upload-artifact@v4
Expand All @@ -328,5 +356,10 @@ jobs:
path: |
testnet-run/n*/*.log
testnet-run/fund.log
testnet-run/contracts.log
testnet-run/n*/log/*.txt
testnet-run/n*/config.ini
testnet-run/resources.tsv
testnet-run/resources-summary.txt
testnet-run/sampler.log
retention-days: 14
20 changes: 18 additions & 2 deletions .github/workflows/testnet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ on:
restart_nodes:
description: 'Nodes stopped and restarted together in the restart test, e.g. "6 7" (empty = the last node)'
default: ''
nodes:
description: 'Nodes in the network (odd, 5 to 15; 15 is the full-mesh limit of max_neighbours=16)'
default: '7'
schedule:
- cron: '17 2 * * *'
pull_request:
Expand All @@ -71,11 +74,17 @@ env:
jobs:
testnet:
if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'testnet')
name: Private network (7 nodes)
name: Private network (${{ inputs.nodes || '7' }} nodes)
runs-on: ubuntu-24.04
timeout-minutes: 340

steps:
- name: Check inputs
env:
NODES: ${{ inputs.nodes || '7' }}
run: |
case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac

- name: Checkout
uses: actions/checkout@v4

Expand Down Expand Up @@ -259,7 +268,7 @@ jobs:
- name: Run private network
timeout-minutes: 300
env:
NODES: '7'
NODES: ${{ inputs.nodes || '7' }}
TARGET_SEQ: ${{ inputs.target_seq || '1100' }}
TIMEOUT_MIN: ${{ inputs.timeout_min || '45' }}
DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }}
Expand All @@ -283,6 +292,9 @@ jobs:
free -m
df -h "${GITHUB_WORKSPACE}"
du -sh testnet-run 2>/dev/null || true
if [ -f testnet-run/resources-summary.txt ]; then
{ echo '### Resources'; echo '```'; cat testnet-run/resources-summary.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
fi

- name: Upload node logs
if: always()
Expand All @@ -295,4 +307,8 @@ jobs:
testnet-run/contracts.log
testnet-run/n*/log/*.txt
testnet-run/n*/config.ini
testnet-run/resources.tsv
testnet-run/resources-summary.txt
testnet-run/sampler.log
testnet-run/progress
retention-days: 14
26 changes: 26 additions & 0 deletions ci/testnet/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
# TARGET_SEQ (default 1100: past the first STATE DIGEST at block 1000)
# TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test)
# WORK (default ./testnet-run)
# SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU
# of the nodes and executors in resources.tsv, and prints a summary at the end
# FUND (default 0; needs a -DCREDITS_TESTNET=ON binary): genesis funds go to a generated master
# key, which funds every node through node 1's API (port 9090) once the chain reaches FUND_AT
# (default 30) and then keeps random transfers flowing for LOAD_SECONDS (default 300)
Expand Down Expand Up @@ -105,6 +107,13 @@ BIN_ALT=${BIN_ALT:+$(realpath "$BIN_ALT")}
HERE=$(cd "$(dirname "$0")" && pwd)
REPO=$(cd "$HERE/../.." && pwd)
NODES=${NODES:-7}
# MinTrustedNodes is 4; an even set of ready nodes drops one at bootstrap
case "$NODES" in
5|7|9|11|13|15|17|19|21|23|25) ;;
*) echo "NODES must be odd, 5 to 25, got '$NODES'"; exit 1 ;;
esac
[ "$NODES" -le 15 ] || echo "warning: $NODES nodes, max_neighbours is 16: not a full mesh"
SAMPLE_SECONDS=${SAMPLE_SECONDS:-60}
TARGET_SEQ=${TARGET_SEQ:-1100}
TIMEOUT_MIN=${TIMEOUT_MIN:-45}
RESTART_AT=${RESTART_AT:-300}
Expand Down Expand Up @@ -377,6 +386,13 @@ net_fault_stats() {
}

stop_all() {
# the sampler's last sample still sees the nodes running
if [ -n "${SAMPLER_PID:-}" ]; then
kill -TERM "$SAMPLER_PID" 2>/dev/null || true
for _ in $(seq 1 20); do kill -0 "$SAMPLER_PID" 2>/dev/null || break; sleep 1; done
kill -KILL "$SAMPLER_PID" 2>/dev/null || true
wait "$SAMPLER_PID" 2>/dev/null || true
fi
[ -z "${API_LOAD_PID:-}" ] || kill -TERM "$API_LOAD_PID" 2>/dev/null || true
for i in $(seq 1 "$ALL_NODES"); do
[ -n "${PIDS[$i]:-}" ] && kill -TERM "${PIDS[$i]}" 2>/dev/null || true
Expand All @@ -397,6 +413,7 @@ stop_all() {
fi
[ -z "$NET_FAULT" ] || net_fault_off
[ -z "${GARBAGE_PID:-}" ] || kill "$GARBAGE_PID" 2>/dev/null || true
[ ! -f "$WORK/resources-summary.txt" ] || cat "$WORK/resources-summary.txt"
}
trap stop_all EXIT

Expand Down Expand Up @@ -452,6 +469,12 @@ fi

for i in $(seq 1 "$NODES"); do start_node "$i"; done
echo "started $NODES nodes in $WORK"
SAMPLER_PID=
if [ "$SAMPLE_SECONDS" -gt 0 ]; then
python3 "$HERE/sampler.py" "$WORK" "$SAMPLE_SECONDS" "$WORK/resources.tsv" --nodes "$NODES" \
--summary-file "$WORK/resources-summary.txt" > "$WORK/sampler.log" 2>&1 &
SAMPLER_PID=$!
fi

deadline=$(( $(date +%s) + TIMEOUT_MIN * 60 ))
restarted=0
Expand Down Expand Up @@ -479,6 +502,7 @@ while :; do
sleep 20
line="t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) ))s"
min=-1
max=0
for i in $(seq 1 "$NODES"); do
s=$(seq_of "$i"); s=${s:-0}
line+=" n$i=$s"
Expand All @@ -498,13 +522,15 @@ while :; do
fi
fi
if [ "$min" -lt 0 ] || [ "$s" -lt "$min" ]; then min=$s; fi
if [ "$s" -gt "$max" ]; then max=$s; fi
done
join_seq=-1
if [ "$JOIN_NODE" -gt 0 ] && [ -n "${PIDS[$JOIN_NODE]:-}" ]; then
join_seq=$(seq_of "$JOIN_NODE"); join_seq=${join_seq:-0}
line+=" n$JOIN_NODE=$join_seq(joined)"
fi
echo "$line"
echo "t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) )) min=$min max=$max" > "$WORK/progress"

if [ "$JOIN_PRUNED_AT" -gt 0 ] && [ -z "${PIDS[$JOIN_NODE]:-}" ] && [ "$min" -ge "$JOIN_PRUNED_AT" ]; then
echo "join test: n$JOIN_NODE starts from block 0 with pruned storage at sequence $min"
Expand Down
Loading
Loading