Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ci/official/*.patch -text
41 changes: 37 additions & 4 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ on:
description: 'Minutes each scenario has to reach its target sequence'
default: '75'
node_alt_ref:
description: 'Branch or commit of this repository built and run by the even nodes (empty = all the same)'
description: 'Branch or commit of this repository built and run by the even nodes; "official" = the official 4.2.532 node with the CI test-network hooks (empty = all the same)'
default: ''

permissions:
Expand Down Expand Up @@ -130,8 +130,16 @@ jobs:
CCACHE_BASEDIR: ${{ github.workspace }}
NODE_ALT_REF: ${{ inputs.node_alt_ref }}
run: |
git fetch -q --no-recurse-submodules origin -- "$NODE_ALT_REF"
# "official": the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI
# test network needs (its starter key and genesis recipient, ci/official/testnet-hooks.patch)
ref=$NODE_ALT_REF
[ "$ref" != official ] || ref=4a95aaaf602daead3ad70ecba113da92374b14f4
git fetch -q --no-recurse-submodules origin -- "$ref"
git worktree add -q node-alt FETCH_HEAD
if [ "$NODE_ALT_REF" = official ]; then
git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch"
git -C node-alt diff --stat
fi
# the submodules are copied from this run's checkout, which must point to the same commits
for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \
third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do
Expand All @@ -150,7 +158,7 @@ jobs:
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \
-DWITH_HASKELL=OFF \
-DCREDITS_TESTNET=ON
-DCREDITS_TESTNET=$([ "$NODE_ALT_REF" = official ] && echo OFF || echo ON)
cmake --build build-alt -j"$(nproc)" --target node
echo "node under comparison: $(git -C node-alt log -1 --format='%h %s')"

Expand Down Expand Up @@ -194,22 +202,28 @@ jobs:
env: DB_BACKEND=lmdb DISK_FULL_NODE=6 DISK_FULL_AT=700
# pruning starts at the periodic checkpoint #2000 (checkpoint_every + prune_keep_blocks <= 2000)
- name: rocksdb-pruned-join
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=rocksdb PRUNE_NODE=6 JOIN_PRUNED_AT=500 TARGET_SEQ=2300
# a full node syncs from block 0 among mostly pruned peers, with special order 35 active: the pruned
# peers must answer NoSuchBlocks, not AwaitAnswer and then nothing
- name: pruned-peers-join
official: 'no' # needs a feature of this fork on an even node
env: NODE_STORAGE=lmdb,lmdb,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000
fixed: true
# kills that land while the node prunes, or right after a checkpoint it prunes from
- name: rocksdb-pruned-crash
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=rocksdb PRUNE_NODE=6 CRASH_NODE=6 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=2800
- name: lmdb-pruned-crash
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 CRASH_NODE=6 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=2800
# API threads read blocks while the node prunes (the pool cache is shared)
- name: lmdb-pruned-api
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 API_LOAD_NODE=6 TARGET_SEQ=3000
# docker stop with a short timeout: SIGTERM, then SIGKILL while the node shuts down
- name: lmdb-pruned-term-kill
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 TARGET_SEQ=3000
- name: berkeleydb-term-kill
env: DB_BACKEND=berkeleydb TERM_KILL_NODE=3
Expand All @@ -223,6 +237,7 @@ jobs:
env: DB_BACKEND=rocksdb APPLY_FAIL_AT=400
# a node that lost its transaction index rebuilds it from its blocks after a quick start
- name: lmdb-pruned-index-loss
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 INDEX_LOSS_AT=2300 TARGET_SEQ=3000
# a full node without its transaction index: quick start, then the index from its blocks
- name: berkeleydb-index-loss
Expand All @@ -231,6 +246,7 @@ jobs:
- name: hashdb-damage
env: DB_BACKEND=lmdb HASHDB_DAMAGE_AT=500 TARGET_SEQ=1100
- name: hashdb-loss-pruned
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 HASHDB_DAMAGE_NODE=6 HASHDB_DAMAGE=remove HASHDB_DAMAGE_AT=2300 TARGET_SEQ=3000
# network faults on two nodes (tc netem on everything they send and on what is sent to them)
- name: net-loss
Expand Down Expand Up @@ -264,47 +280,60 @@ jobs:
# snapshot of n5); then the same network with crashes of the pruned BerkeleyDB node, and
# with a power cut and packet corruption
- name: all-storage
official: 'no' # needs a feature of this fork on an even node
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=5 SNAPSHOT_AT=2300 TARGET_SEQ=3300
fixed: true
- name: all-storage-crash
official: 'no' # needs a feature of this fork on an even node
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_NODE=4 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=3300
fixed: true
- name: all-storage-power-cut
official: 'no' # needs a feature of this fork on an even node
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_ALL_AT=2500 NET_FAULT=corrupt NET_FAULT_PCT=3 NET_FAULT_NODES=2,6 TARGET_SEQ=3300
fixed: true
# a node joining from a snapshot taken after the first pruning, and from a truncated one
- name: snapshot-join
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3100
- name: snapshot-corrupt
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_CORRUPT=1 TARGET_SEQ=3100
# format 2 across block databases: exported by a pruned BerkeleyDB node into a pruned LMDB one,
# and by a pruned RocksDB node into a full RocksDB one; format 1 kept for the previous image
- name: snapshot-join-from-berkeleydb
official: 'no' # needs a feature of this fork on an even node
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=4 SNAPSHOT_AT=2300 TARGET_SEQ=3300
fixed: true
- name: snapshot-join-rocksdb-full
official: 'no' # needs a feature of this fork on an even node
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,rocksdb PRUNE_NODE=6 JOIN_FULL=1 SNAPSHOT_AT=2300 TARGET_SEQ=3300
fixed: true
- name: snapshot-join-format1
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_FORMAT=1 TARGET_SEQ=3100
# node 5 from full to pruned, node 6 from pruned to full, after node 6 pruned once
- name: storage-switch
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 STORAGE_SWITCH_AT=2300 TARGET_SEQ=3300
- name: db-convert
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=berkeleydb CONVERT_AT=1500 TARGET_SEQ=2500
# smart contracts (each node with its executor, contracts.py through n1) across the faults:
# six SIGKILLs of n6 and its executor while contracts are deployed and called
# a node that syncs from block 0 through plays of a contract reading its own balance
# (docs/specs/2026-10-24-contract-state-sync-design.md): its balances and the contract's
# state must end as the live nodes', the state fetched from the confidants once it is live
- name: contracts-dice-join
official: 'no' # an official joiner (n8) re-executes Dice while syncing and ends on another state: the official behaviour #93-#96 fixed (reproduced 2026-10-25)
env: DB_BACKEND=lmdb CONTRACTS=1 CONTRACT_DICE=1 JOIN_PRUNED_AT=500 JOIN_FULL=1 CONTRACTS_AFTER_AT=700 TARGET_SEQ=1300
contracts: true
# the same with sync_contract_execution=false: no re-execution while syncing, every state fetched
- name: contracts-dice-join-nosync-exec
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb CONTRACTS=1 CONTRACT_DICE=1 JOIN_PRUNED_AT=500 JOIN_FULL=1 CONTRACTS_AFTER_AT=700 TARGET_SEQ=1300 SYNC_CONTRACT_EXECUTION=0
contracts: true
- name: contracts-crash
official: 'no' # n6 is official: killed while saving a checkpoint it never restarts (checkpoint ahead of its blocks, official behaviour this fork handles; reproduced 2026-10-25)
env: DB_BACKEND=lmdb CRASH_NODE=6 CONTRACTS=1 TARGET_SEQ=1600
contracts: true
# every node and executor killed after the first calls, calls again after the restart
Expand All @@ -327,22 +356,25 @@ jobs:
manual: true
# a pruned node quick-starting from checkpoints that hold the contract states
- name: contracts-pruned-term-kill
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000
contracts: true
# a node joining from block 0 past the deploys, then following live calls
- name: contracts-pruned-join
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=lmdb PRUNE_NODE=6 JOIN_PRUNED_AT=500 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=2300
contracts: true
# block databases converted with contract records in them
- name: contracts-db-convert
official: 'no' # needs a feature of this fork on an even node
env: DB_BACKEND=berkeleydb CONVERT_AT=1500 CONTRACTS=1 CONTRACTS_AFTER_AT=1700 TARGET_SEQ=2500
contracts: true
name: ${{ matrix.name }}
# a manual run can pick scenarios; the others stop at once
env:
# the scenarios with a NODE_STORAGE list are laid out for 7 nodes
# manual scenarios (known failures kept as reproductions) run only when named in `only`
SELECTED: ${{ ((inputs.only == '' && matrix.manual != true) || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }}
SELECTED: ${{ ((inputs.only == '' && matrix.manual != true) || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') && (inputs.node_alt_ref != 'official' || matrix.official != 'no') }}
steps:
- name: Skip unselected scenario
if: env.SELECTED != 'true'
Expand Down Expand Up @@ -402,6 +434,7 @@ jobs:
if [ -f build-alt/node ]; then
chmod +x build-alt/node
export BIN_ALT="$PWD/build-alt/node"
[ "${{ inputs.node_alt_ref }}" != official ] || export BIN_ALT_OFFICIAL=1
fi
# the scenario's variables come from the matrix above, NAME=value pairs only
for kv in $SCENARIO; do
Expand Down
66 changes: 62 additions & 4 deletions .github/workflows/scale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ on:
scenario:
description: 'More run.sh variables as NAME=value pairs, as a robustness scenario (e.g. PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3300)'
default: ''
official:
description: '1 = the even nodes run the official 4.2.532 node with the CI test-network hooks (ci/official/testnet-hooks.patch)'
default: '0'

permissions:
contents: read
Expand All @@ -52,6 +55,7 @@ jobs:
trusted: ${{ steps.counts.outputs.trusted }}
dpos_at: ${{ steps.counts.outputs.dpos_at }}
low_stake: ${{ steps.counts.outputs.low_stake }}
official: ${{ steps.counts.outputs.official }}
steps:
- name: Node counts
id: counts
Expand All @@ -62,18 +66,22 @@ jobs:
LOW: ${{ inputs.low_stake_nodes || '0' }}
TIMEOUT: ${{ inputs.timeout_min || '60' }}
LOAD: ${{ inputs.contract_load || '120,2,20' }}
OFFICIAL: ${{ inputs.official || '0' }}
run: |
# a tag scale-t<trusted>-n<count>_<count>...[-d<sequence>][-l<nodes>] sets them too
# (scale-t7-n33_49-d200-l10: 7 initial confidants, 33 and 49 nodes, stake-based election from
# block 200, the last 10 nodes below the minimum stake)
# a tag scale-t<trusted>-n<count>_<count>...[-d<sequence>][-l<nodes>][-o] sets them too
# (scale-t7-n33_49-d200-l10-o: 7 initial confidants, 33 and 49 nodes, stake-based election from
# block 200, the last 10 nodes below the minimum stake, the even nodes official)
DPOS=${DPOS:-0}
LOW=${LOW:-0}
if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?(-l([0-9]+))?$ ]]; then
if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?(-l([0-9]+))?(-o)?$ ]]; then
TRUSTED=${BASH_REMATCH[1]}
LIST=${BASH_REMATCH[2]//_/,}
DPOS=${BASH_REMATCH[4]:-0}
LOW=${BASH_REMATCH[6]:-0}
OFFICIAL=$([ -n "${BASH_REMATCH[7]}" ] && echo 1 || echo 0)
fi
case "$OFFICIAL" in 0|1) ;; *) echo "official must be 0 or 1, got '$OFFICIAL'"; exit 1 ;; esac
echo "official=$OFFICIAL" >> "$GITHUB_OUTPUT"
if [ -n "$TRUSTED" ]; then
case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac
fi
Expand Down Expand Up @@ -164,6 +172,45 @@ jobs:
path: build/client/node
retention-days: 7

# the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI test network
# needs (its starter key and genesis recipient), for the even nodes
- name: Build the official node
if: needs.plan.outputs.official == '1'
env:
CCACHE_BASEDIR: ${{ github.workspace }}
run: |
git fetch -q --no-recurse-submodules origin -- 4a95aaaf602daead3ad70ecba113da92374b14f4
git worktree add -q node-alt FETCH_HEAD
git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch"
git -C node-alt diff --stat
# the submodules are copied from this run's checkout, which must point to the same commits
for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \
third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do
if [ "$(git ls-tree HEAD -- "$m")" != "$(git -C node-alt ls-tree HEAD -- "$m")" ]; then
echo "the official node points $m to another commit than this run"; exit 1
fi
rm -rf "node-alt/$m"
cp -a "$m" "node-alt/$m"
done
cmake -S node-alt -B build-alt -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DENABLE_TESTS=OFF \
-DBoost_ROOT="${BOOST_PREFIX}" \
-DBoost_NO_SYSTEM_PATHS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \
-DWITH_HASKELL=OFF
cmake --build build-alt -j"$(nproc)" --target node

- name: Upload the official node
if: needs.plan.outputs.official == '1'
uses: actions/upload-artifact@v4
with:
name: testnet-node-official-${{ github.sha }}
path: build-alt/client/node
retention-days: 7

network:
needs: [plan, build]
runs-on: ubuntu-24.04
Expand Down Expand Up @@ -198,6 +245,13 @@ jobs:
name: testnet-node-${{ github.sha }}
path: build

- name: Download the official node
if: needs.plan.outputs.official == '1'
uses: actions/download-artifact@v4
with:
name: testnet-node-official-${{ github.sha }}
path: build-alt

- name: Run private network
timeout-minutes: 110
env:
Expand All @@ -216,6 +270,10 @@ jobs:
SCENARIO: ${{ inputs.scenario || '' }}
run: |
chmod +x build/node
if [ "${{ needs.plan.outputs.official }}" = 1 ]; then
chmod +x build-alt/node
export BIN_ALT="$PWD/build-alt/node" BIN_ALT_OFFICIAL=1
fi
IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD"
export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE
# NAME=value pairs only, as the robustness scenarios
Expand Down
Loading
Loading