diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..4618eff54 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +ci/official/*.patch -text diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index fb057c733..c1a228fb2 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -31,7 +31,7 @@ on: description: 'Minutes each scenario has to reach its target sequence' default: '75' node_alt_ref: - description: 'Branch or commit of this repository built and run by the even nodes (empty = all the same)' + description: 'Branch or commit of this repository built and run by the even nodes; "official" = the official 4.2.532 node with the CI test-network hooks (empty = all the same)' default: '' permissions: @@ -130,8 +130,16 @@ jobs: CCACHE_BASEDIR: ${{ github.workspace }} NODE_ALT_REF: ${{ inputs.node_alt_ref }} run: | - git fetch -q --no-recurse-submodules origin -- "$NODE_ALT_REF" + # "official": the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI + # test network needs (its starter key and genesis recipient, ci/official/testnet-hooks.patch) + ref=$NODE_ALT_REF + [ "$ref" != official ] || ref=4a95aaaf602daead3ad70ecba113da92374b14f4 + git fetch -q --no-recurse-submodules origin -- "$ref" git worktree add -q node-alt FETCH_HEAD + if [ "$NODE_ALT_REF" = official ]; then + git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch" + git -C node-alt diff --stat + fi # the submodules are copied from this run's checkout, which must point to the same commits for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \ third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do @@ -150,7 +158,7 @@ jobs: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \ -DWITH_HASKELL=OFF \ - -DCREDITS_TESTNET=ON + -DCREDITS_TESTNET=$([ "$NODE_ALT_REF" = official ] && echo OFF || echo ON) cmake --build build-alt -j"$(nproc)" --target node echo "node under comparison: $(git -C node-alt log -1 --format='%h %s')" @@ -194,22 +202,28 @@ jobs: env: DB_BACKEND=lmdb DISK_FULL_NODE=6 DISK_FULL_AT=700 # pruning starts at the periodic checkpoint #2000 (checkpoint_every + prune_keep_blocks <= 2000) - name: rocksdb-pruned-join + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=rocksdb PRUNE_NODE=6 JOIN_PRUNED_AT=500 TARGET_SEQ=2300 # a full node syncs from block 0 among mostly pruned peers, with special order 35 active: the pruned # peers must answer NoSuchBlocks, not AwaitAnswer and then nothing - name: pruned-peers-join + official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=lmdb,lmdb,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000 fixed: true # kills that land while the node prunes, or right after a checkpoint it prunes from - name: rocksdb-pruned-crash + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=rocksdb PRUNE_NODE=6 CRASH_NODE=6 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=2800 - name: lmdb-pruned-crash + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 CRASH_NODE=6 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=2800 # API threads read blocks while the node prunes (the pool cache is shared) - name: lmdb-pruned-api + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 API_LOAD_NODE=6 TARGET_SEQ=3000 # docker stop with a short timeout: SIGTERM, then SIGKILL while the node shuts down - name: lmdb-pruned-term-kill + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 TARGET_SEQ=3000 - name: berkeleydb-term-kill env: DB_BACKEND=berkeleydb TERM_KILL_NODE=3 @@ -223,6 +237,7 @@ jobs: env: DB_BACKEND=rocksdb APPLY_FAIL_AT=400 # a node that lost its transaction index rebuilds it from its blocks after a quick start - name: lmdb-pruned-index-loss + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 INDEX_LOSS_AT=2300 TARGET_SEQ=3000 # a full node without its transaction index: quick start, then the index from its blocks - name: berkeleydb-index-loss @@ -231,6 +246,7 @@ jobs: - name: hashdb-damage env: DB_BACKEND=lmdb HASHDB_DAMAGE_AT=500 TARGET_SEQ=1100 - name: hashdb-loss-pruned + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 HASHDB_DAMAGE_NODE=6 HASHDB_DAMAGE=remove HASHDB_DAMAGE_AT=2300 TARGET_SEQ=3000 # network faults on two nodes (tc netem on everything they send and on what is sent to them) - name: net-loss @@ -264,33 +280,43 @@ jobs: # snapshot of n5); then the same network with crashes of the pruned BerkeleyDB node, and # with a power cut and packet corruption - name: all-storage + official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=5 SNAPSHOT_AT=2300 TARGET_SEQ=3300 fixed: true - name: all-storage-crash + official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_NODE=4 CRASH_FROM=2000 CRASH_TIMES=3 TARGET_SEQ=3300 fixed: true - name: all-storage-power-cut + official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb CRASH_ALL_AT=2500 NET_FAULT=corrupt NET_FAULT_PCT=3 NET_FAULT_NODES=2,6 TARGET_SEQ=3300 fixed: true # a node joining from a snapshot taken after the first pruning, and from a truncated one - name: snapshot-join + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3100 - name: snapshot-corrupt + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_CORRUPT=1 TARGET_SEQ=3100 # format 2 across block databases: exported by a pruned BerkeleyDB node into a pruned LMDB one, # and by a pruned RocksDB node into a full RocksDB one; format 1 kept for the previous image - name: snapshot-join-from-berkeleydb + official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=4 SNAPSHOT_AT=2300 TARGET_SEQ=3300 fixed: true - name: snapshot-join-rocksdb-full + official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,rocksdb PRUNE_NODE=6 JOIN_FULL=1 SNAPSHOT_AT=2300 TARGET_SEQ=3300 fixed: true - name: snapshot-join-format1 + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_FORMAT=1 TARGET_SEQ=3100 # node 5 from full to pruned, node 6 from pruned to full, after node 6 pruned once - name: storage-switch + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 STORAGE_SWITCH_AT=2300 TARGET_SEQ=3300 - name: db-convert + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=berkeleydb CONVERT_AT=1500 TARGET_SEQ=2500 # smart contracts (each node with its executor, contracts.py through n1) across the faults: # six SIGKILLs of n6 and its executor while contracts are deployed and called @@ -298,13 +324,16 @@ jobs: # (docs/specs/2026-10-24-contract-state-sync-design.md): its balances and the contract's # state must end as the live nodes', the state fetched from the confidants once it is live - name: contracts-dice-join + official: 'no' # an official joiner (n8) re-executes Dice while syncing and ends on another state: the official behaviour #93-#96 fixed (reproduced 2026-10-25) env: DB_BACKEND=lmdb CONTRACTS=1 CONTRACT_DICE=1 JOIN_PRUNED_AT=500 JOIN_FULL=1 CONTRACTS_AFTER_AT=700 TARGET_SEQ=1300 contracts: true # the same with sync_contract_execution=false: no re-execution while syncing, every state fetched - name: contracts-dice-join-nosync-exec + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb CONTRACTS=1 CONTRACT_DICE=1 JOIN_PRUNED_AT=500 JOIN_FULL=1 CONTRACTS_AFTER_AT=700 TARGET_SEQ=1300 SYNC_CONTRACT_EXECUTION=0 contracts: true - name: contracts-crash + official: 'no' # n6 is official: killed while saving a checkpoint it never restarts (checkpoint ahead of its blocks, official behaviour this fork handles; reproduced 2026-10-25) env: DB_BACKEND=lmdb CRASH_NODE=6 CONTRACTS=1 TARGET_SEQ=1600 contracts: true # every node and executor killed after the first calls, calls again after the restart @@ -327,14 +356,17 @@ jobs: manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000 contracts: true # a node joining from block 0 past the deploys, then following live calls - name: contracts-pruned-join + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=lmdb PRUNE_NODE=6 JOIN_PRUNED_AT=500 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=2300 contracts: true # block databases converted with contract records in them - name: contracts-db-convert + official: 'no' # needs a feature of this fork on an even node env: DB_BACKEND=berkeleydb CONVERT_AT=1500 CONTRACTS=1 CONTRACTS_AFTER_AT=1700 TARGET_SEQ=2500 contracts: true name: ${{ matrix.name }} @@ -342,7 +374,7 @@ jobs: env: # the scenarios with a NODE_STORAGE list are laid out for 7 nodes # manual scenarios (known failures kept as reproductions) run only when named in `only` - SELECTED: ${{ ((inputs.only == '' && matrix.manual != true) || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') }} + SELECTED: ${{ ((inputs.only == '' && matrix.manual != true) || contains(format(',{0},', inputs.only), format(',{0},', matrix.name))) && (matrix.fixed != true || inputs.nodes == '' || inputs.nodes == '7') && (inputs.node_alt_ref != 'official' || matrix.official != 'no') }} steps: - name: Skip unselected scenario if: env.SELECTED != 'true' @@ -402,6 +434,7 @@ jobs: if [ -f build-alt/node ]; then chmod +x build-alt/node export BIN_ALT="$PWD/build-alt/node" + [ "${{ inputs.node_alt_ref }}" != official ] || export BIN_ALT_OFFICIAL=1 fi # the scenario's variables come from the matrix above, NAME=value pairs only for kv in $SCENARIO; do diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index 2db3e5607..55a4260cf 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -34,6 +34,9 @@ on: scenario: description: 'More run.sh variables as NAME=value pairs, as a robustness scenario (e.g. PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3300)' default: '' + official: + description: '1 = the even nodes run the official 4.2.532 node with the CI test-network hooks (ci/official/testnet-hooks.patch)' + default: '0' permissions: contents: read @@ -52,6 +55,7 @@ jobs: trusted: ${{ steps.counts.outputs.trusted }} dpos_at: ${{ steps.counts.outputs.dpos_at }} low_stake: ${{ steps.counts.outputs.low_stake }} + official: ${{ steps.counts.outputs.official }} steps: - name: Node counts id: counts @@ -62,18 +66,22 @@ jobs: LOW: ${{ inputs.low_stake_nodes || '0' }} TIMEOUT: ${{ inputs.timeout_min || '60' }} LOAD: ${{ inputs.contract_load || '120,2,20' }} + OFFICIAL: ${{ inputs.official || '0' }} run: | - # a tag scale-t-n_...[-d][-l] sets them too - # (scale-t7-n33_49-d200-l10: 7 initial confidants, 33 and 49 nodes, stake-based election from - # block 200, the last 10 nodes below the minimum stake) + # a tag scale-t-n_...[-d][-l][-o] sets them too + # (scale-t7-n33_49-d200-l10-o: 7 initial confidants, 33 and 49 nodes, stake-based election from + # block 200, the last 10 nodes below the minimum stake, the even nodes official) DPOS=${DPOS:-0} LOW=${LOW:-0} - if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?(-l([0-9]+))?$ ]]; then + if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?(-l([0-9]+))?(-o)?$ ]]; then TRUSTED=${BASH_REMATCH[1]} LIST=${BASH_REMATCH[2]//_/,} DPOS=${BASH_REMATCH[4]:-0} LOW=${BASH_REMATCH[6]:-0} + OFFICIAL=$([ -n "${BASH_REMATCH[7]}" ] && echo 1 || echo 0) fi + case "$OFFICIAL" in 0|1) ;; *) echo "official must be 0 or 1, got '$OFFICIAL'"; exit 1 ;; esac + echo "official=$OFFICIAL" >> "$GITHUB_OUTPUT" if [ -n "$TRUSTED" ]; then case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac fi @@ -164,6 +172,45 @@ jobs: path: build/client/node retention-days: 7 + # the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI test network + # needs (its starter key and genesis recipient), for the even nodes + - name: Build the official node + if: needs.plan.outputs.official == '1' + env: + CCACHE_BASEDIR: ${{ github.workspace }} + run: | + git fetch -q --no-recurse-submodules origin -- 4a95aaaf602daead3ad70ecba113da92374b14f4 + git worktree add -q node-alt FETCH_HEAD + git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch" + git -C node-alt diff --stat + # the submodules are copied from this run's checkout, which must point to the same commits + for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \ + third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do + if [ "$(git ls-tree HEAD -- "$m")" != "$(git -C node-alt ls-tree HEAD -- "$m")" ]; then + echo "the official node points $m to another commit than this run"; exit 1 + fi + rm -rf "node-alt/$m" + cp -a "$m" "node-alt/$m" + done + cmake -S node-alt -B build-alt -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTS=OFF \ + -DBoost_ROOT="${BOOST_PREFIX}" \ + -DBoost_NO_SYSTEM_PATHS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \ + -DWITH_HASKELL=OFF + cmake --build build-alt -j"$(nproc)" --target node + + - name: Upload the official node + if: needs.plan.outputs.official == '1' + uses: actions/upload-artifact@v4 + with: + name: testnet-node-official-${{ github.sha }} + path: build-alt/client/node + retention-days: 7 + network: needs: [plan, build] runs-on: ubuntu-24.04 @@ -198,6 +245,13 @@ jobs: name: testnet-node-${{ github.sha }} path: build + - name: Download the official node + if: needs.plan.outputs.official == '1' + uses: actions/download-artifact@v4 + with: + name: testnet-node-official-${{ github.sha }} + path: build-alt + - name: Run private network timeout-minutes: 110 env: @@ -216,6 +270,10 @@ jobs: SCENARIO: ${{ inputs.scenario || '' }} run: | chmod +x build/node + if [ "${{ needs.plan.outputs.official }}" = 1 ]; then + chmod +x build-alt/node + export BIN_ALT="$PWD/build-alt/node" BIN_ALT_OFFICIAL=1 + fi IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD" export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE # NAME=value pairs only, as the robustness scenarios diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index d100f5b4c..9d283a829 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -44,7 +44,7 @@ on: description: 'With contracts: branch or commit of EnzinoBB/contract-executor built and run by the even nodes, the odd nodes keep the release executor (empty = all release)' default: '' node_alt_ref: - description: 'Branch or commit of this repository built and run by the even nodes, the odd nodes run this run''s node (empty = all the same)' + description: 'Branch or commit of this repository built and run by the even nodes, the odd nodes run this run''s node; "official" = the official 4.2.532 node with the CI test-network hooks (the second nightly run uses it; empty = all the same)' default: '' restart_nodes: description: 'Nodes stopped and restarted together in the restart test, e.g. "6 7" (empty = the last node)' @@ -59,13 +59,15 @@ on: description: 'With contracts: load phase as "seconds,writes per second,reads per second", e.g. 120,2,20 (empty = none)' default: '' schedule: + # every night: one run of this node alone, one with official nodes on the even positions - cron: '17 2 * * *' + - cron: '47 3 * * *' pull_request: branches: [master] types: [labeled, synchronize, opened, reopened] concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.schedule }} cancel-in-progress: true permissions: @@ -161,13 +163,21 @@ jobs: # mixed nodes: the even nodes run a node built from another ref, so a block or contract result # that differs from this run's node shows up as a stalled or diverging network - name: Build node under test - if: inputs.node_alt_ref != '' + if: inputs.node_alt_ref != '' || github.event.schedule == '47 3 * * *' env: CCACHE_BASEDIR: ${{ github.workspace }} - NODE_ALT_REF: ${{ inputs.node_alt_ref }} + NODE_ALT_REF: ${{ inputs.node_alt_ref || (github.event.schedule == '47 3 * * *' && 'official' || '') }} run: | - git fetch -q --no-recurse-submodules origin -- "$NODE_ALT_REF" + # "official": the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI + # test network needs (its starter key and genesis recipient, ci/official/testnet-hooks.patch) + ref=$NODE_ALT_REF + [ "$ref" != official ] || ref=4a95aaaf602daead3ad70ecba113da92374b14f4 + git fetch -q --no-recurse-submodules origin -- "$ref" git worktree add -q node-alt FETCH_HEAD + if [ "$NODE_ALT_REF" = official ]; then + git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch" + git -C node-alt diff --stat + fi # the submodules are copied from this run's checkout, which must point to the same commits for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \ third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do @@ -186,9 +196,10 @@ jobs: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \ -DWITH_HASKELL=OFF \ - -DCREDITS_TESTNET=ON + -DCREDITS_TESTNET=$([ "$NODE_ALT_REF" = official ] && echo OFF || echo ON) cmake --build build-alt -j"$(nproc)" --target node echo "BIN_ALT=$PWD/build-alt/client/node" >> "$GITHUB_ENV" + [ "$NODE_ALT_REF" != official ] || echo "BIN_ALT_OFFICIAL=1" >> "$GITHUB_ENV" echo "node under test: $(git -C node-alt log -1 --format='%h %s')" # the contract executor of the official mainnet release (the one operators run) diff --git a/ci/official/testnet-hooks.patch b/ci/official/testnet-hooks.patch new file mode 100644 index 000000000..58224559e --- /dev/null +++ b/ci/official/testnet-hooks.patch @@ -0,0 +1,44 @@ +diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp +index 274d3a30..71714d25 100644 +--- a/csnode/src/blockchain.cpp ++++ b/csnode/src/blockchain.cpp +@@ -1,5 +1,6 @@ + #include + #include ++#include + #include + #include + #include +@@ -367,6 +368,11 @@ void BlockChain::writeGenesisBlock() { + csdb::Transaction transaction; + + std::string strAddr = "5B3YXqDTcWQFGAqEJQJP3Bg1ZK8FFtHtgCiFLT5VAxpe"; ++ // CI test networks only (patch ci/official/testnet-hooks.patch): genesis funds to the network's key ++ if (const char* key = std::getenv("CS_TESTNET_GENESIS_KEY"); key != nullptr && *key != '\0') { ++ strAddr = key; ++ cswarning() << kLogPrefix << "TESTNET hooks: genesis funds go to " << strAddr; ++ } + std::vector pub_key; + DecodeBase58(strAddr, pub_key); + +diff --git a/net/src/packetvalidator.cpp b/net/src/packetvalidator.cpp +index fbaa4f88..70565c23 100644 +--- a/net/src/packetvalidator.cpp ++++ b/net/src/packetvalidator.cpp +@@ -1,3 +1,5 @@ ++#include ++ + #include + #include + +@@ -18,6 +20,10 @@ std::string bcStr("81p93jgrHqA9L4Vkdut9ESSCV1XNoge7LXBW96cuA7sm"); + + struct BlockChainKey { + BlockChainKey() { ++ // CI test networks only (patch ci/official/testnet-hooks.patch): the starter key of the network ++ if (const char* key = std::getenv("CS_TESTNET_STARTER_KEY"); key != nullptr && *key != '\0') { ++ bcStr = key; ++ } + cs::Bytes keyBytes; + DecodeBase58(bcStr, keyBytes); + std::copy(keyBytes.begin(), keyBytes.end(), key_.begin()); diff --git a/ci/testnet/blockhashes.py b/ci/testnet/blockhashes.py new file mode 100644 index 000000000..e93626a7b --- /dev/null +++ b/ci/testnet/blockhashes.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Compare the blocks of several nodes through their public APIs: the hash of every 100th block up to +the lowest head, and of that head, must be the same on every node as on the first one given. + +Used by run.sh with BIN_ALT_OFFICIAL=1, where the official nodes log no state digest: holding the +same blocks as the community nodes is what they are checked on. + + blockhashes.py --head 1100 1:9090 2:9102 3:9103 ... +""" +import argparse +import sys + +from thriftpy2.protocol import TBinaryProtocolFactory +from thriftpy2.rpc import make_client +from thriftpy2.transport import TBufferedTransportFactory + +from fund import api + + +def block_hash(client, seq): + result = client.PoolListGetStable(seq, 1) + for pool in result.pools or []: + if pool.poolNumber == seq: + return bytes(pool.hash).hex() + return None + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--head", type=int, required=True, help="lowest head of the nodes compared") + parser.add_argument("nodes", nargs="+", help="node:port, the first is the reference") + args = parser.parse_args() + + sequences = list(range(100, args.head, 100)) + [args.head] + reference = None + failed = False + for entry in args.nodes: + node, port = entry.split(":") + try: + client = make_client(api.API, host="127.0.0.1", port=int(port), proto_factory=TBinaryProtocolFactory(), + trans_factory=TBufferedTransportFactory(), timeout=60000) + hashes = {seq: block_hash(client, seq) for seq in sequences} + except Exception as e: # a node that does not answer is a failure, with the reason + print(f"n{node}: API on port {port} not answering: {e}") + if reference is None: + sys.exit(1) # no reference to compare the others with + failed = True + continue + if reference is None: + reference = hashes + print(f"blocks: n{node} is the reference, {len(sequences)} blocks up to #{args.head}") + continue + missing = [seq for seq, h in hashes.items() if h is None] + differ = [seq for seq, h in hashes.items() if h is not None and h != reference[seq]] + if missing or differ: + failed = True + print(f"n{node}: {len(differ)} blocks differ from the reference (first {differ[:3]}), " + f"{len(missing)} not found (first {missing[:3]})") + else: + print(f"n{node}: the same {len(sequences)} blocks as the reference") + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index d536ffa40..55efe732f 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -46,6 +46,16 @@ # same JDK, so any result that differs from the release executor's fails the contract consensus # BIN_ALT (optional): the even nodes run this node binary instead, so a block or contract result # that differs from BIN's shows up as a diverging state +# BIN_ALT_OFFICIAL=1: BIN_ALT is the official node (ci/official/testnet-hooks.patch), which logs +# no state digest: every node opens its API and the even nodes must hold the same blocks as n1 +# (hash at every 100th sequence up to the lowest head); the digests are compared among the others. +# The official node runs BerkeleyDB whatever DB_BACKEND says, and a scenario that needs on an +# even node a feature only this fork has (pruning, snapshots, conversion, damage and failure +# injection, NODE_STORAGE other than berkeleydb, SYNC_CONTRACT_EXECUTION=0) is refused at once. +# A fork of an official node shows only as a block hash that differs (it logs no fork line), and +# an official node restarted after DPOS_AT goes back to StartingDPOS 10000 (it does not keep +# order 9 across a quick start): official behaviour, so restarts after DPOS_AT are refused for +# even nodes # RESTART_NODES (default: the last initial confidant): the nodes the restart test stops and restarts together # NET_FAULT (default off): loss, corrupt, delay, reorder (with 5% duplicates), duplicate or partition, applied with # tc netem to the p2p traffic sent and received by NET_FAULT_NODES (comma separated) from @@ -128,6 +138,9 @@ BIN=$(realpath "$1") # mixed nodes: the even nodes run BIN_ALT, built from another ref BIN_ALT=${BIN_ALT:+$(realpath "$BIN_ALT")} [ -z "$BIN_ALT" ] || echo "mixed nodes: even nodes run $BIN_ALT" +BIN_ALT_OFFICIAL=${BIN_ALT_OFFICIAL:-0} +[ -n "$BIN_ALT" ] || BIN_ALT_OFFICIAL=0 +[ "$BIN_ALT_OFFICIAL" != 1 ] || echo "mixed nodes: the even nodes are official nodes" HERE=$(cd "$(dirname "$0")" && pwd) REPO=$(cd "$HERE/../.." && pwd) NODES=${NODES:-7} @@ -269,6 +282,34 @@ BASE_PORT=6000 declare -a PIDS KEYS REPORTED_DOWN +# official even nodes: refuse what they cannot take part in (see BIN_ALT_OFFICIAL above) +official_node() { [ "$BIN_ALT_OFFICIAL" = 1 ] && [ $(( $1 % 2 )) -eq 0 ]; } +if [ "$BIN_ALT_OFFICIAL" = 1 ]; then + refuse() { echo "FAIL: $1 needs a feature the official node lacks (BIN_ALT_OFFICIAL=1: the even nodes are official)"; exit 1; } + for p in $PRUNED_NODES; do ! official_node "$p" || refuse "pruned storage on n$p"; done + [ "$JOIN_NODE" -eq 0 ] || [ "$SNAPSHOT_AT" -eq 0 ] || ! official_node "$JOIN_NODE" || refuse "a snapshot import into n$JOIN_NODE" + [ "$SNAPSHOT_AT" -eq 0 ] || [ "$PRUNE_NODE" -eq 0 ] || ! official_node "$PRUNE_NODE" || refuse "a snapshot export from n$PRUNE_NODE" + [ "$STORAGE_SWITCH_AT" -eq 0 ] || refuse "STORAGE_SWITCH_AT" + [ "$CONVERT_AT" -eq 0 ] || refuse "CONVERT_AT" + [ "$INDEX_LOSS_AT" -eq 0 ] || ! official_node "$INDEX_LOSS_NODE" || refuse "INDEX_LOSS on n$INDEX_LOSS_NODE" + [ "$HASHDB_DAMAGE_AT" -eq 0 ] || ! official_node "$HASHDB_DAMAGE_NODE" || refuse "HASHDB_DAMAGE on n$HASHDB_DAMAGE_NODE" + [ "$CHECKPOINT_AHEAD_AT" -eq 0 ] || ! official_node "$CHECKPOINT_AHEAD_NODE" || refuse "CHECKPOINT_AHEAD on n$CHECKPOINT_AHEAD_NODE" + [ "$APPLY_FAIL_AT" -eq 0 ] || ! official_node "$APPLY_FAIL_NODE" || refuse "APPLY_FAIL on n$APPLY_FAIL_NODE" + for i in ${SNAPSHOT_SERVE_NODES//,/ }; do ! official_node "$i" || refuse "snapshot serving on n$i"; done + [ "$SYNC_CONTRACT_EXECUTION" != 0 ] || refuse "SYNC_CONTRACT_EXECUTION=0" + if [ -n "$NODE_STORAGE" ]; then + for i in $(seq 1 "$ALL_NODES"); do + ! official_node "$i" || [ "${STORE_BACKEND[$i]}" = berkeleydb ] || refuse "NODE_STORAGE ${STORE_BACKEND[$i]} on n$i" + done + fi + if [ "$DPOS_AT" -gt 0 ]; then + restarted="$RESTART_NODES $CRASH_NODE $TERM_KILL_NODE" + [ "$CRASH_ALL_AT" -eq 0 ] || refuse "CRASH_ALL_AT with DPOS_AT (order 9 is not kept across a restart)" + for i in $restarted; do [ "$i" -eq 0 ] || ! official_node "$i" || refuse "a restart of n$i with DPOS_AT (order 9 is not kept across a restart)"; done + fi + [ -z "$DB_BACKEND" ] || echo "mixed nodes: the official nodes run BerkeleyDB whatever DB_BACKEND ($DB_BACKEND) says" +fi + mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$ALL_NODES" "$WORK") if [ "$FUND" = "1" ]; then @@ -300,6 +341,11 @@ fi # public API on node 1 for fund.py; with CONTRACTS every node gets one (contracts.py compares their # contract states) and its own executor, started by the node through executor.sh +# the public API port of node i when it has one: 9090 for n1 with FUND or CONTRACTS, else 9100 + i +api_port() { + if [ "$1" -eq 1 ] && { [ "$FUND" = "1" ] || [ "$CONTRACTS" = "1" ]; }; then echo 9090; else echo $((9100 + $1)); fi +} + api_settings() { local i=$1 if [ "$CONTRACTS" != "1" ]; then @@ -308,7 +354,12 @@ api_settings() { echo "apiexec_port=0" return fi - echo "port=$([ "$FUND" = "1" ] && [ "$i" -eq 1 ] && echo 9090 || echo 0)" + if [ "$BIN_ALT_OFFICIAL" = 1 ]; then + # every node answers block queries: the official ones are compared with n1 by block hash + echo "port=$(api_port "$i")" + else + echo "port=$([ "$FUND" = "1" ] && [ "$i" -eq 1 ] && echo 9090 || echo 0)" + fi echo "apiexec_port=0" return fi @@ -413,6 +464,11 @@ Format="[%TimeStamp%] %Severity% %Message%" CFG done +# the binary node i runs +bin_of() { + if [ -n "$BIN_ALT" ] && [ $(($1 % 2)) -eq 0 ]; then echo "$BIN_ALT"; else echo "$BIN"; fi +} + start_node() { local i=$1 # core dumps (see core_pattern in the workflow) let a crash be traced afterwards @@ -605,7 +661,7 @@ while :; do for core in "$WORK"/core.*."${PIDS[$i]}"; do [ -f "$core" ] || continue echo "n$i core dump $core:" - gdb -q -batch -ex "bt 60" -ex "thread apply all bt 6" "$BIN" "$core" 2>/dev/null | grep -E '^(#|Thread)' | head -n 200 + gdb -q -batch -ex "bt 60" -ex "thread apply all bt 6" "$(bin_of "$i")" "$core" 2>/dev/null | grep -E '^(#|Thread)' | head -n 200 mv "$core" "$core.traced" done fi @@ -1123,6 +1179,7 @@ fi if [ -n "$DB_BACKEND" ] || [ -n "$NODE_STORAGE" ]; then for i in $(seq 1 "$ALL_NODES"); do used=$(grep -ah 'block database: ' "$WORK/n$i"/*.log | tail -1 | sed 's/.*block database: //' || true) + if official_node "$i"; then used="berkeleydb (official node)"; fi echo "n$i block database: $used" if [ -n "$NODE_STORAGE" ] && [ "${used%% *}" != "${STORE_BACKEND[$i]}" ]; then echo "FAIL: n$i was to run on ${STORE_BACKEND[$i]}"; exit 1 @@ -1148,7 +1205,10 @@ fi for core in "$WORK"/core.* "$WORK"/n*/core*; do [ -f "$core" ] || continue echo "core dump $core:" - gdb -q -batch -ex "bt 40" "$BIN" "$core" 2>/dev/null | grep -E '^#' | head -n 40 + # the binary of the node whose process left it (core..) + core_bin=$BIN + for i in $(seq 1 "$ALL_NODES"); do [ "${PIDS[$i]:-}" != "${core##*.}" ] || core_bin=$(bin_of "$i"); done + gdb -q -batch -ex "bt 40" "$core_bin" "$core" 2>/dev/null | grep -E '^#' | head -n 40 done for p in $PRUNED_NODES; do @@ -1260,13 +1320,18 @@ done # every node logs "STATE DIGEST # wallets "; at a sequence logged by several nodes the # digests must be identical -python3 - "$WORK" "$ALL_NODES" "$FUND" "$NODES" "$SNAPSHOT_AT" <<'PY' +python3 - "$WORK" "$ALL_NODES" "$FUND" "$NODES" "$SNAPSHOT_AT" "$BIN_ALT_OFFICIAL" <<'PY' import collections, glob, re, sys work, nodes, funded = sys.argv[1], int(sys.argv[2]), sys.argv[3] == "1" +# official nodes (the even ones with BIN_ALT_OFFICIAL) log no digest: their blocks are compared below +official = {i for i in range(2, nodes + 1, 2)} if sys.argv[6] == "1" else set() # a node that joins from a snapshot starts past block 1000: only the initial nodes log it initial = int(sys.argv[4]) if int(sys.argv[5]) > 0 else nodes +initial -= len([i for i in official if i <= initial]) seen = collections.defaultdict(dict) for i in range(1, nodes + 1): + if i in official: + continue for path in glob.glob(f"{work}/n{i}/*.log"): for m in re.finditer(r"STATE DIGEST #(\d+) ([0-9a-fA-F]+) wallets (\d+)", open(path, errors="replace").read()): seen[int(m.group(1))][i] = (m.group(2).lower(), int(m.group(3))) @@ -1285,6 +1350,18 @@ if funded and 1000 in seen and max(w for _, w in seen[1000].values()) <= 2: ok = False sys.exit(0 if ok else 1) PY +# official nodes: the same block hashes as n1 (every 100th sequence up to the lowest head) +if [ "$BIN_ALT_OFFICIAL" = 1 ]; then + ports="" + head="" + for i in $(seq 1 "$ALL_NODES"); do + ports="$ports $i:$(api_port "$i")" + s=$(seq_of "$i"); s=${s:-0} + if [ -z "$head" ] || [ "$s" -lt "$head" ]; then head=$s; fi + done + (cd "$HERE" && python3 blockhashes.py --head "$head" $ports) || { echo "FAIL: official and community nodes hold different blocks"; exit 1; } +fi + # two blocks of one sequence: a node kept its own block while the round went on on the writer's # (csnode/src/blockchain.cpp updateLastBlock logs it), a fork that nothing reconciles forks=$(grep -al 'this node keeps its own (fork)' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index c2596ee84..5c6aa62b3 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -39,6 +39,7 @@ lines under the new release. | #101 | net: `host_data` announces snapshot serving (bit 24, `[snapshot] serve=on` at start) and the community version (bits 32-47, `MINOR_NODE_VERSION`, now 13); `known_peers.json` at a clean stop lists the known peers with what they announce | | #102 | snapshot: a node with `[snapshot] serve=on` builds the snapshot of each periodic checkpoint and serves it to its p2p neighbours, within `serve_rate_kbps` (KB/s), `serve_max_peers` and per-peer limits, at half budget while trusted; snapshots use 512 KiB chunks and no contracts file | | #104 | snapshot: the p2p fetcher (not started yet): finds the neighbours that serve snapshots, agrees on one (a configured `sources=` key first, then the most independent sources by /24 or /48, the highest block only as a tie-break), downloads its pieces from several sources with every piece and file checked against the signed manifest, an adaptive piece timeout for slow links, limits per network prefix against made-up keys, and the fallback to syncing from block 0 after `fetch_wait` (or a bounded total time) | +| #103 | ci: the official 4.2.532 node takes part in the test networks (`node_alt_ref=official`, `scale.yml` `official=1` or tag suffix `-o`, and the second nightly `testnet.yml` run), built with only the two hooks a test network needs (`ci/official/testnet-hooks.patch`); official nodes are checked by block hash, and scenarios needing a feature only this fork has are refused or skipped. First runs: mixed consensus works (13 nodes, official and community confidants); an official node that syncs a balance-reading contract ends on another state, and one killed while saving a checkpoint does not restart | ## community.12 — 2026-10-24