From a5bc87f7ee8c344a0ff71b74ef50909e1925f536 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 15:47:44 +0200 Subject: [PATCH 01/15] ci(testnet): contract reads on every node, a contract load phase, up to 25 nodes - contracts.py: getters (TransactionFlow with forgetNewState: run on the node's own executor, no transaction) of Counter.get and Ledger.entries on every node after the calls before and after the restart, each node until it returns the expected value; 20 reads on n1 must leave the contract's transaction count, its state and the master's inner id unchanged; optional load phase (--load-seconds, --write-rate, --read-rate): Counter.next writes from every node's account through its own API and getters over every node, latency p50/p95/max in --stats-file; after it the counter must lie between the confirmed writes and all of them, the same on every node - run.sh: CONTRACT_LOAD_SECONDS, CONTRACT_WRITE_RATE, CONTRACT_READ_RATE; up to 25 nodes (MaxTrustedNodes); the full-mesh warning is gone (initial confidants are permanent neighbours) - testnet.yml: nodes 5 to 25, contract_load "seconds,writes,reads"; robustness.yml: nodes 5 to 25 - sampler: executor relaunches in the summary Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 4 +- .github/workflows/testnet.yml | 17 ++- ci/testnet/contracts.py | 201 ++++++++++++++++++++++++++++++- ci/testnet/run.sh | 15 ++- ci/testnet/sampler.py | 3 + 5 files changed, 232 insertions(+), 8 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 97a33cbd6..1bcacac6e 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -25,7 +25,7 @@ on: description: 'Comma-separated scenario names to run (empty = all)' default: '' nodes: - description: 'Nodes in the network (odd, 5 to 15); the NODE_STORAGE scenarios run only at 7' + description: 'Nodes in the network (odd, 5 to 25 = MaxTrustedNodes); the NODE_STORAGE scenarios run only at 7' default: '7' timeout_min: description: 'Minutes each scenario has to reach its target sequence' @@ -50,7 +50,7 @@ jobs: NODES: ${{ inputs.nodes || '7' }} TIMEOUT: ${{ inputs.timeout_min || '75' }} run: | - case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac + case "$NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 5 to 25, got '$NODES'"; exit 1 ;; esac [ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 120 ] || { echo "timeout_min must be 10 to 120, got '$TIMEOUT'"; exit 1; } - name: Checkout diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index 0890efb0f..f745894a3 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -50,8 +50,11 @@ on: description: 'Nodes stopped and restarted together in the restart test, e.g. "6 7" (empty = the last node)' default: '' nodes: - description: 'Nodes in the network (odd, 5 to 15; 15 is the full-mesh limit of max_neighbours=16)' + description: 'Nodes in the network (odd, 5 to 25 = MaxTrustedNodes)' default: '7' + contract_load: + description: 'With contracts: load phase as "seconds,writes per second,reads per second", e.g. 120,2,20 (empty = none)' + default: '' schedule: - cron: '17 2 * * *' pull_request: @@ -83,7 +86,7 @@ jobs: env: NODES: ${{ inputs.nodes || '7' }} run: | - case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac + case "$NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 5 to 25, got '$NODES'"; exit 1 ;; esac - name: Checkout uses: actions/checkout@v4 @@ -246,11 +249,18 @@ jobs: DELEGATE: ${{ inputs.delegate || '0' }} MIN_STAKE: ${{ inputs.min_stake || '0' }} CONTRACTS: ${{ inputs.contracts || '0' }} + CONTRACT_LOAD: ${{ inputs.contract_load || '' }} RESTART_NODES: ${{ inputs.restart_nodes || '' }} FUND: '1' LOAD_SECONDS: '300' WORK: ${{ github.workspace }}/testnet-run - run: ci/testnet/run.sh build/client/node + run: | + if [ -n "$CONTRACT_LOAD" ]; then + echo "$CONTRACT_LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } + IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD" + export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE + fi + ci/testnet/run.sh build/client/node - name: Resource usage if: always() @@ -274,6 +284,7 @@ jobs: testnet-run/n*/log/*.txt testnet-run/n*/config.ini testnet-run/resources.tsv + testnet-run/contracts-stats.json testnet-run/resources-summary.txt testnet-run/sampler.log testnet-run/progress diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index fa732d246..9e2db3294 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -28,10 +28,14 @@ """ import argparse import hashlib +import json import os +import statistics import struct import sys +import threading import time +from concurrent.futures import ThreadPoolExecutor import base58 from thriftpy2.protocol import TBinaryProtocolFactory @@ -39,7 +43,7 @@ from thriftpy2.transport import TBufferedTransportFactory from thriftpy2.utils import serialize -from fund import Account, api, encode_max_fee, general, pack_inner_id +from fund import Account, api, encode_max_fee, general, node_account, pack_inner_id CURRENCY_CS = 1 TT_DEPLOY = 1 # api.TransactionType: contract deployment @@ -196,6 +200,38 @@ def smart_transaction(src, target, sci, tx_type): return t +def read_transaction(src, contract, method): + """A getter: TransactionFlow with forgetNewState runs the method on the node's own executor + against the contract's current state and returns its result; nothing goes to consensus, no + transaction, no fee, no signature check (api/src/apihandler.cpp). A constant signature keeps the + node's getter bookkeeping from growing.""" + sci = api.SmartContractInvocation(method=method, params=[], usedContracts=[], forgetNewState=True, + smartContractDeploy=None, version=1) + t = api.Transaction() + t.id = 0 + t.source = src.pk + t.target = contract + t.amount = general.Amount(integral=0, fraction=0) + t.balance = general.Amount(integral=0, fraction=0) + t.currency = CURRENCY_CS + t.signature = bytes(64) + t.smartContract = sci + t.fee = api.AmountCommission(commission=SMART_MAX_FEE) + t.timeCreation = int(time.time() * 1000) + t.type = TT_EXECUTE + return t + + +def read_value(client, src, contract, method): + """One getter call: (value or None, latency ms, status message).""" + t0 = time.time() + result = client.TransactionFlow(read_transaction(src, contract, method)) + ms = (time.time() - t0) * 1000 + # an executor that is not reachable answers status 0 without a result + value = result.smart_contract_result.v_int if result.status.code == 0 and result.smart_contract_result else None + return value, ms, result.status.message + + def next_inner_id(client, account): last = client.WalletTransactionsCountGet(account.pk).lastTransactionInnerId or 0 account.next_inner_id = int(last) + 1 @@ -242,6 +278,97 @@ def call(client, master, contract, method, label, expected=None): return ok +def percentile(values, p): + if not values: + return None + values = sorted(values) + return values[min(len(values) - 1, int(round(p / 100 * (len(values) - 1))))] + + +def load(args, client_for, master, counter, nodes): + """args.load_seconds of Counter.next writes (args.write_rate per second in all, one writer + thread per funded node account, each through its own node's API) and Counter.get reads + (args.read_rate per second, spread over every node). Returns (writes that returned a value, + writes that did not, 1 if more than 1 % of the reads failed).""" + stats = {"write": [], "read": []} + errors = {"write": 0, "read": 0} + lock = threading.Lock() + stop_at = time.time() + args.load_seconds + writers = list(range(1, nodes + 1)) + + def writer(i): + account = node_account(args.work, i) + period = len(writers) / args.write_rate if args.write_rate > 0 else None + while period and time.time() < stop_at: + t0 = time.time() + ok = False + try: + c = client_for(i) + next_inner_id(c, account) + sci = invocation(method="next") + result = c.TransactionFlow(smart_transaction(account, counter, sci, TT_EXECUTE)) + ok = result.status.code == 0 and result.smart_contract_result is not None + msg = result.status.message + except Exception as e: + msg = type(e).__name__ + ms = (time.time() - t0) * 1000 + with lock: + if ok: + stats["write"].append(ms) + else: + errors["write"] += 1 + if errors["write"] <= 5: + print(f"load write via n{i} FAILED: {msg!r}", flush=True) + time.sleep(max(0.0, period - (time.time() - t0))) + + def reader(k): + period = 1.0 / args.read_rate * 4 if args.read_rate > 0 else None + n = k + while period and time.time() < stop_at: + t0 = time.time() + i = n % nodes + 1 + n += 4 + try: + value, ms, msg = read_value(client_for(i), master, counter, "get") + ok = value is not None + except Exception as e: + ok, ms, msg = False, (time.time() - t0) * 1000, type(e).__name__ + with lock: + if ok: + stats["read"].append(ms) + else: + errors["read"] += 1 + if errors["read"] <= 5: + print(f"load read via n{i} FAILED: {msg!r}", flush=True) + time.sleep(max(0.0, period - (time.time() - t0))) + + print(f"load: {args.load_seconds} s, {args.write_rate} writes/s from {len(writers)} accounts, " + f"{args.read_rate} reads/s on {nodes} nodes", flush=True) + threads = [threading.Thread(target=writer, args=(i,)) for i in writers] + \ + [threading.Thread(target=reader, args=(k,)) for k in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + summary = {"nodes": nodes, "seconds": args.load_seconds} + for kind in ("write", "read"): + ms = stats[kind] + sent = len(ms) + errors[kind] + summary[kind] = {"sent": sent, "ok": len(ms), "failed": errors[kind], + "rate": round(len(ms) / args.load_seconds, 2), + "p50_ms": percentile(ms, 50), "p95_ms": percentile(ms, 95), "max_ms": max(ms) if ms else None} + print(f"load {kind}s: {len(ms)} ok of {sent} ({summary[kind]['rate']}/s), latency " + f"p50 {summary[kind]['p50_ms'] or 0:.0f} ms, p95 {summary[kind]['p95_ms'] or 0:.0f} ms, " + f"max {summary[kind]['max_ms'] or 0:.0f} ms", flush=True) + if args.stats_file: + with open(args.stats_file, "w") as f: + json.dump(summary, f, indent=1) + read_sent = summary["read"]["sent"] + bad_reads = 1 if read_sent and errors["read"] > 0.01 * read_sent else 0 + return len(stats["write"]), errors["write"], bad_reads + + def main(): parser = argparse.ArgumentParser() parser.add_argument("work") @@ -254,7 +381,12 @@ def main(): parser.add_argument("--wait-timeout", type=int, default=1800) parser.add_argument("--check-nodes", type=int, default=0) parser.add_argument("--final-file", default="") + parser.add_argument("--load-seconds", type=int, default=0) + parser.add_argument("--write-rate", type=float, default=1.0) + parser.add_argument("--read-rate", type=float, default=5.0) + parser.add_argument("--stats-file", default="") args = parser.parse_args() + check_nodes = args.check_nodes or args.nodes def client_for(i): port = 9090 if i == 1 else 9100 + i @@ -283,6 +415,48 @@ def call_callers(when, via): calls += 1 failures += 0 if call(client_for(1), master, counter, "next", f"call {n + 1} before restart", calls) else 1 failures += 0 if call(client_for(1), master, ledger, "record", f"Ledger call {calls} before restart", calls) else 1 + def read_all(contract, method, expected, label, timeout=60): + """The getter on every node, until each returns `expected` (a lagging node needs a round or + two); prints one line, returns the number of nodes that did not.""" + def one(i): + deadline = time.time() + timeout + first_ms = None + value, msg = None, "" + while True: + try: + value, ms, msg = read_value(client_for(i), master, contract, method) + first_ms = ms if first_ms is None else first_ms + except Exception as e: # a node without a reachable API + value, msg = None, type(e).__name__ + if value == expected or time.time() >= deadline: + return i, value, first_ms, msg + time.sleep(2) + with ThreadPoolExecutor(max_workers=check_nodes) as pool: + results = list(pool.map(one, range(1, check_nodes + 1))) + bad = [r for r in results if r[1] != expected] + cells = " ".join(f"n{i}={v}" + (f"/{ms:.0f}ms" if ms is not None else "") for i, v, ms, _ in results) + print(f"read {label}: {'ok' if not bad else 'FAILED'} (expected {expected}) {cells}", flush=True) + for i, v, _, msg in bad: + print(f" n{i}: {v!r} {msg!r}", flush=True) + return len(bad) + + failures += read_all(counter, "get", calls, "Counter.get before restart") + failures += read_all(ledger, "entries", calls, "Ledger.entries before restart") + + # reads change nothing: no transaction on the contract, the same state, the same inner id + def footprint(): + c = client_for(1) + got = c.SmartContractGet(counter).smartContract + return got.transactionsCount, hashlib.sha256(got.objectState).hexdigest(), \ + c.WalletTransactionsCountGet(master.pk).lastTransactionInnerId + before = footprint() + for _ in range(20): + read_value(client_for(1), master, counter, "get") + after = footprint() + print(f"20 reads on n1: contract transactions, state, master inner id {before} -> {after}: " + f"{'unchanged' if before == after else 'CHANGED'}", flush=True) + failures += 0 if before == after else 1 + if args.ready_file: open(args.ready_file, "w").close() if args.wait_file: @@ -295,6 +469,31 @@ def call_callers(when, via): failures += 0 if call(client_for(1), master, counter, "next", f"call {n + 1} after restart", calls) else 1 failures += 0 if call(client_for(1), master, ledger, "record", f"Ledger call {calls} after restart", calls) else 1 failures += 0 if call(client_for(1), master, ledger, "entries", "Ledger entries", calls) else 1 + failures += read_all(counter, "get", calls, "Counter.get after restart") + failures += read_all(ledger, "entries", calls, "Ledger.entries after restart") + + if args.load_seconds > 0: + writes_ok, writes_failed, bad_reads = load(args, client_for, master, counter, check_nodes) + failures += writes_failed + bad_reads + # a write that failed on its way back (a timeout) may still have landed: the counter is + # between the confirmed writes and all of them, and every node must read that same value + time.sleep(30) + value = None + for _ in range(10): + try: + value = read_value(client_for(1), master, counter, "get")[0] + except Exception: + value = None + if value is not None: + break + time.sleep(5) + low, high = calls + writes_ok, calls + writes_ok + writes_failed + in_range = value is not None and low <= value <= high + print(f"Counter after the load: {value} (confirmed writes give {low}..{high}): {'ok' if in_range else 'FAILED'}", + flush=True) + failures += 0 if in_range else 1 + calls = value if value is not None else calls + failures += read_all(counter, "get", calls, "Counter.get after the load", timeout=120) call_callers("after restart", 1) call_callers("after restart", 2) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index b18d2aa78..84091877c 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -28,6 +28,10 @@ # node reached CONTRACTS_AFTER_AT (default 0); the states are compared once every node, # a joining one too, reached TARGET_SEQ. contracts.py sends everything through n1: a # contract scenario never kills, partitions or slows n1 +# CONTRACT_LOAD_SECONDS (default 0, with CONTRACTS): after the calls, a load phase of that many +# seconds: CONTRACT_WRITE_RATE (default 1) Counter.next writes a second from every node's +# account through its own API, and CONTRACT_READ_RATE (default 5) Counter.get reads (getters, +# no transaction) a second over every node; latencies in contracts-stats.json # EXECUTOR_JAR_ALT (optional, with CONTRACTS): the even nodes run this executor jar instead, on the # same JDK, so any result that differs from the release executor's fails the contract consensus # BIN_ALT (optional): the even nodes run this node binary instead, so a block or contract result @@ -116,7 +120,12 @@ case "$NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "NODES must be odd, 5 to 25, got '$NODES'"; exit 1 ;; esac -[ "$NODES" -le 15 ] || echo "warning: $NODES nodes, max_neighbours is 16: not a full mesh" +# 25 is Consensus::MaxTrustedNodes: every node here is an initial confidant, so no larger table can +# bootstrap. Above 15 the neighbour map (16) is not a full mesh, but initial confidants are permanent +# neighbours of each other and consensus messages are routed, so that does not stop the network +CONTRACT_LOAD_SECONDS=${CONTRACT_LOAD_SECONDS:-0} +CONTRACT_WRITE_RATE=${CONTRACT_WRITE_RATE:-1} +CONTRACT_READ_RATE=${CONTRACT_READ_RATE:-5} SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} @@ -905,7 +914,9 @@ while :; do echo "contracts: deploying and calling them through node 1" (cd "$HERE" && python3 contracts.py "$WORK" "$NODES" "$MASTER_SEED" \ --ready-file "$WORK/contracts-ready" --wait-file "$WORK/restart-done" --wait-timeout $((TIMEOUT_MIN * 60)) \ - --check-nodes "$ALL_NODES" --final-file "$WORK/contracts-final") > "$WORK/contracts.log" 2>&1 & + --check-nodes "$ALL_NODES" --final-file "$WORK/contracts-final" \ + --load-seconds "$CONTRACT_LOAD_SECONDS" --write-rate "$CONTRACT_WRITE_RATE" --read-rate "$CONTRACT_READ_RATE" \ + --stats-file "$WORK/contracts-stats.json") > "$WORK/contracts.log" 2>&1 & CONTRACTS_PID=$! fi # the calls after the restart must run on the restarted nodes too, so let them catch up first diff --git a/ci/testnet/sampler.py b/ci/testnet/sampler.py index c9862ec5f..ae08e0ba6 100644 --- a/ci/testnet/sampler.py +++ b/ci/testnet/sampler.py @@ -164,6 +164,9 @@ def key(k): out.append(f" {k[1]:<16} {max(r['rss_mb'] for r in rs):>6} {statistics.mean(cpus) if cpus else 0:>6.0f} " f"{max(cpus) if cpus else 0:>6.0f} {len({r['pid'] for r in rs}):>4}") out.append(f" sum of peak RSS: {sum(max(r['rss_mb'] for r in procs[k]) for k in keys)} MB") + if kind == "exec": + # a node that stops stops every executor of the machine (jps), and each node relaunches its own + out.append(f" executor relaunches: {sum(len({r['pid'] for r in procs[k]}) - 1 for k in keys)}") per_ts = {} for r in rows: if r["kind"] in ("node", "exec"): From f2f8e4f047fcabb03024489fd60f6427f94a2383 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 15:48:55 +0200 Subject: [PATCH 02/15] ci: scale workflow, one runner per node count with contracts read and written A row per node count in the step summary: result, block rate, runner CPU and memory, write and read counts and latency. Runs on a scale-* tag with the defaults (9,13,17,21,25 nodes, 120 s of 2 writes and 20 reads a second) or by hand once on master. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 231 ++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 .github/workflows/scale.yml diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml new file mode 100644 index 000000000..afd228d78 --- /dev/null +++ b/.github/workflows/scale.yml @@ -0,0 +1,231 @@ +name: Scale + +# How many nodes a private network runs, with smart contracts read and written: one runner per node +# count (a standard 4-vCPU, 16-GB GitHub runner), each running ci/testnet/run.sh with CONTRACTS=1 and +# a contract load phase (writes from every node's account, getters on every node). Each job writes +# one row to the step summary: result, block rate, CPU and memory of the runner, write and read +# latency. A node count passes when every node reaches the target with the same state digests and +# contracts.py finds the expected values and states everywhere. +on: + # a scale-* tag runs it with the defaults on the tagged commit (a manual run needs the workflow on + # the default branch) + push: + tags: ['scale-*'] + workflow_dispatch: + inputs: + nodes_list: + description: 'Comma-separated node counts, odd, 5 to 25 (25 = MaxTrustedNodes)' + default: '9,13,17,21,25' + contract_load: + description: 'Load phase as "seconds,writes per second,reads per second"' + default: '120,2,20' + timeout_min: + description: 'Minutes each network has to reach its target sequence' + default: '60' + +permissions: + contents: read + +env: + BOOST_VERSION: 1.83.0 + BOOST_SHA256: 6478edfe2f3305127cffe8caf73ea0176c53769f4bf1585be237eb30798c3b8e + BOOST_PREFIX: /home/runner/boost + CCACHE_DIR: /home/runner/.ccache + +jobs: + plan: + runs-on: ubuntu-24.04 + outputs: + counts: ${{ steps.counts.outputs.counts }} + steps: + - name: Node counts + id: counts + env: + LIST: ${{ inputs.nodes_list || '9,13,17,21,25' }} + LOAD: ${{ inputs.contract_load || '120,2,20' }} + run: | + echo "$LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } + json="[" + for n in ${LIST//,/ }; do + case "$n" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "node counts must be odd, 5 to 25, got '$n'"; exit 1 ;; esac + json+="$n," + done + echo "counts=${json%,}]" >> "$GITHUB_OUTPUT" + + build: + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Checkout submodules + run: | + git submodule update --init -- cscrypto p2p \ + third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \ + third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb + git -C cscrypto submodule update --init + git -C p2p submodule update --init + + - name: Install build tools + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq cmake ninja-build flex bison autoconf automake libtool pkg-config ccache + + - name: Restore Boost + id: cache-boost + uses: actions/cache/restore@v4 + with: + path: ${{ env.BOOST_PREFIX }} + key: boost-${{ env.BOOST_VERSION }}-static-rt-ubuntu-24.04-v1 + + - name: Build Boost + if: steps.cache-boost.outputs.cache-hit != 'true' + run: | + v=${BOOST_VERSION//./_} + curl -sSfL -o boost.tar.bz2 "https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${v}.tar.bz2" + echo "${BOOST_SHA256} boost.tar.bz2" | sha256sum -c + tar xjf boost.tar.bz2 + cd "boost_${v}" + ./bootstrap.sh --with-libraries=system,filesystem,program_options,date_time,regex,log,iostreams,thread,serialization,atomic,chrono + ./b2 -j"$(nproc)" -d0 variant=release link=static runtime-link=static threading=multi \ + cxxflags=-fPIC --layout=tagged -sNO_BZIP2=1 -sNO_ZSTD=1 -sNO_LZMA=1 \ + install --prefix="${BOOST_PREFIX}" + + - name: Restore ccache + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-ubuntu-24.04-${{ github.sha }} + restore-keys: ccache-ubuntu-24.04- + + # the test-network build of testnet.yml + - name: Build node + run: | + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTS=OFF \ + -DBoost_ROOT="${BOOST_PREFIX}" \ + -DBoost_NO_SYSTEM_PATHS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \ + -DWITH_HASKELL=OFF \ + -DCREDITS_TESTNET=ON + ccache --max-size=2G + cmake --build build -j"$(nproc)" --target node + + - name: Upload node + uses: actions/upload-artifact@v4 + with: + name: testnet-node-${{ github.sha }} + path: build/client/node + retention-days: 7 + + network: + needs: [plan, build] + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + # a few runners at a time: heavy use got Actions disabled once + max-parallel: 3 + matrix: + nodes: ${{ fromJSON(needs.plan.outputs.counts) }} + name: ${{ matrix.nodes }} nodes + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Checkout API definitions + run: git submodule update --init -- third-party/thrift-interface-definitions + + - name: Install tools + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq python3-nacl python3-base58 gdb + python3 -m pip install --user --quiet --break-system-packages thriftpy2 + sudo sysctl -w kernel.core_pattern="${{ github.workspace }}/testnet-run/core.%e.%p" + + - name: Contract executor + uses: ./.github/actions/contract-executor + + - name: Download node + uses: actions/download-artifact@v4 + with: + name: testnet-node-${{ github.sha }} + path: build + + - name: Run private network + timeout-minutes: 110 + env: + NODES: ${{ matrix.nodes }} + CONTRACTS: '1' + CONTRACT_LOAD: ${{ inputs.contract_load || '120,2,20' }} + TARGET_SEQ: '1100' + TIMEOUT_MIN: ${{ inputs.timeout_min || '60' }} + RESTART_AT: '300' + FUND: '1' + LOAD_SECONDS: '300' + WORK: ${{ github.workspace }}/testnet-run + run: | + chmod +x build/node + IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD" + export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE + ci/testnet/run.sh build/node 2>&1 | tee testnet-run.log + exit "${PIPESTATUS[0]}" + + - name: Summary + if: always() + env: + N: ${{ matrix.nodes }} + OUTCOME: ${{ job.status }} + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json, os, re + n, outcome = os.environ["N"], os.environ["OUTCOME"] + def read(p): + try: + return open(p).read() + except OSError: + return "" + log, res = read("testnet-run.log"), read("testnet-run/resources-summary.txt") + def grab(pattern, text): + m = re.search(pattern, text) + return m.group(1) if m else "?" + passed = "PASS" if re.search(r"^PASS: \d+ nodes", log, re.M) else "FAIL" + try: + s = json.load(open("testnet-run/contracts-stats.json")) + w, r = s["write"], s["read"] + lat = (f"{w['ok']}/{w['sent']} at {w['rate']}/s, p50 {w['p50_ms'] or 0:.0f} p95 {w['p95_ms'] or 0:.0f} ms | " + f"{r['ok']}/{r['sent']} at {r['rate']}/s, p50 {r['p50_ms'] or 0:.0f} p95 {r['p95_ms'] or 0:.0f} ms") + except Exception: + lat = "? | ?" + print("| nodes | result | blocks/s | CPU mean / peak | peak RAM used | writes | reads |") + print("|---|---|---|---|---|---|---|") + print(f"| {n} | {passed} | {grab(r'blocks: ([0-9.]+) per second', res)} " + f"| {grab(r'busy mean ([0-9]+) %', res)} / {grab(r'peak ([0-9]+) %, ', res)} % " + f"| {grab(r'peak used ([0-9]+) MB', res)} MB | {lat} |") + print() + print("```") + print(res) + print("```") + PY + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: scale-${{ matrix.nodes }}-nodes + path: | + testnet-run.log + testnet-run/n*/*.log + testnet-run/fund.log + testnet-run/contracts.log + testnet-run/contracts-stats.json + testnet-run/n*/log/*.txt + testnet-run/n*/config.ini + testnet-run/resources.tsv + testnet-run/resources-summary.txt + retention-days: 14 From 393c3b609f4dacc99e5dbd9648beca9b4c744729 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 16:16:49 +0200 Subject: [PATCH 03/15] ci(testnet): a contract write consensus rejects under contention is counted apart Several accounts calling one contract in the same round: consensus takes some calls and rejects the others (7 nodes, 2 writes/s: 8 of 211). That is an answer, not a fault: such writes are reported, may still land, and widen the range the final counter must lie in. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 2 +- ci/testnet/contracts.py | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index afd228d78..79edc1a4d 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -198,7 +198,7 @@ jobs: try: s = json.load(open("testnet-run/contracts-stats.json")) w, r = s["write"], s["read"] - lat = (f"{w['ok']}/{w['sent']} at {w['rate']}/s, p50 {w['p50_ms'] or 0:.0f} p95 {w['p95_ms'] or 0:.0f} ms | " + lat = (f"{w['ok']}/{w['sent']} at {w['rate']}/s ({s.get('writes_rejected', 0)} rejected), p50 {w['p50_ms'] or 0:.0f} p95 {w['p95_ms'] or 0:.0f} ms | " f"{r['ok']}/{r['sent']} at {r['rate']}/s, p50 {r['p50_ms'] or 0:.0f} p95 {r['p95_ms'] or 0:.0f} ms") except Exception: lat = "? | ?" diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index 9e2db3294..c8f6d483d 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -289,9 +289,12 @@ def load(args, client_for, master, counter, nodes): """args.load_seconds of Counter.next writes (args.write_rate per second in all, one writer thread per funded node account, each through its own node's API) and Counter.get reads (args.read_rate per second, spread over every node). Returns (writes that returned a value, - writes that did not, 1 if more than 1 % of the reads failed).""" + writes that failed, writes consensus rejected, 1 if more than 1 % of the reads failed).""" stats = {"write": [], "read": []} errors = {"write": 0, "read": 0} + # concurrent calls of one contract from several accounts: consensus takes some and rejects the + # others in a round; a rejected call is an answer, not a fault, and is counted on its own + rejected = [0] lock = threading.Lock() stop_at = time.time() + args.load_seconds writers = list(range(1, nodes + 1)) @@ -315,6 +318,8 @@ def writer(i): with lock: if ok: stats["write"].append(ms) + elif "rejected by consensus" in (msg or ""): + rejected[0] += 1 else: errors["write"] += 1 if errors["write"] <= 5: @@ -351,22 +356,23 @@ def reader(k): for t in threads: t.join() - summary = {"nodes": nodes, "seconds": args.load_seconds} + summary = {"nodes": nodes, "seconds": args.load_seconds, "writes_rejected": rejected[0]} for kind in ("write", "read"): ms = stats[kind] - sent = len(ms) + errors[kind] + sent = len(ms) + errors[kind] + (rejected[0] if kind == "write" else 0) summary[kind] = {"sent": sent, "ok": len(ms), "failed": errors[kind], "rate": round(len(ms) / args.load_seconds, 2), "p50_ms": percentile(ms, 50), "p95_ms": percentile(ms, 95), "max_ms": max(ms) if ms else None} print(f"load {kind}s: {len(ms)} ok of {sent} ({summary[kind]['rate']}/s), latency " f"p50 {summary[kind]['p50_ms'] or 0:.0f} ms, p95 {summary[kind]['p95_ms'] or 0:.0f} ms, " f"max {summary[kind]['max_ms'] or 0:.0f} ms", flush=True) + print(f"load writes rejected by consensus (concurrent calls of one contract): {rejected[0]}", flush=True) if args.stats_file: with open(args.stats_file, "w") as f: json.dump(summary, f, indent=1) read_sent = summary["read"]["sent"] bad_reads = 1 if read_sent and errors["read"] > 0.01 * read_sent else 0 - return len(stats["write"]), errors["write"], bad_reads + return len(stats["write"]), errors["write"], rejected[0], bad_reads def main(): @@ -473,7 +479,7 @@ def footprint(): failures += read_all(ledger, "entries", calls, "Ledger.entries after restart") if args.load_seconds > 0: - writes_ok, writes_failed, bad_reads = load(args, client_for, master, counter, check_nodes) + writes_ok, writes_failed, writes_rejected, bad_reads = load(args, client_for, master, counter, check_nodes) failures += writes_failed + bad_reads # a write that failed on its way back (a timeout) may still have landed: the counter is # between the confirmed writes and all of them, and every node must read that same value @@ -487,9 +493,10 @@ def footprint(): if value is not None: break time.sleep(5) - low, high = calls + writes_ok, calls + writes_ok + writes_failed + low, high = calls + writes_ok, calls + writes_ok + writes_failed + writes_rejected in_range = value is not None and low <= value <= high - print(f"Counter after the load: {value} (confirmed writes give {low}..{high}): {'ok' if in_range else 'FAILED'}", + print(f"Counter after the load: {value} (confirmed writes give {low}..{high}, unconfirmed ones may land): " + f"{'ok' if in_range else 'FAILED'}", flush=True) failures += 0 if in_range else 1 calls = value if value is not None else calls From b0f7c4cff8a24ddfec3fd97ee2b52cca13587437 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 16:18:34 +0200 Subject: [PATCH 04/15] ci(testnet): TRUSTED_NODES, a network of up to 61 nodes with 5 to 25 initial confidants Only the first TRUSTED_NODES nodes are in trusted.txt; the others are ordinary nodes, as most of mainnet, that follow the rounds and may be elected trusted later. testnet.yml and scale.yml get a trusted_nodes input. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 15 ++++++++++++--- .github/workflows/testnet.yml | 14 ++++++++++++-- ci/testnet/run.sh | 16 ++++++++++++---- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index 79edc1a4d..e082fd4cf 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -14,8 +14,11 @@ on: workflow_dispatch: inputs: nodes_list: - description: 'Comma-separated node counts, odd, 5 to 25 (25 = MaxTrustedNodes)' + description: 'Comma-separated node counts: odd, 5 to 25 (25 = MaxTrustedNodes) when every node is trusted; up to 61 with trusted_nodes' default: '9,13,17,21,25' + trusted_nodes: + description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node)' + default: '' contract_load: description: 'Load phase as "seconds,writes per second,reads per second"' default: '120,2,20' @@ -42,12 +45,17 @@ jobs: id: counts env: LIST: ${{ inputs.nodes_list || '9,13,17,21,25' }} + TRUSTED: ${{ inputs.trusted_nodes || '' }} LOAD: ${{ inputs.contract_load || '120,2,20' }} run: | echo "$LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } json="[" for n in ${LIST//,/ }; do - case "$n" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "node counts must be odd, 5 to 25, got '$n'"; exit 1 ;; esac + if [ -n "$TRUSTED" ]; then + [ "$n" -ge "$TRUSTED" ] && [ "$n" -le 61 ] || { echo "node counts must be $TRUSTED to 61, got '$n'"; exit 1; } + else + case "$n" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "node counts must be odd, 5 to 25, got '$n'"; exit 1 ;; esac + fi json+="$n," done echo "counts=${json%,}]" >> "$GITHUB_OUTPUT" @@ -133,7 +141,7 @@ jobs: max-parallel: 3 matrix: nodes: ${{ fromJSON(needs.plan.outputs.counts) }} - name: ${{ matrix.nodes }} nodes + name: ${{ matrix.nodes }} nodes${{ inputs.trusted_nodes && format(', {0} trusted', inputs.trusted_nodes) || '' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -161,6 +169,7 @@ jobs: timeout-minutes: 110 env: NODES: ${{ matrix.nodes }} + TRUSTED_NODES: ${{ inputs.trusted_nodes || '' }} CONTRACTS: '1' CONTRACT_LOAD: ${{ inputs.contract_load || '120,2,20' }} TARGET_SEQ: '1100' diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index c20c044fa..d100f5b4c 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -52,6 +52,9 @@ on: nodes: description: 'Nodes in the network (odd, 5 to 25 = MaxTrustedNodes)' default: '7' + trusted_nodes: + description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node); then nodes may go up to 61' + default: '' contract_load: description: 'With contracts: load phase as "seconds,writes per second,reads per second", e.g. 120,2,20 (empty = none)' default: '' @@ -77,7 +80,7 @@ env: jobs: testnet: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'testnet') - name: Private network (${{ inputs.nodes || '7' }} nodes) + name: Private network (${{ inputs.nodes || '7' }} nodes${{ inputs.trusted_nodes && format(', {0} trusted', inputs.trusted_nodes) || '' }}) runs-on: ubuntu-24.04 timeout-minutes: 340 @@ -85,8 +88,14 @@ jobs: - name: Check inputs env: NODES: ${{ inputs.nodes || '7' }} + TRUSTED: ${{ inputs.trusted_nodes || '' }} run: | - case "$NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 5 to 25, got '$NODES'"; exit 1 ;; esac + if [ -n "$TRUSTED" ]; then + case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted_nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac + [ "$NODES" -ge "$TRUSTED" ] && [ "$NODES" -le 61 ] || { echo "nodes must be trusted_nodes to 61, got '$NODES'"; exit 1; } + else + case "$NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 5 to 25, got '$NODES'"; exit 1 ;; esac + fi - name: Checkout uses: actions/checkout@v4 @@ -238,6 +247,7 @@ jobs: timeout-minutes: 300 env: NODES: ${{ inputs.nodes || '7' }} + TRUSTED_NODES: ${{ inputs.trusted_nodes || '' }} TARGET_SEQ: ${{ inputs.target_seq || '1100' }} TIMEOUT_MIN: ${{ inputs.timeout_min || '45' }} DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }} diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index a0e74d089..1cd4513f4 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -6,6 +6,8 @@ # TARGET_SEQ (default 1100: past the first STATE DIGEST at block 1000) # TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test) # WORK (default ./testnet-run) +# TRUSTED_NODES (default NODES): only the first TRUSTED_NODES nodes are initial confidants +# (odd, 5 to 25); the others, up to 61 nodes in all, are ordinary nodes # SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU # of the nodes and executors in resources.tsv, and prints a summary at the end # FUND (default 0; needs a -DCREDITS_TESTNET=ON binary): genesis funds go to a generated master @@ -116,11 +118,17 @@ BIN_ALT=${BIN_ALT:+$(realpath "$BIN_ALT")} HERE=$(cd "$(dirname "$0")" && pwd) REPO=$(cd "$HERE/../.." && pwd) NODES=${NODES:-7} -# MinTrustedNodes is 4; an even set of ready nodes drops one at bootstrap -case "$NODES" in +# the initial confidants (trusted.txt): the first TRUSTED_NODES nodes, by default all of them; the +# others are ordinary nodes, as most nodes on mainnet. MinTrustedNodes is 4, MaxTrustedNodes 25, and +# an even set of ready confidants drops one at bootstrap +TRUSTED_NODES=${TRUSTED_NODES:-$NODES} +case "$TRUSTED_NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; - *) echo "NODES must be odd, 5 to 25, got '$NODES'"; exit 1 ;; + *) echo "TRUSTED_NODES must be odd, 5 to 25, got '$TRUSTED_NODES'"; exit 1 ;; esac +# ports: p2p 6000 + i (offline copies 6500 + i), API 9100 + i, apiexec 9200 + i, executor 9300 + i +[ "$NODES" -ge "$TRUSTED_NODES" ] && [ "$NODES" -le 61 ] \ + || { echo "NODES must be TRUSTED_NODES ($TRUSTED_NODES) to 61, got '$NODES'"; exit 1; } # 25 is Consensus::MaxTrustedNodes: every node here is an initial confidant, so no larger table can # bootstrap. Above 15 the neighbour map (16) is not a full mesh, but initial confidants are permanent # neighbours of each other and consensus messages are routed, so that does not stop the network @@ -325,7 +333,7 @@ for i in $(seq 1 "$ALL_NODES"); do : > "$dir/hosts.txt" : > "$dir/trusted.txt" for j in $(seq 1 "$NODES"); do - echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" + [ "$j" -gt "$TRUSTED_NODES" ] || echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" [ "$j" -ne "$i" ] && echo "127.0.0.1:$((BASE_PORT + j)) ${KEYS[$((j - 1))]}" >> "$dir/hosts.txt" done cat > "$dir/config.ini" < Date: Thu, 22 Oct 2026 16:18:55 +0200 Subject: [PATCH 05/15] ci(scale): trusted nodes and node counts from a scale-t-n_ tag Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index e082fd4cf..edfa2cc8f 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -7,8 +7,8 @@ name: Scale # latency. A node count passes when every node reaches the target with the same state digests and # contracts.py finds the expected values and states everywhere. on: - # a scale-* tag runs it with the defaults on the tagged commit (a manual run needs the workflow on - # the default branch) + # a scale-* tag runs it on the tagged commit with the defaults, or scale-t-n (a manual run needs the workflow on the default branch) push: tags: ['scale-*'] workflow_dispatch: @@ -40,6 +40,7 @@ jobs: runs-on: ubuntu-24.04 outputs: counts: ${{ steps.counts.outputs.counts }} + trusted: ${{ steps.counts.outputs.trusted }} steps: - name: Node counts id: counts @@ -48,6 +49,12 @@ jobs: TRUSTED: ${{ inputs.trusted_nodes || '' }} LOAD: ${{ inputs.contract_load || '120,2,20' }} run: | + # a tag scale-t-n_... sets them too (scale-t7-n25_51) + if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)$ ]]; then + TRUSTED=${BASH_REMATCH[1]} + LIST=${BASH_REMATCH[2]//_/,} + fi + echo "trusted=$TRUSTED" >> "$GITHUB_OUTPUT" echo "$LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } json="[" for n in ${LIST//,/ }; do @@ -141,7 +148,7 @@ jobs: max-parallel: 3 matrix: nodes: ${{ fromJSON(needs.plan.outputs.counts) }} - name: ${{ matrix.nodes }} nodes${{ inputs.trusted_nodes && format(', {0} trusted', inputs.trusted_nodes) || '' }} + name: ${{ matrix.nodes }} nodes${{ needs.plan.outputs.trusted && format(', {0} trusted', needs.plan.outputs.trusted) || '' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -169,7 +176,7 @@ jobs: timeout-minutes: 110 env: NODES: ${{ matrix.nodes }} - TRUSTED_NODES: ${{ inputs.trusted_nodes || '' }} + TRUSTED_NODES: ${{ needs.plan.outputs.trusted }} CONTRACTS: '1' CONTRACT_LOAD: ${{ inputs.contract_load || '120,2,20' }} TARGET_SEQ: '1100' From e4439706d0a21639af577ca820af1508a8cd4f39 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 16:20:54 +0200 Subject: [PATCH 06/15] ci: who took part in consensus; stake-based election from a chosen block in scale runs run.sh prints the trusted count per round (n1) and how many nodes were trusted at least once. scale.yml: dpos_at input and -d in the tag (special order 9 moves StartingDPOS, so that trusted nodes are elected by stake among the funded nodes, every one above the minimum stake). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 17 +++++++++++++++-- ci/testnet/run.sh | 9 +++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index edfa2cc8f..54a011afe 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -19,6 +19,9 @@ on: trusted_nodes: description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node)' default: '' + dpos_at: + description: 'Sequence from which trusted nodes are elected by stake (special order 9; 0 = StartingDPOS 10000, after the run)' + default: '0' contract_load: description: 'Load phase as "seconds,writes per second,reads per second"' default: '120,2,20' @@ -41,20 +44,26 @@ jobs: outputs: counts: ${{ steps.counts.outputs.counts }} trusted: ${{ steps.counts.outputs.trusted }} + dpos_at: ${{ steps.counts.outputs.dpos_at }} steps: - name: Node counts id: counts env: LIST: ${{ inputs.nodes_list || '9,13,17,21,25' }} TRUSTED: ${{ inputs.trusted_nodes || '' }} + DPOS: ${{ inputs.dpos_at || '0' }} LOAD: ${{ inputs.contract_load || '120,2,20' }} run: | - # a tag scale-t-n_... sets them too (scale-t7-n25_51) - if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)$ ]]; then + # a tag scale-t-n_...[-d] sets them too (scale-t7-n33_49-d200: + # 7 initial confidants, 33 and 49 nodes, stake-based election from block 200) + DPOS=${DPOS:-0} + if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?$ ]]; then TRUSTED=${BASH_REMATCH[1]} LIST=${BASH_REMATCH[2]//_/,} + DPOS=${BASH_REMATCH[4]:-0} fi echo "trusted=$TRUSTED" >> "$GITHUB_OUTPUT" + echo "dpos_at=$DPOS" >> "$GITHUB_OUTPUT" echo "$LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } json="[" for n in ${LIST//,/ }; do @@ -177,6 +186,7 @@ jobs: env: NODES: ${{ matrix.nodes }} TRUSTED_NODES: ${{ needs.plan.outputs.trusted }} + DPOS_AT: ${{ needs.plan.outputs.dpos_at }} CONTRACTS: '1' CONTRACT_LOAD: ${{ inputs.contract_load || '120,2,20' }} TARGET_SEQ: '1100' @@ -224,6 +234,9 @@ jobs: f"| {grab(r'busy mean ([0-9]+) %', res)} / {grab(r'peak ([0-9]+) %, ', res)} % " f"| {grab(r'peak used ([0-9]+) MB', res)} MB | {lat} |") print() + m = re.search(r"^(consensus: .*)$", log, re.M) + print(m.group(1) if m else "consensus: ?") + print() print("```") print(res) print("```") diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 1cd4513f4..c3aea54b1 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -1186,3 +1186,12 @@ if funded and 1000 in seen and max(w for _, w in seen[1000].values()) <= 2: sys.exit(0 if ok else 1) PY echo "PASS: $ALL_NODES nodes reached sequence $TARGET_SEQ with matching state digests" +# who took part in consensus: trusted per round (from n1's round headers) and how many nodes were +# trusted at least once (a node logs TRUSTED [index] in the rounds it is trusted in) +per_round=$(grep -ah 'Trusted count:' "$WORK/n1/node.log" | sed -E 's/.*Trusted count: ([0-9]+).*/\1/' | sort -n | uniq -c \ + | awk '{printf "%s%s trusted in %s rounds", sep, $2, $1; sep=", "}') +ever=0 +for i in $(seq 1 "$ALL_NODES"); do + grep -aq 'TRUSTED \[' "$WORK/n$i/node.log" && ever=$((ever + 1)) +done +echo "consensus: $per_round; $ever of $ALL_NODES nodes were trusted at least once (initial confidants: $TRUSTED_NODES)" From 67516caf91c824c635c1648325f276bc204cce65 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 16:22:51 +0200 Subject: [PATCH 07/15] ci(testnet): nodes below the minimum stake, never elected trusted once stake counts fund.py --low-stake-nodes: the last K nodes get 1'000 CS instead of 60'000 (minimum stake 50'000). run.sh LOW_STAKE_NODES (not initial confidants): after StartingDPOS (DPOS_AT, 100 rounds of margin) none of them may log a TRUSTED round, or the run fails. scale.yml: low_stake_nodes input and -l in the tag. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 22 +++++++++++++++++----- ci/testnet/fund.py | 14 +++++++++++--- ci/testnet/run.sh | 23 +++++++++++++++++++++-- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index 54a011afe..34c3c1936 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -19,6 +19,9 @@ on: trusted_nodes: description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node)' default: '' + low_stake_nodes: + description: 'The last that many nodes get 1000 CS, below the minimum stake; never trusted after dpos_at' + default: '0' dpos_at: description: 'Sequence from which trusted nodes are elected by stake (special order 9; 0 = StartingDPOS 10000, after the run)' default: '0' @@ -45,6 +48,7 @@ jobs: counts: ${{ steps.counts.outputs.counts }} trusted: ${{ steps.counts.outputs.trusted }} dpos_at: ${{ steps.counts.outputs.dpos_at }} + low_stake: ${{ steps.counts.outputs.low_stake }} steps: - name: Node counts id: counts @@ -52,18 +56,23 @@ jobs: LIST: ${{ inputs.nodes_list || '9,13,17,21,25' }} TRUSTED: ${{ inputs.trusted_nodes || '' }} DPOS: ${{ inputs.dpos_at || '0' }} + LOW: ${{ inputs.low_stake_nodes || '0' }} LOAD: ${{ inputs.contract_load || '120,2,20' }} run: | - # a tag scale-t-n_...[-d] sets them too (scale-t7-n33_49-d200: - # 7 initial confidants, 33 and 49 nodes, stake-based election from block 200) + # a tag scale-t-n_...[-d][-l] sets them too + # (scale-t7-n33_49-d200-l10: 7 initial confidants, 33 and 49 nodes, stake-based election from + # block 200, the last 10 nodes below the minimum stake) DPOS=${DPOS:-0} - if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?$ ]]; then + LOW=${LOW:-0} + if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?(-l([0-9]+))?$ ]]; then TRUSTED=${BASH_REMATCH[1]} LIST=${BASH_REMATCH[2]//_/,} DPOS=${BASH_REMATCH[4]:-0} + LOW=${BASH_REMATCH[6]:-0} fi echo "trusted=$TRUSTED" >> "$GITHUB_OUTPUT" echo "dpos_at=$DPOS" >> "$GITHUB_OUTPUT" + echo "low_stake=$LOW" >> "$GITHUB_OUTPUT" echo "$LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } json="[" for n in ${LIST//,/ }; do @@ -187,6 +196,7 @@ jobs: NODES: ${{ matrix.nodes }} TRUSTED_NODES: ${{ needs.plan.outputs.trusted }} DPOS_AT: ${{ needs.plan.outputs.dpos_at }} + LOW_STAKE_NODES: ${{ needs.plan.outputs.low_stake }} CONTRACTS: '1' CONTRACT_LOAD: ${{ inputs.contract_load || '120,2,20' }} TARGET_SEQ: '1100' @@ -234,8 +244,10 @@ jobs: f"| {grab(r'busy mean ([0-9]+) %', res)} / {grab(r'peak ([0-9]+) %, ', res)} % " f"| {grab(r'peak used ([0-9]+) MB', res)} MB | {lat} |") print() - m = re.search(r"^(consensus: .*)$", log, re.M) - print(m.group(1) if m else "consensus: ?") + for key in ("consensus", "stake"): + m = re.search(r"^(" + key + r": .*)$", log, re.M) + if m: + print(m.group(1) + " ") print() print("```") print(res) diff --git a/ci/testnet/fund.py b/ci/testnet/fund.py index 62c555e6a..9e54922e1 100644 --- a/ci/testnet/fund.py +++ b/ci/testnet/fund.py @@ -3,10 +3,13 @@ Usage: fund.py [--port 9090] [--amount 60000] [--load-seconds 0] [--dpos-at SEQ --reward-round SEQ] [--min-stake CS] [--delegations] + [--low-stake-nodes K --low-amount 1000] The genesis funds of a CREDITS_TESTNET build go to the key given in CS_TESTNET_GENESIS_KEY; this script holds its seed. It sends CS from it to every node (above the 50'000 minimum stake), then, for --load-seconds, small random transfers between nodes, through the Thrift API of node 1. +With --low-stake-nodes the last K nodes get --low-amount instead, below the minimum stake: once +stake counts (StartingDPOS) they must never be elected trusted. With --dpos-at and/or --reward-round the master key, which is also the starter key of the test network (CS_TESTNET_STARTER_KEY), sends two special "managing" transactions: order 9 moves StartingDPOS to @@ -206,6 +209,8 @@ def main(): parser.add_argument("master_seed") parser.add_argument("--port", type=int, default=9090) parser.add_argument("--amount", type=int, default=60000) + parser.add_argument("--low-stake-nodes", type=int, default=0) + parser.add_argument("--low-amount", type=int, default=1000) parser.add_argument("--load-seconds", type=int, default=0) parser.add_argument("--dpos-at", type=int, default=0) parser.add_argument("--reward-round", type=int, default=0) @@ -222,9 +227,12 @@ def main(): print(f"master balance before funding: {balance(client, master.pk):.4f}", flush=True) failures = 0 + def funded(i): + return args.low_amount if i > len(nodes) - args.low_stake_nodes else args.amount + for i, node in enumerate(nodes, 1): - ok, message = send(client, master, node.pk, args.amount) - print(f"fund n{i}: {'ok' if ok else 'FAILED ' + message}", flush=True) + ok, message = send(client, master, node.pk, funded(i)) + print(f"fund n{i} with {funded(i)}: {'ok' if ok else 'FAILED ' + message}", flush=True) failures += 0 if ok else 1 # Node::processSpecialInfo: u16 order, then the order's fields (cs::IDataStream, little endian); @@ -256,7 +264,7 @@ def delegate(name, src, dst, amount, value): if args.delegations: # a delegation is checked against the confirmed balance, so wait for the funding to land wait_until = time.time() + 180 - while time.time() < wait_until and any(balance(client, n.pk) < args.amount for n in nodes): + while time.time() < wait_until and any(balance(client, n.pk) < funded(i) for i, n in enumerate(nodes, 1)): time.sleep(5) expiry = int(time.time()) + args.delegation_seconds for i, src, dst in pairs: diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index c3aea54b1..8089fec30 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -18,6 +18,8 @@ # SYNCRO_ROUND (default 0 = off, needs FUND=1): special order 35 makes the sync answers count # from that round (Consensus::syncroChangeRound), as on a network that issued it # JOIN_FULL=1: the node of JOIN_PRUNED_AT syncs from block 0 as a full node instead +# LOW_STAKE_NODES (default 0, needs FUND=1): the last that many nodes get 1'000 CS, below the +# minimum stake; after StartingDPOS (DPOS_AT) none of them may be trusted # MIN_STAKE (default 0 = off, needs FUND=1): special order 22 sets the minimum stake; with the # restart test, the restarted node must apply it again after its quick start # DELEGATE (default 0, needs FUND=1): nodes delegate to each other, with and without a time @@ -155,6 +157,10 @@ DPOS_AT=${DPOS_AT:-0} REWARD_ROUND=${REWARD_ROUND:-0} DELEGATE=${DELEGATE:-0} MIN_STAKE=${MIN_STAKE:-0} +# the last LOW_STAKE_NODES nodes (never initial confidants) get 1'000 CS, below the minimum stake +LOW_STAKE_NODES=${LOW_STAKE_NODES:-0} +[ "$LOW_STAKE_NODES" -le $((NODES - TRUSTED_NODES)) ] \ + || { echo "LOW_STAKE_NODES must be at most NODES - TRUSTED_NODES ($((NODES - TRUSTED_NODES)))"; exit 1; } SYNCRO_ROUND=${SYNCRO_ROUND:-0} JOIN_FULL=${JOIN_FULL:-0} CONTRACTS=${CONTRACTS:-0} @@ -919,6 +925,7 @@ while :; do echo "funding nodes from the genesis key, then $LOAD_SECONDS s of transfers" python3 "$HERE/fund.py" "$WORK" "$NODES" "$MASTER_SEED" --load-seconds "$LOAD_SECONDS" \ --dpos-at "$DPOS_AT" --reward-round "$REWARD_ROUND" --delegation-seconds "$DELEGATION_SECONDS" --min-stake "$MIN_STAKE" --syncro-round "$SYNCRO_ROUND" \ + --low-stake-nodes "$LOW_STAKE_NODES" \ $([ "$DELEGATE" = "1" ] && echo --delegations) > "$WORK/fund.log" 2>&1 & FUND_PID=$! fi @@ -1185,13 +1192,25 @@ if funded and 1000 in seen and max(w for _, w in seen[1000].values()) <= 2: ok = False sys.exit(0 if ok else 1) PY -echo "PASS: $ALL_NODES nodes reached sequence $TARGET_SEQ with matching state digests" # who took part in consensus: trusted per round (from n1's round headers) and how many nodes were # trusted at least once (a node logs TRUSTED [index] in the rounds it is trusted in) per_round=$(grep -ah 'Trusted count:' "$WORK/n1/node.log" | sed -E 's/.*Trusted count: ([0-9]+).*/\1/' | sort -n | uniq -c \ - | awk '{printf "%s%s trusted in %s rounds", sep, $2, $1; sep=", "}') + | awk '{printf "%s%s trusted in %s rounds", sep, $2, $1; sep=", "}' || true) ever=0 for i in $(seq 1 "$ALL_NODES"); do grep -aq 'TRUSTED \[' "$WORK/n$i/node.log" && ever=$((ever + 1)) done echo "consensus: $per_round; $ever of $ALL_NODES nodes were trusted at least once (initial confidants: $TRUSTED_NODES)" +if [ "$LOW_STAKE_NODES" -gt 0 ]; then + # once stake counts (StartingDPOS, moved by DPOS_AT; 100 rounds of margin for the order to land), + # a node below the minimum stake must never be trusted + from=$(( ${DPOS_AT:-0} > 0 ? DPOS_AT + 100 : 10100 )) + low_trusted=0 + for i in $(seq $((NODES - LOW_STAKE_NODES + 1)) "$NODES"); do + n=$(grep -ah 'R-[0-9]*\.[0-9]* TRUSTED \[' "$WORK/n$i/node.log" | sed -E 's/.*R-([0-9]+)\..*/\1/' | awk -v f="$from" '$1 > f' | wc -l || true) + [ "$n" -eq 0 ] || { echo "n$i (low stake) was trusted in $n rounds after round $from"; low_trusted=$((low_trusted + 1)); } + done + [ "$low_trusted" -eq 0 ] || { echo "FAIL: nodes below the minimum stake were elected trusted"; exit 1; } + echo "stake: none of the $LOW_STAKE_NODES nodes below the minimum stake was trusted after round $from" +fi +echo "PASS: $ALL_NODES nodes reached sequence $TARGET_SEQ with matching state digests" From d97b0f32bfe808971704e07c373ed955c557cd1b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 17:37:56 +0200 Subject: [PATCH 08/15] ci(testnet): contracts wait for a starting executor; a contracts.py that ends early fails the run 33 nodes start 33 executor JVMs at once: n1's was not reachable yet when contracts.py compiled Counter ('Called write on non-open socket'), contracts.py exited, and run.sh waited for its ready file until the timeout. The compile is retried on a transport error for up to 10 min, and run.sh fails at once when contracts.py ends before its first calls. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 10 +++++++++- ci/testnet/run.sh | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index c8f6d483d..30a2c4a8d 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -238,7 +238,15 @@ def next_inner_id(client, account): def deploy(client, master, name, source): - compiled = client.SmartContractCompile(source) + # the node's executor JVM may still be starting (many nodes start theirs at once): the API + # answers with a transport error then, so the compile is tried again for a while + deadline = time.time() + 600 + while True: + compiled = client.SmartContractCompile(source) + if compiled.status.code == 0 or time.time() >= deadline or "socket" not in (compiled.status.message or ""): + break + print(f"compile {name}: executor not reachable yet ({compiled.status.message!r}), retrying", flush=True) + time.sleep(10) if compiled.status.code != 0: sys.exit(f"compile {name} FAILED: {compiled.status.message}") next_inner_id(client, master) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 8089fec30..0ca1be127 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -939,6 +939,12 @@ while :; do --stats-file "$WORK/contracts-stats.json") > "$WORK/contracts.log" 2>&1 & CONTRACTS_PID=$! fi + # contracts.py ended before its first calls were done: nothing would release the restart test + if [ -n "$CONTRACTS_PID" ] && [ ! -e "$WORK/contracts-ready" ] && ! kill -0 "$CONTRACTS_PID" 2>/dev/null; then + cat "$WORK/contracts.log" + echo "FAIL: contracts.py ended before its first calls" + exit 1 + fi # the calls after the restart must run on the restarted nodes too, so let them catch up first if [ "$CONTRACTS" = "1" ] && [ ! -e "$WORK/restart-done" ] && contracts_go_ready 2>/dev/null; then touch "$WORK/restart-done" From 18e7164774aa141e438d5f51da3ceeb166ce220f Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 17:38:57 +0200 Subject: [PATCH 09/15] ci(testnet): a write refused while the account's previous call is in progress is not a fault Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index 30a2c4a8d..c2c28e36b 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -326,7 +326,9 @@ def writer(i): with lock: if ok: stats["write"].append(ms) - elif "rejected by consensus" in (msg or ""): + # refused by consensus (concurrent calls of one contract in a round), or by the API while + # the account's previous call is still in progress: answers, not faults + elif "rejected by consensus" in (msg or "") or "Transaction in progress" in (msg or ""): rejected[0] += 1 else: errors["write"] += 1 @@ -374,7 +376,8 @@ def reader(k): print(f"load {kind}s: {len(ms)} ok of {sent} ({summary[kind]['rate']}/s), latency " f"p50 {summary[kind]['p50_ms'] or 0:.0f} ms, p95 {summary[kind]['p95_ms'] or 0:.0f} ms, " f"max {summary[kind]['max_ms'] or 0:.0f} ms", flush=True) - print(f"load writes rejected by consensus (concurrent calls of one contract): {rejected[0]}", flush=True) + print(f"load writes refused (by consensus for concurrent calls of one contract, or while the account's " + f"previous call was in progress): {rejected[0]}", flush=True) if args.stats_file: with open(args.stats_file, "w") as f: json.dump(summary, f, indent=1) From 9015867c86bbca2029f0a57d4859557c02ce5b4e Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 18:04:26 +0200 Subject: [PATCH 10/15] ci(testnet): contract load from writer accounts of its own; executors get 60 s to answer at start - fund.py's transfers from the node accounts competed with the load's contract calls from the same accounts for inner ids (consensus rejected some transfers, 9 nodes): the load funds and uses accounts of its own - 33 nodes: n1 asked its executor for its version after the default 7 s while 33 JVMs were starting, got none and disabled the executor for the rest of the run: executor_check_version_delay 60 s when contracts run Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 18 +++++++++++++++--- ci/testnet/run.sh | 4 ++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index c2c28e36b..bcb67f7ca 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -43,7 +43,7 @@ from thriftpy2.transport import TBufferedTransportFactory from thriftpy2.utils import serialize -from fund import Account, api, encode_max_fee, general, node_account, pack_inner_id +from fund import Account, Client, api, balance, encode_max_fee, general, pack_inner_id, send CURRENCY_CS = 1 TT_DEPLOY = 1 # api.TransactionType: contract deployment @@ -296,7 +296,9 @@ def percentile(values, p): def load(args, client_for, master, counter, nodes): """args.load_seconds of Counter.next writes (args.write_rate per second in all, one writer thread per funded node account, each through its own node's API) and Counter.get reads - (args.read_rate per second, spread over every node). Returns (writes that returned a value, + (args.read_rate per second, spread over every node). The writers are accounts of their own, + funded here by the master key: the nodes' accounts are fund.py's, whose transfers would compete + with these calls for their inner ids. Returns (writes that returned a value, writes that failed, writes consensus rejected, 1 if more than 1 % of the reads failed).""" stats = {"write": [], "read": []} errors = {"write": 0, "read": 0} @@ -306,9 +308,19 @@ def load(args, client_for, master, counter, nodes): lock = threading.Lock() stop_at = time.time() + args.load_seconds writers = list(range(1, nodes + 1)) + accounts = {i: Account(hashlib.sha256(master.pk + b"contract writer %d" % i).digest()) for i in writers} + funder = Client(9090, 300) + master.next_inner_id = None + for i in writers: + ok, message = send(funder, master, accounts[i].pk, 1000) + if not ok: + print(f"load: funding writer {i} FAILED: {message}", flush=True) + deadline = time.time() + 300 + while time.time() < deadline and any(balance(funder, a.pk) < 1000 for a in accounts.values()): + time.sleep(5) def writer(i): - account = node_account(args.work, i) + account = accounts[i] period = len(writers) / args.write_rate if args.write_rate > 0 else None while period and time.time() < stop_at: t0 = time.time() diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 0ca1be127..ad38a4ab8 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -291,6 +291,10 @@ api_settings() { # the first contract compile in a fresh JVM takes longer than the 4 s default echo "executor_send_timeout=60000" echo "executor_receive_timeout=60000" + # the node asks its new executor for its version this long after starting it, and disables it + # for good if it does not answer: dozens of JVMs starting at once on one machine take longer + # than the 7 s default + echo "executor_check_version_delay=60000" cat > "$WORK/n$i/settings.properties" < Date: Thu, 22 Oct 2026 21:57:30 +0200 Subject: [PATCH 11/15] docs: CHANGES.md, #68, #69 and #72 Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index b39102762..a1bfbb890 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -24,7 +24,7 @@ lines under the new release. nodes and builds compare their state. - **Continuous integration.** - unit tests; - - a private 7-node network; + - private networks of 5 to 61 nodes, with smart contracts read and written; - a robustness suite: crashes, power cuts, full disks, network faults, mixed binaries, every storage backend, conversions; - upgrades from the official node, tested on mainnet data; @@ -37,6 +37,9 @@ lines under the new release. | #67 | csdb, ci, docs: conversion follow-ups from the review of #65 (contract-records message, keep a copy of `db/` before converting from berkeleydb, sturdier db-convert scenario) | | #70 | csnode: bootstrap tables of 9 to 25 confidants are accepted (the official node silently drops them, so a network of 9 or more initial confidants never starts); the same choice as the official node up to 8 | | #71 | csnode: a bootstrap round table is taken only for a round next to the node's own, and, unless signed with the blockchain key, only once the node's own round has stalled; the block-removal request it set (never effective) is gone | +| #68 | ci(testnet): resource sampling (host and per-process memory and CPU, block rate) in every private-network run; a `nodes` input for testnet and robustness | +| #69 | ci: smart contracts in the robustness suite (crashes, a power cut, packet loss, a pruned node killed and joining, a database conversion) | +| #72 | ci: contract reads on every node and a contract load phase; networks of up to 61 nodes with 5 to 25 initial confidants, stake-based election and nodes below the minimum stake; a scale workflow | ## community.9 — 2026-10-22 From dc42a31ad1ffc065cc5e3d6c4f81b5038f54026b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 22:08:07 +0200 Subject: [PATCH 12/15] =?UTF-8?q?ci:=20review=20of=20#72=20=E2=80=94=20rea?= =?UTF-8?q?ds=20and=20load=20on=20the=20starting=20nodes,=20round=20number?= =?UTF-8?q?s=20with=20digit=20groups,=20DPOS=20for=20low=20stake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - read_all and the load go to the nodes the network started with (a joining node is compared at the end only); 300 s for a node to read the expected value (a node killed by the crash test has to restart, and every executor of the machine is relaunched) - the low-stake check reads rounds printed as R-1'234.0 and requires DPOS_AT (at most TARGET_SEQ - 300), or it passed without checking anything - the restart test restarts the last initial confidant by default - scale.yml checks trusted nodes and timeout in its plan job; docs and comments Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 5 +++++ ci/testnet/contracts.py | 13 ++++++++----- ci/testnet/run.sh | 22 ++++++++++++++-------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index 34c3c1936..c21d6cbc0 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -57,6 +57,7 @@ jobs: TRUSTED: ${{ inputs.trusted_nodes || '' }} DPOS: ${{ inputs.dpos_at || '0' }} LOW: ${{ inputs.low_stake_nodes || '0' }} + TIMEOUT: ${{ inputs.timeout_min || '60' }} LOAD: ${{ inputs.contract_load || '120,2,20' }} run: | # a tag scale-t-n_...[-d][-l] sets them too @@ -70,6 +71,10 @@ jobs: DPOS=${BASH_REMATCH[4]:-0} LOW=${BASH_REMATCH[6]:-0} fi + if [ -n "$TRUSTED" ]; then + case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac + fi + [ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 100 ] || { echo "timeout_min must be 10 to 100, got '$TIMEOUT'"; exit 1; } echo "trusted=$TRUSTED" >> "$GITHUB_OUTPUT" echo "dpos_at=$DPOS" >> "$GITHUB_OUTPUT" echo "low_stake=$LOW" >> "$GITHUB_OUTPUT" diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index bcb67f7ca..8f69e421c 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -4,6 +4,7 @@ Usage: contracts.py [--calls-before 3] [--calls-after 5] [--ready-file PATH] [--wait-file PATH] [--wait-timeout 1800] [--check-nodes N] [--final-file PATH] + [--load-seconds 0 --write-rate 1 --read-rate 5 --stats-file PATH] The master key deploys Counter, whose next() adds one to its state and returns it, and calls it --calls-before times. It then creates --ready-file and, once --wait-file exists (run.sh creates @@ -444,7 +445,9 @@ def call_callers(when, via): calls += 1 failures += 0 if call(client_for(1), master, counter, "next", f"call {n + 1} before restart", calls) else 1 failures += 0 if call(client_for(1), master, ledger, "record", f"Ledger call {calls} before restart", calls) else 1 - def read_all(contract, method, expected, label, timeout=60): + # reads and the load go to the nodes the network started with; a node that joins later is in + # check_nodes, compared once it reached the target (--final-file) + def read_all(contract, method, expected, label, timeout=300): """The getter on every node, until each returns `expected` (a lagging node needs a round or two); prints one line, returns the number of nodes that did not.""" def one(i): @@ -460,8 +463,8 @@ def one(i): if value == expected or time.time() >= deadline: return i, value, first_ms, msg time.sleep(2) - with ThreadPoolExecutor(max_workers=check_nodes) as pool: - results = list(pool.map(one, range(1, check_nodes + 1))) + with ThreadPoolExecutor(max_workers=args.nodes) as pool: + results = list(pool.map(one, range(1, args.nodes + 1))) bad = [r for r in results if r[1] != expected] cells = " ".join(f"n{i}={v}" + (f"/{ms:.0f}ms" if ms is not None else "") for i, v, ms, _ in results) print(f"read {label}: {'ok' if not bad else 'FAILED'} (expected {expected}) {cells}", flush=True) @@ -502,7 +505,7 @@ def footprint(): failures += read_all(ledger, "entries", calls, "Ledger.entries after restart") if args.load_seconds > 0: - writes_ok, writes_failed, writes_rejected, bad_reads = load(args, client_for, master, counter, check_nodes) + writes_ok, writes_failed, writes_rejected, bad_reads = load(args, client_for, master, counter, args.nodes) failures += writes_failed + bad_reads # a write that failed on its way back (a timeout) may still have landed: the counter is # between the confirmed writes and all of them, and every node must read that same value @@ -523,7 +526,7 @@ def footprint(): flush=True) failures += 0 if in_range else 1 calls = value if value is not None else calls - failures += read_all(counter, "get", calls, "Counter.get after the load", timeout=120) + failures += read_all(counter, "get", calls, "Counter.get after the load") call_callers("after restart", 1) call_callers("after restart", 2) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index a79867041..74102b01d 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2,7 +2,8 @@ # Run a private Credits network of N nodes on this machine and check that it works. # # Usage: run.sh -# Env: NODES (default 7, keep it odd: an even set of ready nodes drops one at bootstrap) +# Env: NODES (default 7; keep the initial confidants, TRUSTED_NODES, odd: an even set of ready ones +# drops one at bootstrap) # TARGET_SEQ (default 1100: past the first STATE DIGEST at block 1000) # TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test) # WORK (default ./testnet-run) @@ -40,7 +41,7 @@ # same JDK, so any result that differs from the release executor's fails the contract consensus # BIN_ALT (optional): the even nodes run this node binary instead, so a block or contract result # that differs from BIN's shows up as a diverging state -# RESTART_NODES (default: the last node): the nodes the restart test stops and restarts together +# RESTART_NODES (default: the last initial confidant): the nodes the restart test stops and restarts together # NET_FAULT (default off): loss, corrupt, delay, reorder (with 5% duplicates), duplicate or partition, applied with # tc netem to the p2p traffic sent and received by NET_FAULT_NODES (comma separated) from # sequence NET_FAULT_AT (default 150) for NET_FAULT_SECONDS (default 0 = until the end), at @@ -131,8 +132,7 @@ esac # ports: p2p 6000 + i (offline copies 6500 + i), API 9100 + i, apiexec 9200 + i, executor 9300 + i [ "$NODES" -ge "$TRUSTED_NODES" ] && [ "$NODES" -le 61 ] \ || { echo "NODES must be TRUSTED_NODES ($TRUSTED_NODES) to 61, got '$NODES'"; exit 1; } -# 25 is Consensus::MaxTrustedNodes: every node here is an initial confidant, so no larger table can -# bootstrap. Above 15 the neighbour map (16) is not a full mesh, but initial confidants are permanent +# 25 is Consensus::MaxTrustedNodes: no larger set of initial confidants can bootstrap. Above 15 the neighbour map (16) is not a full mesh, but initial confidants are permanent # neighbours of each other and consensus messages are routed, so that does not stop the network CONTRACT_LOAD_SECONDS=${CONTRACT_LOAD_SECONDS:-0} CONTRACT_WRITE_RATE=${CONTRACT_WRITE_RATE:-1} @@ -141,7 +141,8 @@ SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} RESTART_AT=${RESTART_AT:-300} -RESTART_NODES=${RESTART_NODES:-$NODES} +# by default the last initial confidant (the last node when every node is one) +RESTART_NODES=${RESTART_NODES:-$TRUSTED_NODES} echo "$RESTART_NODES" | grep -qE '^[0-9]+( [0-9]+)*$' || { echo "bad RESTART_NODES '$RESTART_NODES'"; exit 1; } FUND=${FUND:-0} FUND_AT=${FUND_AT:-30} @@ -161,6 +162,9 @@ MIN_STAKE=${MIN_STAKE:-0} LOW_STAKE_NODES=${LOW_STAKE_NODES:-0} [ "$LOW_STAKE_NODES" -le $((NODES - TRUSTED_NODES)) ] \ || { echo "LOW_STAKE_NODES must be at most NODES - TRUSTED_NODES ($((NODES - TRUSTED_NODES)))"; exit 1; } +# stake counts only from StartingDPOS: the check needs it well before the target +[ "$LOW_STAKE_NODES" -eq 0 ] || { [ "$DPOS_AT" -gt 0 ] && [ $((DPOS_AT + 300)) -le "$TARGET_SEQ" ]; } \ + || { echo "LOW_STAKE_NODES needs DPOS_AT, at most TARGET_SEQ - 300"; exit 1; } SYNCRO_ROUND=${SYNCRO_ROUND:-0} JOIN_FULL=${JOIN_FULL:-0} CONTRACTS=${CONTRACTS:-0} @@ -293,7 +297,8 @@ api_settings() { echo "executor_receive_timeout=60000" # the node asks its new executor for its version this long after starting it, and disables it # for good if it does not answer: dozens of JVMs starting at once on one machine take longer - # than the 7 s default + # than the 7 s default. They do start at once: a node that stops kills every contract-executor + # JVM of the machine (ExecutorManager, jps), and each node relaunches its own echo "executor_check_version_delay=60000" cat > "$WORK/n$i/settings.properties" < 0 ? DPOS_AT + 100 : 10100 )) + from=$((DPOS_AT + 100)) low_trusted=0 for i in $(seq $((NODES - LOW_STAKE_NODES + 1)) "$NODES"); do - n=$(grep -ah 'R-[0-9]*\.[0-9]* TRUSTED \[' "$WORK/n$i/node.log" | sed -E 's/.*R-([0-9]+)\..*/\1/' | awk -v f="$from" '$1 > f' | wc -l || true) + # rounds print with digit groups (R-1'234.0) + n=$(grep -a 'TRUSTED \[' "$WORK/n$i/node.log" | sed -E "s/.*R-([0-9']+)\..*/\1/" | tr -d "'" | awk -v f="$from" '$1 > f' | wc -l || true) [ "$n" -eq 0 ] || { echo "n$i (low stake) was trusted in $n rounds after round $from"; low_trusted=$((low_trusted + 1)); } done [ "$low_trusted" -eq 0 ] || { echo "FAIL: nodes below the minimum stake were elected trusted"; exit 1; } From 22fa882f83dc077caaaafdde9bbfa6ed3258b2ea Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 23:02:59 +0200 Subject: [PATCH 13/15] ci(testnet): contract reads skip pruned nodes (known limitation), their states are still compared A pruned node whose contracts' deploy blocks were pruned answers getters with status 0 and no result (contracts-pruned-join, contracts-pruned-term-kill: n6 after pruning blocks before 1'700). Being analysed as a node limitation; until it is fixed the reads and the load's readers leave the pruned nodes out and say so, and the final state comparison still covers them. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 20 ++++++++++++++------ ci/testnet/run.sh | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index 8f69e421c..d52fad17e 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -294,7 +294,7 @@ def percentile(values, p): return values[min(len(values) - 1, int(round(p / 100 * (len(values) - 1))))] -def load(args, client_for, master, counter, nodes): +def load(args, client_for, master, counter, nodes, read_nodes): """args.load_seconds of Counter.next writes (args.write_rate per second in all, one writer thread per funded node account, each through its own node's API) and Counter.get reads (args.read_rate per second, spread over every node). The writers are accounts of their own, @@ -354,7 +354,7 @@ def reader(k): n = k while period and time.time() < stop_at: t0 = time.time() - i = n % nodes + 1 + i = read_nodes[n % len(read_nodes)] n += 4 try: value, ms, msg = read_value(client_for(i), master, counter, "get") @@ -371,7 +371,7 @@ def reader(k): time.sleep(max(0.0, period - (time.time() - t0))) print(f"load: {args.load_seconds} s, {args.write_rate} writes/s from {len(writers)} accounts, " - f"{args.read_rate} reads/s on {nodes} nodes", flush=True) + f"{args.read_rate} reads/s on {len(read_nodes)} nodes", flush=True) threads = [threading.Thread(target=writer, args=(i,)) for i in writers] + \ [threading.Thread(target=reader, args=(k,)) for k in range(4)] for t in threads: @@ -415,8 +415,16 @@ def main(): parser.add_argument("--write-rate", type=float, default=1.0) parser.add_argument("--read-rate", type=float, default=5.0) parser.add_argument("--stats-file", default="") + # nodes with pruned storage: once the contracts' deploy blocks are pruned they answer getters with + # no result (a known limitation of the node, being analysed): not read from, still compared + parser.add_argument("--pruned-nodes", default="") args = parser.parse_args() check_nodes = args.check_nodes or args.nodes + pruned = {int(n) for n in args.pruned_nodes.split() if n.strip()} + read_nodes = [i for i in range(1, args.nodes + 1) if i not in pruned] + if pruned: + print(f"reads skip the pruned nodes {sorted(pruned)}: getters need the contracts' deploy blocks " + f"(known limitation), their states are still compared", flush=True) def client_for(i): port = 9090 if i == 1 else 9100 + i @@ -463,8 +471,8 @@ def one(i): if value == expected or time.time() >= deadline: return i, value, first_ms, msg time.sleep(2) - with ThreadPoolExecutor(max_workers=args.nodes) as pool: - results = list(pool.map(one, range(1, args.nodes + 1))) + with ThreadPoolExecutor(max_workers=len(read_nodes)) as pool: + results = list(pool.map(one, read_nodes)) bad = [r for r in results if r[1] != expected] cells = " ".join(f"n{i}={v}" + (f"/{ms:.0f}ms" if ms is not None else "") for i, v, ms, _ in results) print(f"read {label}: {'ok' if not bad else 'FAILED'} (expected {expected}) {cells}", flush=True) @@ -505,7 +513,7 @@ def footprint(): failures += read_all(ledger, "entries", calls, "Ledger.entries after restart") if args.load_seconds > 0: - writes_ok, writes_failed, writes_rejected, bad_reads = load(args, client_for, master, counter, args.nodes) + writes_ok, writes_failed, writes_rejected, bad_reads = load(args, client_for, master, counter, args.nodes, read_nodes) failures += writes_failed + bad_reads # a write that failed on its way back (a timeout) may still have landed: the counter is # between the confirmed writes and all of them, and every node must read that same value diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 74102b01d..e890ba521 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -945,7 +945,7 @@ while :; do --ready-file "$WORK/contracts-ready" --wait-file "$WORK/restart-done" --wait-timeout $((TIMEOUT_MIN * 60)) \ --check-nodes "$ALL_NODES" --final-file "$WORK/contracts-final" \ --load-seconds "$CONTRACT_LOAD_SECONDS" --write-rate "$CONTRACT_WRITE_RATE" --read-rate "$CONTRACT_READ_RATE" \ - --stats-file "$WORK/contracts-stats.json") > "$WORK/contracts.log" 2>&1 & + --stats-file "$WORK/contracts-stats.json" --pruned-nodes "$PRUNED_NODES") > "$WORK/contracts.log" 2>&1 & CONTRACTS_PID=$! fi # contracts.py ended before its first calls were done: nothing would release the restart test From 40aa2ef1fb16ce5a413b3e34bac9d20d74153ab9 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 00:04:29 +0200 Subject: [PATCH 14/15] ci(testnet): a call node 1 refuses while the master's previous one is in progress is sent again contracts-pruned-term-kill: two calls after the restart came back 'Transaction in progress' (nothing accepted), so the counter ended one short on every node. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index d52fad17e..5cb30f059 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -24,7 +24,8 @@ Transactions go through node 1's API (port 9090), so a test scenario must never kill, partition or slow node 1: a call during an outage ends the script. Only a transaction node 1 refuses because it -is not synchronized yet (after a power cut) is sent again, for up to 120 s: nothing was accepted. +is not synchronized yet (after a power cut) or because the master's previous transaction is still +in progress is sent again, for up to 120 s: nothing was accepted. The final check reads every node's API (node i > 1 listens on 9100 + i). """ import argparse @@ -275,10 +276,13 @@ def call(client, master, contract, method, label, expected=None): next_inner_id(client, master) sci = invocation(method=method) result = client.TransactionFlow(smart_transaction(master, contract, sci, TT_EXECUTE)) - # refused before it was accepted (api/src/apihandler.cpp "Node is not syncronized"): send it again - if result.status.code == 0 or "not syncronized" not in (result.status.message or "") or time.time() >= deadline: + # refused before it was accepted: the node is not synchronized yet (after a power cut), or the + # account's previous transaction is still in progress there (api/src/apihandler.cpp): send it again + msg = result.status.message or "" + if result.status.code == 0 or ("not syncronized" not in msg and "Transaction in progress" not in msg) \ + or time.time() >= deadline: break - print(f"{label}: node not synchronized yet, sending again", flush=True) + print(f"{label}: refused ({msg.strip()!r}), sending again", flush=True) time.sleep(5) value = result.smart_contract_result.v_int if result.smart_contract_result else None ok = result.status.code == 0 and value == expected From 68445f77fa5d3c0a7754addc1c0f512e3aa7ffbc Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 00:41:02 +0200 Subject: [PATCH 15/15] ci(testnet): reads from pruned nodes again (#74), executor comment after #73 Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 14 ++++++-------- ci/testnet/run.sh | 9 ++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index 5cb30f059..a81a9cf68 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -419,16 +419,14 @@ def main(): parser.add_argument("--write-rate", type=float, default=1.0) parser.add_argument("--read-rate", type=float, default=5.0) parser.add_argument("--stats-file", default="") - # nodes with pruned storage: once the contracts' deploy blocks are pruned they answer getters with - # no result (a known limitation of the node, being analysed): not read from, still compared - parser.add_argument("--pruned-nodes", default="") + # nodes left out of the reads (still compared at the end) + parser.add_argument("--skip-read-nodes", default="") args = parser.parse_args() check_nodes = args.check_nodes or args.nodes - pruned = {int(n) for n in args.pruned_nodes.split() if n.strip()} - read_nodes = [i for i in range(1, args.nodes + 1) if i not in pruned] - if pruned: - print(f"reads skip the pruned nodes {sorted(pruned)}: getters need the contracts' deploy blocks " - f"(known limitation), their states are still compared", flush=True) + skipped = {int(n) for n in args.skip_read_nodes.split() if n.strip()} + read_nodes = [i for i in range(1, args.nodes + 1) if i not in skipped] + if skipped: + print(f"reads skip the nodes {sorted(skipped)}, their states are still compared", flush=True) def client_for(i): port = 9090 if i == 1 else 9100 + i diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index e890ba521..ba5d4c1e8 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -295,10 +295,9 @@ api_settings() { # the first contract compile in a fresh JVM takes longer than the 4 s default echo "executor_send_timeout=60000" echo "executor_receive_timeout=60000" - # the node asks its new executor for its version this long after starting it, and disables it - # for good if it does not answer: dozens of JVMs starting at once on one machine take longer - # than the 7 s default. They do start at once: a node that stops kills every contract-executor - # JVM of the machine (ExecutorManager, jps), and each node relaunches its own + # the node asks its new executor for its version every second for this long, and disables it for + # good if it never answers: dozens of JVMs starting at once on one machine (a power cut, a big + # network) can take longer than the 7 s default echo "executor_check_version_delay=60000" cat > "$WORK/n$i/settings.properties" < "$WORK/contracts.log" 2>&1 & + --stats-file "$WORK/contracts-stats.json") > "$WORK/contracts.log" 2>&1 & CONTRACTS_PID=$! fi # contracts.py ended before its first calls were done: nothing would release the restart test