Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
a5bc87f
ci(testnet): contract reads on every node, a contract load phase, up …
EnzinoBB Oct 22, 2026
f2f8e4f
ci: scale workflow, one runner per node count with contracts read and…
EnzinoBB Oct 22, 2026
393c3b6
ci(testnet): a contract write consensus rejects under contention is c…
EnzinoBB Oct 22, 2026
1069ee5
Merge remote-tracking branch 'origin/fix/bootstrap-weight' into ci/sc…
EnzinoBB Oct 22, 2026
b0f7c4c
ci(testnet): TRUSTED_NODES, a network of up to 61 nodes with 5 to 25 …
EnzinoBB Oct 22, 2026
0276084
ci(scale): trusted nodes and node counts from a scale-t<T>-n<a>_<b> tag
EnzinoBB Oct 22, 2026
e443970
ci: who took part in consensus; stake-based election from a chosen bl…
EnzinoBB Oct 22, 2026
67516ca
ci(testnet): nodes below the minimum stake, never elected trusted onc…
EnzinoBB Oct 22, 2026
d97b0f3
ci(testnet): contracts wait for a starting executor; a contracts.py t…
EnzinoBB Oct 22, 2026
18e7164
ci(testnet): a write refused while the account's previous call is in …
EnzinoBB Oct 22, 2026
9015867
ci(testnet): contract load from writer accounts of its own; executors…
EnzinoBB Oct 22, 2026
60cf0bc
Merge remote-tracking branch 'origin/ci/contracts-robustness' into ci…
EnzinoBB Oct 22, 2026
d0e0dc7
docs: CHANGES.md, #68, #69 and #72
EnzinoBB Oct 22, 2026
865985f
Merge remote-tracking branch 'origin/ci/contracts-robustness' into ci…
EnzinoBB Oct 22, 2026
dc42a31
ci: review of #72 — reads and load on the starting nodes, round numbe…
EnzinoBB Oct 22, 2026
22fa882
ci(testnet): contract reads skip pruned nodes (known limitation), the…
EnzinoBB Oct 22, 2026
40aa2ef
ci(testnet): a call node 1 refuses while the master's previous one is…
EnzinoBB Oct 22, 2026
0883168
Merge remote-tracking branch 'origin/ci/contracts-robustness' into ci…
EnzinoBB Oct 22, 2026
68445f7
ci(testnet): reads from pruned nodes again (#74), executor comment af…
EnzinoBB Oct 22, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ on:
description: 'Comma-separated scenario names to run (empty = all)'
default: ''
nodes:
description: 'Nodes in the network (odd, 7 to 15); the NODE_STORAGE scenarios run only at 7'
description: 'Nodes in the network (odd, 7 to 25 = MaxTrustedNodes); the NODE_STORAGE scenarios run only at 7'
default: '7'
timeout_min:
description: 'Minutes each scenario has to reach its target sequence'
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
TIMEOUT: ${{ inputs.timeout_min || '75' }}
run: |
# the scenarios name nodes up to n6 (crashes, pruning, faults), so at least 7
case "$NODES" in 7|9|11|13|15) ;; *) echo "nodes must be odd, 7 to 15, got '$NODES'"; exit 1 ;; esac
case "$NODES" in 7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 7 to 25, got '$NODES'"; exit 1 ;; esac
[ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 120 ] || { echo "timeout_min must be 10 to 120, got '$TIMEOUT'"; exit 1; }

- name: Checkout
Expand Down
277 changes: 277 additions & 0 deletions .github/workflows/scale.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,277 @@
name: Scale

# How many nodes a private network runs, with smart contracts read and written: one runner per node
# count (a standard 4-vCPU, 16-GB GitHub runner), each running ci/testnet/run.sh with CONTRACTS=1 and
# a contract load phase (writes from every node's account, getters on every node). Each job writes
# one row to the step summary: result, block rate, CPU and memory of the runner, write and read
# latency. A node count passes when every node reaches the target with the same state digests and
# contracts.py finds the expected values and states everywhere.
on:
# a scale-* tag runs it on the tagged commit with the defaults, or scale-t<trusted>-n<counts
# joined by _> (a manual run needs the workflow on the default branch)
push:
tags: ['scale-*']
workflow_dispatch:
inputs:
nodes_list:
description: 'Comma-separated node counts: odd, 5 to 25 (25 = MaxTrustedNodes) when every node is trusted; up to 61 with trusted_nodes'
default: '9,13,17,21,25'
trusted_nodes:
description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node)'
default: ''
low_stake_nodes:
description: 'The last that many nodes get 1000 CS, below the minimum stake; never trusted after dpos_at'
default: '0'
dpos_at:
description: 'Sequence from which trusted nodes are elected by stake (special order 9; 0 = StartingDPOS 10000, after the run)'
default: '0'
contract_load:
description: 'Load phase as "seconds,writes per second,reads per second"'
default: '120,2,20'
timeout_min:
description: 'Minutes each network has to reach its target sequence'
default: '60'

permissions:
contents: read

env:
BOOST_VERSION: 1.83.0
BOOST_SHA256: 6478edfe2f3305127cffe8caf73ea0176c53769f4bf1585be237eb30798c3b8e
BOOST_PREFIX: /home/runner/boost
CCACHE_DIR: /home/runner/.ccache

jobs:
plan:
runs-on: ubuntu-24.04
outputs:
counts: ${{ steps.counts.outputs.counts }}
trusted: ${{ steps.counts.outputs.trusted }}
dpos_at: ${{ steps.counts.outputs.dpos_at }}
low_stake: ${{ steps.counts.outputs.low_stake }}
steps:
- name: Node counts
id: counts
env:
LIST: ${{ inputs.nodes_list || '9,13,17,21,25' }}
TRUSTED: ${{ inputs.trusted_nodes || '' }}
DPOS: ${{ inputs.dpos_at || '0' }}
LOW: ${{ inputs.low_stake_nodes || '0' }}
TIMEOUT: ${{ inputs.timeout_min || '60' }}
LOAD: ${{ inputs.contract_load || '120,2,20' }}
run: |
# a tag scale-t<trusted>-n<count>_<count>...[-d<sequence>][-l<nodes>] sets them too
# (scale-t7-n33_49-d200-l10: 7 initial confidants, 33 and 49 nodes, stake-based election from
# block 200, the last 10 nodes below the minimum stake)
DPOS=${DPOS:-0}
LOW=${LOW:-0}
if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?(-l([0-9]+))?$ ]]; then
TRUSTED=${BASH_REMATCH[1]}
LIST=${BASH_REMATCH[2]//_/,}
DPOS=${BASH_REMATCH[4]:-0}
LOW=${BASH_REMATCH[6]:-0}
fi
if [ -n "$TRUSTED" ]; then
case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac
fi
[ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 100 ] || { echo "timeout_min must be 10 to 100, got '$TIMEOUT'"; exit 1; }
echo "trusted=$TRUSTED" >> "$GITHUB_OUTPUT"
echo "dpos_at=$DPOS" >> "$GITHUB_OUTPUT"
echo "low_stake=$LOW" >> "$GITHUB_OUTPUT"
echo "$LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; }
json="["
for n in ${LIST//,/ }; do
if [ -n "$TRUSTED" ]; then
[ "$n" -ge "$TRUSTED" ] && [ "$n" -le 61 ] || { echo "node counts must be $TRUSTED to 61, got '$n'"; exit 1; }
else
case "$n" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "node counts must be odd, 5 to 25, got '$n'"; exit 1 ;; esac
fi
json+="$n,"
done
echo "counts=${json%,}]" >> "$GITHUB_OUTPUT"

build:
needs: plan
runs-on: ubuntu-24.04
timeout-minutes: 180
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Checkout submodules
run: |
git submodule update --init -- cscrypto p2p \
third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \
third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb
git -C cscrypto submodule update --init
git -C p2p submodule update --init

- name: Install build tools
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq cmake ninja-build flex bison autoconf automake libtool pkg-config ccache

- name: Restore Boost
id: cache-boost
uses: actions/cache/restore@v4
with:
path: ${{ env.BOOST_PREFIX }}
key: boost-${{ env.BOOST_VERSION }}-static-rt-ubuntu-24.04-v1

- name: Build Boost
if: steps.cache-boost.outputs.cache-hit != 'true'
run: |
v=${BOOST_VERSION//./_}
curl -sSfL -o boost.tar.bz2 "https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${v}.tar.bz2"
echo "${BOOST_SHA256} boost.tar.bz2" | sha256sum -c
tar xjf boost.tar.bz2
cd "boost_${v}"
./bootstrap.sh --with-libraries=system,filesystem,program_options,date_time,regex,log,iostreams,thread,serialization,atomic,chrono
./b2 -j"$(nproc)" -d0 variant=release link=static runtime-link=static threading=multi \
cxxflags=-fPIC --layout=tagged -sNO_BZIP2=1 -sNO_ZSTD=1 -sNO_LZMA=1 \
install --prefix="${BOOST_PREFIX}"

- name: Restore ccache
uses: actions/cache/restore@v4
with:
path: ${{ env.CCACHE_DIR }}
key: ccache-ubuntu-24.04-${{ github.sha }}
restore-keys: ccache-ubuntu-24.04-

# the test-network build of testnet.yml
- name: Build node
run: |
cmake -S . -B build -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DENABLE_TESTS=OFF \
-DBoost_ROOT="${BOOST_PREFIX}" \
-DBoost_NO_SYSTEM_PATHS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \
-DWITH_HASKELL=OFF \
-DCREDITS_TESTNET=ON
ccache --max-size=2G
cmake --build build -j"$(nproc)" --target node

- name: Upload node
uses: actions/upload-artifact@v4
with:
name: testnet-node-${{ github.sha }}
path: build/client/node
retention-days: 7

network:
needs: [plan, build]
runs-on: ubuntu-24.04
timeout-minutes: 120
strategy:
fail-fast: false
# a few runners at a time: heavy use got Actions disabled once
max-parallel: 3
matrix:
nodes: ${{ fromJSON(needs.plan.outputs.counts) }}
name: ${{ matrix.nodes }} nodes${{ needs.plan.outputs.trusted && format(', {0} trusted', needs.plan.outputs.trusted) || '' }}
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Checkout API definitions
run: git submodule update --init -- third-party/thrift-interface-definitions

- name: Install tools
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq python3-nacl python3-base58 gdb
python3 -m pip install --user --quiet --break-system-packages thriftpy2
sudo sysctl -w kernel.core_pattern="${{ github.workspace }}/testnet-run/core.%e.%p"

- name: Contract executor
uses: ./.github/actions/contract-executor

- name: Download node
uses: actions/download-artifact@v4
with:
name: testnet-node-${{ github.sha }}
path: build

- name: Run private network
timeout-minutes: 110
env:
NODES: ${{ matrix.nodes }}
TRUSTED_NODES: ${{ needs.plan.outputs.trusted }}
DPOS_AT: ${{ needs.plan.outputs.dpos_at }}
LOW_STAKE_NODES: ${{ needs.plan.outputs.low_stake }}
CONTRACTS: '1'
CONTRACT_LOAD: ${{ inputs.contract_load || '120,2,20' }}
TARGET_SEQ: '1100'
TIMEOUT_MIN: ${{ inputs.timeout_min || '60' }}
RESTART_AT: '300'
FUND: '1'
LOAD_SECONDS: '300'
WORK: ${{ github.workspace }}/testnet-run
run: |
chmod +x build/node
IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD"
export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE
ci/testnet/run.sh build/node 2>&1 | tee testnet-run.log
exit "${PIPESTATUS[0]}"

- name: Summary
if: always()
env:
N: ${{ matrix.nodes }}
OUTCOME: ${{ job.status }}
run: |
python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY"
import json, os, re
n, outcome = os.environ["N"], os.environ["OUTCOME"]
def read(p):
try:
return open(p).read()
except OSError:
return ""
log, res = read("testnet-run.log"), read("testnet-run/resources-summary.txt")
def grab(pattern, text):
m = re.search(pattern, text)
return m.group(1) if m else "?"
passed = "PASS" if re.search(r"^PASS: \d+ nodes", log, re.M) else "FAIL"
try:
s = json.load(open("testnet-run/contracts-stats.json"))
w, r = s["write"], s["read"]
lat = (f"{w['ok']}/{w['sent']} at {w['rate']}/s ({s.get('writes_rejected', 0)} rejected), p50 {w['p50_ms'] or 0:.0f} p95 {w['p95_ms'] or 0:.0f} ms | "
f"{r['ok']}/{r['sent']} at {r['rate']}/s, p50 {r['p50_ms'] or 0:.0f} p95 {r['p95_ms'] or 0:.0f} ms")
except Exception:
lat = "? | ?"
print("| nodes | result | blocks/s | CPU mean / peak | peak RAM used | writes | reads |")
print("|---|---|---|---|---|---|---|")
print(f"| {n} | {passed} | {grab(r'blocks: ([0-9.]+) per second', res)} "
f"| {grab(r'busy mean ([0-9]+) %', res)} / {grab(r'peak ([0-9]+) %, ', res)} % "
f"| {grab(r'peak used ([0-9]+) MB', res)} MB | {lat} |")
print()
for key in ("consensus", "stake"):
m = re.search(r"^(" + key + r": .*)$", log, re.M)
if m:
print(m.group(1) + " ")
print()
print("```")
print(res)
print("```")
PY

- name: Upload logs
if: always()
uses: actions/upload-artifact@v4
with:
name: scale-${{ matrix.nodes }}-nodes
path: |
testnet-run.log
testnet-run/n*/*.log
testnet-run/fund.log
testnet-run/contracts.log
testnet-run/contracts-stats.json
testnet-run/n*/log/*.txt
testnet-run/n*/config.ini
testnet-run/resources.tsv
testnet-run/resources-summary.txt
retention-days: 14
29 changes: 25 additions & 4 deletions .github/workflows/testnet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,14 @@ on:
description: 'Nodes stopped and restarted together in the restart test, e.g. "6 7" (empty = the last node)'
default: ''
nodes:
description: 'Nodes in the network (odd, 5 to 15; 15 is the full-mesh limit of max_neighbours=16)'
description: 'Nodes in the network (odd, 5 to 25 = MaxTrustedNodes)'
default: '7'
trusted_nodes:
description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node); then nodes may go up to 61'
default: ''
contract_load:
description: 'With contracts: load phase as "seconds,writes per second,reads per second", e.g. 120,2,20 (empty = none)'
default: ''
schedule:
- cron: '17 2 * * *'
pull_request:
Expand All @@ -74,16 +80,22 @@ env:
jobs:
testnet:
if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'testnet')
name: Private network (${{ inputs.nodes || '7' }} nodes)
name: Private network (${{ inputs.nodes || '7' }} nodes${{ inputs.trusted_nodes && format(', {0} trusted', inputs.trusted_nodes) || '' }})
runs-on: ubuntu-24.04
timeout-minutes: 340

steps:
- name: Check inputs
env:
NODES: ${{ inputs.nodes || '7' }}
TRUSTED: ${{ inputs.trusted_nodes || '' }}
run: |
case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac
if [ -n "$TRUSTED" ]; then
case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted_nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac
[ "$NODES" -ge "$TRUSTED" ] && [ "$NODES" -le 61 ] || { echo "nodes must be trusted_nodes to 61, got '$NODES'"; exit 1; }
else
case "$NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 5 to 25, got '$NODES'"; exit 1 ;; esac
fi

- name: Checkout
uses: actions/checkout@v4
Expand Down Expand Up @@ -235,6 +247,7 @@ jobs:
timeout-minutes: 300
env:
NODES: ${{ inputs.nodes || '7' }}
TRUSTED_NODES: ${{ inputs.trusted_nodes || '' }}
TARGET_SEQ: ${{ inputs.target_seq || '1100' }}
TIMEOUT_MIN: ${{ inputs.timeout_min || '45' }}
DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }}
Expand All @@ -246,11 +259,18 @@ jobs:
DELEGATE: ${{ inputs.delegate || '0' }}
MIN_STAKE: ${{ inputs.min_stake || '0' }}
CONTRACTS: ${{ inputs.contracts || '0' }}
CONTRACT_LOAD: ${{ inputs.contract_load || '' }}
RESTART_NODES: ${{ inputs.restart_nodes || '' }}
FUND: '1'
LOAD_SECONDS: '300'
WORK: ${{ github.workspace }}/testnet-run
run: ci/testnet/run.sh build/client/node
run: |
if [ -n "$CONTRACT_LOAD" ]; then
echo "$CONTRACT_LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; }
IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD"
export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE
fi
ci/testnet/run.sh build/client/node

- name: Resource usage
if: always()
Expand All @@ -274,6 +294,7 @@ jobs:
testnet-run/n*/log/*.txt
testnet-run/n*/config.ini
testnet-run/resources.tsv
testnet-run/contracts-stats.json
testnet-run/resources-summary.txt
testnet-run/sampler.log
testnet-run/progress
Expand Down
Loading
Loading