diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 782fff53f..3034092a9 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -25,7 +25,7 @@ on: description: 'Comma-separated scenario names to run (empty = all)' default: '' nodes: - description: 'Nodes in the network (odd, 7 to 15); the NODE_STORAGE scenarios run only at 7' + description: 'Nodes in the network (odd, 7 to 25 = MaxTrustedNodes); the NODE_STORAGE scenarios run only at 7' default: '7' timeout_min: description: 'Minutes each scenario has to reach its target sequence' @@ -54,7 +54,7 @@ jobs: TIMEOUT: ${{ inputs.timeout_min || '75' }} run: | # the scenarios name nodes up to n6 (crashes, pruning, faults), so at least 7 - case "$NODES" in 7|9|11|13|15) ;; *) echo "nodes must be odd, 7 to 15, got '$NODES'"; exit 1 ;; esac + case "$NODES" in 7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 7 to 25, got '$NODES'"; exit 1 ;; esac [ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 120 ] || { echo "timeout_min must be 10 to 120, got '$TIMEOUT'"; exit 1; } - name: Checkout diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml new file mode 100644 index 000000000..c21d6cbc0 --- /dev/null +++ b/.github/workflows/scale.yml @@ -0,0 +1,277 @@ +name: Scale + +# How many nodes a private network runs, with smart contracts read and written: one runner per node +# count (a standard 4-vCPU, 16-GB GitHub runner), each running ci/testnet/run.sh with CONTRACTS=1 and +# a contract load phase (writes from every node's account, getters on every node). Each job writes +# one row to the step summary: result, block rate, CPU and memory of the runner, write and read +# latency. A node count passes when every node reaches the target with the same state digests and +# contracts.py finds the expected values and states everywhere. +on: + # a scale-* tag runs it on the tagged commit with the defaults, or scale-t-n (a manual run needs the workflow on the default branch) + push: + tags: ['scale-*'] + workflow_dispatch: + inputs: + nodes_list: + description: 'Comma-separated node counts: odd, 5 to 25 (25 = MaxTrustedNodes) when every node is trusted; up to 61 with trusted_nodes' + default: '9,13,17,21,25' + trusted_nodes: + description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node)' + default: '' + low_stake_nodes: + description: 'The last that many nodes get 1000 CS, below the minimum stake; never trusted after dpos_at' + default: '0' + dpos_at: + description: 'Sequence from which trusted nodes are elected by stake (special order 9; 0 = StartingDPOS 10000, after the run)' + default: '0' + contract_load: + description: 'Load phase as "seconds,writes per second,reads per second"' + default: '120,2,20' + timeout_min: + description: 'Minutes each network has to reach its target sequence' + default: '60' + +permissions: + contents: read + +env: + BOOST_VERSION: 1.83.0 + BOOST_SHA256: 6478edfe2f3305127cffe8caf73ea0176c53769f4bf1585be237eb30798c3b8e + BOOST_PREFIX: /home/runner/boost + CCACHE_DIR: /home/runner/.ccache + +jobs: + plan: + runs-on: ubuntu-24.04 + outputs: + counts: ${{ steps.counts.outputs.counts }} + trusted: ${{ steps.counts.outputs.trusted }} + dpos_at: ${{ steps.counts.outputs.dpos_at }} + low_stake: ${{ steps.counts.outputs.low_stake }} + steps: + - name: Node counts + id: counts + env: + LIST: ${{ inputs.nodes_list || '9,13,17,21,25' }} + TRUSTED: ${{ inputs.trusted_nodes || '' }} + DPOS: ${{ inputs.dpos_at || '0' }} + LOW: ${{ inputs.low_stake_nodes || '0' }} + TIMEOUT: ${{ inputs.timeout_min || '60' }} + LOAD: ${{ inputs.contract_load || '120,2,20' }} + run: | + # a tag scale-t-n_...[-d][-l] sets them too + # (scale-t7-n33_49-d200-l10: 7 initial confidants, 33 and 49 nodes, stake-based election from + # block 200, the last 10 nodes below the minimum stake) + DPOS=${DPOS:-0} + LOW=${LOW:-0} + if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" =~ ^scale-t([0-9]+)-n([0-9_]+)(-d([0-9]+))?(-l([0-9]+))?$ ]]; then + TRUSTED=${BASH_REMATCH[1]} + LIST=${BASH_REMATCH[2]//_/,} + DPOS=${BASH_REMATCH[4]:-0} + LOW=${BASH_REMATCH[6]:-0} + fi + if [ -n "$TRUSTED" ]; then + case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac + fi + [ "$TIMEOUT" -ge 10 ] && [ "$TIMEOUT" -le 100 ] || { echo "timeout_min must be 10 to 100, got '$TIMEOUT'"; exit 1; } + echo "trusted=$TRUSTED" >> "$GITHUB_OUTPUT" + echo "dpos_at=$DPOS" >> "$GITHUB_OUTPUT" + echo "low_stake=$LOW" >> "$GITHUB_OUTPUT" + echo "$LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } + json="[" + for n in ${LIST//,/ }; do + if [ -n "$TRUSTED" ]; then + [ "$n" -ge "$TRUSTED" ] && [ "$n" -le 61 ] || { echo "node counts must be $TRUSTED to 61, got '$n'"; exit 1; } + else + case "$n" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "node counts must be odd, 5 to 25, got '$n'"; exit 1 ;; esac + fi + json+="$n," + done + echo "counts=${json%,}]" >> "$GITHUB_OUTPUT" + + build: + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Checkout submodules + run: | + git submodule update --init -- cscrypto p2p \ + third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \ + third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb + git -C cscrypto submodule update --init + git -C p2p submodule update --init + + - name: Install build tools + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq cmake ninja-build flex bison autoconf automake libtool pkg-config ccache + + - name: Restore Boost + id: cache-boost + uses: actions/cache/restore@v4 + with: + path: ${{ env.BOOST_PREFIX }} + key: boost-${{ env.BOOST_VERSION }}-static-rt-ubuntu-24.04-v1 + + - name: Build Boost + if: steps.cache-boost.outputs.cache-hit != 'true' + run: | + v=${BOOST_VERSION//./_} + curl -sSfL -o boost.tar.bz2 "https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${v}.tar.bz2" + echo "${BOOST_SHA256} boost.tar.bz2" | sha256sum -c + tar xjf boost.tar.bz2 + cd "boost_${v}" + ./bootstrap.sh --with-libraries=system,filesystem,program_options,date_time,regex,log,iostreams,thread,serialization,atomic,chrono + ./b2 -j"$(nproc)" -d0 variant=release link=static runtime-link=static threading=multi \ + cxxflags=-fPIC --layout=tagged -sNO_BZIP2=1 -sNO_ZSTD=1 -sNO_LZMA=1 \ + install --prefix="${BOOST_PREFIX}" + + - name: Restore ccache + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-ubuntu-24.04-${{ github.sha }} + restore-keys: ccache-ubuntu-24.04- + + # the test-network build of testnet.yml + - name: Build node + run: | + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTS=OFF \ + -DBoost_ROOT="${BOOST_PREFIX}" \ + -DBoost_NO_SYSTEM_PATHS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \ + -DWITH_HASKELL=OFF \ + -DCREDITS_TESTNET=ON + ccache --max-size=2G + cmake --build build -j"$(nproc)" --target node + + - name: Upload node + uses: actions/upload-artifact@v4 + with: + name: testnet-node-${{ github.sha }} + path: build/client/node + retention-days: 7 + + network: + needs: [plan, build] + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + # a few runners at a time: heavy use got Actions disabled once + max-parallel: 3 + matrix: + nodes: ${{ fromJSON(needs.plan.outputs.counts) }} + name: ${{ matrix.nodes }} nodes${{ needs.plan.outputs.trusted && format(', {0} trusted', needs.plan.outputs.trusted) || '' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Checkout API definitions + run: git submodule update --init -- third-party/thrift-interface-definitions + + - name: Install tools + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq python3-nacl python3-base58 gdb + python3 -m pip install --user --quiet --break-system-packages thriftpy2 + sudo sysctl -w kernel.core_pattern="${{ github.workspace }}/testnet-run/core.%e.%p" + + - name: Contract executor + uses: ./.github/actions/contract-executor + + - name: Download node + uses: actions/download-artifact@v4 + with: + name: testnet-node-${{ github.sha }} + path: build + + - name: Run private network + timeout-minutes: 110 + env: + NODES: ${{ matrix.nodes }} + TRUSTED_NODES: ${{ needs.plan.outputs.trusted }} + DPOS_AT: ${{ needs.plan.outputs.dpos_at }} + LOW_STAKE_NODES: ${{ needs.plan.outputs.low_stake }} + CONTRACTS: '1' + CONTRACT_LOAD: ${{ inputs.contract_load || '120,2,20' }} + TARGET_SEQ: '1100' + TIMEOUT_MIN: ${{ inputs.timeout_min || '60' }} + RESTART_AT: '300' + FUND: '1' + LOAD_SECONDS: '300' + WORK: ${{ github.workspace }}/testnet-run + run: | + chmod +x build/node + IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD" + export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE + ci/testnet/run.sh build/node 2>&1 | tee testnet-run.log + exit "${PIPESTATUS[0]}" + + - name: Summary + if: always() + env: + N: ${{ matrix.nodes }} + OUTCOME: ${{ job.status }} + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json, os, re + n, outcome = os.environ["N"], os.environ["OUTCOME"] + def read(p): + try: + return open(p).read() + except OSError: + return "" + log, res = read("testnet-run.log"), read("testnet-run/resources-summary.txt") + def grab(pattern, text): + m = re.search(pattern, text) + return m.group(1) if m else "?" + passed = "PASS" if re.search(r"^PASS: \d+ nodes", log, re.M) else "FAIL" + try: + s = json.load(open("testnet-run/contracts-stats.json")) + w, r = s["write"], s["read"] + lat = (f"{w['ok']}/{w['sent']} at {w['rate']}/s ({s.get('writes_rejected', 0)} rejected), p50 {w['p50_ms'] or 0:.0f} p95 {w['p95_ms'] or 0:.0f} ms | " + f"{r['ok']}/{r['sent']} at {r['rate']}/s, p50 {r['p50_ms'] or 0:.0f} p95 {r['p95_ms'] or 0:.0f} ms") + except Exception: + lat = "? | ?" + print("| nodes | result | blocks/s | CPU mean / peak | peak RAM used | writes | reads |") + print("|---|---|---|---|---|---|---|") + print(f"| {n} | {passed} | {grab(r'blocks: ([0-9.]+) per second', res)} " + f"| {grab(r'busy mean ([0-9]+) %', res)} / {grab(r'peak ([0-9]+) %, ', res)} % " + f"| {grab(r'peak used ([0-9]+) MB', res)} MB | {lat} |") + print() + for key in ("consensus", "stake"): + m = re.search(r"^(" + key + r": .*)$", log, re.M) + if m: + print(m.group(1) + " ") + print() + print("```") + print(res) + print("```") + PY + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: scale-${{ matrix.nodes }}-nodes + path: | + testnet-run.log + testnet-run/n*/*.log + testnet-run/fund.log + testnet-run/contracts.log + testnet-run/contracts-stats.json + testnet-run/n*/log/*.txt + testnet-run/n*/config.ini + testnet-run/resources.tsv + testnet-run/resources-summary.txt + retention-days: 14 diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index efeb6724c..d100f5b4c 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -50,8 +50,14 @@ on: description: 'Nodes stopped and restarted together in the restart test, e.g. "6 7" (empty = the last node)' default: '' nodes: - description: 'Nodes in the network (odd, 5 to 15; 15 is the full-mesh limit of max_neighbours=16)' + description: 'Nodes in the network (odd, 5 to 25 = MaxTrustedNodes)' default: '7' + trusted_nodes: + description: 'Initial confidants among the nodes (odd, 5 to 25; empty = every node); then nodes may go up to 61' + default: '' + contract_load: + description: 'With contracts: load phase as "seconds,writes per second,reads per second", e.g. 120,2,20 (empty = none)' + default: '' schedule: - cron: '17 2 * * *' pull_request: @@ -74,7 +80,7 @@ env: jobs: testnet: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'testnet') - name: Private network (${{ inputs.nodes || '7' }} nodes) + name: Private network (${{ inputs.nodes || '7' }} nodes${{ inputs.trusted_nodes && format(', {0} trusted', inputs.trusted_nodes) || '' }}) runs-on: ubuntu-24.04 timeout-minutes: 340 @@ -82,8 +88,14 @@ jobs: - name: Check inputs env: NODES: ${{ inputs.nodes || '7' }} + TRUSTED: ${{ inputs.trusted_nodes || '' }} run: | - case "$NODES" in 5|7|9|11|13|15) ;; *) echo "nodes must be odd, 5 to 15, got '$NODES'"; exit 1 ;; esac + if [ -n "$TRUSTED" ]; then + case "$TRUSTED" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "trusted_nodes must be odd, 5 to 25, got '$TRUSTED'"; exit 1 ;; esac + [ "$NODES" -ge "$TRUSTED" ] && [ "$NODES" -le 61 ] || { echo "nodes must be trusted_nodes to 61, got '$NODES'"; exit 1; } + else + case "$NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "nodes must be odd, 5 to 25, got '$NODES'"; exit 1 ;; esac + fi - name: Checkout uses: actions/checkout@v4 @@ -235,6 +247,7 @@ jobs: timeout-minutes: 300 env: NODES: ${{ inputs.nodes || '7' }} + TRUSTED_NODES: ${{ inputs.trusted_nodes || '' }} TARGET_SEQ: ${{ inputs.target_seq || '1100' }} TIMEOUT_MIN: ${{ inputs.timeout_min || '45' }} DISK_FULL_NODE: ${{ inputs.disk_full_node || '0' }} @@ -246,11 +259,18 @@ jobs: DELEGATE: ${{ inputs.delegate || '0' }} MIN_STAKE: ${{ inputs.min_stake || '0' }} CONTRACTS: ${{ inputs.contracts || '0' }} + CONTRACT_LOAD: ${{ inputs.contract_load || '' }} RESTART_NODES: ${{ inputs.restart_nodes || '' }} FUND: '1' LOAD_SECONDS: '300' WORK: ${{ github.workspace }}/testnet-run - run: ci/testnet/run.sh build/client/node + run: | + if [ -n "$CONTRACT_LOAD" ]; then + echo "$CONTRACT_LOAD" | grep -qE '^[0-9]+,[0-9.]+,[0-9.]+$' || { echo "contract_load must be seconds,writes,reads"; exit 1; } + IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD" + export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE + fi + ci/testnet/run.sh build/client/node - name: Resource usage if: always() @@ -274,6 +294,7 @@ jobs: testnet-run/n*/log/*.txt testnet-run/n*/config.ini testnet-run/resources.tsv + testnet-run/contracts-stats.json testnet-run/resources-summary.txt testnet-run/sampler.log testnet-run/progress diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index fa732d246..a81a9cf68 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -4,6 +4,7 @@ Usage: contracts.py [--calls-before 3] [--calls-after 5] [--ready-file PATH] [--wait-file PATH] [--wait-timeout 1800] [--check-nodes N] [--final-file PATH] + [--load-seconds 0 --write-rate 1 --read-rate 5 --stats-file PATH] The master key deploys Counter, whose next() adds one to its state and returns it, and calls it --calls-before times. It then creates --ready-file and, once --wait-file exists (run.sh creates @@ -23,15 +24,20 @@ Transactions go through node 1's API (port 9090), so a test scenario must never kill, partition or slow node 1: a call during an outage ends the script. Only a transaction node 1 refuses because it -is not synchronized yet (after a power cut) is sent again, for up to 120 s: nothing was accepted. +is not synchronized yet (after a power cut) or because the master's previous transaction is still +in progress is sent again, for up to 120 s: nothing was accepted. The final check reads every node's API (node i > 1 listens on 9100 + i). """ import argparse import hashlib +import json import os +import statistics import struct import sys +import threading import time +from concurrent.futures import ThreadPoolExecutor import base58 from thriftpy2.protocol import TBinaryProtocolFactory @@ -39,7 +45,7 @@ from thriftpy2.transport import TBufferedTransportFactory from thriftpy2.utils import serialize -from fund import Account, api, encode_max_fee, general, pack_inner_id +from fund import Account, Client, api, balance, encode_max_fee, general, pack_inner_id, send CURRENCY_CS = 1 TT_DEPLOY = 1 # api.TransactionType: contract deployment @@ -196,13 +202,53 @@ def smart_transaction(src, target, sci, tx_type): return t +def read_transaction(src, contract, method): + """A getter: TransactionFlow with forgetNewState runs the method on the node's own executor + against the contract's current state and returns its result; nothing goes to consensus, no + transaction, no fee, no signature check (api/src/apihandler.cpp). A constant signature keeps the + node's getter bookkeeping from growing.""" + sci = api.SmartContractInvocation(method=method, params=[], usedContracts=[], forgetNewState=True, + smartContractDeploy=None, version=1) + t = api.Transaction() + t.id = 0 + t.source = src.pk + t.target = contract + t.amount = general.Amount(integral=0, fraction=0) + t.balance = general.Amount(integral=0, fraction=0) + t.currency = CURRENCY_CS + t.signature = bytes(64) + t.smartContract = sci + t.fee = api.AmountCommission(commission=SMART_MAX_FEE) + t.timeCreation = int(time.time() * 1000) + t.type = TT_EXECUTE + return t + + +def read_value(client, src, contract, method): + """One getter call: (value or None, latency ms, status message).""" + t0 = time.time() + result = client.TransactionFlow(read_transaction(src, contract, method)) + ms = (time.time() - t0) * 1000 + # an executor that is not reachable answers status 0 without a result + value = result.smart_contract_result.v_int if result.status.code == 0 and result.smart_contract_result else None + return value, ms, result.status.message + + def next_inner_id(client, account): last = client.WalletTransactionsCountGet(account.pk).lastTransactionInnerId or 0 account.next_inner_id = int(last) + 1 def deploy(client, master, name, source): - compiled = client.SmartContractCompile(source) + # the node's executor JVM may still be starting (many nodes start theirs at once): the API + # answers with a transport error then, so the compile is tried again for a while + deadline = time.time() + 600 + while True: + compiled = client.SmartContractCompile(source) + if compiled.status.code == 0 or time.time() >= deadline or "socket" not in (compiled.status.message or ""): + break + print(f"compile {name}: executor not reachable yet ({compiled.status.message!r}), retrying", flush=True) + time.sleep(10) if compiled.status.code != 0: sys.exit(f"compile {name} FAILED: {compiled.status.message}") next_inner_id(client, master) @@ -230,10 +276,13 @@ def call(client, master, contract, method, label, expected=None): next_inner_id(client, master) sci = invocation(method=method) result = client.TransactionFlow(smart_transaction(master, contract, sci, TT_EXECUTE)) - # refused before it was accepted (api/src/apihandler.cpp "Node is not syncronized"): send it again - if result.status.code == 0 or "not syncronized" not in (result.status.message or "") or time.time() >= deadline: + # refused before it was accepted: the node is not synchronized yet (after a power cut), or the + # account's previous transaction is still in progress there (api/src/apihandler.cpp): send it again + msg = result.status.message or "" + if result.status.code == 0 or ("not syncronized" not in msg and "Transaction in progress" not in msg) \ + or time.time() >= deadline: break - print(f"{label}: node not synchronized yet, sending again", flush=True) + print(f"{label}: refused ({msg.strip()!r}), sending again", flush=True) time.sleep(5) value = result.smart_contract_result.v_int if result.smart_contract_result else None ok = result.status.code == 0 and value == expected @@ -242,6 +291,118 @@ def call(client, master, contract, method, label, expected=None): return ok +def percentile(values, p): + if not values: + return None + values = sorted(values) + return values[min(len(values) - 1, int(round(p / 100 * (len(values) - 1))))] + + +def load(args, client_for, master, counter, nodes, read_nodes): + """args.load_seconds of Counter.next writes (args.write_rate per second in all, one writer + thread per funded node account, each through its own node's API) and Counter.get reads + (args.read_rate per second, spread over every node). The writers are accounts of their own, + funded here by the master key: the nodes' accounts are fund.py's, whose transfers would compete + with these calls for their inner ids. Returns (writes that returned a value, + writes that failed, writes consensus rejected, 1 if more than 1 % of the reads failed).""" + stats = {"write": [], "read": []} + errors = {"write": 0, "read": 0} + # concurrent calls of one contract from several accounts: consensus takes some and rejects the + # others in a round; a rejected call is an answer, not a fault, and is counted on its own + rejected = [0] + lock = threading.Lock() + stop_at = time.time() + args.load_seconds + writers = list(range(1, nodes + 1)) + accounts = {i: Account(hashlib.sha256(master.pk + b"contract writer %d" % i).digest()) for i in writers} + funder = Client(9090, 300) + master.next_inner_id = None + for i in writers: + ok, message = send(funder, master, accounts[i].pk, 1000) + if not ok: + print(f"load: funding writer {i} FAILED: {message}", flush=True) + deadline = time.time() + 300 + while time.time() < deadline and any(balance(funder, a.pk) < 1000 for a in accounts.values()): + time.sleep(5) + + def writer(i): + account = accounts[i] + period = len(writers) / args.write_rate if args.write_rate > 0 else None + while period and time.time() < stop_at: + t0 = time.time() + ok = False + try: + c = client_for(i) + next_inner_id(c, account) + sci = invocation(method="next") + result = c.TransactionFlow(smart_transaction(account, counter, sci, TT_EXECUTE)) + ok = result.status.code == 0 and result.smart_contract_result is not None + msg = result.status.message + except Exception as e: + msg = type(e).__name__ + ms = (time.time() - t0) * 1000 + with lock: + if ok: + stats["write"].append(ms) + # refused by consensus (concurrent calls of one contract in a round), or by the API while + # the account's previous call is still in progress: answers, not faults + elif "rejected by consensus" in (msg or "") or "Transaction in progress" in (msg or ""): + rejected[0] += 1 + else: + errors["write"] += 1 + if errors["write"] <= 5: + print(f"load write via n{i} FAILED: {msg!r}", flush=True) + time.sleep(max(0.0, period - (time.time() - t0))) + + def reader(k): + period = 1.0 / args.read_rate * 4 if args.read_rate > 0 else None + n = k + while period and time.time() < stop_at: + t0 = time.time() + i = read_nodes[n % len(read_nodes)] + n += 4 + try: + value, ms, msg = read_value(client_for(i), master, counter, "get") + ok = value is not None + except Exception as e: + ok, ms, msg = False, (time.time() - t0) * 1000, type(e).__name__ + with lock: + if ok: + stats["read"].append(ms) + else: + errors["read"] += 1 + if errors["read"] <= 5: + print(f"load read via n{i} FAILED: {msg!r}", flush=True) + time.sleep(max(0.0, period - (time.time() - t0))) + + print(f"load: {args.load_seconds} s, {args.write_rate} writes/s from {len(writers)} accounts, " + f"{args.read_rate} reads/s on {len(read_nodes)} nodes", flush=True) + threads = [threading.Thread(target=writer, args=(i,)) for i in writers] + \ + [threading.Thread(target=reader, args=(k,)) for k in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + summary = {"nodes": nodes, "seconds": args.load_seconds, "writes_rejected": rejected[0]} + for kind in ("write", "read"): + ms = stats[kind] + sent = len(ms) + errors[kind] + (rejected[0] if kind == "write" else 0) + summary[kind] = {"sent": sent, "ok": len(ms), "failed": errors[kind], + "rate": round(len(ms) / args.load_seconds, 2), + "p50_ms": percentile(ms, 50), "p95_ms": percentile(ms, 95), "max_ms": max(ms) if ms else None} + print(f"load {kind}s: {len(ms)} ok of {sent} ({summary[kind]['rate']}/s), latency " + f"p50 {summary[kind]['p50_ms'] or 0:.0f} ms, p95 {summary[kind]['p95_ms'] or 0:.0f} ms, " + f"max {summary[kind]['max_ms'] or 0:.0f} ms", flush=True) + print(f"load writes refused (by consensus for concurrent calls of one contract, or while the account's " + f"previous call was in progress): {rejected[0]}", flush=True) + if args.stats_file: + with open(args.stats_file, "w") as f: + json.dump(summary, f, indent=1) + read_sent = summary["read"]["sent"] + bad_reads = 1 if read_sent and errors["read"] > 0.01 * read_sent else 0 + return len(stats["write"]), errors["write"], rejected[0], bad_reads + + def main(): parser = argparse.ArgumentParser() parser.add_argument("work") @@ -254,7 +415,18 @@ def main(): parser.add_argument("--wait-timeout", type=int, default=1800) parser.add_argument("--check-nodes", type=int, default=0) parser.add_argument("--final-file", default="") + parser.add_argument("--load-seconds", type=int, default=0) + parser.add_argument("--write-rate", type=float, default=1.0) + parser.add_argument("--read-rate", type=float, default=5.0) + parser.add_argument("--stats-file", default="") + # nodes left out of the reads (still compared at the end) + parser.add_argument("--skip-read-nodes", default="") args = parser.parse_args() + check_nodes = args.check_nodes or args.nodes + skipped = {int(n) for n in args.skip_read_nodes.split() if n.strip()} + read_nodes = [i for i in range(1, args.nodes + 1) if i not in skipped] + if skipped: + print(f"reads skip the nodes {sorted(skipped)}, their states are still compared", flush=True) def client_for(i): port = 9090 if i == 1 else 9100 + i @@ -283,6 +455,50 @@ def call_callers(when, via): calls += 1 failures += 0 if call(client_for(1), master, counter, "next", f"call {n + 1} before restart", calls) else 1 failures += 0 if call(client_for(1), master, ledger, "record", f"Ledger call {calls} before restart", calls) else 1 + # reads and the load go to the nodes the network started with; a node that joins later is in + # check_nodes, compared once it reached the target (--final-file) + def read_all(contract, method, expected, label, timeout=300): + """The getter on every node, until each returns `expected` (a lagging node needs a round or + two); prints one line, returns the number of nodes that did not.""" + def one(i): + deadline = time.time() + timeout + first_ms = None + value, msg = None, "" + while True: + try: + value, ms, msg = read_value(client_for(i), master, contract, method) + first_ms = ms if first_ms is None else first_ms + except Exception as e: # a node without a reachable API + value, msg = None, type(e).__name__ + if value == expected or time.time() >= deadline: + return i, value, first_ms, msg + time.sleep(2) + with ThreadPoolExecutor(max_workers=len(read_nodes)) as pool: + results = list(pool.map(one, read_nodes)) + bad = [r for r in results if r[1] != expected] + cells = " ".join(f"n{i}={v}" + (f"/{ms:.0f}ms" if ms is not None else "") for i, v, ms, _ in results) + print(f"read {label}: {'ok' if not bad else 'FAILED'} (expected {expected}) {cells}", flush=True) + for i, v, _, msg in bad: + print(f" n{i}: {v!r} {msg!r}", flush=True) + return len(bad) + + failures += read_all(counter, "get", calls, "Counter.get before restart") + failures += read_all(ledger, "entries", calls, "Ledger.entries before restart") + + # reads change nothing: no transaction on the contract, the same state, the same inner id + def footprint(): + c = client_for(1) + got = c.SmartContractGet(counter).smartContract + return got.transactionsCount, hashlib.sha256(got.objectState).hexdigest(), \ + c.WalletTransactionsCountGet(master.pk).lastTransactionInnerId + before = footprint() + for _ in range(20): + read_value(client_for(1), master, counter, "get") + after = footprint() + print(f"20 reads on n1: contract transactions, state, master inner id {before} -> {after}: " + f"{'unchanged' if before == after else 'CHANGED'}", flush=True) + failures += 0 if before == after else 1 + if args.ready_file: open(args.ready_file, "w").close() if args.wait_file: @@ -295,6 +511,32 @@ def call_callers(when, via): failures += 0 if call(client_for(1), master, counter, "next", f"call {n + 1} after restart", calls) else 1 failures += 0 if call(client_for(1), master, ledger, "record", f"Ledger call {calls} after restart", calls) else 1 failures += 0 if call(client_for(1), master, ledger, "entries", "Ledger entries", calls) else 1 + failures += read_all(counter, "get", calls, "Counter.get after restart") + failures += read_all(ledger, "entries", calls, "Ledger.entries after restart") + + if args.load_seconds > 0: + writes_ok, writes_failed, writes_rejected, bad_reads = load(args, client_for, master, counter, args.nodes, read_nodes) + failures += writes_failed + bad_reads + # a write that failed on its way back (a timeout) may still have landed: the counter is + # between the confirmed writes and all of them, and every node must read that same value + time.sleep(30) + value = None + for _ in range(10): + try: + value = read_value(client_for(1), master, counter, "get")[0] + except Exception: + value = None + if value is not None: + break + time.sleep(5) + low, high = calls + writes_ok, calls + writes_ok + writes_failed + writes_rejected + in_range = value is not None and low <= value <= high + print(f"Counter after the load: {value} (confirmed writes give {low}..{high}, unconfirmed ones may land): " + f"{'ok' if in_range else 'FAILED'}", + flush=True) + failures += 0 if in_range else 1 + calls = value if value is not None else calls + failures += read_all(counter, "get", calls, "Counter.get after the load") call_callers("after restart", 1) call_callers("after restart", 2) diff --git a/ci/testnet/fund.py b/ci/testnet/fund.py index 62c555e6a..9e54922e1 100644 --- a/ci/testnet/fund.py +++ b/ci/testnet/fund.py @@ -3,10 +3,13 @@ Usage: fund.py [--port 9090] [--amount 60000] [--load-seconds 0] [--dpos-at SEQ --reward-round SEQ] [--min-stake CS] [--delegations] + [--low-stake-nodes K --low-amount 1000] The genesis funds of a CREDITS_TESTNET build go to the key given in CS_TESTNET_GENESIS_KEY; this script holds its seed. It sends CS from it to every node (above the 50'000 minimum stake), then, for --load-seconds, small random transfers between nodes, through the Thrift API of node 1. +With --low-stake-nodes the last K nodes get --low-amount instead, below the minimum stake: once +stake counts (StartingDPOS) they must never be elected trusted. With --dpos-at and/or --reward-round the master key, which is also the starter key of the test network (CS_TESTNET_STARTER_KEY), sends two special "managing" transactions: order 9 moves StartingDPOS to @@ -206,6 +209,8 @@ def main(): parser.add_argument("master_seed") parser.add_argument("--port", type=int, default=9090) parser.add_argument("--amount", type=int, default=60000) + parser.add_argument("--low-stake-nodes", type=int, default=0) + parser.add_argument("--low-amount", type=int, default=1000) parser.add_argument("--load-seconds", type=int, default=0) parser.add_argument("--dpos-at", type=int, default=0) parser.add_argument("--reward-round", type=int, default=0) @@ -222,9 +227,12 @@ def main(): print(f"master balance before funding: {balance(client, master.pk):.4f}", flush=True) failures = 0 + def funded(i): + return args.low_amount if i > len(nodes) - args.low_stake_nodes else args.amount + for i, node in enumerate(nodes, 1): - ok, message = send(client, master, node.pk, args.amount) - print(f"fund n{i}: {'ok' if ok else 'FAILED ' + message}", flush=True) + ok, message = send(client, master, node.pk, funded(i)) + print(f"fund n{i} with {funded(i)}: {'ok' if ok else 'FAILED ' + message}", flush=True) failures += 0 if ok else 1 # Node::processSpecialInfo: u16 order, then the order's fields (cs::IDataStream, little endian); @@ -256,7 +264,7 @@ def delegate(name, src, dst, amount, value): if args.delegations: # a delegation is checked against the confirmed balance, so wait for the funding to land wait_until = time.time() + 180 - while time.time() < wait_until and any(balance(client, n.pk) < args.amount for n in nodes): + while time.time() < wait_until and any(balance(client, n.pk) < funded(i) for i, n in enumerate(nodes, 1)): time.sleep(5) expiry = int(time.time()) + args.delegation_seconds for i, src, dst in pairs: diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 69b7c028d..ba5d4c1e8 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2,10 +2,13 @@ # Run a private Credits network of N nodes on this machine and check that it works. # # Usage: run.sh -# Env: NODES (default 7, keep it odd: an even set of ready nodes drops one at bootstrap) +# Env: NODES (default 7; keep the initial confidants, TRUSTED_NODES, odd: an even set of ready ones +# drops one at bootstrap) # TARGET_SEQ (default 1100: past the first STATE DIGEST at block 1000) # TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test) # WORK (default ./testnet-run) +# TRUSTED_NODES (default NODES): only the first TRUSTED_NODES nodes are initial confidants +# (odd, 5 to 25); the others, up to 61 nodes in all, are ordinary nodes # SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU # of the nodes and executors in resources.tsv, and prints a summary at the end # FUND (default 0; needs a -DCREDITS_TESTNET=ON binary): genesis funds go to a generated master @@ -16,6 +19,8 @@ # SYNCRO_ROUND (default 0 = off, needs FUND=1): special order 35 makes the sync answers count # from that round (Consensus::syncroChangeRound), as on a network that issued it # JOIN_FULL=1: the node of JOIN_PRUNED_AT syncs from block 0 as a full node instead +# LOW_STAKE_NODES (default 0, needs FUND=1): the last that many nodes get 1'000 CS, below the +# minimum stake; after StartingDPOS (DPOS_AT) none of them may be trusted # MIN_STAKE (default 0 = off, needs FUND=1): special order 22 sets the minimum stake; with the # restart test, the restarted node must apply it again after its quick start # DELEGATE (default 0, needs FUND=1): nodes delegate to each other, with and without a time @@ -28,11 +33,15 @@ # node reached CONTRACTS_AFTER_AT (default 0); the states are compared once every node, # a joining one too, reached TARGET_SEQ. contracts.py sends everything through n1: a # contract scenario never kills, partitions or slows n1 +# CONTRACT_LOAD_SECONDS (default 0, with CONTRACTS): after the calls, a load phase of that many +# seconds: CONTRACT_WRITE_RATE (default 1) Counter.next writes a second from every node's +# account through its own API, and CONTRACT_READ_RATE (default 5) Counter.get reads (getters, +# no transaction) a second over every node; latencies in contracts-stats.json # EXECUTOR_JAR_ALT (optional, with CONTRACTS): the even nodes run this executor jar instead, on the # same JDK, so any result that differs from the release executor's fails the contract consensus # BIN_ALT (optional): the even nodes run this node binary instead, so a block or contract result # that differs from BIN's shows up as a diverging state -# RESTART_NODES (default: the last node): the nodes the restart test stops and restarts together +# RESTART_NODES (default: the last initial confidant): the nodes the restart test stops and restarts together # NET_FAULT (default off): loss, corrupt, delay, reorder (with 5% duplicates), duplicate or partition, applied with # tc netem to the p2p traffic sent and received by NET_FAULT_NODES (comma separated) from # sequence NET_FAULT_AT (default 150) for NET_FAULT_SECONDS (default 0 = until the end), at @@ -112,17 +121,28 @@ BIN_ALT=${BIN_ALT:+$(realpath "$BIN_ALT")} HERE=$(cd "$(dirname "$0")" && pwd) REPO=$(cd "$HERE/../.." && pwd) NODES=${NODES:-7} -# MinTrustedNodes is 4; an even set of ready nodes drops one at bootstrap -case "$NODES" in +# the initial confidants (trusted.txt): the first TRUSTED_NODES nodes, by default all of them; the +# others are ordinary nodes, as most nodes on mainnet. MinTrustedNodes is 4, MaxTrustedNodes 25, and +# an even set of ready confidants drops one at bootstrap +TRUSTED_NODES=${TRUSTED_NODES:-$NODES} +case "$TRUSTED_NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; - *) echo "NODES must be odd, 5 to 25, got '$NODES'"; exit 1 ;; + *) echo "TRUSTED_NODES must be odd, 5 to 25, got '$TRUSTED_NODES'"; exit 1 ;; esac -[ "$NODES" -le 15 ] || echo "warning: $NODES nodes, max_neighbours is 16: not a full mesh" +# ports: p2p 6000 + i (offline copies 6500 + i), API 9100 + i, apiexec 9200 + i, executor 9300 + i +[ "$NODES" -ge "$TRUSTED_NODES" ] && [ "$NODES" -le 61 ] \ + || { echo "NODES must be TRUSTED_NODES ($TRUSTED_NODES) to 61, got '$NODES'"; exit 1; } +# 25 is Consensus::MaxTrustedNodes: no larger set of initial confidants can bootstrap. Above 15 the neighbour map (16) is not a full mesh, but initial confidants are permanent +# neighbours of each other and consensus messages are routed, so that does not stop the network +CONTRACT_LOAD_SECONDS=${CONTRACT_LOAD_SECONDS:-0} +CONTRACT_WRITE_RATE=${CONTRACT_WRITE_RATE:-1} +CONTRACT_READ_RATE=${CONTRACT_READ_RATE:-5} SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} RESTART_AT=${RESTART_AT:-300} -RESTART_NODES=${RESTART_NODES:-$NODES} +# by default the last initial confidant (the last node when every node is one) +RESTART_NODES=${RESTART_NODES:-$TRUSTED_NODES} echo "$RESTART_NODES" | grep -qE '^[0-9]+( [0-9]+)*$' || { echo "bad RESTART_NODES '$RESTART_NODES'"; exit 1; } FUND=${FUND:-0} FUND_AT=${FUND_AT:-30} @@ -138,6 +158,13 @@ DPOS_AT=${DPOS_AT:-0} REWARD_ROUND=${REWARD_ROUND:-0} DELEGATE=${DELEGATE:-0} MIN_STAKE=${MIN_STAKE:-0} +# the last LOW_STAKE_NODES nodes (never initial confidants) get 1'000 CS, below the minimum stake +LOW_STAKE_NODES=${LOW_STAKE_NODES:-0} +[ "$LOW_STAKE_NODES" -le $((NODES - TRUSTED_NODES)) ] \ + || { echo "LOW_STAKE_NODES must be at most NODES - TRUSTED_NODES ($((NODES - TRUSTED_NODES)))"; exit 1; } +# stake counts only from StartingDPOS: the check needs it well before the target +[ "$LOW_STAKE_NODES" -eq 0 ] || { [ "$DPOS_AT" -gt 0 ] && [ $((DPOS_AT + 300)) -le "$TARGET_SEQ" ]; } \ + || { echo "LOW_STAKE_NODES needs DPOS_AT, at most TARGET_SEQ - 300"; exit 1; } SYNCRO_ROUND=${SYNCRO_ROUND:-0} JOIN_FULL=${JOIN_FULL:-0} CONTRACTS=${CONTRACTS:-0} @@ -268,6 +295,10 @@ api_settings() { # the first contract compile in a fresh JVM takes longer than the 4 s default echo "executor_send_timeout=60000" echo "executor_receive_timeout=60000" + # the node asks its new executor for its version every second for this long, and disables it for + # good if it never answers: dozens of JVMs starting at once on one machine (a power cut, a big + # network) can take longer than the 7 s default + echo "executor_check_version_delay=60000" cat > "$WORK/n$i/settings.properties" < "$dir/hosts.txt" : > "$dir/trusted.txt" for j in $(seq 1 "$NODES"); do - echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" + [ "$j" -gt "$TRUSTED_NODES" ] || echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" [ "$j" -ne "$i" ] && echo "127.0.0.1:$((BASE_PORT + j)) ${KEYS[$((j - 1))]}" >> "$dir/hosts.txt" done cat > "$dir/config.ini" < "$WORK/fund.log" 2>&1 & FUND_PID=$! fi @@ -910,9 +942,17 @@ while :; do echo "contracts: deploying and calling them through node 1" (cd "$HERE" && python3 contracts.py "$WORK" "$NODES" "$MASTER_SEED" \ --ready-file "$WORK/contracts-ready" --wait-file "$WORK/restart-done" --wait-timeout $((TIMEOUT_MIN * 60)) \ - --check-nodes "$ALL_NODES" --final-file "$WORK/contracts-final") > "$WORK/contracts.log" 2>&1 & + --check-nodes "$ALL_NODES" --final-file "$WORK/contracts-final" \ + --load-seconds "$CONTRACT_LOAD_SECONDS" --write-rate "$CONTRACT_WRITE_RATE" --read-rate "$CONTRACT_READ_RATE" \ + --stats-file "$WORK/contracts-stats.json") > "$WORK/contracts.log" 2>&1 & CONTRACTS_PID=$! fi + # contracts.py ended before its first calls were done: nothing would release the restart test + if [ -n "$CONTRACTS_PID" ] && [ ! -e "$WORK/contracts-ready" ] && ! kill -0 "$CONTRACTS_PID" 2>/dev/null; then + cat "$WORK/contracts.log" + echo "FAIL: contracts.py ended before its first calls" + exit 1 + fi # the calls after the restart must run on the restarted nodes too, so let them catch up first if [ "$CONTRACTS" = "1" ] && [ ! -e "$WORK/restart-done" ] && contracts_go_ready 2>/dev/null; then touch "$WORK/restart-done" @@ -1169,4 +1209,26 @@ if funded and 1000 in seen and max(w for _, w in seen[1000].values()) <= 2: ok = False sys.exit(0 if ok else 1) PY +# who took part in consensus: trusted per round (from n1's round headers) and how many nodes were +# trusted at least once (a node logs TRUSTED [index] in the rounds it is trusted in) +per_round=$(grep -ah 'Trusted count:' "$WORK/n1/node.log" | sed -E 's/.*Trusted count: ([0-9]+).*/\1/' | sort -n | uniq -c \ + | awk '{printf "%s%s trusted in %s rounds", sep, $2, $1; sep=", "}' || true) +ever=0 +for i in $(seq 1 "$ALL_NODES"); do + grep -aq 'TRUSTED \[' "$WORK/n$i/node.log" && ever=$((ever + 1)) +done +echo "consensus: $per_round; $ever of $ALL_NODES nodes were trusted at least once (initial confidants: $TRUSTED_NODES)" +if [ "$LOW_STAKE_NODES" -gt 0 ]; then + # once stake counts (StartingDPOS, moved by DPOS_AT; 100 rounds of margin for the order to land), + # a node below the minimum stake must never be trusted + from=$((DPOS_AT + 100)) + low_trusted=0 + for i in $(seq $((NODES - LOW_STAKE_NODES + 1)) "$NODES"); do + # rounds print with digit groups (R-1'234.0) + n=$(grep -a 'TRUSTED \[' "$WORK/n$i/node.log" | sed -E "s/.*R-([0-9']+)\..*/\1/" | tr -d "'" | awk -v f="$from" '$1 > f' | wc -l || true) + [ "$n" -eq 0 ] || { echo "n$i (low stake) was trusted in $n rounds after round $from"; low_trusted=$((low_trusted + 1)); } + done + [ "$low_trusted" -eq 0 ] || { echo "FAIL: nodes below the minimum stake were elected trusted"; exit 1; } + echo "stake: none of the $LOW_STAKE_NODES nodes below the minimum stake was trusted after round $from" +fi echo "PASS: $ALL_NODES nodes reached sequence $TARGET_SEQ with matching state digests" diff --git a/ci/testnet/sampler.py b/ci/testnet/sampler.py index c66ed6fe6..4fff14c4e 100644 --- a/ci/testnet/sampler.py +++ b/ci/testnet/sampler.py @@ -164,6 +164,9 @@ def key(k): out.append(f" {k[1]:<16} {max(r['rss_mb'] for r in rs):>6} {statistics.mean(cpus) if cpus else 0:>6.0f} " f"{max(cpus) if cpus else 0:>6.0f} {len({r['pid'] for r in rs}):>4}") out.append(f" sum of peak RSS: {sum(max(r['rss_mb'] for r in procs[k]) for k in keys)} MB") + if kind == "exec": + # a node that stops stops every executor of the machine (jps), and each node relaunches its own + out.append(f" executor relaunches: {sum(len({r['pid'] for r in procs[k]}) - 1 for k in keys)}") per_ts = {} for r in rows: if r["kind"] in ("node", "exec"): diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 25ad4d91e..e6730bd8c 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -24,7 +24,7 @@ lines under the new release. nodes and builds compare their state. - **Continuous integration.** - unit tests; - - a private 7-node network; + - private networks of 5 to 61 nodes, with smart contracts read and written; - a robustness suite: crashes, power cuts, full disks, network faults, mixed binaries, every storage backend, conversions; - upgrades from the official node, tested on mainnet data; @@ -37,6 +37,9 @@ lines under the new release. | #67 | csdb, ci, docs: conversion follow-ups from the review of #65 (contract-records message, keep a copy of `db/` before converting from berkeleydb, sturdier db-convert scenario) | | #70 | csnode: bootstrap tables of 9 to 25 confidants are accepted (the official node silently drops them, so a network of 9 or more initial confidants never starts); the same choice as the official node up to 8 | | #71 | csnode: a bootstrap round table is taken only for a round next to the node's own, and, unless signed with the blockchain key, only once the node's own round has stalled; the block-removal request it set (never effective) is gone | +| #68 | ci(testnet): resource sampling (host and per-process memory and CPU, block rate) in every private-network run; a `nodes` input for testnet and robustness | +| #69 | ci: smart contracts in the robustness suite (crashes, a power cut, packet loss, a pruned node killed and joining, a database conversion) | +| #72 | ci: contract reads on every node and a contract load phase; networks of up to 61 nodes with 5 to 25 initial confidants, stake-based election and nodes below the minimum stake; a scale workflow | | #73 | api: the executor version check asks every second up to `executor_check_version_delay` instead of once after it (the node starts as soon as its executor answers); with `executor_multi_instance` a stopping node no longer kills the other nodes' executors | | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails |