diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 0f1ceb2e6..0bf72a7ef 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -392,14 +392,20 @@ jobs: contracts: true # stress on the confidants that serve (n1, n3, n5, n7): four nodes join at once, the first killed # during its fetch and its import; n3 and n9 on a 2 Mbit/s link. The block rate and the round - # times while serving must stay within 10 % / 20 % of the same window before the joins + # times while serving must stay within 10 % / 20 % of the same window before the joins (the rate + # is the median of the one-minute rates) - name: snapshot-p2p-stress official: 'no' # four joining nodes: the even ones would be official env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 TARGET_SEQ=3300 JOIN_EXTRA=3 JOIN_FETCH_RATE=200 JOIN_KILL_DURING_FETCH=1 NET_FAULT=rate NET_FAULT_RATE=2mbit NET_FAULT_NODES=3,9 fixed: true + # 5 % loss on n3 and n9: 20 % / 50 % allowed. The loss reaches the joiner n9 only once it has + # joined, so the second window has more lossy traffic than the first, whatever serving costs; + # and the first window's round p95 alone varies between identical runs (0.57 s or 0.8-0.93 s). + # Measured: a steady fall of 5-16 % (-10 % failed the former 10 % limit); the same logs slowed + # by 30 % (or 20 %) give -37 to -40 % (-26 to -31 %), which still fails - name: snapshot-p2p-stress-loss official: 'no' # four joining nodes: the even ones would be official - env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 TARGET_SEQ=3300 JOIN_EXTRA=3 NET_FAULT=loss NET_FAULT_PCT=5 NET_FAULT_NODES=3,9 + env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 STRESS_RATE_DROP=20 STRESS_ROUND_RISE=50 TARGET_SEQ=3300 JOIN_EXTRA=3 NET_FAULT=loss NET_FAULT_PCT=5 NET_FAULT_NODES=3,9 fixed: true # with contracts running before and during the joins - name: snapshot-p2p-stress-contracts @@ -497,6 +503,61 @@ jobs: - name: bootstrap-livelock-rewards env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 REWARD_ROUND=100 LOAD_SECONDS=2400 TARGET_SEQ=3000 manual: true + # if all official nodes become unavailable at once (node_alt_ref=official; run by name): 5 + # official even nodes with more stake hold the trusted seats, 5 community odd nodes; n1, n2, + # n3, n4 and n6 are the genesis initial confidants (official majority: a table that still + # names the stopped nodes cannot decide). They are killed in a round they hold the majority of, stay + # down 5 minutes and start again. Without extra keys the network must stay stalled until + # they are back; with every community key in the community nodes' own trusted.txt the + # community nodes must restart it, and the official nodes must then take the same blocks + - name: official-stop-stalled + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=none RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true + - name: official-stop-trusted + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=trusted RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true + # the same with the community bootstrap fallback ([community_bootstrap], stall_minutes=2) in + # place of the hand-edited trusted.txt: it must restart the network, and not act before + - name: official-stop-config + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_STOP_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true + # the same with staggered stall_minutes: the starter (the community node with the lowest key) + # 2, the others 3. Its first tables reach nodes whose fallback is still closed: the restart + # needs a repeat of the table in the same round to get through (not dropped as a p2p duplicate) + - name: official-stop-config-staggered + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_STOP_MINUTES=7 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=3 COMMUNITY_STALL_STARTER=2 RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true + # the community nodes cut off from the RUNNING official nodes for 6 minutes (fallback on, + # stall_minutes=2): both sides write blocks; after the reconnection the run reports which + # side the nodes kept, how many blocks and transactions the other lost, and needs one chain + # (refusals and FORK HELD on the community side are expected: DEEP_ROLLBACK_EXPECTED=1) + - name: official-partition + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 + fixed: true + manual: true + # the same, the community nodes restarted after the reconnection (they dial every host again) + - name: official-partition-restart + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 + fixed: true + manual: true + # the operator path after a partition: the community nodes, restarted after the + # reconnection, hold the fork (FORK HELD: the longer chain is not vouched for by their + # confidants); each one is stopped and run once with --resolve-fork, then, if the fork + # point was only an upper bound and the node holds the fork again lower, with --set-bc-top + # at the last common block; every node must end on the official chain + - name: official-partition-restart-resolve + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 RESOLVE_FORK_AFTER=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 + fixed: true + manual: true + # the fallback on (stall_minutes=1) in a network whose official nodes keep running, under + # packet loss and the restart test: it must never act + - name: community-bootstrap-quiet + env: COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=1 NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 TARGET_SEQ=1500 + manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill official: 'no' # needs a feature of this fork on an even node @@ -608,4 +669,8 @@ jobs: testnet-run/resources.tsv testnet-run/resources-summary.txt testnet-run/sampler.log + testnet-run/partition-*.txt + testnet-run/n*/fork-held.json + testnet-run/n*/reorg-episode.json + # branch runs (test variants) are kept briefly: Actions storage is shared retention-days: ${{ github.ref_name == 'master' && 14 || 3 }} diff --git a/ci/testnet/fund.py b/ci/testnet/fund.py index 5622ae66c..e9cc22f68 100644 --- a/ci/testnet/fund.py +++ b/ci/testnet/fund.py @@ -211,7 +211,12 @@ def main(): parser.add_argument("--amount", type=int, default=60000) parser.add_argument("--low-stake-nodes", type=int, default=0) parser.add_argument("--low-amount", type=int, default=1000) + # the even nodes' amount (0 = --amount): with official even nodes, more stake takes them the seats + parser.add_argument("--even-amount", type=int, default=0) parser.add_argument("--load-seconds", type=int, default=0) + # the chain of node 1 is cut back during the load (run.sh RESOLVE_FORK_AFTER): a transfer it took on + # the branch that was cut may come back expired, an answer and not a fault + parser.add_argument("--chain-cut", action="store_true") parser.add_argument("--dpos-at", type=int, default=0) parser.add_argument("--reward-round", type=int, default=0) parser.add_argument("--min-stake", type=int, default=0) @@ -228,7 +233,9 @@ def main(): failures = 0 def funded(i): - return args.low_amount if i > len(nodes) - args.low_stake_nodes else args.amount + if i > len(nodes) - args.low_stake_nodes: + return args.low_amount + return args.even_amount if args.even_amount and i % 2 == 0 else args.amount for i, node in enumerate(nodes, 1): ok, message = send(client, master, node.pk, funded(i)) @@ -320,6 +327,9 @@ def delegate(name, src, dst, amount, value): # the inner id of a transfer whose outcome was unknown: an answer, not a fault pending += 1 print(f"load transfer rejected after an unknown outcome: {message}", flush=True) + elif not ok and args.chain_cut and "transaction is expired" in message: + pending += 1 + print(f"load transfer expired after the chain was cut back: {message}", flush=True) elif not ok and "not syncronized" in message: # the node refuses transfers while it catches up after a restart (power-cut scenario) unsynced += 1 diff --git a/ci/testnet/partition.py b/ci/testnet/partition.py new file mode 100755 index 000000000..dec40b91a --- /dev/null +++ b/ci/testnet/partition.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""The two chains of a partition (run.sh, OFFICIAL_PARTITION_AT) and the chain left after it. + + partition.py dump --from 395 --port 9090 > community.txt # "seq hash transactions" per block + partition.py report --community community.txt --official official.txt --port 9090 --port 9102 + partition.py fork --community community.txt --official official.txt # the first block that differs + +dump: the blocks of one node from a sequence to its head, taken just before the reconnection. +report: where the two sides forked, and which side's blocks the nodes hold after the reconnection +(each port given, every block of both sides compared); the side whose blocks are gone was unwound, +and the transactions of its blocks above the fork with them (unless written again on the other side). +""" +import argparse +import sys + +from thriftpy2.protocol import TBinaryProtocolFactory +from thriftpy2.rpc import make_client +from thriftpy2.transport import TBufferedTransportFactory + +from fund import api + + +def client_of(port): + return make_client(api.API, host="127.0.0.1", port=int(port), proto_factory=TBinaryProtocolFactory(), + trans_factory=TBufferedTransportFactory(), timeout=60000) + + +def blocks(client, first, last): + """{seq: (hash, transactions)} of the blocks first..last a node holds""" + result = {} + seq = first + while seq <= last: + count = min(100, last - seq + 1) + pools = client.PoolListGetStable(seq + count - 1, count).pools or [] + for pool in pools: + if first <= pool.poolNumber <= last: + result[pool.poolNumber] = (bytes(pool.hash).hex(), pool.transactionsCount) + seq += count + return result + + +def head(client): + return client.SyncStateGet().lastBlock + + +def read(path): + result = {} + with open(path) as f: + for line in f: + seq, h, txs = line.split() + result[int(seq)] = (h, int(txs)) + return result + + +def dump(args): + client = client_of(args.port) + last = head(client) + for seq, (h, txs) in sorted(blocks(client, args.first, last).items()): + print(seq, h, txs) + + +def fork_of(community, official): + """the first sequence at which the two sides hold different blocks""" + common = sorted(set(community) & set(official)) + fork = next((s for s in common if community[s][0] != official[s][0]), None) + if fork is None: + # one side may only be shorter: the fork is right above the shorter head + fork = min(max(community, default=0), max(official, default=0)) + 1 + return fork + + +def report(args): + community = read(args.community) + official = read(args.official) + c_head = max(community) if community else 0 + o_head = max(official) if official else 0 + fork = fork_of(community, official) + c_side = {s: v for s, v in community.items() if s >= fork} + o_side = {s: v for s, v in official.items() if s >= fork} + print(f"PARTITION RESULT: fork at #{fork}; before the reconnection the community side held #{fork}..#{c_head} " + f"({len(c_side)} blocks, {sum(t for _, t in c_side.values())} transactions), the official side #{fork}..#{o_head} " + f"({len(o_side)} blocks, {sum(t for _, t in o_side.values())} transactions)") + failed = False + verdicts = set() + for port in args.port: + client = client_of(port) + last = max(c_head, o_head) + now = blocks(client, fork, last) + kept_c = sum(1 for s, v in c_side.items() if now.get(s, (None,))[0] == v[0]) + kept_o = sum(1 for s, v in o_side.items() if now.get(s, (None,))[0] == v[0]) + lost_c_txs = sum(v[1] for s, v in c_side.items() if now.get(s, (None,))[0] != v[0]) + lost_o_txs = sum(v[1] for s, v in o_side.items() if now.get(s, (None,))[0] != v[0]) + if not o_side and c_side and kept_c == len(c_side): + # the official side wrote nothing while cut off (it stalled): no fork, one chain + verdict = f"no fork: the official side wrote no block, the community blocks were kept ({len(c_side)} blocks)" + elif not c_side and o_side and kept_o == len(o_side): + verdict = f"no fork: the community side wrote no block, the official blocks were kept ({len(o_side)} blocks)" + elif c_side and kept_c == 0 and kept_o == len(o_side): + verdict = f"the community side was unwound ({len(c_side)} blocks, {lost_c_txs} transactions reverted)" + elif o_side and kept_o == 0 and kept_c == len(c_side): + verdict = f"the official side was unwound ({len(o_side)} blocks, {lost_o_txs} transactions reverted)" + else: + verdict = f"mixed: {kept_c}/{len(c_side)} community and {kept_o}/{len(o_side)} official blocks kept" + failed = True + verdicts.add(verdict) + print(f"PARTITION RESULT: port {port} (head #{head(client)}): {verdict}") + if len(verdicts) > 1: + print("PARTITION RESULT: the nodes compared did not keep the same side") + failed = True + sys.exit(1 if failed else 0) + + +def main(): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + d = sub.add_parser("dump") + d.add_argument("--from", dest="first", type=int, required=True) + d.add_argument("--port", required=True) + r = sub.add_parser("report") + r.add_argument("--community", required=True) + r.add_argument("--official", required=True) + r.add_argument("--port", action="append", required=True) + f = sub.add_parser("fork") + f.add_argument("--community", required=True) + f.add_argument("--official", required=True) + args = parser.parse_args() + if args.command == "dump": + dump(args) + elif args.command == "fork": + print(fork_of(read(args.community), read(args.official))) + else: + report(args) + + +if __name__ == "__main__": + main() diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 554ae8700..d5d9aa93e 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -8,6 +8,7 @@ # TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test) # WORK (default ./testnet-run) # TRUSTED_NODES (default NODES): only the first TRUSTED_NODES nodes are initial confidants +# (INITIAL_NODES, comma separated: these nodes instead) # (odd, 5 to 25); the others, up to 61 nodes in all, are ordinary nodes # ROUND_ELAPSE_MS (default 20000): round_elapse_time of every node # SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU @@ -71,8 +72,9 @@ # one and configured like it; each must start from a snapshot (or fall back as JOIN_EXPECT says) # and reach the target. STRESS_WINDOW (seconds, default 0 = off): the block rate and the round # times of n1 in that many seconds before the joins and after they started are compared; the -# run fails when the rate falls by more than 10 % or the 95th percentile of the round time rises -# by more than 20 % +# rate of a window is the median of its one-minute rates, so one slow minute does not decide. +# The run fails when the rate falls by more than STRESS_RATE_DROP % (default 10) or the 95th +# percentile of the round time rises by more than STRESS_ROUND_RISE % (default 20) # NET_FAULT (default off): loss, corrupt, delay, reorder (with 5% duplicates), duplicate, partition, # or rate (NET_FAULT_RATE, default 2mbit, for the faulty nodes' traffic together), applied with # tc netem to the p2p traffic sent and received by NET_FAULT_NODES (comma separated) from @@ -197,6 +199,42 @@ # state digest (needs sudo). With DISK_FULL_STOP=1 the node gets SIGTERM halfway through the # full-disk period and must exit within 120 s, as an operator would stop it # +# OFFICIAL_STOP_AT (default 0 = off, needs BIN_ALT_OFFICIAL=1): at that sequence, in a round whose +# table the official (even) nodes hold the majority of, every official node is killed at once; +# they stay down OFFICIAL_STOP_MINUTES (default 5) and are started again, and must end on the +# same blocks as the community nodes (every block compared). OFFICIAL_FUND (FUND amount of the +# even nodes, default the others' 60000): more stake than the community nodes, so that the +# official nodes take the trusted seats as on mainnet. COMMUNITY_BOOTSTRAP says what the +# community nodes may do meanwhile: none (default) - nothing, the network must stay stalled +# until the official nodes are back; trusted - their own trusted.txt also lists every community +# node, the network must restart without them; config - the community nodes run the community +# bootstrap fallback ([community_bootstrap] confidants = every community key, stall_minutes = +# COMMUNITY_STALL_MINUTES, default 2), the network must restart without them. COMMUNITY_BOOTSTRAP=config +# also runs without OFFICIAL_STOP_AT: the fallback must then never act (no "COMMUNITY BOOTSTRAP:" +# table started or taken), and with OFFICIAL_STOP_AT never before the official nodes stop. +# COMMUNITY_STALL_STARTER (minutes, default 0 = off, with COMMUNITY_BOOTSTRAP=config): the +# community node with the lowest key, which starts the community table, gets that stall_minutes +# and the others COMMUNITY_STALL_MINUTES: its first tables reach nodes whose fallback is still +# closed, so the restart needs a repeat of the table to get through (each community node other +# than the starter must drop one table and take a later one) +# OFFICIAL_PARTITION_AT (default 0 = off, needs BIN_ALT_OFFICIAL=1 and Linux; not with +# OFFICIAL_STOP_AT): at that sequence the community (odd) nodes are cut off from the official +# (even) ones, all of them running (iptables: UDP between their p2p ports, TCP by cgroup), for +# OFFICIAL_PARTITION_MINUTES (default 6), then reconnected. With COMMUNITY_BOOTSTRAP=config the +# community side restarts on its own after stall_minutes: two chains until the reconnection. +# partition.py reports where they forked, which side the nodes kept and how many blocks and +# transactions the other side lost; every node must then end on one chain. The nodes only +# reconnect if they look each other up again: 120 s after the cut is lifted the run reports +# how many neighbours each node has. OFFICIAL_PARTITION_RESTART=1: then the community nodes are +# restarted (they dial every node of hosts.txt again), as an operator would. +# RESOLVE_FORK_AFTER=1: then, as an operator choosing the official chain, wait up to +# RESOLVE_FORK_WAIT minutes (default 5) for FORK HELD on the community nodes (the bounded-rollback +# guard refuses the longer chain their confidants do not vouch for), stop every community node +# holding a fork, run it once with --resolve-fork and start it again; a node whose fork point +# was only an upper bound and that holds the fork again lower is cut with --set-bc-top at the +# last common block (docs/quick-start.md, "If your node reports FORK HELD"). A start right after +# a cut that stops at once is reported and repeated. Every node must then end on one chain +# # Checks: every node writes blocks up to TARGET_SEQ; a node stopped and restarted (quick start # from its caches) catches up; all nodes report the same STATE DIGEST at every common sequence. set -euo pipefail @@ -232,6 +270,12 @@ case "$TRUSTED_NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "TRUSTED_NODES must be odd, 5 to 25, got '$TRUSTED_NODES'"; exit 1 ;; esac +# INITIAL_NODES (comma separated, TRUSTED_NODES of them): these nodes are the initial confidants +# instead of the first TRUSTED_NODES, e.g. a set the official (even) nodes hold the majority of +INITIAL_NODES=${INITIAL_NODES:-$(seq -s, 1 "$TRUSTED_NODES")} +[ "$(tr ',' '\n' <<<"$INITIAL_NODES" | sort -un | grep -c .)" -eq "$TRUSTED_NODES" ] \ + || { echo "INITIAL_NODES must name $TRUSTED_NODES distinct nodes, got '$INITIAL_NODES'"; exit 1; } +is_initial() { [[ ",$INITIAL_NODES," == *",$1,"* ]]; } # ports: p2p 6000 + i (offline copies 6500 + i), API 9100 + i, apiexec 9200 + i, executor 9300 + i, diag 9400 + i [ "$NODES" -ge "$TRUSTED_NODES" ] && [ "$NODES" -le 61 ] \ || { echo "NODES must be TRUSTED_NODES ($TRUSTED_NODES) to 61, got '$NODES'"; exit 1; } @@ -296,6 +340,20 @@ TERM_KILL_NODE=${TERM_KILL_NODE:-0} TERM_KILL_TIMES=${TERM_KILL_TIMES:-4} TERM_KILL_FROM=${TERM_KILL_FROM:-150} CRASH_ALL_AT=${CRASH_ALL_AT:-0} +OFFICIAL_STOP_AT=${OFFICIAL_STOP_AT:-0} +OFFICIAL_STOP_MINUTES=${OFFICIAL_STOP_MINUTES:-5} +OFFICIAL_FUND=${OFFICIAL_FUND:-0} +COMMUNITY_BOOTSTRAP=${COMMUNITY_BOOTSTRAP:-none} +case "$COMMUNITY_BOOTSTRAP" in none|trusted|config) ;; *) echo "COMMUNITY_BOOTSTRAP must be none, trusted or config, got '$COMMUNITY_BOOTSTRAP'"; exit 1 ;; esac +COMMUNITY_STALL_MINUTES=${COMMUNITY_STALL_MINUTES:-2} +COMMUNITY_STALL_STARTER=${COMMUNITY_STALL_STARTER:-0} +COMMUNITY_STARTER=0 # the node given COMMUNITY_STALL_STARTER, set with the configs +OFFICIAL_PARTITION_AT=${OFFICIAL_PARTITION_AT:-0} +OFFICIAL_PARTITION_MINUTES=${OFFICIAL_PARTITION_MINUTES:-6} +OFFICIAL_PARTITION_RESTART=${OFFICIAL_PARTITION_RESTART:-0} +RESOLVE_FORK_AFTER=${RESOLVE_FORK_AFTER:-0} +RESOLVE_LOCATE_WAIT=${RESOLVE_LOCATE_WAIT:-6} # minutes to wait for the located fork point +RESOLVE_FORK_WAIT=${RESOLVE_FORK_WAIT:-5} JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} @@ -369,6 +427,8 @@ is_joiner() { [ "$JOIN_NODE" -gt 0 ] && [ "$1" -ge "$JOIN_NODE" ]; } alt_node() { [ -n "$BIN_ALT" ] && [ $(( $1 % 2 )) -eq 0 ] && ! { [ "${JOIN_COMMUNITY:-0}" = 1 ] && [ "$1" -eq "$JOIN_NODE" ]; }; } EXTRA_JOINERS=$( { [ "$JOIN_EXTRA" -gt 0 ] && seq $((JOIN_NODE + 1)) "$ALL_NODES" | tr "\n" " "; } || true) STRESS_WINDOW=${STRESS_WINDOW:-0} +STRESS_RATE_DROP=${STRESS_RATE_DROP:-10} +STRESS_ROUND_RISE=${STRESS_ROUND_RISE:-20} BACKFILL_RATE=${BACKFILL_RATE:-} BACKFILL_BATCH=${BACKFILL_BATCH:-} BACKFILL_PER_TURN=${BACKFILL_PER_TURN:-} @@ -531,6 +591,25 @@ if [ "$BIN_ALT_OFFICIAL" = 1 ]; then [ -z "$DB_BACKEND" ] || echo "mixed nodes: the official nodes run BerkeleyDB whatever DB_BACKEND ($DB_BACKEND) says" fi +# every official node stopped at once: the community nodes alone are left (see OFFICIAL_STOP_AT above) +if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then + [ "$BIN_ALT_OFFICIAL" = 1 ] || { echo "FAIL: OFFICIAL_STOP_AT needs official even nodes (BIN_ALT_OFFICIAL=1)"; exit 1; } + [ "$CRASH_ALL_AT" -eq 0 ] && [ "$DPOS_AT" -eq 0 ] && [ "$JOIN_NODE" -eq 0 ] \ + || { echo "FAIL: OFFICIAL_STOP_AT runs without CRASH_ALL_AT, DPOS_AT and a joining node"; exit 1; } + # the community nodes that could start a bootstrap with the extra keys: MinTrustedNodes is 4 + [ $(( (NODES + 1) / 2 )) -ge 5 ] || { echo "FAIL: OFFICIAL_STOP_AT needs at least 5 community (odd) nodes, NODES >= 9"; exit 1; } +fi +# the community nodes cut off from the running official nodes (see OFFICIAL_PARTITION_AT above) +if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then + [ "$BIN_ALT_OFFICIAL" = 1 ] || { echo "FAIL: OFFICIAL_PARTITION_AT needs official even nodes (BIN_ALT_OFFICIAL=1)"; exit 1; } + [ "$OS_KIND" = linux ] || { echo "FAIL: OFFICIAL_PARTITION_AT needs Linux (iptables, cgroups)"; exit 1; } + [ "$OFFICIAL_STOP_AT" -eq 0 ] && [ "$CRASH_ALL_AT" -eq 0 ] && [ "$DPOS_AT" -eq 0 ] && [ "$JOIN_NODE" -eq 0 ] && [ -z "$NET_FAULT" ] \ + || { echo "FAIL: OFFICIAL_PARTITION_AT runs without OFFICIAL_STOP_AT, CRASH_ALL_AT, DPOS_AT, NET_FAULT and a joining node"; exit 1; } + [ $(( (NODES + 1) / 2 )) -ge 5 ] || { echo "FAIL: OFFICIAL_PARTITION_AT needs at least 5 community (odd) nodes, NODES >= 9"; exit 1; } +fi +[ "$RESOLVE_FORK_AFTER" = 0 ] || [ "$OFFICIAL_PARTITION_AT" -gt 0 ] || { echo "FAIL: RESOLVE_FORK_AFTER needs OFFICIAL_PARTITION_AT"; exit 1; } +[ "$COMMUNITY_STALL_STARTER" -eq 0 ] || [ "$COMMUNITY_BOOTSTRAP" = config ] \ + || { echo "FAIL: COMMUNITY_STALL_STARTER needs COMMUNITY_BOOTSTRAP=config"; exit 1; } mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$ALL_NODES" "$WORK") if [ "$FUND" = "1" ]; then @@ -687,7 +766,7 @@ for i in $(seq 1 "$ALL_NODES"); do : > "$dir/hosts.txt" : > "$dir/trusted.txt" for j in $(seq 1 "$NODES"); do - [ "$j" -gt "$TRUSTED_NODES" ] || echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" + ! is_initial "$j" || echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" [ "$j" -ne "$i" ] && echo "$(node_ip "$j"):$((BASE_PORT + j)) ${KEYS[$((j - 1))]}" >> "$dir/hosts.txt" done cat > "$dir/config.ini" <> "$WORK/n$i/trusted.txt" + done + done + echo "official stop: the community nodes list $(grep -c . "$WORK/n1/trusted.txt") initial confidants, the official nodes $TRUSTED_NODES" +fi + +# COMMUNITY_BOOTSTRAP=config: the community bootstrap fallback on every node of this build, listing +# every node of this build (the official nodes, even, have no such option) +if [ "$COMMUNITY_BOOTSTRAP" = config ]; then + listed="" + for j in $(seq 1 "$NODES"); do alt_node "$j" || listed="$listed${listed:+,}${KEYS[$((j - 1))]}"; done + # COMMUNITY_STALL_STARTER: the node of this build with the lowest key (bytes, as cs::PublicKey + # compares) starts the community table; it waits less than the others + if [ "$COMMUNITY_STALL_STARTER" -gt 0 ]; then + COMMUNITY_STARTER=$(for j in $(seq 1 "$NODES"); do alt_node "$j" || echo "$j ${KEYS[$((j - 1))]}"; done \ + | python3 -c 'import sys, base58; print(min((base58.b58decode(k), int(j)) for j, k in (l.split() for l in sys.stdin))[1])') + fi + for i in $(seq 1 "$NODES"); do + alt_node "$i" && continue + stall=$COMMUNITY_STALL_MINUTES + [ "$i" -ne "$COMMUNITY_STARTER" ] || stall=$COMMUNITY_STALL_STARTER + printf '\n[community_bootstrap]\nconfidants=%s\nstall_minutes=%s\n' "$listed" "$stall" >> "$WORK/n$i/config.ini" + done + echo "community bootstrap: on in every node of this build, $(tr ',' '\n' <<<"$listed" | grep -c .) keys listed, stall_minutes=$COMMUNITY_STALL_MINUTES" \ + "$([ "$COMMUNITY_STARTER" -eq 0 ] || echo "(n$COMMUNITY_STARTER, the starter: $COMMUNITY_STALL_STARTER)")" +fi + # test-build hooks of a forged or corrupt snapshot source, and the joiner's list of serving nodes [ "$FORGE_NODE" -eq 0 ] || : > "$WORK/n$FORGE_NODE/testnet-forge-snapshot" [ "$CORRUPT_NODE" -eq 0 ] || : > "$WORK/n$CORRUPT_NODE/testnet-corrupt-pieces" @@ -763,7 +875,7 @@ start_node() { PIDS[$i]=$! # a slow or faulty node runs in its own cgroup: a CPU quota for a slow one, and its packets are # marked for the network fault (its executor, started later, inherits the cgroup) - if [[ ",$SLOW_NODES,$NET_FAULT_NODES," == *",$i,"* ]]; then + if [[ ",$SLOW_NODES,$NET_FAULT_NODES," == *",$i,"* ]] || [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then sudo mkdir -p "/sys/fs/cgroup/cstest-$i" if [[ ",$SLOW_NODES," == *",$i,"* ]]; then echo "$((SLOW_CPU * 1000)) 100000" | sudo tee "/sys/fs/cgroup/cstest-$i/cpu.max" > /dev/null @@ -791,6 +903,32 @@ net_fault_off() { sudo tc qdisc del dev lo root 2>/dev/null || true sudo iptables -t mangle -F OUTPUT 2>/dev/null || true } +# OFFICIAL_PARTITION_AT: nothing passes between a community and an official node, both ways. UDP by +# the p2p ports (a node sends its datagrams from its own port), TCP by the sender's cgroup +partition_on() { + local i j + partition_off # a chain left by an earlier cut: -N would fail, and its rules would stay + sudo iptables -N CSPART + sudo iptables -I OUTPUT 1 -o lo -j CSPART + for i in $COMMUNITY_NODES; do + for j in $OFFICIAL_NODES; do + sudo iptables -A CSPART -p udp --sport $((BASE_PORT + i)) --dport $((BASE_PORT + j)) -j DROP + sudo iptables -A CSPART -p udp --sport $((BASE_PORT + j)) --dport $((BASE_PORT + i)) -j DROP + sudo iptables -A CSPART -p tcp -m cgroup --path "cstest-$i" --dport $((BASE_PORT + j)) -j DROP \ + || echo "partition: no cgroup match, TCP from n$i to n$j still passes" + sudo iptables -A CSPART -p tcp -m cgroup --path "cstest-$j" --dport $((BASE_PORT + i)) -j DROP \ + || echo "partition: no cgroup match, TCP from n$j to n$i still passes" + done + done +} +partition_off() { + sudo iptables -D OUTPUT -o lo -j CSPART 2>/dev/null || true + sudo iptables -F CSPART 2>/dev/null || true + sudo iptables -X CSPART 2>/dev/null || true +} +partition_stats() { + sudo iptables -L CSPART -v -x -n 2>/dev/null | awk 'NR > 2 { p += $1 } END { print p + 0 }' +} node_nets_on() { for i in $(seq 1 "$ALL_NODES"); do sudo iptables -t nat -A POSTROUTING -o lo -p udp --sport $((BASE_PORT + i)) -j SNAT --to-source "$(node_ip "$i")" \ @@ -833,6 +971,7 @@ stop_all() { echo "net fault: until the end ($(net_fault_stats))" fi [ -z "$NET_FAULT" ] || net_fault_off + [ "$OFFICIAL_PARTITION_AT" -eq 0 ] || partition_off [ "$NODE_NETS" != 1 ] || node_nets_off [ -z "${GARBAGE_PID:-}" ] || kill "$GARBAGE_PID" 2>/dev/null || true [ ! -f "$WORK/resources-summary.txt" ] || cat "$WORK/resources-summary.txt" @@ -983,6 +1122,353 @@ backfill_node_started() { done } +# OFFICIAL_STOP_AT: the community and official nodes, the highest sequence of the community nodes +off_done=0 # 1 once the official nodes were stopped, 2 once they were started again +off_fail="" # what went against the expectation of COMMUNITY_BOOTSTRAP +off_restart_t=0 +off_restart_seq=0 +off_caught=0 +COMMUNITY_NODES="" +OFFICIAL_NODES="" +for i in $(seq 1 "$NODES"); do + if official_node "$i"; then OFFICIAL_NODES+="$i "; else COMMUNITY_NODES+="$i "; fi +done +community_max() { + local m=0 s i + for i in $COMMUNITY_NODES; do s=$(seq_of "$i"); s=${s:-0}; [ "$s" -le "$m" ] || m=$s; done + echo "$m" +} +community_min() { + local m=-1 s i + for i in $COMMUNITY_NODES; do s=$(seq_of "$i"); s=${s:-0}; { [ "$m" -ge 0 ] && [ "$s" -ge "$m" ]; } || m=$s; done + echo "$m" +} +# the header of node i's latest round, "R-1'234.0 TRUSTED" or "R-1'234.0 NORMAL" +round_header() { + tail -n 3000 "$WORK/n$1/node.log" 2>/dev/null | grep -a ' R-[0-9'"'"']*\.[0-9]* \(TRUSTED\|NORMAL\)' | tail -1 \ + | sed -E 's/.* (R-[0-9'"'"']+\.[0-9]+) (TRUSTED|NORMAL).*/\1 \2/' || true +} +bootstrap_lines() { + local n=0 i + for i in $COMMUNITY_NODES; do n=$((n + $(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true))); done + echo "$n" +} +official_stop() { + local t0 round official community h r i hdr seen + # wait for a round whose table the official nodes hold the majority of (seen by every node): without + # them that round cannot be decided (TrustedStage3State::take_urgent_decision) + t0=$(date +%s) + seen="" + while [ $(( $(date +%s) - t0 )) -lt 300 ]; do + round=""; official=0; community=0 + for i in $(seq 1 "$NODES"); do + hdr=$(round_header "$i") + r=${hdr%% *} + [ -n "$r" ] || { round=""; break; } + if [ -z "$round" ]; then round=$r; elif [ "$r" != "$round" ]; then round=""; break; fi + if [ "${hdr##* }" = TRUSTED ]; then + if official_node "$i"; then official=$((official + 1)); else community=$((community + 1)); fi + fi + done + if [ -n "$round" ] && [ $((official + community)) -gt 0 ] && [ $((community * 2)) -lt $((official + community + 1)) ]; then + seen="$round: $official official and $community community confidants" + break + fi + sleep 0.2 + done + [ -n "$seen" ] || seen="no round with an official majority seen in 300 s, stopped anyway" + for i in $OFFICIAL_NODES; do kill -KILL "${PIDS[$i]}" 2>/dev/null || true; done + for i in $OFFICIAL_NODES; do wait "${PIDS[$i]}" 2>/dev/null || true; REPORTED_DOWN[$i]=1; done + local t_kill s0 b0 first_t=0 first_seq=0 c end + t_kill=$(date +%s) + # the fallback must not have acted while the official nodes ran + for i in $COMMUNITY_NODES; do + if grep -aq 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log"; then + off_fail="${off_fail:+$off_fail; }n$i used the community bootstrap while the official nodes ran" + grep -a 'COMMUNITY BOOTSTRAP' "$WORK/n$i/node.log" | head -3 + fi + done + s0=$(community_max) + b0=$(bootstrap_lines) + echo "official stop: killed $(printf 'n%s ' $OFFICIAL_NODES)at once (round $seen); community head $s0, COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" + end=$(( t_kill + OFFICIAL_STOP_MINUTES * 60 )) + while [ "$(date +%s)" -lt "$end" ]; do + sleep 2 + c=$(community_max) + if [ "$first_t" -eq 0 ] && [ "$c" -gt "$s0" ]; then + first_t=$(date +%s); first_seq=$c + echo "official stop: community block $c $((first_t - t_kill)) s after the stop" + fi + done + c=$(community_max) + echo "official stop: after $OFFICIAL_STOP_MINUTES min the community head is $c ($((c - s0)) blocks since the stop), $(( $(bootstrap_lines) - b0 )) 'Bootstrap on' lines on the community nodes" + for i in $COMMUNITY_NODES; do + { grep -a 'NODE> Bootstrap on\|Bootstrap available nodes' "$WORK/n$i/node.log" || true; } | tail -2 | sed "s/^/official stop: n$i: /" + done + if [ "$COMMUNITY_BOOTSTRAP" = none ]; then + [ "$first_t" -eq 0 ] || off_fail="the community nodes wrote blocks without the official nodes and without the extra keys" + else + if [ "$first_t" -eq 0 ]; then + off_fail="the community nodes did not restart the network in $OFFICIAL_STOP_MINUTES min" + else + echo "OFFICIAL STOP RESULT: restart after $((first_t - t_kill)) s (first block $first_seq), $((c - s0)) blocks in $OFFICIAL_STOP_MINUTES min" + fi + fi + # COMMUNITY_STALL_STARTER: the others dropped the starter's first tables (fallback still closed) + # and took a repeat: a repeat in the same round is a new message, not a p2p duplicate + if [ "$COMMUNITY_STARTER" -gt 0 ] && [ "$first_t" -gt 0 ]; then + local dropped taken + for i in $COMMUNITY_NODES; do + [ "$i" -ne "$COMMUNITY_STARTER" ] || continue + dropped=$(grep -ac 'community bootstrap table .* has not stalled long enough' "$WORK/n$i/node.log" || true) + taken=$(grep -ac 'COMMUNITY BOOTSTRAP: taking' "$WORK/n$i/node.log" || true) + echo "OFFICIAL STOP RESULT: n$i dropped $dropped community tables while its fallback was closed, then took $taken" + if [ "$dropped" -eq 0 ] || [ "$taken" -eq 0 ]; then + off_fail="${off_fail:+$off_fail; }n$i: $dropped tables dropped and $taken taken, a repeat taken after a drop expected" + fi + done + grep -a 'COMMUNITY BOOTSTRAP: starting' "$WORK/n$COMMUNITY_STARTER/node.log" | head -3 | sed "s/^/official stop: n$COMMUNITY_STARTER (starter): /" + fi + # the community nodes agree on every block they hold (all of them, compared over the API) + local ports="" head + head=$(community_min) + for i in $COMMUNITY_NODES; do ports="$ports $i:$(api_port "$i")"; done + if (cd "$HERE" && python3 blockhashes.py --head "$head" --step 1 $ports); then + echo "OFFICIAL STOP RESULT: the community nodes hold the same blocks up to $head" + else + off_fail="${off_fail:+$off_fail; }the community nodes hold different blocks while the official nodes were down" + fi + off_restart_seq=$c + for i in $OFFICIAL_NODES; do start_node "$i"; REPORTED_DOWN[$i]=""; done + off_restart_t=$(date +%s) + off_done=2 + echo "official stop: $(printf 'n%s ' $OFFICIAL_NODES)started again, community head $c" +} + +# chain reorganisations (see "REORG DEPTHS" near the end); a partition scenario reports them before its own verdict +reorg_reported=0 +reorg_report() { + reorg_reported=1 + reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) + echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" + # one line per refusal, or (since the lines are throttled) one a minute with the count since the + # previous one + refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } \ + | sed -E 's/.*\(([0-9]+) refused since.*/\1/; t; s/.*/1/' | awk '{ n += $1 } END { print n + 0 }') + refused_lines=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) + echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants ($refused_lines log lines)" + held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) + [ -z "$held" ] || echo "FORK HELD on: $held" +} + +# OFFICIAL_PARTITION_AT: the community nodes cut off from the running official nodes +part_done=0 # 1 while partitioned, 2 once reconnected +part_fail="" +part_from=0 # the community head at the cut: the chains are dumped from a little below it +official_max() { + local m=0 s i + for i in $OFFICIAL_NODES; do s=$(seq_of "$i"); s=${s:-0}; [ "$s" -le "$m" ] || m=$s; done + echo "$m" +} +official_partition() { + local t_cut s0 o0 first_t=0 c o end i + for i in $COMMUNITY_NODES; do + if grep -aq 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log"; then + part_fail="${part_fail:+$part_fail; }n$i used the community bootstrap before the partition" + fi + done + s0=$(community_max); o0=$(official_max) + part_from=$(( s0 > 5 ? s0 - 5 : 0 )) + partition_on + t_cut=$(date +%s) + echo "partition: $(printf 'n%s ' $COMMUNITY_NODES)cut off from $(printf 'n%s ' $OFFICIAL_NODES)for $OFFICIAL_PARTITION_MINUTES min (community head $s0, official head $o0), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" + end=$(( t_cut + OFFICIAL_PARTITION_MINUTES * 60 )) + while [ "$(date +%s)" -lt "$end" ]; do + sleep 5 + c=$(community_max) + if [ "$first_t" -eq 0 ] && [ "$c" -gt $((s0 + 1)) ]; then + first_t=$(date +%s) + echo "partition: community block $c $((first_t - t_cut)) s after the cut" + fi + done + c=$(community_max); o=$(official_max) + echo "PARTITION RESULT: after $OFFICIAL_PARTITION_MINUTES min the community head is $c ($((c - s0)) blocks, restart after" \ + "$([ "$first_t" -eq 0 ] && echo never || echo "$((first_t - t_cut)) s")), the official head $o ($((o - o0)) blocks); $(partition_stats) packets dropped" + for i in $COMMUNITY_NODES; do + { grep -a 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log" || true; } | tail -1 | sed "s/^/partition: n$i: /" + done + # the two chains just before the reconnection: a community and an official node + local cn on + cn=$(echo $COMMUNITY_NODES | cut -d' ' -f1); on=$(echo $OFFICIAL_NODES | cut -d' ' -f1) + (cd "$HERE" && python3 partition.py dump --from "$part_from" --port "$(api_port "$cn")") > "$WORK/partition-community.txt" \ + || part_fail="${part_fail:+$part_fail; }no chain dump of n$cn" + (cd "$HERE" && python3 partition.py dump --from "$part_from" --port "$(api_port "$on")") > "$WORK/partition-official.txt" \ + || part_fail="${part_fail:+$part_fail; }no chain dump of n$on" + partition_off + part_done=2 + echo "partition: reconnected (community head $c, official head $o)" + # have the two sides found each other again? (each node logs its neighbour count with its blocks) + sleep 120 + for i in $COMMUNITY_NODES $OFFICIAL_NODES; do + echo "partition: 120 s after the reconnection n$i has $(grep -a 'neighbour nodes = ' "$WORK/n$i/node.log" | tail -1 | sed -E 's/.*neighbour nodes = ([0-9]+).*/\1/') neighbours, head $(seq_of "$i")" + done + if [ "$OFFICIAL_PARTITION_RESTART" = 1 ]; then + echo "partition: restarting $(printf 'n%s ' $COMMUNITY_NODES)(community head $(community_max), official head $(official_max))" + partition_stop_nodes $COMMUNITY_NODES + for i in $COMMUNITY_NODES; do start_node "$i"; REPORTED_DOWN[$i]=""; done + echo "partition: community nodes started again" + fi + [ "$RESOLVE_FORK_AFTER" != 1 ] || partition_resolve_fork +} + +# stop nodes as an operator would: SIGTERM, then SIGKILL after 60 s (a trusted node waits for its +# round before it exits, and with most of the round's confidants stopping that round never ends) +partition_stop_nodes() { + local i t_stop + for i in "$@"; do kill -TERM "${PIDS[$i]}" 2>/dev/null || true; REPORTED_DOWN[$i]=1; done + t_stop=$(date +%s) + for i in "$@"; do + while kill -0 "${PIDS[$i]}" 2>/dev/null && [ $(( $(date +%s) - t_stop )) -lt 60 ]; do sleep 1; done + if kill -0 "${PIDS[$i]}" 2>/dev/null; then + echo "partition: n$i still running 60 s after SIGTERM, SIGKILL" + kill -KILL "${PIDS[$i]}" 2>/dev/null || true + fi + wait "${PIDS[$i]}" 2>/dev/null || true + pkill -f "cs.testnet.node=n$i " 2>/dev/null || true + done +} + +# RESOLVE_FORK_AFTER: the operator of each community node holding a fork follows the official chain, +# as docs/quick-start.md ("If your node reports FORK HELD") says: --resolve-fork; if the node knows +# only an upper bound of the fork point and reports FORK HELD again lower, --set-bc-top at the last +# common block (here from the chains dumped before the reconnection, as an operator would take it +# from the monitor) +part_resolved="" # community nodes cut back to the official chain +part_restart_fail=0 # starts right after a cut that stopped with an error (started again) + +# nodes holding a fork: fork-held.json and a FORK HELD line after line $2 of node.log +partition_held() { + local i from=$1 held="" + for i in $COMMUNITY_NODES; do + [ -f "$WORK/n$i/fork-held.json" ] && tail -n +"$((from + 1))" "$WORK/n$i/node.log" | grep -aq 'FORK HELD since' && held="$held $i" + done + echo $held +} + +# start a node after its chain was cut; if it stops at once (the first start after a cut can fail +# on a quick-start checkpoint above the new head), report it and start it once more +partition_start_after_cut() { + local i t0 status + for i in "$@"; do start_node "$i"; REPORTED_DOWN[$i]=""; done + t0=$(date +%s) + while [ $(( $(date +%s) - t0 )) -lt 60 ]; do sleep 2; done + for i in "$@"; do + kill -0 "${PIDS[$i]}" 2>/dev/null && continue + status=0; wait "${PIDS[$i]}" 2>/dev/null || status=$? + part_restart_fail=$((part_restart_fail + 1)) + echo "RESOLVE FORK: n$i stopped at its first start after the cut (exit $status):" + { grep -a ' error ' "$WORK/n$i/node.log" || true; } | tail -4 | sed "s/^/partition: n$i: /" | cut -c1-260 + start_node "$i"; REPORTED_DOWN[$i]="" + echo "RESOLVE FORK: n$i started again" + done +} + +partition_resolve_fork() { + local t0 held i n_comm rc fork lines upper="" again + local -a lines_i=() + n_comm=$(echo $COMMUNITY_NODES | wc -w) + t0=$(date +%s) + while :; do + held=$(partition_held 0) + [ "$(echo $held | wc -w)" -lt "$n_comm" ] || break + [ $(( $(date +%s) - t0 )) -lt $(( RESOLVE_FORK_WAIT * 60 )) ] || break + sleep 10 + done + echo "RESOLVE FORK: FORK HELD on $(echo $held | wc -w) of $n_comm community nodes${held:+$(printf ' n%s' $held)} after $(( $(date +%s) - t0 )) s (community head $(community_max), official head $(official_max))" + for i in $COMMUNITY_NODES; do + { grep -a 'FORK HELD since' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E "s/^.*(FORK HELD since)/partition: n$i: \\1/; s/ What to do:.*//" + done + if [ -z "$held" ]; then + part_fail="${part_fail:+$part_fail; }no community node reported FORK HELD within $RESOLVE_FORK_WAIT min, nothing to resolve" + return 0 + fi + # 0. while FORK HELD the node looks for the last common block itself: wait until every node holding + # the fork knows it ("fork_point_status": "exact" in fork-held.json; "deeper" is an answer too) + local pending status to_located="" extra + t0=$(date +%s) + while :; do + pending="" + for i in $held; do + grep -aqE '"fork_point_status": "(exact|deeper|located \(unverified\))"' "$WORK/n$i/fork-held.json" 2>/dev/null || pending="$pending $i" + done + [ -n "$pending" ] || break + [ $(( $(date +%s) - t0 )) -lt $(( RESOLVE_LOCATE_WAIT * 60 )) ] || break + sleep 10 + done + echo "RESOLVE FORK: fork point located on$(printf ' n%s' $held) but$([ -n "$pending" ] && printf ' n%s' $pending || echo ' none') after $(( $(date +%s) - t0 )) s" + for i in $held; do + status=$(sed -nE 's/.*"fork_point_status": "([a-z ()]+)".*/\1/p' "$WORK/n$i/fork-held.json" 2>/dev/null) + echo "RESOLVE FORK: n$i fork point #$(sed -nE 's/.*"fork_point": "([0-9]+)".*/\1/p' "$WORK/n$i/fork-held.json" 2>/dev/null) ($status), upper bound #$(sed -nE 's/.*"fork_point_upper_bound": "([0-9]+)".*/\1/p' "$WORK/n$i/fork-held.json" 2>/dev/null)" + { grep -aE 'fork point search|last common block' "$WORK/n$i/node.log" || true; } | tail -3 | sed "s/^/partition: n$i: /" | cut -c1-260 + case "$status" in + exact) ;; + "located (unverified)") to_located="$to_located $i"; echo "RESOLVE FORK: n$i: located but not verified: the operator confirms it with --to-located" ;; + *) part_fail="${part_fail:+$part_fail; }n$i: the fork point was not located ($status) before --resolve-fork" ;; + esac + done + # 1. --resolve-fork on every node holding the fork + partition_stop_nodes $held + for i in $held; do + lines=$(wc -l < "$WORK/n$i/node.log") + rc=0 + extra=""; case " $to_located " in *" $i "*) extra=--to-located ;; esac + (cd "$WORK/n$i" && exec timeout 600 "$(bin_of "$i")" --db-path db --config-file config.ini --resolve-fork $extra > resolve-fork.log 2>&1) || rc=$? + echo "RESOLVE FORK: n$i --resolve-fork $extra exit $rc" + tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | { grep -a -- '--resolve-fork' || true; } | sed -E "s/^.*(--resolve-fork)/partition: n$i: \\1/" | cut -c1-260 + if [ "$rc" -ne 0 ]; then part_fail="${part_fail:+$part_fail; }--resolve-fork failed on n$i (exit $rc)"; continue; fi + tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | grep -aq 'the fork point is an upper bound' && upper="$upper $i" + done + for i in $held; do lines_i[$i]=$(wc -l < "$WORK/n$i/node.log"); done + partition_start_after_cut $held + echo "RESOLVE FORK: $(printf 'n%s ' $held)started again after --resolve-fork" + # one --resolve-fork, at the exact fork point, and no start-up failure after the cut + [ -z "$upper" ] || part_fail="${part_fail:+$part_fail; }--resolve-fork cut at an upper bound on$(printf ' n%s' $upper)" + [ "$part_restart_fail" -eq 0 ] || part_fail="${part_fail:+$part_fail; }$part_restart_fail start(s) right after the cut stopped with an error" + # 2. the fork point was an upper bound: the node holds the fork again, lower; cut at the last + # common block by hand + if [ -n "$upper" ]; then + fork=$(cd "$HERE" && python3 partition.py fork --community "$WORK/partition-community.txt" --official "$WORK/partition-official.txt") + t0=$(date +%s) + while :; do + again="" + for i in $upper; do + tail -n +"$((${lines_i[$i]} + 1))" "$WORK/n$i/node.log" | grep -aq 'FORK HELD since' && again="$again $i" + done + [ "$(echo $again | wc -w)" -lt "$(echo $upper | wc -w)" ] || break + [ $(( $(date +%s) - t0 )) -lt 180 ] || break + sleep 10 + done + echo "RESOLVE FORK: the fork point was an upper bound on$(printf ' n%s' $upper); FORK HELD again on$([ -n "$again" ] && printf ' n%s' $again || echo ' none') after $(( $(date +%s) - t0 )) s; the chains differ from #$fork: --set-bc-top $((fork - 1))" + for i in $again; do + tail -n +"$((${lines_i[$i]} + 1))" "$WORK/n$i/node.log" | { grep -a 'FORK HELD since' || true; } | tail -1 | sed -E "s/^.*(FORK HELD since)/partition: n$i: \\1/; s/ What to do:.*//" + done + if [ -n "$again" ]; then + partition_stop_nodes $again + for i in $again; do + lines=$(wc -l < "$WORK/n$i/node.log") + rc=0 + (cd "$WORK/n$i" && exec timeout 600 "$(bin_of "$i")" --db-path db --config-file config.ini --set-bc-top "$((fork - 1))" > resolve-fork.log 2>&1) || rc=$? + echo "RESOLVE FORK: n$i --set-bc-top $((fork - 1)) exit $rc" + tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | { grep -aE 'start remove|set-bc-top|error' || true; } | tail -3 | sed "s/^/partition: n$i: /" | cut -c1-260 + [ "$rc" -eq 0 ] || part_fail="${part_fail:+$part_fail; }--set-bc-top failed on n$i (exit $rc)" + done + partition_start_after_cut $again + echo "RESOLVE FORK: $(printf 'n%s ' $again)started again after --set-bc-top" + fi + fi + part_resolved=$held +} + while :; do sleep 20 line="t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) ))s" @@ -1137,6 +1623,24 @@ while :; do echo "crash test: network restarted" fi + if [ "$OFFICIAL_PARTITION_AT" -gt 0 ] && [ "$part_done" -eq 0 ] && [ "$min" -ge "$OFFICIAL_PARTITION_AT" ]; then + part_done=1 + official_partition + fi + if [ "$OFFICIAL_STOP_AT" -gt 0 ] && [ "$off_done" -eq 0 ] && [ "$min" -ge "$OFFICIAL_STOP_AT" ]; then + off_done=1 + official_stop + fi + if [ "$off_done" -eq 2 ] && [ "$off_caught" -eq 0 ]; then + lowest=-1 + for i in $OFFICIAL_NODES; do s=$(seq_of "$i"); s=${s:-0}; { [ "$lowest" -ge 0 ] && [ "$s" -ge "$lowest" ]; } || lowest=$s; done + if [ "$lowest" -ge "$off_restart_seq" ]; then + off_caught=$(( $(date +%s) - off_restart_t )) + [ "$off_caught" -gt 0 ] || off_caught=1 + echo "OFFICIAL STOP RESULT: the official nodes reached the community head of their restart ($off_restart_seq) in about $off_caught s" + fi + fi + if [ "$RESTART_AT" -gt 0 ] && [ "$restarted" -eq 0 ] && [ "$min" -ge "$RESTART_AT" ] \ && { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/contracts-ready" ]; }; then echo "restart test: stopping $(printf 'n%s ' $RESTART_NODES)at sequence $min" @@ -1598,8 +2102,8 @@ PY echo "funding nodes from the genesis key, then $LOAD_SECONDS s of transfers" python3 "$HERE/fund.py" "$WORK" "$NODES" "$MASTER_SEED" --load-seconds "$LOAD_SECONDS" \ --dpos-at "$DPOS_AT" --reward-round "$REWARD_ROUND" --delegation-seconds "$DELEGATION_SECONDS" --min-stake "$MIN_STAKE" --syncro-round "$SYNCRO_ROUND" \ - --low-stake-nodes "$LOW_STAKE_NODES" \ - $([ "$DELEGATE" = "1" ] && echo --delegations) > "$WORK/fund.log" 2>&1 & + --low-stake-nodes "$LOW_STAKE_NODES" --even-amount "$OFFICIAL_FUND" \ + $([ "$DELEGATE" = "1" ] && echo --delegations) $([ "$RESOLVE_FORK_AFTER" = 1 ] && echo --chain-cut) > "$WORK/fund.log" 2>&1 & FUND_PID=$! fi @@ -1649,6 +2153,7 @@ PY && { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/restart-done" ]; } \ && { [ "$SAMPLE_CONTRACTS" != "1" ] || [ -n "$SAMPLES_PID" ]; }; then break; fi if [ "$(date +%s)" -ge "$deadline" ]; then + [ "$OFFICIAL_PARTITION_AT" -eq 0 ] || reorg_report echo "FAIL: not every node reached sequence $TARGET_SEQ within $TIMEOUT_MIN min" exit 1 fi @@ -2205,9 +2710,10 @@ done # the consensus while the snapshots are served: n1's block rate and round times in the same number of # seconds before the joins and after they started if [ "$STRESS_WINDOW" -gt 0 ] && [ -n "${join_started:-}" ]; then - python3 - "$WORK/n1/node.log" "$join_started" "$STRESS_WINDOW" <<'PY' || { echo "FAIL: the consensus slowed down while snapshots were served"; exit 1; } -import re, sys, datetime + python3 - "$WORK/n1/node.log" "$join_started" "$STRESS_WINDOW" "$STRESS_RATE_DROP" "$STRESS_ROUND_RISE" <<'PY' || { echo "FAIL: the consensus slowed down while snapshots were served"; exit 1; } +import re, sys, datetime, statistics log, start, window = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) +max_drop, max_rise = float(sys.argv[4]), float(sys.argv[5]) stamp = re.compile(r"^\[(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d+)\]") def t(line): m = stamp.match(line) @@ -2223,30 +2729,68 @@ for line in open(log, errors="replace"): m = re.search(r"Last written sequence = ([0-9']+)", line) if x is not None and m: seqs.append((x, int(m.group(1).replace("'", "")))) +def rate_in(a, b): + s = [q for x, q in seqs if a <= x < b] + return (s[-1] - s[0]) / (b - a) if len(s) > 1 else 0.0 def window_stats(a, b): r = [y - x for x, y in zip(rounds, rounds[1:]) if a <= x < b] - s = [q for x, q in seqs if a <= x < b] - rate = (s[-1] - s[0]) / (b - a) if len(s) > 1 else 0.0 + # the median of the one-minute rates (at least 3 parts): a single slow minute, e.g. a burst of + # lost packets, moves the median far less than the rate over the whole window + parts = max(3, (b - a) // 60) + step = (b - a) / parts + rate = statistics.median(rate_in(a + k * step, a + (k + 1) * step) for k in range(parts)) r.sort() p95 = r[int(0.95 * (len(r) - 1))] if r else 0.0 return rate, p95, len(r) base = window_stats(start - window, start) load = window_stats(start, start + window) -print(f"consensus: before the joins {base[0]:.2f} blocks/s, round p95 {base[1]:.2f} s ({base[2]} rounds); " +print(f"consensus: before the joins {base[0]:.2f} blocks/s (median minute), round p95 {base[1]:.2f} s ({base[2]} rounds); " f"while serving {load[0]:.2f} blocks/s, round p95 {load[1]:.2f} s ({load[2]} rounds)") -ok = base[2] > 10 and load[2] > 10 +ok = base[2] > 10 and load[2] > 10 and base[0] > 0 and base[1] > 0 if not ok: print("FAIL: too few rounds in a window to compare") -elif load[0] < 0.9 * base[0]: - print(f"FAIL: block rate fell by {100 * (1 - load[0] / base[0]):.0f} % (10 % allowed)") + sys.exit(1) +drop, rise = 100 * (1 - load[0] / base[0]), 100 * (load[1] / base[1] - 1) +print(f"consensus: block rate {-drop:+.1f} % ({max_drop:g} % fall allowed), round p95 {rise:+.1f} % ({max_rise:g} % rise allowed)") +if drop > max_drop: + print(f"FAIL: block rate fell by {drop:.0f} % ({max_drop:g} % allowed)") ok = False -elif load[1] > 1.2 * base[1]: - print(f"FAIL: round time p95 rose by {100 * (load[1] / base[1] - 1):.0f} % (20 % allowed)") +if rise > max_rise: + print(f"FAIL: round time p95 rose by {rise:.0f} % ({max_rise:g} % allowed)") ok = False sys.exit(0 if ok else 1) PY fi +if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then + [ "$off_done" -eq 2 ] || off_fail="${off_fail:+$off_fail; }the official nodes were never stopped" + [ "$off_caught" -gt 0 ] || off_fail="${off_fail:+$off_fail; }the official nodes never reached the community head of their restart" + # what the official nodes made of the blocks written without them + for i in $OFFICIAL_NODES; do + echo "official stop: n$i logged $(grep -ac "Pool Signatures aren't valid" "$WORK/n$i/node.log" || true) invalid pool signatures," \ + "$(grep -ac 'confirmations are not OK' "$WORK/n$i/node.log" || true) refused confirmations," \ + "$(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true) bootstraps" + done + if [ -n "$off_fail" ]; then echo "FAIL: official stop: $off_fail"; exit 1; fi +fi + +if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then + reorg_report + [ "$part_done" -eq 2 ] || part_fail="${part_fail:+$part_fail; }the partition never ended" + if [ "$part_done" -eq 2 ]; then + ports="" + for i in $COMMUNITY_NODES $OFFICIAL_NODES; do ports="$ports --port $(api_port "$i")"; done + (cd "$HERE" && python3 partition.py report --community "$WORK/partition-community.txt" --official "$WORK/partition-official.txt" $ports) \ + || part_fail="${part_fail:+$part_fail; }the nodes did not all keep the blocks of one side" + fi + for i in $COMMUNITY_NODES $OFFICIAL_NODES; do + echo "partition: n$i logged $(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true) bootstraps," \ + "$(grep -ac 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log" || true) community tables started or taken" + done + [ "$RESOLVE_FORK_AFTER" != 1 ] || echo "RESOLVE FORK: cut back to the official chain:$([ -n "$part_resolved" ] && printf ' n%s' $part_resolved || echo ' none'); $part_restart_fail start(s) right after a cut stopped with an error and were repeated" + if [ -n "$part_fail" ]; then echo "FAIL: partition: $part_fail"; exit 1; fi +fi + # official nodes: the same block hashes as n1 (every 100th sequence up to the lowest head) if [ "$BIN_ALT_OFFICIAL" = 1 ]; then ports="" @@ -2258,7 +2802,8 @@ if [ "$BIN_ALT_OFFICIAL" = 1 ]; then done # pruned nodes, and a node that joined from a snapshot, lack the oldest blocks partial=$(for i in $PRUNED_NODES $({ [ "$JOIN_FETCH" = p2p ] || [ "$SNAPSHOT_AT" -gt 0 ]; } && echo "$JOIN_NODE" $EXTRA_JOINERS); do echo "$i"; done | sort -u | paste -sd, -) - (cd "$HERE" && python3 blockhashes.py --head "$head" --partial "$partial" $ports) || { echo "FAIL: official and community nodes hold different blocks"; exit 1; } + (cd "$HERE" && python3 blockhashes.py --head "$head" --partial "$partial" --step "$([ "$OFFICIAL_STOP_AT" -gt 0 ] || [ "$OFFICIAL_PARTITION_AT" -gt 0 ] && echo 1 || echo 100)" $ports) \ + || { echo "FAIL: official and community nodes hold different blocks"; exit 1; } fi # two blocks of one sequence: a node kept its own block while the round went on on the writer's @@ -2274,12 +2819,7 @@ fi # finished rollback episode). In a normal run the guard must never stop a reorg and no competing block # may be refused (a refusal leaves a lagging node waiting for the operator, FORK HELD). # DEEP_ROLLBACK_EXPECTED=1 is for a scenario that provokes either on purpose. -reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) -echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" -refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) -echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" -held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) -[ -z "$held" ] || echo "FORK HELD on: $held" +[ "$reorg_reported" = 1 ] || reorg_report guarded=$(grep -al 'LONGEST-CHAIN GUARD: a competing chain' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) if [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then if [ -n "$guarded" ]; then @@ -2294,6 +2834,15 @@ if [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then fi fi +# the community bootstrap fallback in a network that never lost its official nodes: never used +if [ "$COMMUNITY_BOOTSTRAP" = config ] && [ "$OFFICIAL_STOP_AT" -eq 0 ] && [ "$OFFICIAL_PARTITION_AT" -eq 0 ]; then + used=$(grep -alE 'COMMUNITY BOOTSTRAP: (starting|taking)' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) + [ -z "$used" ] || { grep -ah 'COMMUNITY BOOTSTRAP' "$WORK"/n*/node.log | head -5; echo "FAIL: the community bootstrap acted on $used"; exit 1; } + echo "community bootstrap: never used ($(grep -ah 'NODE> Bootstrap on' "$WORK"/n*/node.log | wc -l) regular bootstrap lines on all nodes)" +fi +[ "$OFFICIAL_PARTITION_AT" -eq 0 ] || echo "PARTITION RESULT: one chain: every node holds the same blocks (all compared), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" +[ "$OFFICIAL_STOP_AT" -eq 0 ] || echo "OFFICIAL STOP RESULT: one chain: every node holds the same blocks (all compared), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" + # a network still restarting at the end is stuck, even if it reached the target before (a bootstrap # round, R-x.1 and up, that never gives way to R-(x+1).0): the last three round headers of n1 stuck=$(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -3 | grep -vc '\.0 ' || true) diff --git a/client/config/config.cpp b/client/config/config.cpp index 78a804e5a..b5e96f7af 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -47,6 +47,7 @@ const std::string BLOCK_NAME_HOST_ADDRESS = "host_address"; const std::string BLOCK_NAME_POOL_SYNC = "pool_sync"; const std::string BLOCK_NAME_STORAGE = "storage"; const std::string BLOCK_NAME_SNAPSHOT = "snapshot"; +const std::string BLOCK_NAME_COMMUNITY_BOOTSTRAP = "community_bootstrap"; const std::string BLOCK_NAME_API = "api"; const std::string BLOCK_NAME_CONVEYER = "conveyer"; const std::string BLOCK_NAME_EVENT_REPORTER = "event_report"; @@ -1089,6 +1090,7 @@ Config Config::readFromFile(const std::string& fileName) { result.readPoolSynchronizerData(config); result.readStorageData(config); result.snapshotData_ = parseSnapshotData(config); + result.communityBootstrapData_ = parseCommunityBootstrapData(config); result.readApiData(config); result.readConveyerData(config); result.readEventsReportData(config); @@ -1365,6 +1367,54 @@ SnapshotData Config::parseSnapshotData(const boost::property_tree::ptree& config return result; } +CommunityBootstrapData Config::parseCommunityBootstrapData(const boost::property_tree::ptree& config) { + CommunityBootstrapData result; + if (!config.count(BLOCK_NAME_COMMUNITY_BOOTSTRAP)) { + return result; + } + const boost::property_tree::ptree& data = config.get_child(BLOCK_NAME_COMMUNITY_BOOTSTRAP); + static const std::set known = {"confidants", "stall_minutes"}; + for (const auto& [key, value] : data) { + if (!known.count(key)) { + cswarning() << "config: [community_bootstrap] " << key << " is not a setting of this version, ignored"; + } + } + if (data.count("confidants")) { + std::vector keys; + boost::split(keys, data.get("confidants"), boost::is_any_of(", \t"), boost::token_compress_on); + for (const auto& key : keys) { + if (key.empty()) { + continue; + } + std::vector bytes; + if (!DecodeBase58(key, bytes) || bytes.size() != cscrypto::kPublicKeySize) { + cswarning() << "config: [community_bootstrap] confidants: " << key << " is not a node key, ignored"; + continue; + } + cs::PublicKey publicKey; + std::copy(bytes.begin(), bytes.end(), publicKey.begin()); + if (std::find(result.confidants.begin(), result.confidants.end(), publicKey) == result.confidants.end()) { + result.confidants.push_back(publicKey); + } + } + } + if (data.count("stall_minutes")) { + const auto text = data.get("stall_minutes"); + try { + size_t pos = 0; + const unsigned long long value = std::stoull(text, &pos); + if (pos != text.size() || value < 1 || value > 10'080) { + throw std::out_of_range(text); + } + result.stallMinutes = static_cast(value); + } + catch (const std::exception&) { + cswarning() << "config: [community_bootstrap] stall_minutes=" << text << " is not 1..10080, using " << result.stallMinutes; + } + } + return result; +} + void Config::readApiData(const boost::property_tree::ptree& config) { if (!config.count(BLOCK_NAME_API)) { return; @@ -1544,6 +1594,14 @@ bool operator!=(const SnapshotData& lhs, const SnapshotData& rhs) { return !(lhs == rhs); } +bool operator==(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs) { + return lhs.confidants == rhs.confidants && lhs.stallMinutes == rhs.stallMinutes; +} + +bool operator!=(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs) { + return !(lhs == rhs); +} + bool operator==(const ConveyerData& lhs, const ConveyerData& rhs) { return lhs.maxPacketLifeTime == rhs.maxPacketLifeTime; } @@ -1593,6 +1651,7 @@ bool operator==(const Config& lhs, const Config& rhs) { lhs.poolSyncData_ == rhs.poolSyncData_ && lhs.apiData_ == rhs.apiData_ && lhs.snapshotData_ == rhs.snapshotData_ && + lhs.communityBootstrapData_ == rhs.communityBootstrapData_ && lhs.alwaysExecuteContracts_ == rhs.alwaysExecuteContracts_ && lhs.syncContractExecution_ == rhs.syncContractExecution_ && lhs.compatibleVersion_ == rhs.compatibleVersion_ && diff --git a/client/config/config.hpp b/client/config/config.hpp index ed7a69afd..689c60c71 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -100,6 +100,15 @@ struct SnapshotData { size_t fetchRateKbps = 0; // 0: unlimited }; +// [community_bootstrap]: a liveness fallback, off unless confidants lists keys +// (csnode/include/csnode/communitybootstrap.hpp). After stall_minutes without a block, a round or a +// regular bootstrap, the node may start and take a bootstrap table of initial confidants and these +// keys, signed by its starter instead of the project. Read again by the config observer. +struct CommunityBootstrapData { + std::vector confidants; // community keys; empty: the fallback is off + size_t stallMinutes = 30; // minutes without progress before it may act, 1..10080 +}; + struct ApiData { // on by default: uint16_t port = 9090; @@ -254,6 +263,10 @@ class Config { return storageData_; } + const CommunityBootstrapData& getCommunityBootstrapSettings() const { + return communityBootstrapData_; + } + const ApiData& getApiSettings() const { return apiData_; } @@ -412,6 +425,8 @@ class Config { public: // the [snapshot] section, defaults for missing keys, a warning and the default for a bad value static SnapshotData parseSnapshotData(const boost::property_tree::ptree& config); + // the [community_bootstrap] section, off when it is missing or lists no valid key + static CommunityBootstrapData parseCommunityBootstrapData(const boost::property_tree::ptree& config); // the backfill keys of a [storage] section (backfill, backfill_batch, backfill_rate) static void readBackfillSettings(const boost::property_tree::ptree& storage, StorageData& data); // the [params] keys of the bound on automatic chain reorganisations: max_auto_rollback (default @@ -473,6 +488,7 @@ class Config { PoolSyncData poolSyncData_; StorageData storageData_; SnapshotData snapshotData_; + CommunityBootstrapData communityBootstrapData_; ApiData apiData_; bool showBalanceChange_ = false; @@ -523,6 +539,9 @@ bool operator!=(const ApiData& lhs, const ApiData& rhs); bool operator==(const SnapshotData& lhs, const SnapshotData& rhs); bool operator!=(const SnapshotData& lhs, const SnapshotData& rhs); +bool operator==(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs); +bool operator!=(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs); + bool operator==(const ConveyerData& lhs, const ConveyerData& rhs); bool operator!=(const ConveyerData& lhs, const ConveyerData& rhs); diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index f32da4a8a..5f6219433 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -351,6 +351,10 @@ class BlockChain { void test_setLastBlock(const csdb::Pool& block) { deferredBlock_ = block; lastSequence_ = block.sequence(); } // let the episode end as if a block above the reference had been recorded void test_onHeadAdvanced() { onHeadAdvanced(); checkApprovedBranch(); } + bool test_recordForkHeld(const char* reason) { return recordForkHeld(reason); } + uint64_t test_onRefusedBlock(std::chrono::steady_clock::time_point now) { + std::lock_guard l(guardMutex_); return refusedLine_.onEvent(now); + } void test_setSelectionUnfinished(const csdb::PoolHash& lastPrev) { selectionFinished_ = false; lastPrevHash_ = lastPrev; antiForkMode_ = true; emittingRequest_ = 1; } bool test_selectionFinished() const { return selectionFinished_; } bool test_antiForkMode() const { return antiForkMode_; } @@ -726,6 +730,8 @@ public slots: mutable std::mutex guardMutex_; std::optional forkRecord_; std::chrono::steady_clock::time_point lastDismissedLog_{}; // dismissed branch offered again (hourly) + // refused competing blocks: one line, then at most one a minute with the count since (guardMutex_) + cs::forkguard::RepeatedLine refusedLine_; // --resolve-fork's approved branch (exact fork point only), pending until the node is past it std::optional approvedBranch_; std::string resolveCompetingHash_; diff --git a/csnode/include/csnode/communitybootstrap.hpp b/csnode/include/csnode/communitybootstrap.hpp new file mode 100644 index 000000000..dc250636d --- /dev/null +++ b/csnode/include/csnode/communitybootstrap.hpp @@ -0,0 +1,278 @@ +#ifndef COMMUNITYBOOTSTRAP_HPP +#define COMMUNITYBOOTSTRAP_HPP + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace cs { + +// Community bootstrap: a liveness fallback, off by default ([community_bootstrap] in config.ini). +// +// Only the initial confidants (initialConfidants_) start a bootstrap, and a table with any other key +// needs the project's signature. If the initial confidants that are still running are too few, the +// network stays stalled whatever the other nodes do. The fallback lets a node that lists a set of +// community keys start and take, after a long stall only, a bootstrap table of keys from the initial +// confidants and that list, signed by its starter instead of the project. +// +// What keeps it out of normal operation: +// - the node itself has seen no block, no round package whose signatures it accepted and no regular +// bootstrap table for stall_minutes (a jump of its clocks counts as progress), and no neighbour +// reports a block two above its own, or one above for less than a further stall_minutes +// (isNeighbourAhead: this node would sync); +// - the table: 4 to 25 keys in ascending order, each an initial confidant or listed, at least one +// of them listed and not an initial confidant (else it is a regular table), more than half of the +// list in it (two parts of a split community cannot both restart), and signed by its first key, the +// starter; +// - the round window (#71) and the stall check (#70) of every unsigned table still apply; +// - a node takes one such table per half round_elapse_time; within that window only a heavier table +// whose starter key is not above the current one replaces it, and a regular table always replaces +// it (it goes through the unchanged path, whose weight it does not touch). A regular table, even +// a project-signed one, is progress: a community table may follow it only after a further +// stall_minutes without a block. +// Official nodes, and community nodes without the list, drop such a table: one of its keys is not an +// initial confidant and its signature is not the project's. + +constexpr size_t kCommunityMinTable = 4; // Consensus::MinTrustedNodes +constexpr size_t kCommunityMaxTable = 25; // Consensus::MaxTrustedNodes + +using CommunityWeight = uint64_t; + +// The bytes a project-signed table and a community table sign: the round, then the table as sent +// (the count and the keys). The round is in them, so a table of another round does not verify +inline cs::Bytes bootstrapSignedBytes(cs::RoundNumber round, const uint8_t* table, size_t tableLen) { + cs::Bytes buf; + buf.reserve(sizeof(cs::RoundNumber) + tableLen); + const auto* ptr = reinterpret_cast(&round); + buf.insert(buf.end(), ptr, ptr + sizeof(cs::RoundNumber)); + buf.insert(buf.end(), table, table + tableLen); + return buf; +} + +// A community table as its starter sends it: the count, the keys in ascending order, the starter's +// signature over bootstrapSignedBytes, and 4 random bytes as in a regular table. Without them a +// repeat in the same round would be the same message, and the p2p layer drops a broadcast whose +// bytes it has seen (Host::IsDuplicate): a node whose fallback opens after the first one would never +// get one. Receivers read the signature and ignore the bytes after it +inline cs::Bytes makeCommunityTable(cs::RoundNumber round, const std::set& keys, const cs::PrivateKey& starterKey, + uint32_t random) { + cs::Bytes bin; + cs::ODataStream out(bin); + out << uint8_t(keys.size()); + for (const auto& key : keys) { + out << key; + } + const cs::Bytes signedPart = bootstrapSignedBytes(round, bin.data(), bin.size()); + out << cscrypto::generateSignature(starterKey, signedPart.data(), signedPart.size()); + out << random; + return bin; +} + +// the starter's table is a std::set, sent in order: each key above the previous one +inline bool isAscending(const std::vector& table) { + for (size_t i = 1; i < table.size(); ++i) { + if (!(table[i - 1] < table[i])) { + return false; + } + } + return true; +} + +inline bool isCommunityTable(const std::set& initial, const std::set& community, + const std::vector& table) { + if (community.empty() || table.size() < kCommunityMinTable || table.size() > kCommunityMaxTable || !isAscending(table)) { + return false; + } + size_t listed = 0; + bool outsideInitial = false; + for (const auto& key : table) { + const bool isInitial = initial.count(key) > 0; + const bool isListed = community.count(key) > 0; + if (!isInitial && !isListed) { + return false; + } + outsideInitial = outsideInitial || !isInitial; + listed += isListed ? 1 : 0; + } + return outsideInitial && listed * 2 > community.size(); +} + +// The starter of an even set drops its last key (the official rule, for an odd table). Not when that +// would leave a community table without more than half of the list: with a list of 6 and 4 of its +// nodes left, the 3 kept would never be enough. A community table whose last key is its only one +// outside the initial confidants drops it: the rest is a regular table, as the official rule gives +inline bool dropForOddTable(const std::set& initial, const std::set& community, + const std::vector& table) { + if (table.empty() || table.size() % 2 != 0) { + return false; + } + if (!isCommunityTable(initial, community, table)) { + return true; + } + const std::vector rest(table.begin(), table.end() - 1); + return isCommunityTable(initial, community, rest) + || std::none_of(rest.cbegin(), rest.cend(), [&initial](const PublicKey& key) { return initial.count(key) == 0; }); +} + +// The table a starter sends from the keys at the highest sequence it sees (its own included), after +// the odd rule. A set with a listed key outside the initial confidants that is not a community table +// (not more than half of the list) cannot start anything as it is; it must not hold back the +// regular restart either, which the initial confidants among it would make without the list: the +// table is then theirs, under the official odd rule, and a node outside them waits as before +inline std::set bootstrapTableOf(const std::set& initial, const std::set& community, + const std::set& seen) { + const auto oddTable = [&initial](const std::set& listed, std::set keys) { + if (dropForOddTable(initial, listed, std::vector(keys.cbegin(), keys.cend()))) { + keys.erase(std::prev(keys.end())); + } + return keys; + }; + std::set table = oddTable(community, seen); + const std::vector asSent(table.cbegin(), table.cend()); + const bool outsideInitial = std::any_of(asSent.cbegin(), asSent.cend(), [&initial](const PublicKey& key) { + return initial.count(key) == 0; + }); + if (!outsideInitial || isCommunityTable(initial, community, asSent)) { + return table; + } + std::set regular; + for (const auto& key : seen) { + if (initial.count(key) > 0) { + regular.insert(key); + } + } + return oddTable({}, regular); +} + +// A neighbour ahead of this node: it syncs from it, the network is not stalled. Two blocks ahead +// always; one block ahead only for a further stall_minutes (sync takes one block in seconds, so a +// neighbour that stays one block ahead cannot give it, and must not keep the fallback closed for ever) +inline bool isNeighbourAhead(Sequence neighbourSeq, Sequence lastSeq, uint64_t msSinceProgress, uint64_t stallMs) { + if (neighbourSeq >= lastSeq + 2) { + return true; + } + return neighbourSeq == lastSeq + 1 && msSinceProgress < 2 * stallMs; +} + +// More keys weigh more; among tables of as many keys, the one with the keys earlier in the sorted +// union of the two sets. Only compared between community tables: the regular weight is not involved +constexpr size_t kCommunityWeightPositions = 58; + +inline CommunityWeight communityTableWeight(const std::set& initial, const std::set& community, + const std::vector& table) { + std::set all(initial); + all.insert(community.begin(), community.end()); + CommunityWeight weight = CommunityWeight(table.size()) << kCommunityWeightPositions; + size_t index = 0; + for (const auto& key : all) { + if (index >= kCommunityWeightPositions) { + break; + } + for (const auto& member : table) { + if (member == key) { + weight += CommunityWeight(1) << (kCommunityWeightPositions - 1 - index); + break; + } + } + ++index; + } + return weight; +} + +// The fallback's state on one node; times in ms of a steady clock +class CommunityBootstrapGate { +public: + // a block written, a round package with accepted signatures, a regular bootstrap table: the network is alive. + // The node's start counts too, so that it has watched for stall_minutes before it may act + void onProgress(uint64_t nowMs) { + lastProgressMs_ = nowMs; + taken_ = false; + lastWeight_ = 0; + lastTakenMs_ = 0; + } + + // the clocks whenever the fallback is considered: a jump of either over maxGapMs since the last + // look (the host suspended and resumed, the wall clock set) is progress, so that the node watches + // again for stall_minutes before it acts on heights it saw before the jump + void onClock(uint64_t steadyMs, uint64_t wallMs, uint64_t maxGapMs) { + const bool jump = checked_ && (gap(lastCheckSteadyMs_, steadyMs) > maxGapMs || gap(lastCheckWallMs_, wallMs) > maxGapMs); + checked_ = true; + lastCheckSteadyMs_ = steadyMs; + lastCheckWallMs_ = wallMs; + if (jump) { + onProgress(steadyMs); + } + } + + // the node's last block, read whenever the fallback is considered: a new one is progress + void onLastSequence(Sequence seq, uint64_t nowMs) { + if (seq != lastSeq_) { + lastSeq_ = seq; + onProgress(nowMs); + } + } + + uint64_t msSinceProgress(uint64_t nowMs) const { + return nowMs > lastProgressMs_ ? nowMs - lastProgressMs_ : 0; + } + + bool isStalled(uint64_t nowMs, uint64_t stallMs) const { + return stallMs > 0 && msSinceProgress(nowMs) >= stallMs; + } + + // a community table of that weight and starter (its first key) may be started or taken now: the + // first one in this stall, any once half a round_elapse_time has passed since the last (a starter + // repeats its table every round_elapse_time), and within that half only a heavier one whose + // starter key is not above the last one's. The starter of a table is its lowest key, and among + // the nodes left the lowest key starts anyway: a node with a lower key that saw more nodes + // replaces the table at once, but a listed node with a higher key cannot displace a running table + // by padding its own with keys of stopped nodes (heavier, and unable to decide a round) + bool mayTake(CommunityWeight weight, const PublicKey& starter, uint64_t nowMs, uint64_t roundElapseMs) const { + if (!taken_) { + return true; + } + if (nowMs >= lastTakenMs_ && nowMs - lastTakenMs_ >= roundElapseMs / 2) { + return true; + } + return weight > lastWeight_ && !(lastStarter_ < starter); + } + + void onTaken(CommunityWeight weight, const PublicKey& starter, uint64_t nowMs) { + taken_ = true; + lastWeight_ = weight; + lastStarter_ = starter; + lastTakenMs_ = nowMs; + } + +private: + static uint64_t gap(uint64_t a, uint64_t b) { + return a > b ? a - b : b - a; + } + + uint64_t lastProgressMs_ = 0; + Sequence lastSeq_ = 0; + bool taken_ = false; + CommunityWeight lastWeight_ = 0; + PublicKey lastStarter_{}; + uint64_t lastTakenMs_ = 0; + bool checked_ = false; + uint64_t lastCheckSteadyMs_ = 0; + uint64_t lastCheckWallMs_ = 0; +}; + +// the fallback may act on this node: a list configured, the node stalled for stall_minutes, and no +// neighbour ahead of it +inline bool isCommunityFallbackOpen(bool configured, bool stalled, bool neighbourAhead) { + return configured && stalled && !neighbourAhead; +} + +} // namespace cs + +#endif // COMMUNITYBOOTSTRAP_HPP diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp index d3445260d..a5238c341 100644 --- a/csnode/include/csnode/forkguardstate.hpp +++ b/csnode/include/csnode/forkguardstate.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -124,4 +125,32 @@ std::string remedy(const ForkRecord& record); // remedy std::string summary(const ForkRecord& record); +// A line for an event that may repeat many times a minute (a node holding a fork refuses every block +// its peers offer of the other chain, and asks for them again): the first event gets a line at once, +// later ones are counted and reported with the next line, at most one per `period`. onEvent returns +// the number of events the line due now reports (this one included), or 0 when no line is due. +class RepeatedLine { +public: + explicit RepeatedLine(std::chrono::steady_clock::duration period = std::chrono::minutes(1)) + : period_(period) {} + + uint64_t onEvent(std::chrono::steady_clock::time_point now) { + ++pending_; + if (logged_ && now - last_ < period_) { + return 0; + } + const uint64_t count = pending_; + pending_ = 0; + last_ = now; + logged_ = true; + return count; + } + +private: + std::chrono::steady_clock::duration period_; + std::chrono::steady_clock::time_point last_{}; + bool logged_ = false; + uint64_t pending_ = 0; +}; + } // namespace cs::forkguard diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index 6fe51844c..9f8b946cf 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -446,6 +447,13 @@ public slots: void deepBlockValidation(const csdb::Pool& block, bool* shouldStop); void sendBlockAlarmSignal(cs::Sequence seq); void onRoundTimeElapsed(); + // the community bootstrap fallback ([community_bootstrap], off by default): see communitybootstrap.hpp + std::set communityConfidants() const; + // progress seen (a round package, a regular bootstrap table): the fallback closes again + void onCommunityProgress(); + // the fallback may act now; also notes a new last block as progress + bool isCommunityFallbackOpen(); + bool takeCommunityTable(const cs::PublicKeys& table, bool own); void onNeighbourAdded(const cs::PublicKey& neighbour, cs::Sequence lastSeq, cs::RoundNumber lastRound); void onNeighbourRemoved(const cs::PublicKey& neighbour); @@ -691,6 +699,9 @@ public slots: std::set initialConfidants_; bool isBootstrapRound_ = false; + // guarded: read by the round elapse handler and the processor thread + std::mutex communityMutex_; + cs::CommunityBootstrapGate communityGate_; cs::CachesSerializationManager cachesSerializationManager_; size_t notInRound_ = 0; diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index deb00f63f..50e827cf3 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1061,6 +1061,7 @@ void BlockChain::holdFork(cs::forkguard::ForkRecord r) { { std::lock_guard l(guardMutex_); forkRecord_ = r; + refusedLine_ = cs::forkguard::RepeatedLine{}; // a new episode: its first refusal gets a line at once } cs::forkguard::saveForkRecord(guardStateDir_, r); @@ -2937,8 +2938,22 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { // last block uncertain (which would let a later replacement remove it). The node // reports FORK HELD until its head moves on or the operator decides. if (recordForkHeld("authority")) { - cslog() << kLogPrefix << "competing block #" << poolSequence - << " is not authenticated by this node's confidants, keep own last block"; + // a node holding a fork is offered the other chain's blocks over and over: one + // line, then one a minute with the count (the FORK HELD detail line says the rest) + uint64_t refused = 0; + { + std::lock_guard l(guardMutex_); + refused = refusedLine_.onEvent(std::chrono::steady_clock::now()); + } + if (refused > 0) { + cslog() << kLogPrefix << "competing block #" << poolSequence + << " is not authenticated by this node's confidants, keep own last block (" << refused + << " refused since the previous such line)"; + } + else { + csdebug() << kLogPrefix << "competing block #" << poolSequence + << " is not authenticated by this node's confidants, keep own last block"; + } } else { csdebug() << kLogPrefix << "competing block #" << poolSequence diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 6a11ba977..b07291586 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -1176,6 +1176,68 @@ cs::BootstrapWeight Node::calculateBootStrapWeight(const cs::PublicKeys& confida } +std::set Node::communityConfidants() const { + const auto& keys = cs::ConfigHolder::instance().config()->getCommunityBootstrapSettings().confidants; + return std::set(keys.begin(), keys.end()); +} + +void Node::onCommunityProgress() { + std::lock_guard lock(communityMutex_); + communityGate_.onProgress(stat_.msSinceStart()); +} + +bool Node::isCommunityFallbackOpen() { + const auto& settings = cs::ConfigHolder::instance().config()->getCommunityBootstrapSettings(); + const cs::Sequence lastSeq = blockChain_.getLastSeq(); + const uint64_t stallMs = uint64_t(settings.stallMinutes) * 60'000ULL; + // a jump of the clocks between two looks (host suspended, clock set) is progress: the node then + // watches again before it acts on what it saw before. The looks come every round_elapse_time + const uint64_t maxGapMs = std::max(5 * 60'000ULL, 3 * cs::ConfigHolder::instance().config()->roundElapseTime()); + const uint64_t wallMs = uint64_t(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + bool stalled = false; + uint64_t sinceProgress = 0; + { + std::lock_guard lock(communityMutex_); + const uint64_t now = stat_.msSinceStart(); + communityGate_.onClock(now, wallMs, maxGapMs); + communityGate_.onLastSequence(lastSeq, now); + stalled = communityGate_.isStalled(now, stallMs); + sinceProgress = communityGate_.msSinceProgress(now); + } + if (settings.confidants.empty() || !stalled) { + return false; + } + // a neighbour with a later block: this node is behind and syncs, the network is not stalled + bool ahead = false; + transport_->forEachNeighbour([&ahead, lastSeq, sinceProgress, stallMs](const cs::PublicKey&, cs::Sequence seq, cs::RoundNumber) { + if (cs::isNeighbourAhead(seq, lastSeq, sinceProgress, stallMs)) { + ahead = true; + } + }); + return cs::isCommunityFallbackOpen(true, stalled, ahead); +} + +bool Node::takeCommunityTable(const cs::PublicKeys& table, bool own) { + const cs::CommunityWeight weight = cs::communityTableWeight(initialConfidants_, communityConfidants(), table); + const uint64_t elapse = cs::ConfigHolder::instance().config()->roundElapseTime(); + std::lock_guard lock(communityMutex_); + const uint64_t now = stat_.msSinceStart(); + if (table.empty()) { + cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "not starting" : "dropping") << " an empty table"; + return false; + } + if (!communityGate_.mayTake(weight, table.front(), now, elapse)) { + cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "not starting" : "dropping") << " a table of " << table.size() + << " keys: another was taken less than half a round_elapse_time ago, and this one is not heavier with a starter key not above its starter's"; + return false; + } + communityGate_.onTaken(weight, table.front(), now); + cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "starting" : "taking") << " a table of " << table.size() + << " keys after " << communityGate_.msSinceProgress(now) / 1000 << " s without a block, a round or a regular bootstrap"; + return true; +} + void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::RoundNumber rNum) { cslog() << "NODE> get Boot strap Round Table #" << rNum; // a table for a round no honest starter would send to this node changes nothing here @@ -1219,6 +1281,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R bool signedTable = false; const size_t signed_subdata_len = 1 + size_t(confSize) * cscrypto::kPublicKeySize; + // a table with a key that is not an initial confidant: signed by the project, or, with the + // community bootstrap fallback open on this node, a community table signed by its starter + bool communityTable = false; if (unknown) { cs::Signature sig; stream >> sig; @@ -1226,19 +1291,26 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R cswarning() << "malformed bootstrap round table, ignore"; return; } - const size_t signed_buf_len = sizeof(cs::RoundNumber) + signed_subdata_len; - cs::Bytes buf; - buf.reserve(signed_buf_len); - auto* ptr = reinterpret_cast(&rNum); - std::copy(ptr, ptr + sizeof(cs::RoundNumber), std::back_inserter(buf)); - ptr = payload.data(); - std::copy(ptr, ptr + signed_subdata_len, std::back_inserter(buf)); - if (!cscrypto::verifySignature(sig, cs::PacketValidator::getBlockChainKey(), buf.data(), buf.size())) { + // the same bytes as the community starter signs (cs::makeCommunityTable); the stream read + // signed_subdata_len bytes of the payload already, so they are there + const cs::Bytes buf = cs::bootstrapSignedBytes(rNum, payload.data(), signed_subdata_len); + if (cscrypto::verifySignature(sig, cs::PacketValidator::getBlockChainKey(), buf.data(), buf.size())) { + currentWeight = cs::kSignedBootstrapWeight; + signedTable = true; + } + else if (cs::isCommunityTable(initialConfidants_, communityConfidants(), confidants) + && cscrypto::verifySignature(sig, confidants.front(), buf.data(), buf.size())) { + if (!isCommunityFallbackOpen()) { + cslog() << kLogPrefix_ << "community bootstrap table #" << rNum << " from " << KeyToBase58(confidants.front()) + << ", but this node has not stalled long enough or a neighbour is ahead, drop"; + return; + } + communityTable = true; + } + else { cswarning() << kLogPrefix_ << "failed to test bootstrap signature, drop"; return; } - currentWeight = cs::kSignedBootstrapWeight; - signedTable = true; } else if (payload.size() == signed_subdata_len + cscrypto::kSignatureSize) { // a table of initial confidants signed through the diag API (Node::bootstrap): the official @@ -1270,7 +1342,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R } } - if (currentWeight <= bootStrapWeight_) { + // a community table is weighed against community tables only (takeCommunityTable): a regular + // table always replaces it + if (!communityTable && currentWeight <= bootStrapWeight_) { cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " with " << confidants.size() << " confidants weighs " << currentWeight << ", not above " << bootStrapWeight_ << ", drop"; return; @@ -1280,11 +1354,20 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R cswarning() << "Bad round table format, ignore"; return; } + if (communityTable) { + if (!takeCommunityTable(confidants, false)) { + return; + } + } + else { + onCommunityProgress(); + } subRound_ = 1; if (!isBootstrapRound_) { isBootstrapRound_ = true; - cslog() << "NODE> Bootstrap on, table of " << confidants.size() << " confidants, weight " << currentWeight; + cslog() << "NODE> Bootstrap on, table of " << confidants.size() << " confidants, " + << (communityTable ? "community table" : "weight " + std::to_string(currentWeight)); } roundTable.confidants = std::move(confidants); @@ -2532,7 +2615,13 @@ void Node::getBlockRequest(const uint8_t* data, const size_t size, const cs::Pub } void Node::getBlockReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { - csinfo() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + // while FORK HELD the replies bring the other chain's blocks again and again (see PoolSynchronizer) + if (blockChain_.forkHeld()) { + csdebug() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + } + else { + csinfo() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + } // the blocks of this reply come from `sender`: named in the fork record should the node refuse one blockChain_.setSyncSource(sender, poolSynchronizer_->announcedSequence(sender)); struct ClearSource { @@ -3958,6 +4047,7 @@ void Node::sendRoundTable(cs::RoundPackage& rPackage) { csdebug() << "Round " << rPackage.roundTable().round << ", Confidants count " << rPackage.roundTable().confidants.size(); csdebug() << "Hashes count: " << rPackage.roundTable().hashes.size(); + onCommunityProgress(); // this node's own round package performRoundPackage(rPackage, solver_->getPublicKey(), false); } @@ -4226,6 +4316,10 @@ void Node::getRoundTable(const uint8_t* data, const size_t size, const cs::Round cs::Signatures poolSignatures; cs::PublicKeys confidants; + // progress for the community bootstrap fallback: a package whose signatures this node checked + // and accepted; one taken unchecked (a bootstrap round, the first rounds, a node lagging behind) + // is not, and the blocks it leads to are progress anyway + bool signaturesAccepted = false; if (!isBootstrapRound()) { if (rPackage.roundTable().round > 2/* && confirmationList_.size() > 0*/) { //Here we have problems when the trusted have the first block and the others do not!!! @@ -4249,7 +4343,8 @@ void Node::getRoundTable(const uint8_t* data, const size_t size, const cs::Round } } - if (!receivingSignatures(rPackage, confidants) && storedRound == getBlockChain().getLastSeq()) { + signaturesAccepted = receivingSignatures(rPackage, confidants); + if (!signaturesAccepted && storedRound == getBlockChain().getLastSeq()) { return; } } @@ -4263,6 +4358,9 @@ void Node::getRoundTable(const uint8_t* data, const size_t size, const cs::Round currentRoundTableMessage_.round = rPackage.roundTable().round; currentRoundTableMessage_.sender = sender; currentRoundTableMessage_.message = cs::Bytes(data, data + size); + if (signaturesAccepted) { + onCommunityProgress(); + } performRoundPackage(rPackage, sender, updateRound); } @@ -5462,7 +5560,14 @@ void Node::onRoundTimeElapsed() { } solver_->resetGrayList(); const cs::PublicKey& own_key = solver_->getPublicKey(); - if (initialConfidants_.find(own_key) == initialConfidants_.end()) { + // the community bootstrap fallback, open only after a long stall: the listed keys count as + // initial confidants here, and a table with any of them is a community table (see below) + const bool community = isCommunityFallbackOpen(); + const std::set listed = community ? communityConfidants() : std::set{}; + const auto isStarterKey = [this, &listed](const cs::PublicKey& key) { + return initialConfidants_.count(key) > 0 || listed.count(key) > 0; + }; + if (!isStarterKey(own_key)) { cslog() << "Waiting for next round..."; myLevel_ = Level::Normal; @@ -5484,28 +5589,38 @@ void Node::onRoundTimeElapsed() { const cs::Sequence maxLocalBlock = blockChain_.getLastSeq(); cs::Sequence maxGlobalBlock = maxLocalBlock; - auto callback = [&maxGlobalBlock, &actualConfidants, this] + auto callback = [&maxGlobalBlock, &actualConfidants, &isStarterKey] (const cs::PublicKey& neighbour, cs::Sequence lastSeq, cs::RoundNumber) { - const auto it = initialConfidants_.find(neighbour); - if (it == initialConfidants_.end()) { + if (!isStarterKey(neighbour)) { return; } if (lastSeq > maxGlobalBlock) { maxGlobalBlock = lastSeq; actualConfidants.clear(); - actualConfidants.insert(*it); + actualConfidants.insert(neighbour); } else if (lastSeq == maxGlobalBlock) { - actualConfidants.insert(*it); + actualConfidants.insert(neighbour); } }; transport_->forEachNeighbour(std::move(callback)); - if (actualConfidants.size() % 2 == 0) { - auto it = actualConfidants.end(); - --it; - it = actualConfidants.erase(it); - + // an odd table: the last key of an even set goes, unless a community table would lose its + // majority of the list with it (cs::dropForOddTable; without the fallback open, always). A set + // with listed keys that is not a community table: the initial confidants in it, as without the + // list, so that it never holds back a regular restart (cs::bootstrapTableOf) + { + const auto isOutsideInitial = [this](const cs::PublicKey& key) { + return initialConfidants_.count(key) == 0; + }; + const size_t seen = actualConfidants.size(); + const bool seenListed = std::any_of(actualConfidants.cbegin(), actualConfidants.cend(), isOutsideInitial); + actualConfidants = cs::bootstrapTableOf(initialConfidants_, listed, actualConfidants); + if (seenListed && std::none_of(actualConfidants.cbegin(), actualConfidants.cend(), isOutsideInitial)) { + cslog() << "COMMUNITY BOOTSTRAP: " << seen << " confidants with max sequence " << maxGlobalBlock + << " do not hold more than half of the " << listed.size() + << " listed keys, regular table of the initial confidants among them"; + } } initBootstrapRP(actualConfidants); @@ -5521,6 +5636,9 @@ void Node::onRoundTimeElapsed() { << initialConfidants_.size() << ", start lookup..."; transport_->addToNeighbours(initialConfidants_); + if (community) { + transport_->addToNeighbours(listed); + } } if (actualConfidants.size() < Consensus::MinTrustedNodes) { @@ -5544,28 +5662,51 @@ void Node::onRoundTimeElapsed() { // do not increment, only "mark" default round start subRound_ = 1; + // a table with a key that is not an initial confidant: only as a community table (no regular + // starter would send it, and only nodes with the fallback open take it) + cs::PublicKeys confs(actualConfidants.cbegin(), actualConfidants.cend()); + const bool communityTable = std::any_of(confs.cbegin(), confs.cend(), [this](const cs::PublicKey& key) { + return initialConfidants_.count(key) == 0; + }); + if (communityTable && !cs::isCommunityTable(initialConfidants_, listed, confs)) { + // cs::bootstrapTableOf never gives such a set; kept as a guard + cslog() << "COMMUNITY BOOTSTRAP: " << confs.size() << " confidants with max sequence " << maxGlobalBlock + << " do not hold more than half of the " << listed.size() << " listed keys, wait"; + return; + } + if (*actualConfidants.cbegin() == own_key) { cslog() << "Starting round..."; - cs::Bytes bin; - cs::ODataStream out(bin); - out << uint8_t(actualConfidants.size()); - - for (const auto& item : actualConfidants) { - out << item; + if (communityTable && !takeCommunityTable(confs, true)) { + return; } - cs::PublicKeys confs; - for (auto it : actualConfidants) { - confs.push_back(it); + + cs::Bytes bin; + if (communityTable) { + // signed by this node, the starter: the round, the count and the keys, as a table + // signed by the project, then random bytes so that its repeat is a new message; its + // weight stays out of bootStrapWeight_ + bin = cs::makeCommunityTable(roundPackageCache_.back().roundTable().round, actualConfidants, solver_->getPrivateKey(), + std::random_device{}()); } - cs::BootstrapWeight currentWeight = calculateBootStrapWeight(confs); - if (currentWeight > bootStrapWeight_) { - bootStrapWeight_ = currentWeight; + else { + cs::ODataStream out(bin); + out << uint8_t(actualConfidants.size()); + + for (const auto& item : actualConfidants) { + out << item; + } + cs::BootstrapWeight currentWeight = calculateBootStrapWeight(confs); + if (currentWeight > bootStrapWeight_) { + bootStrapWeight_ = currentWeight; + } + // when we try to start rounds several times, we will not send duplicates + auto random = std::random_device{}(); + out << random; + onCommunityProgress(); } - // when we try to start rounds several times, we will not send duplicates - auto random = std::random_device{}(); - out << random; auto& conveyer = cs::Conveyer::instance(); conveyer.updateRoundTable(roundPackageCache_.back().roundTable().round, roundPackageCache_.back().roundTable()); @@ -5629,6 +5770,7 @@ bool Node::bootstrap(const cs::Bytes& bytes, cs::RoundNumber round) { if (currentWeight > bootStrapWeight_) { bootStrapWeight_ = currentWeight; } + onCommunityProgress(); sendBroadcast(MsgTypes::BootstrapTable, roundPackageCache_.back().roundTable().round, bytes); if (!isBootstrapRound_) { diff --git a/csnode/src/poolsynchronizer.cpp b/csnode/src/poolsynchronizer.cpp index cb0a4495f..06bc303fb 100644 --- a/csnode/src/poolsynchronizer.cpp +++ b/csnode/src/poolsynchronizer.cpp @@ -86,7 +86,14 @@ void PoolSynchronizer::sync(cs::RoundNumber roundNum, cs::RoundNumber difference << "\n Last written seq: " << lastWrittenSequence << "\n Cached blocks: " << cachedBlocksSize << "\n Total blocks: " << totalBlocks; - cslog() << "SYNC: Blocks remaining: " << blocksRemaining; + // while FORK HELD the node asks for the other chain's blocks again and again and refuses them: + // the FORK HELD lines say so, these go to the debug log + if (blockChain_->forkHeld()) { + csdebug() << "SYNC: Blocks remaining: " << blocksRemaining; + } + else { + cslog() << "SYNC: Blocks remaining: " << blocksRemaining; + } if (blocksRemaining == 0) { showSyncronizationProgress(lastWrittenSequence); @@ -372,9 +379,16 @@ void PoolSynchronizer::sendBlockRequest() { else { maxRequestedSequence_ = neededSequences.back(); if (checkSynchroLog(neighbour->first)) { - cslog() << "SYNC: requesting for " << neededSequences.size() - << " blocks [" << neededSequences.front() << ", " << neededSequences.back() - << "] from " << cs::Utils::byteStreamToHex(neighbour->first); + if (blockChain_->forkHeld()) { // asked again and again while FORK HELD: debug log + csdebug() << "SYNC: requesting for " << neededSequences.size() + << " blocks [" << neededSequences.front() << ", " << neededSequences.back() + << "] from " << cs::Utils::byteStreamToHex(neighbour->first); + } + else { + cslog() << "SYNC: requesting for " << neededSequences.size() + << " blocks [" << neededSequences.front() << ", " << neededSequences.back() + << "] from " << cs::Utils::byteStreamToHex(neighbour->first); + } emit sendRequest(neighbour->first, neededSequences); pacing_.onRequest(neighbour->first, neededSequences.back(), now); addSynchroLog(neighbour->first, neededSequences, SyncroMessage::NoAnswer); diff --git a/docs/CHANGES.md b/docs/CHANGES.md index a8369230a..af28d55eb 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,6 +44,7 @@ lines under the new release. | PR | Change | |---|---| +| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. After a partition the community side reports FORK HELD (#189) and its operator chooses `--resolve-fork` / `--set-bc-top` (official chain) or `dismiss-fork` (community chain). CI: `official-stop-*`, `official-partition*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge). A set at the highest block whose listed keys are no majority of the list falls back to the regular table of its initial confidants, so the option never holds back a regular restart. FORK HELD (#189): a new episode's first refused block is logged at once. CI: the snapshot stress check takes the median one-minute block rate, and `snapshot-p2p-stress-loss` allows a 20 % rate fall and a 50 % round p95 rise (its loss reaches a joiner only after the joins; 10 % failed on noise, a 30 % slowdown still fails) | | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | | #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). A block not vouched for is refused at once but reported only if the head then stays put (no flapping); unreadable guard files fail safe; the files go when the database is replaced. `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | diff --git a/docs/differences.md b/docs/differences.md index b8e22d884..6f5cfdaee 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -122,6 +122,147 @@ The official node has it only in its monitor and wallet builds. The cache follow standard, including v1/v2 tokens whose amounts are BigDecimals, and it skips failed calls. It is rebuilt from the chain when the checkpoint holds none. See [tokens.md](tokens.md). +### Community bootstrap fallback (expert option, off by default) + +Only the initial confidants (the `init_trusted` list, or special order 2) may restart a stalled +network with a bootstrap table, and a table with any other key needs the project's signature. If +all official nodes become unavailable at once, the initial confidants still running may be too few +(fewer than 4), and the network stays stalled until the official nodes are back. + +The `[community_bootstrap]` section of `config.ini` adds a liveness fallback. It is off unless +`confidants` lists keys: + +```ini +[community_bootstrap] +; community node keys (base58), comma or space separated; empty or missing: off +confidants=,,... +; minutes without progress before the fallback may act (1 to 10080, default 30) +stall_minutes=30 +``` + +With the list set, and only after `stall_minutes` in which the node saw no new block, no round +package whose signatures it checked and accepted, and no regular bootstrap table, and while no +neighbour reports a block two or more above its own (one above only holds it for a further +`stall_minutes`, since sync would have fetched that block long before): + +- a node whose key is an initial confidant or listed may start a bootstrap table of the initial + confidants and listed nodes that hold the highest block, as an initial confidant does; the table + is signed by its starter, not by the project; +- a node with the list takes such a table only if it has more than half of the listed keys, each + key is an initial confidant or listed, and the starter's signature checks. The round window and + the stall check of every unsigned table still apply. A node takes one such table per half + `round_elapse_time`; within that time only a larger table whose starter (its lowest key) is not + above the current table's starter replaces it, so a listed node cannot displace a running table + by padding its own with keys of stopped nodes. The starter repeats its table every + `round_elapse_time` with new random bytes after the signature, so that a node whose fallback + opens later gets the repeat (the transport drops a message it has seen before); +- of an even set of nodes the starter drops the last key, as the official code does, unless that + would leave the table without more than half of the listed keys (it does drop it when the rest + is a regular table of initial confidants); +- a jump of the clock between two checks (the host suspended and resumed, the clock set) counts + as progress: the node watches again for `stall_minutes` before it acts. + +In normal operation nothing changes: blocks and round packages keep the fallback closed, a regular +bootstrap table is handled exactly as before and always replaces a community one, and blocks, +hashes and validation are unchanged. A regular table, even one signed by the project, is progress +like a block: if no block follows it for a further `stall_minutes`, a community table may replace it. Official nodes, and community nodes without the list, drop a +community table (it holds a key that is not an initial confidant and is not signed by the +project); they then take the blocks written meanwhile through ordinary sync once they are back, +as the `official-stop-*` scenarios of the robustness workflow check. + +**An expert option, not in the set-up.** Every node that should take part needs the same list: it +only helps when more than half of the listed nodes are running, and mismatched lists weaken that +majority rule. So `setup.sh`, `setup.ps1` and the Docker environment do not offer it until the +option has been reviewed and a coordinated list exists (possibly shipped like `known_hosts.txt`); +edit `config.ini` by hand (in Docker, `~/.credits/config.ini`, which the entrypoint writes only +when it is missing). + +The risk it brings is a partition. If a network partition cuts more than half of the listed nodes +off from the official nodes for longer than `stall_minutes`, while the official nodes keep +producing blocks, the listed nodes restart on their own and the two sides write different blocks. +There is no rule that makes either side yield to the other, and the threshold stays "more than +half of the list". What happens at reconnection (option A of the bounded-rollback guard, #189: the +shorter side waits for its operator): + +- official nodes follow the longer chain, as for any two chains, and unwind their blocks above the + fork; the transactions of those blocks are reverted and must be sent again unless the other + side also wrote them; +- community nodes never follow a longer chain on their own unless its blocks are vouched for by + confidants they hold (the bounded-rollback guard, see "Consensus fixes"). The blocks of the other + side are not, so each community node on the shorter side keeps its chain and reports **FORK + HELD** (on every round line, a detail line once a minute, `fork-held.json` next to the + database, an unhealthy Docker healthcheck). Its operator checks the other nodes or the monitor + and chooses (docs/quick-start.md, "If your node reports FORK HELD"): + - **follow the official chain**: after a partition the block the node refused does not continue + any block it holds, so at first it knows only an upper bound of the fork point. While FORK + HELD it looks for the last common block itself (#192). Wait until `fork-held.json` shows + `"fork_point_status": "exact"`, then stop the node, run it once with `--resolve-fork` (Docker: + `resolve-fork`) and start it again: it cuts at that block in one run. If the status is + `"located (unverified)"`, check the point against other nodes or the monitor and add + `--to-located`; if it stays `"upper bound"` or `"deeper"`, cut at the last common block by hand + with `--set-bc-top `. The blocks above the fork, and their transactions, are + reverted; the quick-start checkpoints above the new top go with them; + - **keep the community chain**: create `dismiss-fork` next to the database; the node keeps + refusing that branch and no longer reports it. + +The fallback's own blocks pass that guard on the community side: a community bootstrap block is +signed by the confidants of the community table, and a node that took that table holds it as a +round table it accepted, which is what the guard asks of a bootstrap block. Nothing in this option +changes what may unwind a node's chain: it does not touch block removal, the rollback bound or the +fork records. With the option on, the `official-stop-*` scenarios show no refused block and no +reorganisation on any node. + +Measured in the `official-partition` scenarios of the robustness workflow (community side cut off +for 6 minutes, `stall_minutes=2`, the official nodes run the official build), with the guard: + +- the community side restarts by the fallback 120-180 s after its last block; both sides then + write blocks (fork at #533-#563); +- after the cut is lifted the two sides do not look each other up again: every node keeps only + the 4 neighbours of its own side, nothing is refused, and both chains go on until the end of the + run (`official-partition`); +- with the community nodes restarted after the reconnection (they dial every host again, + `official-partition-restart`), all five report FORK HELD within 20 s and refuse every block of + the official chain: the community side is no longer unwound part of the way and left stuck + above the fork, as before the guard; it stays on its own chain until its operator decides. In + one run the official side had stalled during the cut (3 live initial confidants, fewer than 4), + wrote no block of its own, and took the community blocks once the community nodes were + restarted: every node then held all 217 community blocks and the same head; +- with the operator path (`official-partition-restart-resolve`) and the fork locator of #192: the + five community nodes held the fork within 20 s and found the exact fork point (#506) 30 s later; + one `--resolve-fork` each cut there, every start after it succeeded, and every node then held + the official chain: one chain, 275 community blocks and their 63 transactions reverted. Without + the locator, `--resolve-fork` cut a single block (the upper bound), FORK HELD came back a block + lower, and `--set-bc-top` at the last common block was needed (179-215 blocks reverted); the + first start after each such cut stopped once on a quick-start checkpoint above the new top, + which #192 removes. The scenario now fails if a cut lands on an upper bound or a start after a + cut fails. + +A partition with this option on therefore needs operator attention. Keep `stall_minutes` long. + +**Known limitations** (accepted; the operators control the list): + +- **A hostile lowest listed key.** The starter of a table is its lowest key, and a node takes a new + community table once half a `round_elapse_time` has passed since the last one. A listed node with + the lowest key that turns hostile can therefore send, every half `round_elapse_time`, a table + padded with keys of stopped nodes that cannot decide a round, and keep the community restart + from deciding while the stall lasts. It cannot write blocks the others would not sign, and a + regular table still replaces its table at once; the remedy is to take its key off the list. +- **Replay across a chain cut.** A community table is signed over the round and the keys only. + After `--set-bc-top` or `--resolve-fork` below the round of a table sent earlier, that table + verifies again for the same round on a node whose fallback is open. It changes nothing unless its + confidants are themselves running and stalled at that height. Binding the last block hash into + the signed bytes would change the format a project-signed table shares; it is left to the + BootstrapSignature protocol of the decentralisation plan. + +A set at the highest block with listed keys that hold no more than half of the list is no +community table; the initial confidants in it then make the regular table they would make without +the list (the official odd rule applies), so a node with the list never holds back a regular +restart, for example when the official nodes come back while a minority of the listed nodes runs. + +**Decentralisation plan.** When `H_activate_decentralization` of BK's plan lands, the fallback is to +be gated on `lastSeq + 1 < H_activate` (it acts only before the activation height) and later folded +into the plan's BootstrapSignature protocol. + ## Start-up behaviour - **A banner names the release.** At start the console and the log show the CREDITS logo with @@ -343,9 +484,9 @@ consensus rules. reconnects. The same holds for a node that lags behind with a divergent last block it never replaced (for example, it missed a bootstrap round): when the network's next block is not confirmed by the confidants of that block, the node waits for the operator instead of unwinding - it. This is the deliberate choice of this change; `--resolve-fork` is the remedy. The opt-in - community bootstrap fallback (PR #188, not merged yet) expects the longer chain to win after a - partition; with this change that switch is an operator decision too. + it. This is the deliberate choice of this change; `--resolve-fork` is the remedy. After a + partition with the opt-in community bootstrap fallback, the community side holds the fork in the + same way and its operator decides (see "Community bootstrap fallback"). - **Two blocks of one round.** When a trusted node does not get one stage-2 in time, it fetches it with a request that takes exactly as long as the others wait for its stage-3. Depending on a few diff --git a/docs/quick-start.md b/docs/quick-start.md index b77e5931d..dac716e41 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -444,6 +444,9 @@ describe that database: if you delete `db/` by hand to start again, delete these A block of another chain that the node refuses while its own chain goes on is not reported: FORK HELD appears only when the node stays at the same block after the refusal (two rounds, or a minute). +While the fork is held, peers keep offering their chain: the node logs the refused blocks once a +minute with a count ("N refused since the previous such line"), and its block requests and replies +go to the debug log only. **1. Check before acting.** Compare the node's head (`Last written sequence`) and the block at the fork point with other nodes: the [monitor](https://monitor.credits.com), the diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index 6b04cf97f..3ca69a82f 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -544,6 +544,75 @@ TEST(BlockChain, RollbackReferenceSurvivesRestart) { chain.setGuardStateDir(""); } +// a node holding a fork is offered the other chain's blocks many times a minute: one line at once, +// then at most one a minute with the count since the previous one +TEST(BlockChain, RefusedBlocksAreCountedNotLoggedOneByOne) { + using namespace std::chrono_literals; + cs::forkguard::RepeatedLine line; + const auto t0 = std::chrono::steady_clock::time_point{} + 1h; + EXPECT_EQ(line.onEvent(t0), 1u); // the first refusal: a line at once + for (int i = 1; i < 2000; ++i) { + EXPECT_EQ(line.onEvent(t0 + std::chrono::milliseconds(25 * i)), 0u); // 2000 within 50 s + } + EXPECT_EQ(line.onEvent(t0 + 60s), 2000u); // 1999 counted + this one + EXPECT_EQ(line.onEvent(t0 + 61s), 0u); + EXPECT_EQ(line.onEvent(t0 + 10min), 2u); // a quiet spell: the next one at once, with the one held back + EXPECT_EQ(line.onEvent(t0 + 10min + 1s), 0u); +} + +// once FORK HELD, further refusals change nothing on disk: no file is written per refused block +TEST(BlockChain, RefusalsWhileHeldDoNotTouchTheForkFile) { + installDefaultConfig(); + TempDir dir("refusals"); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(5000); + chain.setGuardStateDir(dir.str()); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); + ASSERT_FALSE(chain.guardAutoRollback()); // beyond the depth bound: FORK HELD + ASSERT_TRUE(chain.forkHeld()); + const auto file = dir.path / cs::forkguard::kForkFile; + ASSERT_TRUE(std::filesystem::exists(file)); + const auto written = std::filesystem::last_write_time(file); + const auto size = std::filesystem::file_size(file); + for (int i = 0; i < 1000; ++i) { + EXPECT_TRUE(chain.test_recordForkHeld("authority")); + } + EXPECT_EQ(std::filesystem::last_write_time(file), written); + EXPECT_EQ(std::filesystem::file_size(file), size); + EXPECT_TRUE(chain.forkHeld()); + chain.setGuardStateDir(""); +} + +// a new FORK HELD episode starts its refusal line afresh: its first refusal is logged at once, even +// within a minute of the previous episode's last line +TEST(BlockChain, ANewForkHeldEpisodeLogsItsFirstRefusalAtOnce) { + using namespace std::chrono_literals; + installDefaultConfig(); + TempDir dir("refusalepisodes"); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.setGuardStateDir(dir.str()); + const auto t0 = std::chrono::steady_clock::now(); + for (int episode = 0; episode < 2; ++episode) { + chain.test_resetRollbackEpisode(); // the previous episode ended + chain.test_setLastSequence(5000); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); + ASSERT_FALSE(chain.guardAutoRollback()); // FORK HELD + ASSERT_TRUE(chain.forkHeld()); + const auto t = t0 + std::chrono::seconds(episode * 5); + EXPECT_EQ(chain.test_onRefusedBlock(t), 1u) << "episode " << episode; + EXPECT_EQ(chain.test_onRefusedBlock(t + 1s), 0u); + EXPECT_EQ(chain.test_onRefusedBlock(t + 2s), 0u); + } + chain.setGuardStateDir(""); +} + // the fork record and the episode in their files, and the --resolve-fork plan TEST(BlockChain, ForkGuardFilesRoundTrip) { TempDir dir("files"); diff --git a/tests/csnode/communitybootstrap_tests.cpp b/tests/csnode/communitybootstrap_tests.cpp new file mode 100644 index 000000000..80d7c0865 --- /dev/null +++ b/tests/csnode/communitybootstrap_tests.cpp @@ -0,0 +1,454 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +using cs::CommunityBootstrapGate; +using cs::communityTableWeight; +using cs::isCommunityFallbackOpen; +using cs::isCommunityTable; + +namespace { +// key k: every byte k, so that keys sort as their numbers +cs::PublicKey key(uint8_t k) { + cs::PublicKey result{}; + result.fill(k); + return result; +} + +std::set keys(std::initializer_list ks) { + std::set result; + for (int k : ks) { + result.insert(key(uint8_t(k))); + } + return result; +} + +std::vector table(std::initializer_list ks) { + std::vector result; + for (int k : ks) { + result.push_back(key(uint8_t(k))); + } + return result; +} + +// the CI layout: initial confidants 1-5 (2 and 4 official), community nodes 1, 3, 5, 7, 9 +const std::set kInitial = keys({1, 2, 3, 4, 5}); +const std::set kCommunity = keys({1, 3, 5, 7, 9}); + +constexpr uint64_t kMinute = 60'000; +constexpr uint64_t kStall = 30 * kMinute; +constexpr uint64_t kElapse = 20'000; +} // namespace + +TEST(CommunityBootstrap, TheCommunityNodesLeftAfterTheOfficialOnesMakeATable) { + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 3, 5, 7, 9}))); + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 3, 7, 9}))); + // with initial confidants that are still there + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 2, 3, 7, 9}))); +} + +TEST(CommunityBootstrap, ATableOfInitialConfidantsOnlyIsARegularOne) { + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 2, 3, 4, 5}))); + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 4, 5}))); +} + +TEST(CommunityBootstrap, NoListNoTable) { + EXPECT_FALSE(isCommunityTable(kInitial, {}, table({1, 3, 5, 7, 9}))); +} + +TEST(CommunityBootstrap, EveryKeyMustBeInitialOrListed) { + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 5, 7, 8}))); + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 5, 7, 9, 10}))); +} + +TEST(CommunityBootstrap, MoreThanHalfOfTheListMustBeInIt) { + // 3 of 5 listed: a majority + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 2, 4, 7, 9}))); + // 2 of 5 listed: another part of a split community could hold the other 3 + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({2, 3, 4, 7}))); + const auto six = keys({1, 3, 5, 7, 9, 11}); + EXPECT_FALSE(isCommunityTable(kInitial, six, table({2, 4, 7, 9, 11}))); // 3 of 6 + EXPECT_TRUE(isCommunityTable(kInitial, six, table({4, 5, 7, 9, 11}))); // 4 of 6 +} + +TEST(CommunityBootstrap, SizeAndOrderAsAStarterSendsIt) { + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 7, 9}))); // below MinTrustedNodes + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 9, 7}))); // not ascending + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 7, 7, 9}))); // a repeated key + std::set many; + std::vector big; + for (int k = 1; k <= 26; ++k) { + many.insert(key(uint8_t(k))); + big.push_back(key(uint8_t(k))); + } + EXPECT_FALSE(isCommunityTable(kInitial, many, big)); // above MaxTrustedNodes + big.pop_back(); + EXPECT_TRUE(isCommunityTable(kInitial, many, big)); +} + +TEST(CommunityBootstrap, MoreKeysWeighMoreThenEarlierKeys) { + const auto w5 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7, 9})); + const auto w4 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7})); + const auto w4late = communityTableWeight(kInitial, kCommunity, table({3, 5, 7, 9})); + EXPECT_GT(w5, w4); + EXPECT_GT(w4, w4late); + // a table of fewer keys never outweighs a larger one, whatever its keys + EXPECT_GT(communityTableWeight(kInitial, kCommunity, table({5, 7, 9, 3, 1})), communityTableWeight(kInitial, kCommunity, table({1, 2, 3, 4}))); + EXPECT_EQ(w5, communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7, 9}))); +} + +TEST(CommunityBootstrap, ClosedUntilTheNodeStalledForStallMinutes) { + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + EXPECT_FALSE(gate.isStalled(0, kStall)); + EXPECT_FALSE(gate.isStalled(kStall - 1, kStall)); + EXPECT_TRUE(gate.isStalled(kStall, kStall)); + EXPECT_TRUE(gate.isStalled(kStall + kMinute, kStall)); + // stall_minutes 0 is never stalled (the config allows 1..10080) + EXPECT_FALSE(gate.isStalled(kStall, 0)); +} + +TEST(CommunityBootstrap, ANodeJustStartedHasNotWatchedLongEnough) { + // the clock is ms since the node started: no progress yet counts as progress at the start + CommunityBootstrapGate gate; + EXPECT_FALSE(gate.isStalled(kStall - 1, kStall)); + gate.onLastSequence(500, kStall - 1); // the first look at the chain is progress too + EXPECT_FALSE(gate.isStalled(2 * kStall - 2, kStall)); + EXPECT_TRUE(gate.isStalled(2 * kStall - 1, kStall)); +} + +TEST(CommunityBootstrap, ABlockARoundOrARegularBootstrapClosesIt) { + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + ASSERT_TRUE(gate.isStalled(kStall, kStall)); + gate.onProgress(kStall); // a round package, a regular table + EXPECT_FALSE(gate.isStalled(kStall + 1, kStall)); + ASSERT_TRUE(gate.isStalled(2 * kStall, kStall)); + gate.onLastSequence(500, 2 * kStall); // the same last block: no progress + EXPECT_TRUE(gate.isStalled(2 * kStall, kStall)); + gate.onLastSequence(501, 2 * kStall); // a block (synced or written) + EXPECT_FALSE(gate.isStalled(2 * kStall + 1, kStall)); +} + +// The safety property: while blocks are produced (official nodes alive), the fallback never opens. +// A day of rounds, one a second or one every round_elapse_time at worst, a block each +TEST(CommunityBootstrap, NeverOpenWhileTheNetworkProducesBlocks) { + for (uint64_t gap : {uint64_t(1000), kElapse, 10 * kMinute, kStall - 1}) { + CommunityBootstrapGate gate; + cs::Sequence seq = 100; + for (uint64_t now = 0; now < 24 * 60 * kMinute; now += gap) { + gate.onLastSequence(++seq, now); + for (uint64_t t = now; t < now + gap; t += std::max(gap / 4, 1)) { + ASSERT_FALSE(isCommunityFallbackOpen(true, gate.isStalled(t, kStall), false)) << "gap " << gap << " at " << t; + } + } + } + // bootstrap rounds that end in blocks are progress just as well (a regular table every round_elapse_time) + CommunityBootstrapGate gate; + gate.onLastSequence(100, 0); + for (uint64_t now = 0; now < 4 * kStall; now += kElapse) { + gate.onProgress(now); + ASSERT_FALSE(gate.isStalled(now + kElapse - 1, kStall)); + } +} + +TEST(CommunityBootstrap, OpenOnlyWhenConfiguredStalledAndNoNeighbourAhead) { + EXPECT_TRUE(isCommunityFallbackOpen(true, true, false)); + EXPECT_FALSE(isCommunityFallbackOpen(false, true, false)); // off by default: no list + EXPECT_FALSE(isCommunityFallbackOpen(true, false, false)); + EXPECT_FALSE(isCommunityFallbackOpen(true, true, true)); // behind a neighbour: sync instead +} + +TEST(CommunityBootstrap, OneTablePerHalfRoundElapseUnlessHeavierFromALowerStarter) { + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + const auto w5 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7, 9})); + const auto w4 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7})); + const uint64_t t = kStall; + EXPECT_TRUE(gate.mayTake(w4, key(1), t, kElapse)); + gate.onTaken(w4, key(1), t); + // a repeat or a lighter table right after: dropped, so a listed key cannot force restarts + EXPECT_FALSE(gate.mayTake(w4, key(1), t + 1, kElapse)); + EXPECT_FALSE(gate.mayTake(w4 - 1, key(1), t + kElapse / 2 - 1, kElapse)); + // a heavier one of the same starter at once: the table of more nodes wins + EXPECT_TRUE(gate.mayTake(w5, key(1), t + 1, kElapse)); + gate.onTaken(w5, key(1), t + 1); + EXPECT_FALSE(gate.mayTake(w4, key(1), t + 2, kElapse)); + // the starter's repeat, a round_elapse_time later + EXPECT_TRUE(gate.mayTake(w5, key(1), t + 1 + kElapse, kElapse)); + EXPECT_TRUE(gate.mayTake(w4, key(1), t + 1 + kElapse / 2, kElapse)); + // progress forgets the tables of this stall + gate.onProgress(t + 3); + EXPECT_TRUE(gate.mayTake(w4, key(3), t + 4, kElapse)); +} + +// A hostile listed key above the starter pads its table with keys of stopped nodes: heavier, but it +// cannot displace the running table; a lower starter that saw more nodes can +TEST(CommunityBootstrap, AHigherListedKeyCannotDisplaceATableByPaddingIt) { + const std::set initial = keys({1, 2, 4, 6, 8}); + const std::set community = keys({3, 5, 7, 9, 11, 13, 15}); // 4 make a majority + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + const uint64_t t = kStall; + const auto running = table({3, 5, 7, 9}); + ASSERT_TRUE(isCommunityTable(initial, community, running)); + const auto wRunning = communityTableWeight(initial, community, running); + ASSERT_TRUE(gate.mayTake(wRunning, running.front(), t, kElapse)); + gate.onTaken(wRunning, running.front(), t); + // key 5 pads with the stopped 2, 4, 6, 8, 11, 13: more keys, heavier, refused within the window + const auto padded = table({2, 4, 5, 6, 7, 8, 9, 11, 13}); + ASSERT_TRUE(isCommunityTable(initial, community, padded)); + ASSERT_GT(communityTableWeight(initial, community, padded), wRunning); + EXPECT_FALSE(gate.mayTake(communityTableWeight(initial, community, padded), key(5), t + 1, kElapse)); + // the same starter, or a lower one, that saw more nodes: taken + const auto more = table({3, 5, 7, 9, 11}); + EXPECT_TRUE(gate.mayTake(communityTableWeight(initial, community, more), key(3), t + 1, kElapse)); + const auto lower = table({1, 3, 5, 7, 9}); + EXPECT_TRUE(gate.mayTake(communityTableWeight(initial, community, lower), key(1), t + 1, kElapse)); + // after half a round_elapse_time any table: a repeat is needed when a round fails + EXPECT_TRUE(gate.mayTake(communityTableWeight(initial, community, padded), key(5), t + kElapse / 2, kElapse)); +} + +TEST(CommunityBootstrap, ANeighbourOneBlockAheadHoldsTheGateOnlyForAFurtherStall) { + using cs::isNeighbourAhead; + EXPECT_FALSE(isNeighbourAhead(500, 500, kStall, kStall)); + EXPECT_FALSE(isNeighbourAhead(499, 500, kStall, kStall)); + // two blocks ahead: always, this node is behind + EXPECT_TRUE(isNeighbourAhead(502, 500, 100 * kStall, kStall)); + // one block ahead: sync gets it in seconds; while it does not, only for a further stall_minutes + EXPECT_TRUE(isNeighbourAhead(501, 500, kStall, kStall)); + EXPECT_TRUE(isNeighbourAhead(501, 500, 2 * kStall - 1, kStall)); + EXPECT_FALSE(isNeighbourAhead(501, 500, 2 * kStall, kStall)); +} + +TEST(CommunityBootstrap, AClockJumpIsProgress) { + constexpr uint64_t kGap = 5 * kMinute; + constexpr uint64_t kWall = 1'700'000'000'000ULL; + CommunityBootstrapGate gate; + gate.onClock(0, kWall, kGap); + gate.onLastSequence(500, 0); + // looks every round_elapse_time while stalled: no jump + for (uint64_t t = kElapse; t <= kStall; t += kElapse) { + gate.onClock(t, kWall + t, kGap); + } + ASSERT_TRUE(gate.isStalled(kStall, kStall)); + // the host slept for an hour: the wall clock jumped (the steady one may not count the sleep) + gate.onClock(kStall + kElapse, kWall + kStall + kElapse + 60 * kMinute, kGap); + EXPECT_FALSE(gate.isStalled(kStall + kElapse + 1, kStall)); + EXPECT_TRUE(gate.isStalled(2 * kStall + kElapse, kStall)); + // a wall clock set back, and a steady clock jump, as well + for (const bool wallBack : {true, false}) { + CommunityBootstrapGate other; + other.onClock(0, kWall, kGap); + other.onLastSequence(500, 0); + for (uint64_t t = kElapse; t <= kStall; t += kElapse) { + other.onClock(t, kWall + t, kGap); + } + ASSERT_TRUE(other.isStalled(kStall, kStall)); + const uint64_t steady = wallBack ? kStall + kElapse : kStall + 10 * kMinute; + const uint64_t wall = wallBack ? kWall + kStall - 10 * kMinute : kWall + kStall + kElapse; + other.onClock(steady, wall, kGap); + EXPECT_FALSE(other.isStalled(steady, kStall)) << (wallBack ? "wall clock set back" : "steady clock jump"); + } +} + +// lists of even size: the starter must not drop the key that holds the majority of the list +TEST(CommunityBootstrap, AnEvenSetKeepsItsMajorityOfTheList) { + using cs::dropForOddTable; + const std::set initial = keys({2, 4, 6, 8, 10}); + const std::set six = keys({1, 3, 5, 7, 9, 11}); + // 4 of 6 listed left: dropping one leaves 3, not more than half: kept even + EXPECT_FALSE(dropForOddTable(initial, six, table({1, 3, 5, 7}))); + // 6 of 6: 5 still hold the majority, the official rule applies + EXPECT_TRUE(dropForOddTable(initial, six, table({1, 3, 5, 7, 9, 11}))); + // a list of 10, 6 left: kept; 8 left: dropped to 7 + const std::set ten = keys({1, 3, 5, 7, 9, 11, 13, 15, 17, 19}); + EXPECT_FALSE(dropForOddTable(initial, ten, table({1, 3, 5, 7, 9, 11}))); + EXPECT_TRUE(dropForOddTable(initial, ten, table({1, 3, 5, 7, 9, 11, 13, 15}))); + // odd sets are never touched; regular sets (no list, or initial keys only) as in the official code + EXPECT_FALSE(dropForOddTable(initial, six, table({1, 3, 5, 7, 9}))); + EXPECT_TRUE(dropForOddTable(initial, {}, table({2, 4, 6, 8}))); + EXPECT_TRUE(dropForOddTable(initial, six, table({2, 4, 6, 8}))); + // a community table whose only key outside the initial confidants is its last: the rest is a + // regular table, the official rule applies + EXPECT_TRUE(dropForOddTable(kInitial, kCommunity, table({1, 2, 3, 4, 5, 7}))); + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 2, 3, 4, 5}))); +} + +// a set with a listed key that holds no majority of the list must not hold back the regular restart +// the initial confidants in it would make without the list +TEST(CommunityBootstrap, AMixedSetWithoutAMajorityOfTheListFallsBackToTheRegularTable) { + using cs::bootstrapTableOf; + const std::set initial = keys({1, 2, 3, 4, 6}); + const std::set list = keys({5, 7, 9, 11, 13}); + // the 5 initial confidants and listed 5 alive: 1 of 5 listed is no community table; the 5 initial + // ones make the regular table they make on a node without the list + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 2, 3, 4, 5, 6})), keys({1, 2, 3, 4, 6})); + EXPECT_EQ(bootstrapTableOf(initial, {}, keys({1, 2, 3, 4, 6})), keys({1, 2, 3, 4, 6})); + // 4 initial ones and listed 5: even, the official odd rule leaves 3, below MinTrustedNodes: wait + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 2, 3, 4, 5})), keys({1, 2, 3})); + // a community table stays one + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 5, 7, 9})), keys({1, 5, 7, 9})); + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 3, 5, 7, 9, 11})), keys({1, 3, 5, 7, 9})); + // initial keys only: as before + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 2, 3, 4})), keys({1, 2, 3})); +} + +namespace { +struct Starter { + cs::PublicKey pub; + cs::PrivateKey priv; +}; + +// keys live in sodium's guarded memory: the library must be initialised first +std::vector makeStarters(size_t n) { + EXPECT_TRUE(cscrypto::cryptoInit()); + std::vector result(n); + for (auto& s : result) { + s.priv = cscrypto::generateKeyPair(s.pub); + } + std::sort(result.begin(), result.end(), [](const Starter& a, const Starter& b) { return a.pub < b.pub; }); + return result; +} + +// what Node::getBootstrapTable reads of a table with a key that is not an initial confidant: the +// count, the keys, the signature over bootstrapSignedBytes(round, count and keys); the rest ignored +bool readsAsCommunityTable(const cs::Bytes& payload, cs::RoundNumber rNum, const std::set& initial, + const std::set& community) { + cs::IDataStream stream(payload.data(), payload.size()); + uint8_t confSize = 0; + stream >> confSize; + cs::PublicKeys confidants; + for (int i = 0; i < confSize; ++i) { + cs::PublicKey k; + stream >> k; + confidants.push_back(k); + } + cs::Signature sig; + stream >> sig; + if (!stream.isValid() || confidants.empty()) { + return false; + } + const size_t signedLen = 1 + size_t(confSize) * cscrypto::kPublicKeySize; + const cs::Bytes buf = cs::bootstrapSignedBytes(rNum, payload.data(), signedLen); + return isCommunityTable(initial, community, confidants) + && cscrypto::verifySignature(sig, confidants.front(), buf.data(), buf.size()); +} +} // namespace + +TEST(CommunityBootstrap, TheStartersTableVerifiesAsTheReceiverReadsIt) { + const auto s = makeStarters(5); + std::set listed; + for (const auto& st : s) { + listed.insert(st.pub); + } + const std::set initial = keys({200, 201, 202, 203, 204}); + const cs::RoundNumber round = 123'456; + const cs::Bytes payload = cs::makeCommunityTable(round, listed, s.front().priv, 0x01020304); + // count, keys, signature, 4 random bytes + ASSERT_EQ(payload.size(), 1 + 5 * cscrypto::kPublicKeySize + cscrypto::kSignatureSize + sizeof(uint32_t)); + EXPECT_EQ(payload[0], 5); + EXPECT_TRUE(readsAsCommunityTable(payload, round, initial, listed)); + // signed by another listed key than the first: not its starter's table + const cs::Bytes notStarter = cs::makeCommunityTable(round, listed, s[1].priv, 0x01020304); + EXPECT_FALSE(readsAsCommunityTable(notStarter, round, initial, listed)); + // a key changed after signing + cs::Bytes altered = payload; + altered[1 + cscrypto::kPublicKeySize] ^= 1; + EXPECT_FALSE(readsAsCommunityTable(altered, round, initial, listed)); +} + +// The p2p layer drops a broadcast whose bytes it has seen (Host::IsDuplicate, SHA1 of the data): the +// starter's repeat in the same round must differ, and still verify +TEST(CommunityBootstrap, ARepeatInTheSameRoundIsANewMessage) { + const auto s = makeStarters(4); + std::set listed; + for (const auto& st : s) { + listed.insert(st.pub); + } + const std::set initial = keys({200, 201, 202, 203, 204}); + const cs::RoundNumber round = 777; + const cs::Bytes first = cs::makeCommunityTable(round, listed, s.front().priv, 1); + const cs::Bytes repeat = cs::makeCommunityTable(round, listed, s.front().priv, 2); + EXPECT_NE(first, repeat); + EXPECT_NE(cscrypto::calculateHash(first.data(), first.size()), cscrypto::calculateHash(repeat.data(), repeat.size())); + EXPECT_TRUE(readsAsCommunityTable(first, round, initial, listed)); + EXPECT_TRUE(readsAsCommunityTable(repeat, round, initial, listed)); + // only the random bytes differ: the signature is deterministic + const size_t signedEnd = first.size() - sizeof(uint32_t); + EXPECT_TRUE(std::equal(first.begin(), first.begin() + signedEnd, repeat.begin())); +} + +// a table recorded in one round and sent again in another: the round is in the signed bytes +TEST(CommunityBootstrap, ATableReplayedFromAnotherRoundDoesNotVerify) { + const auto s = makeStarters(4); + std::set listed; + for (const auto& st : s) { + listed.insert(st.pub); + } + const std::set initial = keys({200, 201, 202, 203, 204}); + const cs::Bytes payload = cs::makeCommunityTable(1000, listed, s.front().priv, 42); + EXPECT_TRUE(readsAsCommunityTable(payload, 1000, initial, listed)); + EXPECT_FALSE(readsAsCommunityTable(payload, 1001, initial, listed)); + EXPECT_FALSE(readsAsCommunityTable(payload, 999, initial, listed)); +} + +namespace { +CommunityBootstrapData parse(const std::string& ini) { + std::istringstream in(ini); + boost::property_tree::ptree tree; + boost::property_tree::read_ini(in, tree); + return Config::parseCommunityBootstrapData(tree); +} + +const char* const kKeyA = "3rUevsW5xfob6qDxWMDFwwTQCq39SYhzstuyfUGSDvF2"; +const char* const kKeyB = "ESWr7ZThCmP2DVNNgrcsCvZZ2rrhDnFRW8oUFEpKpgdD"; + +cs::PublicKey decoded(const char* text) { + std::vector bytes; + EXPECT_TRUE(DecodeBase58(text, bytes)); + cs::PublicKey result{}; + std::copy(bytes.begin(), bytes.end(), result.begin()); + return result; +} +} // namespace + +TEST(CommunityBootstrapConfig, OffWithoutTheSection) { + const auto c = parse("[params]\nipv6=false\n"); + EXPECT_EQ(c, CommunityBootstrapData{}); + EXPECT_TRUE(c.confidants.empty()); + EXPECT_EQ(c.stallMinutes, 30u); +} + +TEST(CommunityBootstrapConfig, KeysAndStallMinutes) { + const auto c = parse(std::string("[community_bootstrap]\nconfidants=") + kKeyA + ", " + kKeyB + "," + kKeyA + "\nstall_minutes=45\n"); + ASSERT_EQ(c.confidants.size(), 2u); + EXPECT_EQ(c.confidants[0], decoded(kKeyA)); + EXPECT_EQ(c.confidants[1], decoded(kKeyB)); + EXPECT_EQ(c.stallMinutes, 45u); +} + +TEST(CommunityBootstrapConfig, BadValuesAreLeftOut) { + const auto c = parse(std::string("[community_bootstrap]\nconfidants=notakey ") + kKeyB + "\nstall_minutes=0\n"); + ASSERT_EQ(c.confidants.size(), 1u); + EXPECT_EQ(c.confidants[0], decoded(kKeyB)); + EXPECT_EQ(c.stallMinutes, 30u); + EXPECT_EQ(parse("[community_bootstrap]\nstall_minutes=10081\n").stallMinutes, 30u); + EXPECT_EQ(parse("[community_bootstrap]\nstall_minutes=5m\n").stallMinutes, 30u); + EXPECT_EQ(parse("[community_bootstrap]\nstall_minutes=1\n").stallMinutes, 1u); + // an empty list keeps it off + EXPECT_TRUE(parse("[community_bootstrap]\nconfidants=\n").confidants.empty()); +}