From 8acbd30de60e61f21486433f988cba7c53235fc8 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Tue, 17 Nov 2026 15:51:39 +0100 Subject: [PATCH 01/22] ci: official-stop scenarios: every official node stopped at once OFFICIAL_STOP_AT (node_alt_ref=official): the official even nodes, with more stake (OFFICIAL_FUND), are killed in a round they hold the majority of, stay down OFFICIAL_STOP_MINUTES and start again. COMMUNITY_BOOTSTRAP=none expects the network to stay stalled meanwhile; =trusted lists every community key in the community nodes' own trusted.txt and expects them to restart it. Every block is then compared across all nodes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 14 +++ ci/testnet/fund.py | 6 +- ci/testnet/run.sh | 169 ++++++++++++++++++++++++++++++- 3 files changed, 186 insertions(+), 3 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index a13751d86..14fd86475 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -497,6 +497,20 @@ jobs: - name: bootstrap-livelock-rewards env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 REWARD_ROUND=100 LOAD_SECONDS=2400 TARGET_SEQ=3000 manual: true + # if all official nodes become unavailable at once (node_alt_ref=official; run by name): 5 + # official even nodes with more stake hold the trusted seats, 5 community odd nodes; n1-n5 are + # the genesis initial confidants. They are killed in a round they hold the majority of, stay + # down 5 minutes and start again. Without extra keys the network must stay stalled until + # they are back; with every community key in the community nodes' own trusted.txt the + # community nodes must restart it, and the official nodes must then take the same blocks + - name: official-stop-stalled + env: NODES=10 TRUSTED_NODES=5 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=none RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true + - name: official-stop-trusted + env: NODES=10 TRUSTED_NODES=5 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=trusted RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill official: 'no' # needs a feature of this fork on an even node diff --git a/ci/testnet/fund.py b/ci/testnet/fund.py index 5622ae66c..bc81ecc55 100644 --- a/ci/testnet/fund.py +++ b/ci/testnet/fund.py @@ -211,6 +211,8 @@ def main(): parser.add_argument("--amount", type=int, default=60000) parser.add_argument("--low-stake-nodes", type=int, default=0) parser.add_argument("--low-amount", type=int, default=1000) + # the even nodes' amount (0 = --amount): with official even nodes, more stake takes them the seats + parser.add_argument("--even-amount", type=int, default=0) parser.add_argument("--load-seconds", type=int, default=0) parser.add_argument("--dpos-at", type=int, default=0) parser.add_argument("--reward-round", type=int, default=0) @@ -228,7 +230,9 @@ def main(): failures = 0 def funded(i): - return args.low_amount if i > len(nodes) - args.low_stake_nodes else args.amount + if i > len(nodes) - args.low_stake_nodes: + return args.low_amount + return args.even_amount if args.even_amount and i % 2 == 0 else args.amount for i, node in enumerate(nodes, 1): ok, message = send(client, master, node.pk, funded(i)) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index b3cb3931e..372ea99ca 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -197,6 +197,16 @@ # state digest (needs sudo). With DISK_FULL_STOP=1 the node gets SIGTERM halfway through the # full-disk period and must exit within 120 s, as an operator would stop it # +# OFFICIAL_STOP_AT (default 0 = off, needs BIN_ALT_OFFICIAL=1): at that sequence, in a round whose +# table the official (even) nodes hold the majority of, every official node is killed at once; +# they stay down OFFICIAL_STOP_MINUTES (default 5) and are started again, and must end on the +# same blocks as the community nodes (every block compared). OFFICIAL_FUND (FUND amount of the +# even nodes, default the others' 60000): more stake than the community nodes, so that the +# official nodes take the trusted seats as on mainnet. COMMUNITY_BOOTSTRAP says what the +# community nodes may do meanwhile: none (default) - nothing, the network must stay stalled +# until the official nodes are back; trusted - their own trusted.txt also lists every community +# node, the network must restart without them +# # Checks: every node writes blocks up to TARGET_SEQ; a node stopped and restarted (quick start # from its caches) catches up; all nodes report the same STATE DIGEST at every common sequence. set -euo pipefail @@ -296,6 +306,11 @@ TERM_KILL_NODE=${TERM_KILL_NODE:-0} TERM_KILL_TIMES=${TERM_KILL_TIMES:-4} TERM_KILL_FROM=${TERM_KILL_FROM:-150} CRASH_ALL_AT=${CRASH_ALL_AT:-0} +OFFICIAL_STOP_AT=${OFFICIAL_STOP_AT:-0} +OFFICIAL_STOP_MINUTES=${OFFICIAL_STOP_MINUTES:-5} +OFFICIAL_FUND=${OFFICIAL_FUND:-0} +COMMUNITY_BOOTSTRAP=${COMMUNITY_BOOTSTRAP:-none} +case "$COMMUNITY_BOOTSTRAP" in none|trusted) ;; *) echo "COMMUNITY_BOOTSTRAP must be none or trusted, got '$COMMUNITY_BOOTSTRAP'"; exit 1 ;; esac JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} @@ -531,6 +546,14 @@ if [ "$BIN_ALT_OFFICIAL" = 1 ]; then [ -z "$DB_BACKEND" ] || echo "mixed nodes: the official nodes run BerkeleyDB whatever DB_BACKEND ($DB_BACKEND) says" fi +# every official node stopped at once: the community nodes alone are left (see OFFICIAL_STOP_AT above) +if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then + [ "$BIN_ALT_OFFICIAL" = 1 ] || { echo "FAIL: OFFICIAL_STOP_AT needs official even nodes (BIN_ALT_OFFICIAL=1)"; exit 1; } + [ "$CRASH_ALL_AT" -eq 0 ] && [ "$DPOS_AT" -eq 0 ] && [ "$JOIN_NODE" -eq 0 ] \ + || { echo "FAIL: OFFICIAL_STOP_AT runs without CRASH_ALL_AT, DPOS_AT and a joining node"; exit 1; } + # the community nodes that could start a bootstrap with the extra keys: MinTrustedNodes is 4 + [ $(( (NODES + 1) / 2 )) -ge 5 ] || { echo "FAIL: OFFICIAL_STOP_AT needs at least 5 community (odd) nodes, NODES >= 9"; exit 1; } +fi mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$ALL_NODES" "$WORK") if [ "$FUND" = "1" ]; then @@ -724,6 +747,18 @@ Format="[%TimeStamp%] %Severity% %Message%" CFG done +# COMMUNITY_BOOTSTRAP=trusted: each community node lists every community node among its initial +# confidants, next to the genesis ones; the official nodes keep the genesis list +if [ "$OFFICIAL_STOP_AT" -gt 0 ] && [ "$COMMUNITY_BOOTSTRAP" = trusted ]; then + for i in $(seq 1 "$NODES"); do + official_node "$i" && continue + for j in $(seq $((TRUSTED_NODES + 1)) "$NODES"); do + official_node "$j" || echo "${KEYS[$((j - 1))]}" >> "$WORK/n$i/trusted.txt" + done + done + echo "official stop: the community nodes list $(grep -c . "$WORK/n1/trusted.txt") initial confidants, the official nodes $TRUSTED_NODES" +fi + # test-build hooks of a forged or corrupt snapshot source, and the joiner's list of serving nodes [ "$FORGE_NODE" -eq 0 ] || : > "$WORK/n$FORGE_NODE/testnet-forge-snapshot" [ "$CORRUPT_NODE" -eq 0 ] || : > "$WORK/n$CORRUPT_NODE/testnet-corrupt-pieces" @@ -983,6 +1018,107 @@ backfill_node_started() { done } +# OFFICIAL_STOP_AT: the community and official nodes, the highest sequence of the community nodes +off_done=0 # 1 once the official nodes were stopped, 2 once they were started again +off_fail="" # what went against the expectation of COMMUNITY_BOOTSTRAP +off_restart_t=0 +off_restart_seq=0 +off_caught=0 +COMMUNITY_NODES="" +OFFICIAL_NODES="" +for i in $(seq 1 "$NODES"); do + if official_node "$i"; then OFFICIAL_NODES+="$i "; else COMMUNITY_NODES+="$i "; fi +done +community_max() { + local m=0 s i + for i in $COMMUNITY_NODES; do s=$(seq_of "$i"); s=${s:-0}; [ "$s" -le "$m" ] || m=$s; done + echo "$m" +} +community_min() { + local m=-1 s i + for i in $COMMUNITY_NODES; do s=$(seq_of "$i"); s=${s:-0}; { [ "$m" -ge 0 ] && [ "$s" -ge "$m" ]; } || m=$s; done + echo "$m" +} +# the header of node i's latest round, "R-1'234.0 TRUSTED" or "R-1'234.0 NORMAL" +round_header() { + tail -n 3000 "$WORK/n$1/node.log" 2>/dev/null | grep -a ' R-[0-9'"'"']*\.[0-9]* \(TRUSTED\|NORMAL\)' | tail -1 \ + | sed -E 's/.* (R-[0-9'"'"']+\.[0-9]+) (TRUSTED|NORMAL).*/\1 \2/' || true +} +bootstrap_lines() { + local n=0 i + for i in $COMMUNITY_NODES; do n=$((n + $(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true))); done + echo "$n" +} +official_stop() { + local t0 round official community h r i hdr seen + # wait for a round whose table the official nodes hold the majority of (seen by every node): without + # them that round cannot be decided (TrustedStage3State::take_urgent_decision) + t0=$(date +%s) + seen="" + while [ $(( $(date +%s) - t0 )) -lt 300 ]; do + round=""; official=0; community=0 + for i in $(seq 1 "$NODES"); do + hdr=$(round_header "$i") + r=${hdr%% *} + [ -n "$r" ] || { round=""; break; } + if [ -z "$round" ]; then round=$r; elif [ "$r" != "$round" ]; then round=""; break; fi + if [ "${hdr##* }" = TRUSTED ]; then + if official_node "$i"; then official=$((official + 1)); else community=$((community + 1)); fi + fi + done + if [ -n "$round" ] && [ $((official + community)) -gt 0 ] && [ $((community * 2)) -lt $((official + community + 1)) ]; then + seen="$round: $official official and $community community confidants" + break + fi + sleep 0.2 + done + [ -n "$seen" ] || seen="no round with an official majority seen in 300 s, stopped anyway" + for i in $OFFICIAL_NODES; do kill -KILL "${PIDS[$i]}" 2>/dev/null || true; done + for i in $OFFICIAL_NODES; do wait "${PIDS[$i]}" 2>/dev/null || true; REPORTED_DOWN[$i]=1; done + local t_kill s0 b0 first_t=0 first_seq=0 c end + t_kill=$(date +%s) + s0=$(community_max) + b0=$(bootstrap_lines) + echo "official stop: killed $(printf 'n%s ' $OFFICIAL_NODES)at once (round $seen); community head $s0, COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" + end=$(( t_kill + OFFICIAL_STOP_MINUTES * 60 )) + while [ "$(date +%s)" -lt "$end" ]; do + sleep 2 + c=$(community_max) + if [ "$first_t" -eq 0 ] && [ "$c" -gt "$s0" ]; then + first_t=$(date +%s); first_seq=$c + echo "official stop: community block $c $((first_t - t_kill)) s after the stop" + fi + done + c=$(community_max) + echo "official stop: after $OFFICIAL_STOP_MINUTES min the community head is $c ($((c - s0)) blocks since the stop), $(( $(bootstrap_lines) - b0 )) 'Bootstrap on' lines on the community nodes" + for i in $COMMUNITY_NODES; do + { grep -a 'NODE> Bootstrap on\|Bootstrap available nodes' "$WORK/n$i/node.log" || true; } | tail -2 | sed "s/^/official stop: n$i: /" + done + if [ "$COMMUNITY_BOOTSTRAP" = none ]; then + [ "$first_t" -eq 0 ] || off_fail="the community nodes wrote blocks without the official nodes and without the extra keys" + else + if [ "$first_t" -eq 0 ]; then + off_fail="the community nodes did not restart the network in $OFFICIAL_STOP_MINUTES min" + else + echo "OFFICIAL STOP RESULT: restart after $((first_t - t_kill)) s (first block $first_seq), $((c - s0)) blocks in $OFFICIAL_STOP_MINUTES min" + fi + fi + # the community nodes agree on every block they hold (all of them, compared over the API) + local ports="" head + head=$(community_min) + for i in $COMMUNITY_NODES; do ports="$ports $i:$(api_port "$i")"; done + if (cd "$HERE" && python3 blockhashes.py --head "$head" --step 1 $ports); then + echo "OFFICIAL STOP RESULT: the community nodes hold the same blocks up to $head" + else + off_fail="${off_fail:+$off_fail; }the community nodes hold different blocks while the official nodes were down" + fi + off_restart_seq=$c + for i in $OFFICIAL_NODES; do start_node "$i"; REPORTED_DOWN[$i]=""; done + off_restart_t=$(date +%s) + off_done=2 + echo "official stop: $(printf 'n%s ' $OFFICIAL_NODES)started again, community head $c" +} + while :; do sleep 20 line="t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) ))s" @@ -1137,6 +1273,20 @@ while :; do echo "crash test: network restarted" fi + if [ "$OFFICIAL_STOP_AT" -gt 0 ] && [ "$off_done" -eq 0 ] && [ "$min" -ge "$OFFICIAL_STOP_AT" ]; then + off_done=1 + official_stop + fi + if [ "$off_done" -eq 2 ] && [ "$off_caught" -eq 0 ]; then + lowest=-1 + for i in $OFFICIAL_NODES; do s=$(seq_of "$i"); s=${s:-0}; { [ "$lowest" -ge 0 ] && [ "$s" -ge "$lowest" ]; } || lowest=$s; done + if [ "$lowest" -ge "$off_restart_seq" ]; then + off_caught=$(( $(date +%s) - off_restart_t )) + [ "$off_caught" -gt 0 ] || off_caught=1 + echo "OFFICIAL STOP RESULT: the official nodes reached the community head of their restart ($off_restart_seq) in about $off_caught s" + fi + fi + if [ "$RESTART_AT" -gt 0 ] && [ "$restarted" -eq 0 ] && [ "$min" -ge "$RESTART_AT" ] \ && { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/contracts-ready" ]; }; then echo "restart test: stopping $(printf 'n%s ' $RESTART_NODES)at sequence $min" @@ -1598,7 +1748,7 @@ PY echo "funding nodes from the genesis key, then $LOAD_SECONDS s of transfers" python3 "$HERE/fund.py" "$WORK" "$NODES" "$MASTER_SEED" --load-seconds "$LOAD_SECONDS" \ --dpos-at "$DPOS_AT" --reward-round "$REWARD_ROUND" --delegation-seconds "$DELEGATION_SECONDS" --min-stake "$MIN_STAKE" --syncro-round "$SYNCRO_ROUND" \ - --low-stake-nodes "$LOW_STAKE_NODES" \ + --low-stake-nodes "$LOW_STAKE_NODES" --even-amount "$OFFICIAL_FUND" \ $([ "$DELEGATE" = "1" ] && echo --delegations) > "$WORK/fund.log" 2>&1 & FUND_PID=$! fi @@ -2247,6 +2397,18 @@ sys.exit(0 if ok else 1) PY fi +if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then + [ "$off_done" -eq 2 ] || off_fail="${off_fail:+$off_fail; }the official nodes were never stopped" + [ "$off_caught" -gt 0 ] || off_fail="${off_fail:+$off_fail; }the official nodes never reached the community head of their restart" + # what the official nodes made of the blocks written without them + for i in $OFFICIAL_NODES; do + echo "official stop: n$i logged $(grep -ac "Pool Signatures aren't valid" "$WORK/n$i/node.log" || true) invalid pool signatures," \ + "$(grep -ac 'confirmations are not OK' "$WORK/n$i/node.log" || true) refused confirmations," \ + "$(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true) bootstraps" + done + if [ -n "$off_fail" ]; then echo "FAIL: official stop: $off_fail"; exit 1; fi +fi + # official nodes: the same block hashes as n1 (every 100th sequence up to the lowest head) if [ "$BIN_ALT_OFFICIAL" = 1 ]; then ports="" @@ -2258,7 +2420,8 @@ if [ "$BIN_ALT_OFFICIAL" = 1 ]; then done # pruned nodes, and a node that joined from a snapshot, lack the oldest blocks partial=$(for i in $PRUNED_NODES $({ [ "$JOIN_FETCH" = p2p ] || [ "$SNAPSHOT_AT" -gt 0 ]; } && echo "$JOIN_NODE" $EXTRA_JOINERS); do echo "$i"; done | sort -u | paste -sd, -) - (cd "$HERE" && python3 blockhashes.py --head "$head" --partial "$partial" $ports) || { echo "FAIL: official and community nodes hold different blocks"; exit 1; } + (cd "$HERE" && python3 blockhashes.py --head "$head" --partial "$partial" --step "$([ "$OFFICIAL_STOP_AT" -gt 0 ] && echo 1 || echo 100)" $ports) \ + || { echo "FAIL: official and community nodes hold different blocks"; exit 1; } fi # two blocks of one sequence: a node kept its own block while the round went on on the writer's @@ -2270,6 +2433,8 @@ if [ -n "$forks" ]; then exit 1 fi +[ "$OFFICIAL_STOP_AT" -eq 0 ] || echo "OFFICIAL STOP RESULT: one chain: every node holds the same blocks (all compared), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" + # a network still restarting at the end is stuck, even if it reached the target before (a bootstrap # round, R-x.1 and up, that never gives way to R-(x+1).0): the last three round headers of n1 stuck=$(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -3 | grep -vc '\.0 ' || true) From a5e270082fc09d6e9783d88f747e6ffe956ddf14 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Tue, 17 Nov 2026 16:03:49 +0100 Subject: [PATCH 02/22] csnode: community bootstrap fallback, off by default [community_bootstrap] confidants= lists community keys; empty (the default) keeps the node as before. After stall_minutes (default 30) without a block, a round package or a regular bootstrap table, and with no neighbour ahead, a listed or initial node may start, and any node with the list may take, a bootstrap table of initial and listed keys that holds more than half of the list, signed by its starter instead of the project. Round window and stall checks still apply; one such table per half round_elapse_time unless heavier; regular tables keep their own weight and always replace it. Official nodes and nodes without the list drop such tables. Co-Authored-By: Claude Opus 5.5 --- client/config/config.cpp | 59 +++++ client/config/config.hpp | 19 ++ csnode/include/csnode/communitybootstrap.hpp | 153 ++++++++++++ csnode/include/csnode/node.hpp | 11 + csnode/src/node.cpp | 150 ++++++++++-- tests/csnode/communitybootstrap_tests.cpp | 241 +++++++++++++++++++ 6 files changed, 613 insertions(+), 20 deletions(-) create mode 100644 csnode/include/csnode/communitybootstrap.hpp create mode 100644 tests/csnode/communitybootstrap_tests.cpp diff --git a/client/config/config.cpp b/client/config/config.cpp index 1d2050636..edbac280a 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -47,6 +47,7 @@ const std::string BLOCK_NAME_HOST_ADDRESS = "host_address"; const std::string BLOCK_NAME_POOL_SYNC = "pool_sync"; const std::string BLOCK_NAME_STORAGE = "storage"; const std::string BLOCK_NAME_SNAPSHOT = "snapshot"; +const std::string BLOCK_NAME_COMMUNITY_BOOTSTRAP = "community_bootstrap"; const std::string BLOCK_NAME_API = "api"; const std::string BLOCK_NAME_CONVEYER = "conveyer"; const std::string BLOCK_NAME_EVENT_REPORTER = "event_report"; @@ -1084,6 +1085,7 @@ Config Config::readFromFile(const std::string& fileName) { result.readPoolSynchronizerData(config); result.readStorageData(config); result.snapshotData_ = parseSnapshotData(config); + result.communityBootstrapData_ = parseCommunityBootstrapData(config); result.readApiData(config); result.readConveyerData(config); result.readEventsReportData(config); @@ -1355,6 +1357,54 @@ SnapshotData Config::parseSnapshotData(const boost::property_tree::ptree& config return result; } +CommunityBootstrapData Config::parseCommunityBootstrapData(const boost::property_tree::ptree& config) { + CommunityBootstrapData result; + if (!config.count(BLOCK_NAME_COMMUNITY_BOOTSTRAP)) { + return result; + } + const boost::property_tree::ptree& data = config.get_child(BLOCK_NAME_COMMUNITY_BOOTSTRAP); + static const std::set known = {"confidants", "stall_minutes"}; + for (const auto& [key, value] : data) { + if (!known.count(key)) { + cswarning() << "config: [community_bootstrap] " << key << " is not a setting of this version, ignored"; + } + } + if (data.count("confidants")) { + std::vector keys; + boost::split(keys, data.get("confidants"), boost::is_any_of(", \t"), boost::token_compress_on); + for (const auto& key : keys) { + if (key.empty()) { + continue; + } + std::vector bytes; + if (!DecodeBase58(key, bytes) || bytes.size() != cscrypto::kPublicKeySize) { + cswarning() << "config: [community_bootstrap] confidants: " << key << " is not a node key, ignored"; + continue; + } + cs::PublicKey publicKey; + std::copy(bytes.begin(), bytes.end(), publicKey.begin()); + if (std::find(result.confidants.begin(), result.confidants.end(), publicKey) == result.confidants.end()) { + result.confidants.push_back(publicKey); + } + } + } + if (data.count("stall_minutes")) { + const auto text = data.get("stall_minutes"); + try { + size_t pos = 0; + const unsigned long long value = std::stoull(text, &pos); + if (pos != text.size() || value < 1 || value > 10'080) { + throw std::out_of_range(text); + } + result.stallMinutes = static_cast(value); + } + catch (const std::exception&) { + cswarning() << "config: [community_bootstrap] stall_minutes=" << text << " is not 1..10080, using " << result.stallMinutes; + } + } + return result; +} + void Config::readApiData(const boost::property_tree::ptree& config) { if (!config.count(BLOCK_NAME_API)) { return; @@ -1534,6 +1584,14 @@ bool operator!=(const SnapshotData& lhs, const SnapshotData& rhs) { return !(lhs == rhs); } +bool operator==(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs) { + return lhs.confidants == rhs.confidants && lhs.stallMinutes == rhs.stallMinutes; +} + +bool operator!=(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs) { + return !(lhs == rhs); +} + bool operator==(const ConveyerData& lhs, const ConveyerData& rhs) { return lhs.maxPacketLifeTime == rhs.maxPacketLifeTime; } @@ -1583,6 +1641,7 @@ bool operator==(const Config& lhs, const Config& rhs) { lhs.poolSyncData_ == rhs.poolSyncData_ && lhs.apiData_ == rhs.apiData_ && lhs.snapshotData_ == rhs.snapshotData_ && + lhs.communityBootstrapData_ == rhs.communityBootstrapData_ && lhs.alwaysExecuteContracts_ == rhs.alwaysExecuteContracts_ && lhs.syncContractExecution_ == rhs.syncContractExecution_ && lhs.compatibleVersion_ == rhs.compatibleVersion_ && diff --git a/client/config/config.hpp b/client/config/config.hpp index 55c762f50..ae4fa698b 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -98,6 +98,15 @@ struct SnapshotData { size_t fetchRateKbps = 0; // 0: unlimited }; +// [community_bootstrap]: a liveness fallback, off unless confidants lists keys +// (csnode/include/csnode/communitybootstrap.hpp). After stall_minutes without a block, a round or a +// regular bootstrap, the node may start and take a bootstrap table of initial confidants and these +// keys, signed by its starter instead of the project. Read again by the config observer. +struct CommunityBootstrapData { + std::vector confidants; // community keys; empty: the fallback is off + size_t stallMinutes = 30; // minutes without progress before it may act, 1..10080 +}; + struct ApiData { // on by default: uint16_t port = 9090; @@ -252,6 +261,10 @@ class Config { return storageData_; } + const CommunityBootstrapData& getCommunityBootstrapSettings() const { + return communityBootstrapData_; + } + const ApiData& getApiSettings() const { return apiData_; } @@ -393,6 +406,8 @@ class Config { public: // the [snapshot] section, defaults for missing keys, a warning and the default for a bad value static SnapshotData parseSnapshotData(const boost::property_tree::ptree& config); + // the [community_bootstrap] section, off when it is missing or lists no valid key + static CommunityBootstrapData parseCommunityBootstrapData(const boost::property_tree::ptree& config); // the backfill keys of a [storage] section (backfill, backfill_batch, backfill_rate) static void readBackfillSettings(const boost::property_tree::ptree& storage, StorageData& data); private: @@ -448,6 +463,7 @@ class Config { PoolSyncData poolSyncData_; StorageData storageData_; SnapshotData snapshotData_; + CommunityBootstrapData communityBootstrapData_; ApiData apiData_; bool showBalanceChange_ = false; @@ -498,6 +514,9 @@ bool operator!=(const ApiData& lhs, const ApiData& rhs); bool operator==(const SnapshotData& lhs, const SnapshotData& rhs); bool operator!=(const SnapshotData& lhs, const SnapshotData& rhs); +bool operator==(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs); +bool operator!=(const CommunityBootstrapData& lhs, const CommunityBootstrapData& rhs); + bool operator==(const ConveyerData& lhs, const ConveyerData& rhs); bool operator!=(const ConveyerData& lhs, const ConveyerData& rhs); diff --git a/csnode/include/csnode/communitybootstrap.hpp b/csnode/include/csnode/communitybootstrap.hpp new file mode 100644 index 000000000..a6a4cec2c --- /dev/null +++ b/csnode/include/csnode/communitybootstrap.hpp @@ -0,0 +1,153 @@ +#ifndef COMMUNITYBOOTSTRAP_HPP +#define COMMUNITYBOOTSTRAP_HPP + +#include +#include +#include +#include + +#include + +namespace cs { + +// Community bootstrap: a liveness fallback, off by default ([community_bootstrap] in config.ini). +// +// Only the initial confidants (initialConfidants_) start a bootstrap, and a table with any other key +// needs the project's signature. If the initial confidants that are still running are too few, the +// network stays stalled whatever the other nodes do. The fallback lets a node that lists a set of +// community keys start and take, after a long stall only, a bootstrap table of keys from the initial +// confidants and that list, signed by its starter instead of the project. +// +// What keeps it out of normal operation: +// - the node itself has seen no block, no round of a round package and no regular bootstrap table +// for stall_minutes, and no neighbour reports a block above its own (that node would sync); +// - the table: 4 to 25 keys in ascending order, each an initial confidant or listed, at least one +// of them listed and not an initial confidant (else it is a regular table), more than half of the +// list in it (two parts of a split community cannot both restart), and signed by its first key, the +// starter; +// - the round window (#71) and the stall check (#70) of every unsigned table still apply; +// - a node takes one such table per half round_elapse_time, or a heavier one at once, and a regular +// table always replaces it (it goes through the unchanged path, whose weight it does not touch). +// Official nodes, and community nodes without the list, drop such a table: one of its keys is not an +// initial confidant and its signature is not the project's. + +constexpr size_t kCommunityMinTable = 4; // Consensus::MinTrustedNodes +constexpr size_t kCommunityMaxTable = 25; // Consensus::MaxTrustedNodes + +using CommunityWeight = uint64_t; + +// the starter's table is a std::set, sent in order: each key above the previous one +inline bool isAscending(const std::vector& table) { + for (size_t i = 1; i < table.size(); ++i) { + if (!(table[i - 1] < table[i])) { + return false; + } + } + return true; +} + +inline bool isCommunityTable(const std::set& initial, const std::set& community, + const std::vector& table) { + if (community.empty() || table.size() < kCommunityMinTable || table.size() > kCommunityMaxTable || !isAscending(table)) { + return false; + } + size_t listed = 0; + bool outsideInitial = false; + for (const auto& key : table) { + const bool isInitial = initial.count(key) > 0; + const bool isListed = community.count(key) > 0; + if (!isInitial && !isListed) { + return false; + } + outsideInitial = outsideInitial || !isInitial; + listed += isListed ? 1 : 0; + } + return outsideInitial && listed * 2 > community.size(); +} + +// More keys weigh more; among tables of as many keys, the one with the keys earlier in the sorted +// union of the two sets. Only compared between community tables: the regular weight is not involved +constexpr size_t kCommunityWeightPositions = 58; + +inline CommunityWeight communityTableWeight(const std::set& initial, const std::set& community, + const std::vector& table) { + std::set all(initial); + all.insert(community.begin(), community.end()); + CommunityWeight weight = CommunityWeight(table.size()) << kCommunityWeightPositions; + size_t index = 0; + for (const auto& key : all) { + if (index >= kCommunityWeightPositions) { + break; + } + for (const auto& member : table) { + if (member == key) { + weight += CommunityWeight(1) << (kCommunityWeightPositions - 1 - index); + break; + } + } + ++index; + } + return weight; +} + +// The fallback's state on one node; times in ms of a steady clock +class CommunityBootstrapGate { +public: + // a block written, a round of a round package, a regular bootstrap table: the network is alive. + // The node's start counts too, so that it has watched for stall_minutes before it may act + void onProgress(uint64_t nowMs) { + lastProgressMs_ = nowMs; + taken_ = false; + lastWeight_ = 0; + lastTakenMs_ = 0; + } + + // the node's last block, read whenever the fallback is considered: a new one is progress + void onLastSequence(Sequence seq, uint64_t nowMs) { + if (seq != lastSeq_) { + lastSeq_ = seq; + onProgress(nowMs); + } + } + + uint64_t msSinceProgress(uint64_t nowMs) const { + return nowMs > lastProgressMs_ ? nowMs - lastProgressMs_ : 0; + } + + bool isStalled(uint64_t nowMs, uint64_t stallMs) const { + return stallMs > 0 && msSinceProgress(nowMs) >= stallMs; + } + + // a community table of that weight may be started or taken now: the first one in this stall, one + // heavier than the last, or any once half a round_elapse_time has passed since the last (a starter + // repeats its table every round_elapse_time) + bool mayTake(CommunityWeight weight, uint64_t nowMs, uint64_t roundElapseMs) const { + if (!taken_ || weight > lastWeight_) { + return true; + } + return nowMs >= lastTakenMs_ && nowMs - lastTakenMs_ >= roundElapseMs / 2; + } + + void onTaken(CommunityWeight weight, uint64_t nowMs) { + taken_ = true; + lastWeight_ = weight; + lastTakenMs_ = nowMs; + } + +private: + uint64_t lastProgressMs_ = 0; + Sequence lastSeq_ = 0; + bool taken_ = false; + CommunityWeight lastWeight_ = 0; + uint64_t lastTakenMs_ = 0; +}; + +// the fallback may act on this node: a list configured, the node stalled for stall_minutes, and no +// neighbour ahead of it +inline bool isCommunityFallbackOpen(bool configured, bool stalled, bool neighbourAhead) { + return configured && stalled && !neighbourAhead; +} + +} // namespace cs + +#endif // COMMUNITYBOOTSTRAP_HPP diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index ceb1fd570..17b0ad542 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -446,6 +447,13 @@ public slots: void deepBlockValidation(const csdb::Pool& block, bool* shouldStop); void sendBlockAlarmSignal(cs::Sequence seq); void onRoundTimeElapsed(); + // the community bootstrap fallback ([community_bootstrap], off by default): see communitybootstrap.hpp + std::set communityConfidants() const; + // progress seen (a round package, a regular bootstrap table): the fallback closes again + void onCommunityProgress(); + // the fallback may act now; also notes a new last block as progress + bool isCommunityFallbackOpen(); + bool takeCommunityTable(const cs::PublicKeys& table, bool own); void onNeighbourAdded(const cs::PublicKey& neighbour, cs::Sequence lastSeq, cs::RoundNumber lastRound); void onNeighbourRemoved(const cs::PublicKey& neighbour); @@ -682,6 +690,9 @@ public slots: std::set initialConfidants_; bool isBootstrapRound_ = false; + // guarded: read by the round elapse handler and the processor thread + std::mutex communityMutex_; + cs::CommunityBootstrapGate communityGate_; cs::CachesSerializationManager cachesSerializationManager_; size_t notInRound_ = 0; diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index f903688e5..d41ca9700 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -1083,6 +1083,55 @@ cs::BootstrapWeight Node::calculateBootStrapWeight(const cs::PublicKeys& confida } +std::set Node::communityConfidants() const { + const auto& keys = cs::ConfigHolder::instance().config()->getCommunityBootstrapSettings().confidants; + return std::set(keys.begin(), keys.end()); +} + +void Node::onCommunityProgress() { + std::lock_guard lock(communityMutex_); + communityGate_.onProgress(stat_.msSinceStart()); +} + +bool Node::isCommunityFallbackOpen() { + const auto& settings = cs::ConfigHolder::instance().config()->getCommunityBootstrapSettings(); + const cs::Sequence lastSeq = blockChain_.getLastSeq(); + bool stalled = false; + { + std::lock_guard lock(communityMutex_); + const uint64_t now = stat_.msSinceStart(); + communityGate_.onLastSequence(lastSeq, now); + stalled = communityGate_.isStalled(now, uint64_t(settings.stallMinutes) * 60'000ULL); + } + if (settings.confidants.empty() || !stalled) { + return false; + } + // a neighbour with a later block: this node is behind and syncs, the network is not stalled + bool ahead = false; + transport_->forEachNeighbour([&ahead, lastSeq](const cs::PublicKey&, cs::Sequence seq, cs::RoundNumber) { + if (seq > lastSeq) { + ahead = true; + } + }); + return cs::isCommunityFallbackOpen(true, stalled, ahead); +} + +bool Node::takeCommunityTable(const cs::PublicKeys& table, bool own) { + const cs::CommunityWeight weight = cs::communityTableWeight(initialConfidants_, communityConfidants(), table); + const uint64_t elapse = cs::ConfigHolder::instance().config()->roundElapseTime(); + std::lock_guard lock(communityMutex_); + const uint64_t now = stat_.msSinceStart(); + if (!communityGate_.mayTake(weight, now, elapse)) { + cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "not starting" : "dropping") << " a table of " << table.size() + << " keys: one at least as heavy was taken less than half a round_elapse_time ago"; + return false; + } + communityGate_.onTaken(weight, now); + cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "starting" : "taking") << " a table of " << table.size() + << " keys after " << communityGate_.msSinceProgress(now) / 1000 << " s without a block, a round or a regular bootstrap"; + return true; +} + void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::RoundNumber rNum) { cslog() << "NODE> get Boot strap Round Table #" << rNum; // a table for a round no honest starter would send to this node changes nothing here @@ -1126,6 +1175,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R bool signedTable = false; const size_t signed_subdata_len = 1 + size_t(confSize) * cscrypto::kPublicKeySize; + // a table with a key that is not an initial confidant: signed by the project, or, with the + // community bootstrap fallback open on this node, a community table signed by its starter + bool communityTable = false; if (unknown) { cs::Signature sig; stream >> sig; @@ -1140,12 +1192,23 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R std::copy(ptr, ptr + sizeof(cs::RoundNumber), std::back_inserter(buf)); ptr = payload.data(); std::copy(ptr, ptr + signed_subdata_len, std::back_inserter(buf)); - if (!cscrypto::verifySignature(sig, cs::PacketValidator::getBlockChainKey(), buf.data(), buf.size())) { + if (cscrypto::verifySignature(sig, cs::PacketValidator::getBlockChainKey(), buf.data(), buf.size())) { + currentWeight = cs::kSignedBootstrapWeight; + signedTable = true; + } + else if (cs::isCommunityTable(initialConfidants_, communityConfidants(), confidants) + && cscrypto::verifySignature(sig, confidants.front(), buf.data(), buf.size())) { + if (!isCommunityFallbackOpen()) { + cslog() << kLogPrefix_ << "community bootstrap table #" << rNum << " from " << KeyToBase58(confidants.front()) + << ", but this node has not stalled long enough or a neighbour is ahead, drop"; + return; + } + communityTable = true; + } + else { cswarning() << kLogPrefix_ << "failed to test bootstrap signature, drop"; return; } - currentWeight = cs::kSignedBootstrapWeight; - signedTable = true; } else if (payload.size() == signed_subdata_len + cscrypto::kSignatureSize) { // a table of initial confidants signed through the diag API (Node::bootstrap): the official @@ -1177,7 +1240,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R } } - if (currentWeight <= bootStrapWeight_) { + // a community table is weighed against community tables only (takeCommunityTable): a regular + // table always replaces it + if (!communityTable && currentWeight <= bootStrapWeight_) { cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " with " << confidants.size() << " confidants weighs " << currentWeight << ", not above " << bootStrapWeight_ << ", drop"; return; @@ -1187,11 +1252,20 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R cswarning() << "Bad round table format, ignore"; return; } + if (communityTable) { + if (!takeCommunityTable(confidants, false)) { + return; + } + } + else { + onCommunityProgress(); + } subRound_ = 1; if (!isBootstrapRound_) { isBootstrapRound_ = true; - cslog() << "NODE> Bootstrap on, table of " << confidants.size() << " confidants, weight " << currentWeight; + cslog() << "NODE> Bootstrap on, table of " << confidants.size() << " confidants, " + << (communityTable ? "community table" : "weight " + std::to_string(currentWeight)); } roundTable.confidants = std::move(confidants); @@ -4173,6 +4247,7 @@ void Node::setCurrentRP(const cs::RoundPackage& rp) { void Node::performRoundPackage(cs::RoundPackage& rPackage, const cs::PublicKey& sender, bool updateRound) { csdebug() << __func__; + onCommunityProgress(); // got round package in any way, reset default round table flag confirmationList_.add(rPackage.roundTable().round, false, rPackage.roundTable().confidants, rPackage.poolMetaInfo().realTrustedMask, rPackage.trustedSignatures()); @@ -5352,7 +5427,14 @@ void Node::onRoundTimeElapsed() { } solver_->resetGrayList(); const cs::PublicKey& own_key = solver_->getPublicKey(); - if (initialConfidants_.find(own_key) == initialConfidants_.end()) { + // the community bootstrap fallback, open only after a long stall: the listed keys count as + // initial confidants here, and a table with any of them is a community table (see below) + const bool community = isCommunityFallbackOpen(); + const std::set listed = community ? communityConfidants() : std::set{}; + const auto isStarterKey = [this, &listed](const cs::PublicKey& key) { + return initialConfidants_.count(key) > 0 || listed.count(key) > 0; + }; + if (!isStarterKey(own_key)) { cslog() << "Waiting for next round..."; myLevel_ = Level::Normal; @@ -5374,19 +5456,18 @@ void Node::onRoundTimeElapsed() { const cs::Sequence maxLocalBlock = blockChain_.getLastSeq(); cs::Sequence maxGlobalBlock = maxLocalBlock; - auto callback = [&maxGlobalBlock, &actualConfidants, this] + auto callback = [&maxGlobalBlock, &actualConfidants, &isStarterKey] (const cs::PublicKey& neighbour, cs::Sequence lastSeq, cs::RoundNumber) { - const auto it = initialConfidants_.find(neighbour); - if (it == initialConfidants_.end()) { + if (!isStarterKey(neighbour)) { return; } if (lastSeq > maxGlobalBlock) { maxGlobalBlock = lastSeq; actualConfidants.clear(); - actualConfidants.insert(*it); + actualConfidants.insert(neighbour); } else if (lastSeq == maxGlobalBlock) { - actualConfidants.insert(*it); + actualConfidants.insert(neighbour); } }; @@ -5411,6 +5492,9 @@ void Node::onRoundTimeElapsed() { << initialConfidants_.size() << ", start lookup..."; transport_->addToNeighbours(initialConfidants_); + if (community) { + transport_->addToNeighbours(listed); + } } if (actualConfidants.size() < Consensus::MinTrustedNodes) { @@ -5434,10 +5518,26 @@ void Node::onRoundTimeElapsed() { // do not increment, only "mark" default round start subRound_ = 1; + // a table with a key that is not an initial confidant: only as a community table (no regular + // starter would send it, and only nodes with the fallback open take it) + cs::PublicKeys confs(actualConfidants.cbegin(), actualConfidants.cend()); + const bool communityTable = std::any_of(confs.cbegin(), confs.cend(), [this](const cs::PublicKey& key) { + return initialConfidants_.count(key) == 0; + }); + if (communityTable && !cs::isCommunityTable(initialConfidants_, listed, confs)) { + cslog() << "COMMUNITY BOOTSTRAP: " << confs.size() << " confidants with max sequence " << maxGlobalBlock + << " do not hold more than half of the " << listed.size() << " listed keys, wait"; + return; + } + if (*actualConfidants.cbegin() == own_key) { cslog() << "Starting round..."; + if (communityTable && !takeCommunityTable(confs, true)) { + return; + } + cs::Bytes bin; cs::ODataStream out(bin); out << uint8_t(actualConfidants.size()); @@ -5445,17 +5545,26 @@ void Node::onRoundTimeElapsed() { for (const auto& item : actualConfidants) { out << item; } - cs::PublicKeys confs; - for (auto it : actualConfidants) { - confs.push_back(it); + if (communityTable) { + // signed by this node, the starter: the round, the count and the keys, as a table + // signed by the project; its weight stays out of bootStrapWeight_ + const cs::RoundNumber round = roundPackageCache_.back().roundTable().round; + cs::Bytes signedPart; + auto* ptr = reinterpret_cast(&round); + std::copy(ptr, ptr + sizeof(cs::RoundNumber), std::back_inserter(signedPart)); + std::copy(bin.cbegin(), bin.cend(), std::back_inserter(signedPart)); + out << cscrypto::generateSignature(solver_->getPrivateKey(), signedPart.data(), signedPart.size()); } - cs::BootstrapWeight currentWeight = calculateBootStrapWeight(confs); - if (currentWeight > bootStrapWeight_) { - bootStrapWeight_ = currentWeight; + else { + cs::BootstrapWeight currentWeight = calculateBootStrapWeight(confs); + if (currentWeight > bootStrapWeight_) { + bootStrapWeight_ = currentWeight; + } + // when we try to start rounds several times, we will not send duplicates + auto random = std::random_device{}(); + out << random; + onCommunityProgress(); } - // when we try to start rounds several times, we will not send duplicates - auto random = std::random_device{}(); - out << random; auto& conveyer = cs::Conveyer::instance(); conveyer.updateRoundTable(roundPackageCache_.back().roundTable().round, roundPackageCache_.back().roundTable()); @@ -5519,6 +5628,7 @@ bool Node::bootstrap(const cs::Bytes& bytes, cs::RoundNumber round) { if (currentWeight > bootStrapWeight_) { bootStrapWeight_ = currentWeight; } + onCommunityProgress(); sendBroadcast(MsgTypes::BootstrapTable, roundPackageCache_.back().roundTable().round, bytes); if (!isBootstrapRound_) { diff --git a/tests/csnode/communitybootstrap_tests.cpp b/tests/csnode/communitybootstrap_tests.cpp new file mode 100644 index 000000000..e7f0d98fb --- /dev/null +++ b/tests/csnode/communitybootstrap_tests.cpp @@ -0,0 +1,241 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +using cs::CommunityBootstrapGate; +using cs::communityTableWeight; +using cs::isCommunityFallbackOpen; +using cs::isCommunityTable; + +namespace { +// key k: every byte k, so that keys sort as their numbers +cs::PublicKey key(uint8_t k) { + cs::PublicKey result{}; + result.fill(k); + return result; +} + +std::set keys(std::initializer_list ks) { + std::set result; + for (int k : ks) { + result.insert(key(uint8_t(k))); + } + return result; +} + +std::vector table(std::initializer_list ks) { + std::vector result; + for (int k : ks) { + result.push_back(key(uint8_t(k))); + } + return result; +} + +// the CI layout: initial confidants 1-5 (2 and 4 official), community nodes 1, 3, 5, 7, 9 +const std::set kInitial = keys({1, 2, 3, 4, 5}); +const std::set kCommunity = keys({1, 3, 5, 7, 9}); + +constexpr uint64_t kMinute = 60'000; +constexpr uint64_t kStall = 30 * kMinute; +constexpr uint64_t kElapse = 20'000; +} // namespace + +TEST(CommunityBootstrap, TheCommunityNodesLeftAfterTheOfficialOnesMakeATable) { + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 3, 5, 7, 9}))); + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 3, 7, 9}))); + // with initial confidants that are still there + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 2, 3, 7, 9}))); +} + +TEST(CommunityBootstrap, ATableOfInitialConfidantsOnlyIsARegularOne) { + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 2, 3, 4, 5}))); + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 4, 5}))); +} + +TEST(CommunityBootstrap, NoListNoTable) { + EXPECT_FALSE(isCommunityTable(kInitial, {}, table({1, 3, 5, 7, 9}))); +} + +TEST(CommunityBootstrap, EveryKeyMustBeInitialOrListed) { + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 5, 7, 8}))); + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 5, 7, 9, 10}))); +} + +TEST(CommunityBootstrap, MoreThanHalfOfTheListMustBeInIt) { + // 3 of 5 listed: a majority + EXPECT_TRUE(isCommunityTable(kInitial, kCommunity, table({1, 2, 4, 7, 9}))); + // 2 of 5 listed: another part of a split community could hold the other 3 + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({2, 3, 4, 7}))); + const auto six = keys({1, 3, 5, 7, 9, 11}); + EXPECT_FALSE(isCommunityTable(kInitial, six, table({2, 4, 7, 9, 11}))); // 3 of 6 + EXPECT_TRUE(isCommunityTable(kInitial, six, table({4, 5, 7, 9, 11}))); // 4 of 6 +} + +TEST(CommunityBootstrap, SizeAndOrderAsAStarterSendsIt) { + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 7, 9}))); // below MinTrustedNodes + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 9, 7}))); // not ascending + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 3, 7, 7, 9}))); // a repeated key + std::set many; + std::vector big; + for (int k = 1; k <= 26; ++k) { + many.insert(key(uint8_t(k))); + big.push_back(key(uint8_t(k))); + } + EXPECT_FALSE(isCommunityTable(kInitial, many, big)); // above MaxTrustedNodes + big.pop_back(); + EXPECT_TRUE(isCommunityTable(kInitial, many, big)); +} + +TEST(CommunityBootstrap, MoreKeysWeighMoreThenEarlierKeys) { + const auto w5 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7, 9})); + const auto w4 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7})); + const auto w4late = communityTableWeight(kInitial, kCommunity, table({3, 5, 7, 9})); + EXPECT_GT(w5, w4); + EXPECT_GT(w4, w4late); + // a table of fewer keys never outweighs a larger one, whatever its keys + EXPECT_GT(communityTableWeight(kInitial, kCommunity, table({5, 7, 9, 3, 1})), communityTableWeight(kInitial, kCommunity, table({1, 2, 3, 4}))); + EXPECT_EQ(w5, communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7, 9}))); +} + +TEST(CommunityBootstrap, ClosedUntilTheNodeStalledForStallMinutes) { + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + EXPECT_FALSE(gate.isStalled(0, kStall)); + EXPECT_FALSE(gate.isStalled(kStall - 1, kStall)); + EXPECT_TRUE(gate.isStalled(kStall, kStall)); + EXPECT_TRUE(gate.isStalled(kStall + kMinute, kStall)); + // stall_minutes 0 is never stalled (the config allows 1..10080) + EXPECT_FALSE(gate.isStalled(kStall, 0)); +} + +TEST(CommunityBootstrap, ANodeJustStartedHasNotWatchedLongEnough) { + // the clock is ms since the node started: no progress yet counts as progress at the start + CommunityBootstrapGate gate; + EXPECT_FALSE(gate.isStalled(kStall - 1, kStall)); + gate.onLastSequence(500, kStall - 1); // the first look at the chain is progress too + EXPECT_FALSE(gate.isStalled(2 * kStall - 2, kStall)); + EXPECT_TRUE(gate.isStalled(2 * kStall - 1, kStall)); +} + +TEST(CommunityBootstrap, ABlockARoundOrARegularBootstrapClosesIt) { + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + ASSERT_TRUE(gate.isStalled(kStall, kStall)); + gate.onProgress(kStall); // a round package, a regular table + EXPECT_FALSE(gate.isStalled(kStall + 1, kStall)); + ASSERT_TRUE(gate.isStalled(2 * kStall, kStall)); + gate.onLastSequence(500, 2 * kStall); // the same last block: no progress + EXPECT_TRUE(gate.isStalled(2 * kStall, kStall)); + gate.onLastSequence(501, 2 * kStall); // a block (synced or written) + EXPECT_FALSE(gate.isStalled(2 * kStall + 1, kStall)); +} + +// The safety property: while blocks are produced (official nodes alive), the fallback never opens. +// A day of rounds, one a second or one every round_elapse_time at worst, a block each +TEST(CommunityBootstrap, NeverOpenWhileTheNetworkProducesBlocks) { + for (uint64_t gap : {uint64_t(1000), kElapse, 10 * kMinute, kStall - 1}) { + CommunityBootstrapGate gate; + cs::Sequence seq = 100; + for (uint64_t now = 0; now < 24 * 60 * kMinute; now += gap) { + gate.onLastSequence(++seq, now); + for (uint64_t t = now; t < now + gap; t += std::max(gap / 4, 1)) { + ASSERT_FALSE(isCommunityFallbackOpen(true, gate.isStalled(t, kStall), false)) << "gap " << gap << " at " << t; + } + } + } + // bootstrap rounds that end in blocks are progress just as well (a regular table every round_elapse_time) + CommunityBootstrapGate gate; + gate.onLastSequence(100, 0); + for (uint64_t now = 0; now < 4 * kStall; now += kElapse) { + gate.onProgress(now); + ASSERT_FALSE(gate.isStalled(now + kElapse - 1, kStall)); + } +} + +TEST(CommunityBootstrap, OpenOnlyWhenConfiguredStalledAndNoNeighbourAhead) { + EXPECT_TRUE(isCommunityFallbackOpen(true, true, false)); + EXPECT_FALSE(isCommunityFallbackOpen(false, true, false)); // off by default: no list + EXPECT_FALSE(isCommunityFallbackOpen(true, false, false)); + EXPECT_FALSE(isCommunityFallbackOpen(true, true, true)); // behind a neighbour: sync instead +} + +TEST(CommunityBootstrap, OneTablePerHalfRoundElapseUnlessHeavier) { + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + const auto w5 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7, 9})); + const auto w4 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7})); + const uint64_t t = kStall; + EXPECT_TRUE(gate.mayTake(w4, t, kElapse)); + gate.onTaken(w4, t); + // a repeat or a lighter table right after: dropped, so a listed key cannot force restarts + EXPECT_FALSE(gate.mayTake(w4, t + 1, kElapse)); + EXPECT_FALSE(gate.mayTake(w4 - 1, t + kElapse / 2 - 1, kElapse)); + // a heavier one at once: the table of more nodes wins + EXPECT_TRUE(gate.mayTake(w5, t + 1, kElapse)); + gate.onTaken(w5, t + 1); + EXPECT_FALSE(gate.mayTake(w4, t + 2, kElapse)); + // the starter's repeat, a round_elapse_time later + EXPECT_TRUE(gate.mayTake(w5, t + 1 + kElapse, kElapse)); + EXPECT_TRUE(gate.mayTake(w4, t + 1 + kElapse / 2, kElapse)); + // progress forgets the tables of this stall + gate.onProgress(t + 3); + EXPECT_TRUE(gate.mayTake(w4, t + 4, kElapse)); +} + +namespace { +CommunityBootstrapData parse(const std::string& ini) { + std::istringstream in(ini); + boost::property_tree::ptree tree; + boost::property_tree::read_ini(in, tree); + return Config::parseCommunityBootstrapData(tree); +} + +const char* const kKeyA = "3rUevsW5xfob6qDxWMDFwwTQCq39SYhzstuyfUGSDvF2"; +const char* const kKeyB = "ESWr7ZThCmP2DVNNgrcsCvZZ2rrhDnFRW8oUFEpKpgdD"; + +cs::PublicKey decoded(const char* text) { + std::vector bytes; + EXPECT_TRUE(DecodeBase58(text, bytes)); + cs::PublicKey result{}; + std::copy(bytes.begin(), bytes.end(), result.begin()); + return result; +} +} // namespace + +TEST(CommunityBootstrapConfig, OffWithoutTheSection) { + const auto c = parse("[params]\nipv6=false\n"); + EXPECT_EQ(c, CommunityBootstrapData{}); + EXPECT_TRUE(c.confidants.empty()); + EXPECT_EQ(c.stallMinutes, 30u); +} + +TEST(CommunityBootstrapConfig, KeysAndStallMinutes) { + const auto c = parse(std::string("[community_bootstrap]\nconfidants=") + kKeyA + ", " + kKeyB + "," + kKeyA + "\nstall_minutes=45\n"); + ASSERT_EQ(c.confidants.size(), 2u); + EXPECT_EQ(c.confidants[0], decoded(kKeyA)); + EXPECT_EQ(c.confidants[1], decoded(kKeyB)); + EXPECT_EQ(c.stallMinutes, 45u); +} + +TEST(CommunityBootstrapConfig, BadValuesAreLeftOut) { + const auto c = parse(std::string("[community_bootstrap]\nconfidants=notakey ") + kKeyB + "\nstall_minutes=0\n"); + ASSERT_EQ(c.confidants.size(), 1u); + EXPECT_EQ(c.confidants[0], decoded(kKeyB)); + EXPECT_EQ(c.stallMinutes, 30u); + EXPECT_EQ(parse("[community_bootstrap]\nstall_minutes=10081\n").stallMinutes, 30u); + EXPECT_EQ(parse("[community_bootstrap]\nstall_minutes=5m\n").stallMinutes, 30u); + EXPECT_EQ(parse("[community_bootstrap]\nstall_minutes=1\n").stallMinutes, 1u); + // an empty list keeps it off + EXPECT_TRUE(parse("[community_bootstrap]\nconfidants=\n").confidants.empty()); +} From fd5bf96ad5cb5dd9a12b445230b8c999731a0151 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Tue, 17 Nov 2026 16:03:49 +0100 Subject: [PATCH 03/22] ci: official-stop-config and community-bootstrap-quiet scenarios COMMUNITY_BOOTSTRAP=config turns the fallback on in every node of this build with every community key listed. With OFFICIAL_STOP_AT the network must restart without the official nodes and the fallback must not act before they stop; without it the fallback must never act. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 11 +++++++++++ ci/testnet/run.sh | 34 ++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 14fd86475..5c6380411 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -511,6 +511,17 @@ jobs: env: NODES=10 TRUSTED_NODES=5 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=trusted RESTART_AT=0 LOAD_SECONDS=1500 fixed: true manual: true + # the same with the community bootstrap fallback ([community_bootstrap], stall_minutes=2) in + # place of the hand-edited trusted.txt: it must restart the network, and not act before + - name: official-stop-config + env: NODES=10 TRUSTED_NODES=5 OFFICIAL_STOP_AT=400 OFFICIAL_STOP_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true + # the fallback on (stall_minutes=1) in a network whose official nodes keep running, under + # packet loss and the restart test: it must never act + - name: community-bootstrap-quiet + env: COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=1 NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 TARGET_SEQ=1500 + manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill official: 'no' # needs a feature of this fork on an even node diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 372ea99ca..509bf9c08 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -205,7 +205,11 @@ # official nodes take the trusted seats as on mainnet. COMMUNITY_BOOTSTRAP says what the # community nodes may do meanwhile: none (default) - nothing, the network must stay stalled # until the official nodes are back; trusted - their own trusted.txt also lists every community -# node, the network must restart without them +# node, the network must restart without them; config - the community nodes run the community +# bootstrap fallback ([community_bootstrap] confidants = every community key, stall_minutes = +# COMMUNITY_STALL_MINUTES, default 2), the network must restart without them. COMMUNITY_BOOTSTRAP=config +# also runs without OFFICIAL_STOP_AT: the fallback must then never act (no "COMMUNITY BOOTSTRAP:" +# table started or taken), and with OFFICIAL_STOP_AT never before the official nodes stop # # Checks: every node writes blocks up to TARGET_SEQ; a node stopped and restarted (quick start # from its caches) catches up; all nodes report the same STATE DIGEST at every common sequence. @@ -310,7 +314,8 @@ OFFICIAL_STOP_AT=${OFFICIAL_STOP_AT:-0} OFFICIAL_STOP_MINUTES=${OFFICIAL_STOP_MINUTES:-5} OFFICIAL_FUND=${OFFICIAL_FUND:-0} COMMUNITY_BOOTSTRAP=${COMMUNITY_BOOTSTRAP:-none} -case "$COMMUNITY_BOOTSTRAP" in none|trusted) ;; *) echo "COMMUNITY_BOOTSTRAP must be none or trusted, got '$COMMUNITY_BOOTSTRAP'"; exit 1 ;; esac +case "$COMMUNITY_BOOTSTRAP" in none|trusted|config) ;; *) echo "COMMUNITY_BOOTSTRAP must be none, trusted or config, got '$COMMUNITY_BOOTSTRAP'"; exit 1 ;; esac +COMMUNITY_STALL_MINUTES=${COMMUNITY_STALL_MINUTES:-2} JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} @@ -759,6 +764,18 @@ if [ "$OFFICIAL_STOP_AT" -gt 0 ] && [ "$COMMUNITY_BOOTSTRAP" = trusted ]; then echo "official stop: the community nodes list $(grep -c . "$WORK/n1/trusted.txt") initial confidants, the official nodes $TRUSTED_NODES" fi +# COMMUNITY_BOOTSTRAP=config: the community bootstrap fallback on every node of this build, listing +# every node of this build (the official nodes, even, have no such option) +if [ "$COMMUNITY_BOOTSTRAP" = config ]; then + listed="" + for j in $(seq 1 "$NODES"); do alt_node "$j" || listed="$listed${listed:+,}${KEYS[$((j - 1))]}"; done + for i in $(seq 1 "$NODES"); do + alt_node "$i" && continue + printf '\n[community_bootstrap]\nconfidants=%s\nstall_minutes=%s\n' "$listed" "$COMMUNITY_STALL_MINUTES" >> "$WORK/n$i/config.ini" + done + echo "community bootstrap: on in every node of this build, $(tr ',' '\n' <<<"$listed" | grep -c .) keys listed, stall_minutes=$COMMUNITY_STALL_MINUTES" +fi + # test-build hooks of a forged or corrupt snapshot source, and the joiner's list of serving nodes [ "$FORGE_NODE" -eq 0 ] || : > "$WORK/n$FORGE_NODE/testnet-forge-snapshot" [ "$CORRUPT_NODE" -eq 0 ] || : > "$WORK/n$CORRUPT_NODE/testnet-corrupt-pieces" @@ -1077,6 +1094,13 @@ official_stop() { for i in $OFFICIAL_NODES; do wait "${PIDS[$i]}" 2>/dev/null || true; REPORTED_DOWN[$i]=1; done local t_kill s0 b0 first_t=0 first_seq=0 c end t_kill=$(date +%s) + # the fallback must not have acted while the official nodes ran + for i in $COMMUNITY_NODES; do + if grep -aq 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log"; then + off_fail="${off_fail:+$off_fail; }n$i used the community bootstrap while the official nodes ran" + grep -a 'COMMUNITY BOOTSTRAP' "$WORK/n$i/node.log" | head -3 + fi + done s0=$(community_max) b0=$(bootstrap_lines) echo "official stop: killed $(printf 'n%s ' $OFFICIAL_NODES)at once (round $seen); community head $s0, COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" @@ -2433,6 +2457,12 @@ if [ -n "$forks" ]; then exit 1 fi +# the community bootstrap fallback in a network that never lost its official nodes: never used +if [ "$COMMUNITY_BOOTSTRAP" = config ] && [ "$OFFICIAL_STOP_AT" -eq 0 ]; then + used=$(grep -alE 'COMMUNITY BOOTSTRAP: (starting|taking)' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) + [ -z "$used" ] || { grep -ah 'COMMUNITY BOOTSTRAP' "$WORK"/n*/node.log | head -5; echo "FAIL: the community bootstrap acted on $used"; exit 1; } + echo "community bootstrap: never used ($(grep -ah 'NODE> Bootstrap on' "$WORK"/n*/node.log | wc -l) regular bootstrap lines on all nodes)" +fi [ "$OFFICIAL_STOP_AT" -eq 0 ] || echo "OFFICIAL STOP RESULT: one chain: every node holds the same blocks (all compared), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" # a network still restarting at the end is stuck, even if it reached the target before (a bootstrap From b3016697699475ac0a4b77a765dda1d31c2c0108 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Tue, 17 Nov 2026 16:29:38 +0100 Subject: [PATCH 04/22] docs: differences.md: the community bootstrap fallback Co-Authored-By: Claude Opus 5.5 --- docs/differences.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/differences.md b/docs/differences.md index 4f74d09c0..acd9e8e3f 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -122,6 +122,49 @@ The official node has it only in its monitor and wallet builds. The cache follow standard, including v1/v2 tokens whose amounts are BigDecimals, and it skips failed calls. It is rebuilt from the chain when the checkpoint holds none. See [tokens.md](tokens.md). +### Community bootstrap fallback (expert option, off by default) + +Only the initial confidants (the `init_trusted` list, or special order 2) may restart a stalled +network with a bootstrap table, and a table with any other key needs the project's signature. If +all official nodes become unavailable at once, the initial confidants still running may be too few +(fewer than 4), and the network stays stalled until the official nodes are back. + +The `[community_bootstrap]` section of `config.ini` adds a liveness fallback. It is off unless +`confidants` lists keys: + +```ini +[community_bootstrap] +; community node keys (base58), comma or space separated; empty or missing: off +confidants=,,... +; minutes without progress before the fallback may act (1 to 10080, default 30) +stall_minutes=30 +``` + +With the list set, and only after `stall_minutes` in which the node saw no new block, no round +package and no regular bootstrap table, and while no neighbour reports a block above its own: + +- a node whose key is an initial confidant or listed may start a bootstrap table of the initial + confidants and listed nodes that hold the highest block, as an initial confidant does; the table + is signed by its starter, not by the project; +- a node with the list takes such a table only if it has more than half of the listed keys, each + key is an initial confidant or listed, and the starter's signature checks. The round window and + the stall check of every unsigned table still apply, and a node takes at most one such table per + half `round_elapse_time` unless it is larger. + +In normal operation nothing changes: blocks and round packages keep the fallback closed, a regular +bootstrap table is handled exactly as before and always replaces a community one, and blocks, +hashes and validation are unchanged. Official nodes, and community nodes without the list, drop a +community table (it holds a key that is not an initial confidant and is not signed by the +project); they then take the blocks written meanwhile through ordinary sync once they are back, +as the `official-stop-*` scenarios of the robustness workflow check. + +Every node that should take part needs the same list: it only helps when more than half of the +listed nodes are running. The set-up scripts and the Docker image do not offer the option; edit +`config.ini` by hand (in Docker, `~/.credits/config.ini`). The risk it brings: if a network +partition cuts more than half of the listed nodes off from the official nodes for longer than +`stall_minutes`, while the official nodes keep producing blocks, the listed nodes restart on their +own and the two sides write different blocks. Keep `stall_minutes` long. + ## Start-up behaviour - **A banner names the release.** At start the console and the log show the CREDITS logo with From 707146c781da9c226d876d7e5a8a26625594705b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Tue, 17 Nov 2026 16:30:26 +0100 Subject: [PATCH 05/22] docs: CHANGES row for #188 Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index fba892eac..01aa99030 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,6 +44,7 @@ lines under the new release. | PR | Change | |---|---| +| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. CI: `official-stop-*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | From 20fdc174756940b3918b6a20468125984f3e901e Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Tue, 17 Nov 2026 17:11:18 +0100 Subject: [PATCH 06/22] ci: official-stop scenarios with an official majority among the initial confidants With n1-n5 as initial confidants, 3 of them community, a bootstrap table that still names the stopped official nodes (stale neighbours) can decide with the 3 community ones, so the network sometimes restarted through the regular path and the fallback was not exercised. INITIAL_NODES picks the initial set: n1, n2, n3, n4, n6, three of them official. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 11 ++++++----- ci/testnet/run.sh | 13 ++++++++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 5c6380411..ea2b026f0 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -498,23 +498,24 @@ jobs: env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 REWARD_ROUND=100 LOAD_SECONDS=2400 TARGET_SEQ=3000 manual: true # if all official nodes become unavailable at once (node_alt_ref=official; run by name): 5 - # official even nodes with more stake hold the trusted seats, 5 community odd nodes; n1-n5 are - # the genesis initial confidants. They are killed in a round they hold the majority of, stay + # official even nodes with more stake hold the trusted seats, 5 community odd nodes; n1, n2, + # n3, n4 and n6 are the genesis initial confidants (official majority: a table that still + # names the stopped nodes cannot decide). They are killed in a round they hold the majority of, stay # down 5 minutes and start again. Without extra keys the network must stay stalled until # they are back; with every community key in the community nodes' own trusted.txt the # community nodes must restart it, and the official nodes must then take the same blocks - name: official-stop-stalled - env: NODES=10 TRUSTED_NODES=5 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=none RESTART_AT=0 LOAD_SECONDS=1500 + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=none RESTART_AT=0 LOAD_SECONDS=1500 fixed: true manual: true - name: official-stop-trusted - env: NODES=10 TRUSTED_NODES=5 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=trusted RESTART_AT=0 LOAD_SECONDS=1500 + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=trusted RESTART_AT=0 LOAD_SECONDS=1500 fixed: true manual: true # the same with the community bootstrap fallback ([community_bootstrap], stall_minutes=2) in # place of the hand-edited trusted.txt: it must restart the network, and not act before - name: official-stop-config - env: NODES=10 TRUSTED_NODES=5 OFFICIAL_STOP_AT=400 OFFICIAL_STOP_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_STOP_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 fixed: true manual: true # the fallback on (stall_minutes=1) in a network whose official nodes keep running, under diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 509bf9c08..a901f8e93 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -8,6 +8,7 @@ # TIMEOUT_MIN (default 45) RESTART_AT (default 300, 0 disables the restart test) # WORK (default ./testnet-run) # TRUSTED_NODES (default NODES): only the first TRUSTED_NODES nodes are initial confidants +# (INITIAL_NODES, comma separated: these nodes instead) # (odd, 5 to 25); the others, up to 61 nodes in all, are ordinary nodes # ROUND_ELAPSE_MS (default 20000): round_elapse_time of every node # SAMPLE_SECONDS (default 60, 0 = off): sampler.py records host and per-process memory and CPU @@ -246,6 +247,12 @@ case "$TRUSTED_NODES" in 5|7|9|11|13|15|17|19|21|23|25) ;; *) echo "TRUSTED_NODES must be odd, 5 to 25, got '$TRUSTED_NODES'"; exit 1 ;; esac +# INITIAL_NODES (comma separated, TRUSTED_NODES of them): these nodes are the initial confidants +# instead of the first TRUSTED_NODES, e.g. a set the official (even) nodes hold the majority of +INITIAL_NODES=${INITIAL_NODES:-$(seq -s, 1 "$TRUSTED_NODES")} +[ "$(tr ',' '\n' <<<"$INITIAL_NODES" | sort -un | grep -c .)" -eq "$TRUSTED_NODES" ] \ + || { echo "INITIAL_NODES must name $TRUSTED_NODES distinct nodes, got '$INITIAL_NODES'"; exit 1; } +is_initial() { [[ ",$INITIAL_NODES," == *",$1,"* ]]; } # ports: p2p 6000 + i (offline copies 6500 + i), API 9100 + i, apiexec 9200 + i, executor 9300 + i, diag 9400 + i [ "$NODES" -ge "$TRUSTED_NODES" ] && [ "$NODES" -le 61 ] \ || { echo "NODES must be TRUSTED_NODES ($TRUSTED_NODES) to 61, got '$NODES'"; exit 1; } @@ -715,7 +722,7 @@ for i in $(seq 1 "$ALL_NODES"); do : > "$dir/hosts.txt" : > "$dir/trusted.txt" for j in $(seq 1 "$NODES"); do - [ "$j" -gt "$TRUSTED_NODES" ] || echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" + ! is_initial "$j" || echo "${KEYS[$((j - 1))]}" >> "$dir/trusted.txt" [ "$j" -ne "$i" ] && echo "$(node_ip "$j"):$((BASE_PORT + j)) ${KEYS[$((j - 1))]}" >> "$dir/hosts.txt" done cat > "$dir/config.ini" <> "$WORK/n$i/trusted.txt" + for j in $(seq 1 "$NODES"); do + is_initial "$j" || official_node "$j" || echo "${KEYS[$((j - 1))]}" >> "$WORK/n$i/trusted.txt" done done echo "official stop: the community nodes list $(grep -c . "$WORK/n1/trusted.txt") initial confidants, the official nodes $TRUSTED_NODES" From 3a5600080e2ecb30fb4f0a412b038bed375df78b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 07:26:29 +0100 Subject: [PATCH 07/22] csnode: community bootstrap fallback review fixes - a community table ends with random bytes after the signature, as a regular one: a repeat in the same round is no longer a p2p duplicate (Host::IsDuplicate) that a node whose fallback opens later never receives (cs::makeCommunityTable) - within half a round_elapse_time a heavier community table is taken only from a starter key not above the current one: a higher listed key cannot displace a running table by padding it with keys of stopped nodes - the starter keeps an even set when dropping a key would cost the table its majority of the list (cs::dropForOddTable) - a round package is progress only once its signatures were accepted (or it is the node's own), not when taken unchecked while lagging - a neighbour one block ahead holds the gate only for a further stall_minutes; two or more ahead always - a jump of the steady or wall clock between two checks (suspend/resume, clock set) is progress - gtests: signed bytes as getBootstrapTable reads them, the repeat differs and verifies, a table replayed from another round, even sets, padding, clock jumps Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/communitybootstrap.hpp | 114 ++++++++-- csnode/src/node.cpp | 70 +++--- tests/csnode/communitybootstrap_tests.cpp | 214 +++++++++++++++++-- 3 files changed, 345 insertions(+), 53 deletions(-) diff --git a/csnode/include/csnode/communitybootstrap.hpp b/csnode/include/csnode/communitybootstrap.hpp index a6a4cec2c..3b1366fab 100644 --- a/csnode/include/csnode/communitybootstrap.hpp +++ b/csnode/include/csnode/communitybootstrap.hpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include namespace cs { @@ -19,15 +21,20 @@ namespace cs { // confidants and that list, signed by its starter instead of the project. // // What keeps it out of normal operation: -// - the node itself has seen no block, no round of a round package and no regular bootstrap table -// for stall_minutes, and no neighbour reports a block above its own (that node would sync); +// - the node itself has seen no block, no round package whose signatures it accepted and no regular +// bootstrap table for stall_minutes (a jump of its clocks counts as progress), and no neighbour +// reports a block two above its own, or one above for less than a further stall_minutes +// (isNeighbourAhead: this node would sync); // - the table: 4 to 25 keys in ascending order, each an initial confidant or listed, at least one // of them listed and not an initial confidant (else it is a regular table), more than half of the // list in it (two parts of a split community cannot both restart), and signed by its first key, the // starter; // - the round window (#71) and the stall check (#70) of every unsigned table still apply; -// - a node takes one such table per half round_elapse_time, or a heavier one at once, and a regular -// table always replaces it (it goes through the unchanged path, whose weight it does not touch). +// - a node takes one such table per half round_elapse_time; within that window only a heavier table +// whose starter key is not above the current one replaces it, and a regular table always replaces +// it (it goes through the unchanged path, whose weight it does not touch). A regular table, even +// a project-signed one, is progress: a community table may follow it only after a further +// stall_minutes without a block. // Official nodes, and community nodes without the list, drop such a table: one of its keys is not an // initial confidant and its signature is not the project's. @@ -36,6 +43,36 @@ constexpr size_t kCommunityMaxTable = 25; // Consensus::MaxTrustedNodes using CommunityWeight = uint64_t; +// The bytes a project-signed table and a community table sign: the round, then the table as sent +// (the count and the keys). The round is in them, so a table of another round does not verify +inline cs::Bytes bootstrapSignedBytes(cs::RoundNumber round, const uint8_t* table, size_t tableLen) { + cs::Bytes buf; + buf.reserve(sizeof(cs::RoundNumber) + tableLen); + const auto* ptr = reinterpret_cast(&round); + buf.insert(buf.end(), ptr, ptr + sizeof(cs::RoundNumber)); + buf.insert(buf.end(), table, table + tableLen); + return buf; +} + +// A community table as its starter sends it: the count, the keys in ascending order, the starter's +// signature over bootstrapSignedBytes, and 4 random bytes as in a regular table. Without them a +// repeat in the same round would be the same message, and the p2p layer drops a broadcast whose +// bytes it has seen (Host::IsDuplicate): a node whose fallback opens after the first one would never +// get one. Receivers read the signature and ignore the bytes after it +inline cs::Bytes makeCommunityTable(cs::RoundNumber round, const std::set& keys, const cs::PrivateKey& starterKey, + uint32_t random) { + cs::Bytes bin; + cs::ODataStream out(bin); + out << uint8_t(keys.size()); + for (const auto& key : keys) { + out << key; + } + const cs::Bytes signedPart = bootstrapSignedBytes(round, bin.data(), bin.size()); + out << cscrypto::generateSignature(starterKey, signedPart.data(), signedPart.size()); + out << random; + return bin; +} + // the starter's table is a std::set, sent in order: each key above the previous one inline bool isAscending(const std::vector& table) { for (size_t i = 1; i < table.size(); ++i) { @@ -65,6 +102,30 @@ inline bool isCommunityTable(const std::set& initial, const std::set< return outsideInitial && listed * 2 > community.size(); } +// The starter of an even set drops its last key (the official rule, for an odd table). Not when that +// would leave a community table without more than half of the list: with a list of 6 and 4 of its +// nodes left, the 3 kept would never be enough +inline bool dropForOddTable(const std::set& initial, const std::set& community, + const std::vector& table) { + if (table.empty() || table.size() % 2 != 0) { + return false; + } + if (!isCommunityTable(initial, community, table)) { + return true; + } + return isCommunityTable(initial, community, std::vector(table.begin(), table.end() - 1)); +} + +// A neighbour ahead of this node: it syncs from it, the network is not stalled. Two blocks ahead +// always; one block ahead only for a further stall_minutes (sync takes one block in seconds, so a +// neighbour that stays one block ahead cannot give it, and must not keep the fallback closed for ever) +inline bool isNeighbourAhead(Sequence neighbourSeq, Sequence lastSeq, uint64_t msSinceProgress, uint64_t stallMs) { + if (neighbourSeq >= lastSeq + 2) { + return true; + } + return neighbourSeq == lastSeq + 1 && msSinceProgress < 2 * stallMs; +} + // More keys weigh more; among tables of as many keys, the one with the keys earlier in the sorted // union of the two sets. Only compared between community tables: the regular weight is not involved constexpr size_t kCommunityWeightPositions = 58; @@ -93,7 +154,7 @@ inline CommunityWeight communityTableWeight(const std::set& initial, // The fallback's state on one node; times in ms of a steady clock class CommunityBootstrapGate { public: - // a block written, a round of a round package, a regular bootstrap table: the network is alive. + // a block written, a round package with accepted signatures, a regular bootstrap table: the network is alive. // The node's start counts too, so that it has watched for stall_minutes before it may act void onProgress(uint64_t nowMs) { lastProgressMs_ = nowMs; @@ -102,6 +163,19 @@ class CommunityBootstrapGate { lastTakenMs_ = 0; } + // the clocks whenever the fallback is considered: a jump of either over maxGapMs since the last + // look (the host suspended and resumed, the wall clock set) is progress, so that the node watches + // again for stall_minutes before it acts on heights it saw before the jump + void onClock(uint64_t steadyMs, uint64_t wallMs, uint64_t maxGapMs) { + const bool jump = checked_ && (gap(lastCheckSteadyMs_, steadyMs) > maxGapMs || gap(lastCheckWallMs_, wallMs) > maxGapMs); + checked_ = true; + lastCheckSteadyMs_ = steadyMs; + lastCheckWallMs_ = wallMs; + if (jump) { + onProgress(steadyMs); + } + } + // the node's last block, read whenever the fallback is considered: a new one is progress void onLastSequence(Sequence seq, uint64_t nowMs) { if (seq != lastSeq_) { @@ -118,28 +192,44 @@ class CommunityBootstrapGate { return stallMs > 0 && msSinceProgress(nowMs) >= stallMs; } - // a community table of that weight may be started or taken now: the first one in this stall, one - // heavier than the last, or any once half a round_elapse_time has passed since the last (a starter - // repeats its table every round_elapse_time) - bool mayTake(CommunityWeight weight, uint64_t nowMs, uint64_t roundElapseMs) const { - if (!taken_ || weight > lastWeight_) { + // a community table of that weight and starter (its first key) may be started or taken now: the + // first one in this stall, any once half a round_elapse_time has passed since the last (a starter + // repeats its table every round_elapse_time), and within that half only a heavier one whose + // starter key is not above the last one's. The starter of a table is its lowest key, and among + // the nodes left the lowest key starts anyway: a node with a lower key that saw more nodes + // replaces the table at once, but a listed node with a higher key cannot displace a running table + // by padding its own with keys of stopped nodes (heavier, and unable to decide a round) + bool mayTake(CommunityWeight weight, const PublicKey& starter, uint64_t nowMs, uint64_t roundElapseMs) const { + if (!taken_) { + return true; + } + if (nowMs >= lastTakenMs_ && nowMs - lastTakenMs_ >= roundElapseMs / 2) { return true; } - return nowMs >= lastTakenMs_ && nowMs - lastTakenMs_ >= roundElapseMs / 2; + return weight > lastWeight_ && !(lastStarter_ < starter); } - void onTaken(CommunityWeight weight, uint64_t nowMs) { + void onTaken(CommunityWeight weight, const PublicKey& starter, uint64_t nowMs) { taken_ = true; lastWeight_ = weight; + lastStarter_ = starter; lastTakenMs_ = nowMs; } private: + static uint64_t gap(uint64_t a, uint64_t b) { + return a > b ? a - b : b - a; + } + uint64_t lastProgressMs_ = 0; Sequence lastSeq_ = 0; bool taken_ = false; CommunityWeight lastWeight_ = 0; + PublicKey lastStarter_{}; uint64_t lastTakenMs_ = 0; + bool checked_ = false; + uint64_t lastCheckSteadyMs_ = 0; + uint64_t lastCheckWallMs_ = 0; }; // the fallback may act on this node: a list configured, the node stalled for stall_minutes, and no diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index d41ca9700..15a866de8 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -1096,20 +1096,29 @@ void Node::onCommunityProgress() { bool Node::isCommunityFallbackOpen() { const auto& settings = cs::ConfigHolder::instance().config()->getCommunityBootstrapSettings(); const cs::Sequence lastSeq = blockChain_.getLastSeq(); + const uint64_t stallMs = uint64_t(settings.stallMinutes) * 60'000ULL; + // a jump of the clocks between two looks (host suspended, clock set) is progress: the node then + // watches again before it acts on what it saw before. The looks come every round_elapse_time + const uint64_t maxGapMs = std::max(5 * 60'000ULL, 3 * cs::ConfigHolder::instance().config()->roundElapseTime()); + const uint64_t wallMs = uint64_t(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); bool stalled = false; + uint64_t sinceProgress = 0; { std::lock_guard lock(communityMutex_); const uint64_t now = stat_.msSinceStart(); + communityGate_.onClock(now, wallMs, maxGapMs); communityGate_.onLastSequence(lastSeq, now); - stalled = communityGate_.isStalled(now, uint64_t(settings.stallMinutes) * 60'000ULL); + stalled = communityGate_.isStalled(now, stallMs); + sinceProgress = communityGate_.msSinceProgress(now); } if (settings.confidants.empty() || !stalled) { return false; } // a neighbour with a later block: this node is behind and syncs, the network is not stalled bool ahead = false; - transport_->forEachNeighbour([&ahead, lastSeq](const cs::PublicKey&, cs::Sequence seq, cs::RoundNumber) { - if (seq > lastSeq) { + transport_->forEachNeighbour([&ahead, lastSeq, sinceProgress, stallMs](const cs::PublicKey&, cs::Sequence seq, cs::RoundNumber) { + if (cs::isNeighbourAhead(seq, lastSeq, sinceProgress, stallMs)) { ahead = true; } }); @@ -1121,12 +1130,12 @@ bool Node::takeCommunityTable(const cs::PublicKeys& table, bool own) { const uint64_t elapse = cs::ConfigHolder::instance().config()->roundElapseTime(); std::lock_guard lock(communityMutex_); const uint64_t now = stat_.msSinceStart(); - if (!communityGate_.mayTake(weight, now, elapse)) { + if (table.empty() || !communityGate_.mayTake(weight, table.front(), now, elapse)) { cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "not starting" : "dropping") << " a table of " << table.size() - << " keys: one at least as heavy was taken less than half a round_elapse_time ago"; + << " keys: another was taken less than half a round_elapse_time ago, and this one is not heavier with a starter key not above its starter's"; return false; } - communityGate_.onTaken(weight, now); + communityGate_.onTaken(weight, table.front(), now); cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "starting" : "taking") << " a table of " << table.size() << " keys after " << communityGate_.msSinceProgress(now) / 1000 << " s without a block, a round or a regular bootstrap"; return true; @@ -1185,13 +1194,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R cswarning() << "malformed bootstrap round table, ignore"; return; } - const size_t signed_buf_len = sizeof(cs::RoundNumber) + signed_subdata_len; - cs::Bytes buf; - buf.reserve(signed_buf_len); - auto* ptr = reinterpret_cast(&rNum); - std::copy(ptr, ptr + sizeof(cs::RoundNumber), std::back_inserter(buf)); - ptr = payload.data(); - std::copy(ptr, ptr + signed_subdata_len, std::back_inserter(buf)); + // the same bytes as the community starter signs (cs::makeCommunityTable); the stream read + // signed_subdata_len bytes of the payload already, so they are there + const cs::Bytes buf = cs::bootstrapSignedBytes(rNum, payload.data(), signed_subdata_len); if (cscrypto::verifySignature(sig, cs::PacketValidator::getBlockChainKey(), buf.data(), buf.size())) { currentWeight = cs::kSignedBootstrapWeight; signedTable = true; @@ -3933,6 +3938,7 @@ void Node::sendRoundTable(cs::RoundPackage& rPackage) { csdebug() << "Round " << rPackage.roundTable().round << ", Confidants count " << rPackage.roundTable().confidants.size(); csdebug() << "Hashes count: " << rPackage.roundTable().hashes.size(); + onCommunityProgress(); // this node's own round package performRoundPackage(rPackage, solver_->getPublicKey(), false); } @@ -4201,6 +4207,10 @@ void Node::getRoundTable(const uint8_t* data, const size_t size, const cs::Round cs::Signatures poolSignatures; cs::PublicKeys confidants; + // progress for the community bootstrap fallback: a package whose signatures this node checked + // and accepted; one taken unchecked (a bootstrap round, the first rounds, a node lagging behind) + // is not, and the blocks it leads to are progress anyway + bool signaturesAccepted = false; if (!isBootstrapRound()) { if (rPackage.roundTable().round > 2/* && confirmationList_.size() > 0*/) { //Here we have problems when the trusted have the first block and the others do not!!! @@ -4224,7 +4234,8 @@ void Node::getRoundTable(const uint8_t* data, const size_t size, const cs::Round } } - if (!receivingSignatures(rPackage, confidants) && storedRound == getBlockChain().getLastSeq()) { + signaturesAccepted = receivingSignatures(rPackage, confidants); + if (!signaturesAccepted && storedRound == getBlockChain().getLastSeq()) { return; } } @@ -4238,6 +4249,9 @@ void Node::getRoundTable(const uint8_t* data, const size_t size, const cs::Round currentRoundTableMessage_.round = rPackage.roundTable().round; currentRoundTableMessage_.sender = sender; currentRoundTableMessage_.message = cs::Bytes(data, data + size); + if (signaturesAccepted) { + onCommunityProgress(); + } performRoundPackage(rPackage, sender, updateRound); } @@ -4247,7 +4261,6 @@ void Node::setCurrentRP(const cs::RoundPackage& rp) { void Node::performRoundPackage(cs::RoundPackage& rPackage, const cs::PublicKey& sender, bool updateRound) { csdebug() << __func__; - onCommunityProgress(); // got round package in any way, reset default round table flag confirmationList_.add(rPackage.roundTable().round, false, rPackage.roundTable().confidants, rPackage.poolMetaInfo().realTrustedMask, rPackage.trustedSignatures()); @@ -5472,7 +5485,9 @@ void Node::onRoundTimeElapsed() { }; transport_->forEachNeighbour(std::move(callback)); - if (actualConfidants.size() % 2 == 0) { + // an odd table: the last key of an even set goes, unless a community table would lose its + // majority of the list with it (cs::dropForOddTable; without the fallback open, always) + if (cs::dropForOddTable(initialConfidants_, listed, cs::PublicKeys(actualConfidants.cbegin(), actualConfidants.cend()))) { auto it = actualConfidants.end(); --it; it = actualConfidants.erase(it); @@ -5539,23 +5554,20 @@ void Node::onRoundTimeElapsed() { } cs::Bytes bin; - cs::ODataStream out(bin); - out << uint8_t(actualConfidants.size()); - - for (const auto& item : actualConfidants) { - out << item; - } if (communityTable) { // signed by this node, the starter: the round, the count and the keys, as a table - // signed by the project; its weight stays out of bootStrapWeight_ - const cs::RoundNumber round = roundPackageCache_.back().roundTable().round; - cs::Bytes signedPart; - auto* ptr = reinterpret_cast(&round); - std::copy(ptr, ptr + sizeof(cs::RoundNumber), std::back_inserter(signedPart)); - std::copy(bin.cbegin(), bin.cend(), std::back_inserter(signedPart)); - out << cscrypto::generateSignature(solver_->getPrivateKey(), signedPart.data(), signedPart.size()); + // signed by the project, then random bytes so that its repeat is a new message; its + // weight stays out of bootStrapWeight_ + bin = cs::makeCommunityTable(roundPackageCache_.back().roundTable().round, actualConfidants, solver_->getPrivateKey(), + std::random_device{}()); } else { + cs::ODataStream out(bin); + out << uint8_t(actualConfidants.size()); + + for (const auto& item : actualConfidants) { + out << item; + } cs::BootstrapWeight currentWeight = calculateBootStrapWeight(confs); if (currentWeight > bootStrapWeight_) { bootStrapWeight_ = currentWeight; diff --git a/tests/csnode/communitybootstrap_tests.cpp b/tests/csnode/communitybootstrap_tests.cpp index e7f0d98fb..3a3392a3c 100644 --- a/tests/csnode/communitybootstrap_tests.cpp +++ b/tests/csnode/communitybootstrap_tests.cpp @@ -11,7 +11,9 @@ #include #include +#include #include +#include using cs::CommunityBootstrapGate; using cs::communityTableWeight; @@ -170,27 +172,215 @@ TEST(CommunityBootstrap, OpenOnlyWhenConfiguredStalledAndNoNeighbourAhead) { EXPECT_FALSE(isCommunityFallbackOpen(true, true, true)); // behind a neighbour: sync instead } -TEST(CommunityBootstrap, OneTablePerHalfRoundElapseUnlessHeavier) { +TEST(CommunityBootstrap, OneTablePerHalfRoundElapseUnlessHeavierFromALowerStarter) { CommunityBootstrapGate gate; gate.onLastSequence(500, 0); const auto w5 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7, 9})); const auto w4 = communityTableWeight(kInitial, kCommunity, table({1, 3, 5, 7})); const uint64_t t = kStall; - EXPECT_TRUE(gate.mayTake(w4, t, kElapse)); - gate.onTaken(w4, t); + EXPECT_TRUE(gate.mayTake(w4, key(1), t, kElapse)); + gate.onTaken(w4, key(1), t); // a repeat or a lighter table right after: dropped, so a listed key cannot force restarts - EXPECT_FALSE(gate.mayTake(w4, t + 1, kElapse)); - EXPECT_FALSE(gate.mayTake(w4 - 1, t + kElapse / 2 - 1, kElapse)); - // a heavier one at once: the table of more nodes wins - EXPECT_TRUE(gate.mayTake(w5, t + 1, kElapse)); - gate.onTaken(w5, t + 1); - EXPECT_FALSE(gate.mayTake(w4, t + 2, kElapse)); + EXPECT_FALSE(gate.mayTake(w4, key(1), t + 1, kElapse)); + EXPECT_FALSE(gate.mayTake(w4 - 1, key(1), t + kElapse / 2 - 1, kElapse)); + // a heavier one of the same starter at once: the table of more nodes wins + EXPECT_TRUE(gate.mayTake(w5, key(1), t + 1, kElapse)); + gate.onTaken(w5, key(1), t + 1); + EXPECT_FALSE(gate.mayTake(w4, key(1), t + 2, kElapse)); // the starter's repeat, a round_elapse_time later - EXPECT_TRUE(gate.mayTake(w5, t + 1 + kElapse, kElapse)); - EXPECT_TRUE(gate.mayTake(w4, t + 1 + kElapse / 2, kElapse)); + EXPECT_TRUE(gate.mayTake(w5, key(1), t + 1 + kElapse, kElapse)); + EXPECT_TRUE(gate.mayTake(w4, key(1), t + 1 + kElapse / 2, kElapse)); // progress forgets the tables of this stall gate.onProgress(t + 3); - EXPECT_TRUE(gate.mayTake(w4, t + 4, kElapse)); + EXPECT_TRUE(gate.mayTake(w4, key(3), t + 4, kElapse)); +} + +// A hostile listed key above the starter pads its table with keys of stopped nodes: heavier, but it +// cannot displace the running table; a lower starter that saw more nodes can +TEST(CommunityBootstrap, AHigherListedKeyCannotDisplaceATableByPaddingIt) { + const std::set initial = keys({1, 2, 4, 6, 8}); + const std::set community = keys({3, 5, 7, 9, 11, 13, 15}); // 4 make a majority + CommunityBootstrapGate gate; + gate.onLastSequence(500, 0); + const uint64_t t = kStall; + const auto running = table({3, 5, 7, 9}); + ASSERT_TRUE(isCommunityTable(initial, community, running)); + const auto wRunning = communityTableWeight(initial, community, running); + ASSERT_TRUE(gate.mayTake(wRunning, running.front(), t, kElapse)); + gate.onTaken(wRunning, running.front(), t); + // key 5 pads with the stopped 2, 4, 6, 8, 11, 13: more keys, heavier, refused within the window + const auto padded = table({2, 4, 5, 6, 7, 8, 9, 11, 13}); + ASSERT_TRUE(isCommunityTable(initial, community, padded)); + ASSERT_GT(communityTableWeight(initial, community, padded), wRunning); + EXPECT_FALSE(gate.mayTake(communityTableWeight(initial, community, padded), key(5), t + 1, kElapse)); + // the same starter, or a lower one, that saw more nodes: taken + const auto more = table({3, 5, 7, 9, 11}); + EXPECT_TRUE(gate.mayTake(communityTableWeight(initial, community, more), key(3), t + 1, kElapse)); + const auto lower = table({1, 3, 5, 7, 9}); + EXPECT_TRUE(gate.mayTake(communityTableWeight(initial, community, lower), key(1), t + 1, kElapse)); + // after half a round_elapse_time any table: a repeat is needed when a round fails + EXPECT_TRUE(gate.mayTake(communityTableWeight(initial, community, padded), key(5), t + kElapse / 2, kElapse)); +} + +TEST(CommunityBootstrap, ANeighbourOneBlockAheadHoldsTheGateOnlyForAFurtherStall) { + using cs::isNeighbourAhead; + EXPECT_FALSE(isNeighbourAhead(500, 500, kStall, kStall)); + EXPECT_FALSE(isNeighbourAhead(499, 500, kStall, kStall)); + // two blocks ahead: always, this node is behind + EXPECT_TRUE(isNeighbourAhead(502, 500, 100 * kStall, kStall)); + // one block ahead: sync gets it in seconds; while it does not, only for a further stall_minutes + EXPECT_TRUE(isNeighbourAhead(501, 500, kStall, kStall)); + EXPECT_TRUE(isNeighbourAhead(501, 500, 2 * kStall - 1, kStall)); + EXPECT_FALSE(isNeighbourAhead(501, 500, 2 * kStall, kStall)); +} + +TEST(CommunityBootstrap, AClockJumpIsProgress) { + constexpr uint64_t kGap = 5 * kMinute; + constexpr uint64_t kWall = 1'700'000'000'000ULL; + CommunityBootstrapGate gate; + gate.onClock(0, kWall, kGap); + gate.onLastSequence(500, 0); + // looks every round_elapse_time while stalled: no jump + for (uint64_t t = kElapse; t <= kStall; t += kElapse) { + gate.onClock(t, kWall + t, kGap); + } + ASSERT_TRUE(gate.isStalled(kStall, kStall)); + // the host slept for an hour: the wall clock jumped (the steady one may not count the sleep) + gate.onClock(kStall + kElapse, kWall + kStall + kElapse + 60 * kMinute, kGap); + EXPECT_FALSE(gate.isStalled(kStall + kElapse + 1, kStall)); + EXPECT_TRUE(gate.isStalled(2 * kStall + kElapse, kStall)); + // a wall clock set back, and a steady clock jump, as well + for (const bool wallBack : {true, false}) { + CommunityBootstrapGate other; + other.onClock(0, kWall, kGap); + other.onLastSequence(500, 0); + for (uint64_t t = kElapse; t <= kStall; t += kElapse) { + other.onClock(t, kWall + t, kGap); + } + ASSERT_TRUE(other.isStalled(kStall, kStall)); + const uint64_t steady = wallBack ? kStall + kElapse : kStall + 10 * kMinute; + const uint64_t wall = wallBack ? kWall + kStall - 10 * kMinute : kWall + kStall + kElapse; + other.onClock(steady, wall, kGap); + EXPECT_FALSE(other.isStalled(steady, kStall)) << (wallBack ? "wall clock set back" : "steady clock jump"); + } +} + +// lists of even size: the starter must not drop the key that holds the majority of the list +TEST(CommunityBootstrap, AnEvenSetKeepsItsMajorityOfTheList) { + using cs::dropForOddTable; + const std::set initial = keys({2, 4, 6, 8, 10}); + const std::set six = keys({1, 3, 5, 7, 9, 11}); + // 4 of 6 listed left: dropping one leaves 3, not more than half: kept even + EXPECT_FALSE(dropForOddTable(initial, six, table({1, 3, 5, 7}))); + // 6 of 6: 5 still hold the majority, the official rule applies + EXPECT_TRUE(dropForOddTable(initial, six, table({1, 3, 5, 7, 9, 11}))); + // a list of 10, 6 left: kept; 8 left: dropped to 7 + const std::set ten = keys({1, 3, 5, 7, 9, 11, 13, 15, 17, 19}); + EXPECT_FALSE(dropForOddTable(initial, ten, table({1, 3, 5, 7, 9, 11}))); + EXPECT_TRUE(dropForOddTable(initial, ten, table({1, 3, 5, 7, 9, 11, 13, 15}))); + // odd sets are never touched; regular sets (no list, or initial keys only) as in the official code + EXPECT_FALSE(dropForOddTable(initial, six, table({1, 3, 5, 7, 9}))); + EXPECT_TRUE(dropForOddTable(initial, {}, table({2, 4, 6, 8}))); + EXPECT_TRUE(dropForOddTable(initial, six, table({2, 4, 6, 8}))); +} + +namespace { +struct Starter { + cs::PublicKey pub; + cs::PrivateKey priv; +}; + +// keys live in sodium's guarded memory: the library must be initialised first +std::vector makeStarters(size_t n) { + EXPECT_TRUE(cscrypto::cryptoInit()); + std::vector result(n); + for (auto& s : result) { + s.priv = cscrypto::generateKeyPair(s.pub); + } + std::sort(result.begin(), result.end(), [](const Starter& a, const Starter& b) { return a.pub < b.pub; }); + return result; +} + +// what Node::getBootstrapTable reads of a table with a key that is not an initial confidant: the +// count, the keys, the signature over bootstrapSignedBytes(round, count and keys); the rest ignored +bool readsAsCommunityTable(const cs::Bytes& payload, cs::RoundNumber rNum, const std::set& initial, + const std::set& community) { + cs::IDataStream stream(payload.data(), payload.size()); + uint8_t confSize = 0; + stream >> confSize; + cs::PublicKeys confidants; + for (int i = 0; i < confSize; ++i) { + cs::PublicKey k; + stream >> k; + confidants.push_back(k); + } + cs::Signature sig; + stream >> sig; + if (!stream.isValid() || confidants.empty()) { + return false; + } + const size_t signedLen = 1 + size_t(confSize) * cscrypto::kPublicKeySize; + const cs::Bytes buf = cs::bootstrapSignedBytes(rNum, payload.data(), signedLen); + return isCommunityTable(initial, community, confidants) + && cscrypto::verifySignature(sig, confidants.front(), buf.data(), buf.size()); +} +} // namespace + +TEST(CommunityBootstrap, TheStartersTableVerifiesAsTheReceiverReadsIt) { + const auto s = makeStarters(5); + std::set listed; + for (const auto& st : s) { + listed.insert(st.pub); + } + const std::set initial = keys({200, 201, 202, 203, 204}); + const cs::RoundNumber round = 123'456; + const cs::Bytes payload = cs::makeCommunityTable(round, listed, s.front().priv, 0x01020304); + // count, keys, signature, 4 random bytes + ASSERT_EQ(payload.size(), 1 + 5 * cscrypto::kPublicKeySize + cscrypto::kSignatureSize + sizeof(uint32_t)); + EXPECT_EQ(payload[0], 5); + EXPECT_TRUE(readsAsCommunityTable(payload, round, initial, listed)); + // signed by another listed key than the first: not its starter's table + const cs::Bytes notStarter = cs::makeCommunityTable(round, listed, s[1].priv, 0x01020304); + EXPECT_FALSE(readsAsCommunityTable(notStarter, round, initial, listed)); + // a key changed after signing + cs::Bytes altered = payload; + altered[1 + cscrypto::kPublicKeySize] ^= 1; + EXPECT_FALSE(readsAsCommunityTable(altered, round, initial, listed)); +} + +// The p2p layer drops a broadcast whose bytes it has seen (Host::IsDuplicate, SHA1 of the data): the +// starter's repeat in the same round must differ, and still verify +TEST(CommunityBootstrap, ARepeatInTheSameRoundIsANewMessage) { + const auto s = makeStarters(4); + std::set listed; + for (const auto& st : s) { + listed.insert(st.pub); + } + const std::set initial = keys({200, 201, 202, 203, 204}); + const cs::RoundNumber round = 777; + const cs::Bytes first = cs::makeCommunityTable(round, listed, s.front().priv, 1); + const cs::Bytes repeat = cs::makeCommunityTable(round, listed, s.front().priv, 2); + EXPECT_NE(first, repeat); + EXPECT_NE(cscrypto::calculateHash(first.data(), first.size()), cscrypto::calculateHash(repeat.data(), repeat.size())); + EXPECT_TRUE(readsAsCommunityTable(first, round, initial, listed)); + EXPECT_TRUE(readsAsCommunityTable(repeat, round, initial, listed)); + // only the random bytes differ: the signature is deterministic + const size_t signedEnd = first.size() - sizeof(uint32_t); + EXPECT_TRUE(std::equal(first.begin(), first.begin() + signedEnd, repeat.begin())); +} + +// a table recorded in one round and sent again in another: the round is in the signed bytes +TEST(CommunityBootstrap, ATableReplayedFromAnotherRoundDoesNotVerify) { + const auto s = makeStarters(4); + std::set listed; + for (const auto& st : s) { + listed.insert(st.pub); + } + const std::set initial = keys({200, 201, 202, 203, 204}); + const cs::Bytes payload = cs::makeCommunityTable(1000, listed, s.front().priv, 42); + EXPECT_TRUE(readsAsCommunityTable(payload, 1000, initial, listed)); + EXPECT_FALSE(readsAsCommunityTable(payload, 1001, initial, listed)); + EXPECT_FALSE(readsAsCommunityTable(payload, 999, initial, listed)); } namespace { From 03973a5a0c38b6ab2263e949f094b045a26d07b9 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 07:26:29 +0100 Subject: [PATCH 08/22] ci: staggered community stall and official-partition scenarios - COMMUNITY_STALL_STARTER: the community starter (lowest key) gets a shorter stall_minutes than the others; each of them must drop a table and take a repeat (official-stop-config-staggered) - OFFICIAL_PARTITION_AT: community nodes cut off from the running official nodes (iptables, UDP by port, TCP by cgroup) for OFFICIAL_PARTITION_MINUTES, then reconnected; partition.py reports the fork, which side was kept and what the other lost; one chain required at the end (official-partition) - logs and node binaries of branch runs kept 3 days Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 22 ++++- ci/testnet/partition.py | 119 +++++++++++++++++++++++ ci/testnet/run.sh | 156 +++++++++++++++++++++++++++++-- 3 files changed, 288 insertions(+), 9 deletions(-) create mode 100755 ci/testnet/partition.py diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index ea2b026f0..9b323f523 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -121,7 +121,7 @@ jobs: with: name: testnet-node-${{ github.sha }} path: build/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} # mixed nodes, as in testnet.yml: the even nodes run a node built from another ref - name: Build the node under comparison @@ -182,7 +182,7 @@ jobs: with: name: testnet-node-alt-${{ github.sha }} path: build-alt/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} scenario: needs: build @@ -518,6 +518,20 @@ jobs: env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_STOP_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 fixed: true manual: true + # the same with staggered stall_minutes: the starter (the community node with the lowest key) + # 2, the others 3. Its first tables reach nodes whose fallback is still closed: the restart + # needs a repeat of the table in the same round to get through (not dropped as a p2p duplicate) + - name: official-stop-config-staggered + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_STOP_AT=400 OFFICIAL_STOP_MINUTES=7 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=3 COMMUNITY_STALL_STARTER=2 RESTART_AT=0 LOAD_SECONDS=1500 + fixed: true + manual: true + # the community nodes cut off from the RUNNING official nodes for 6 minutes (fallback on, + # stall_minutes=2): both sides write blocks; after the reconnection the run reports which + # side the nodes kept, how many blocks and transactions the other lost, and needs one chain + - name: official-partition + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 + fixed: true + manual: true # the fallback on (stall_minutes=1) in a network whose official nodes keep running, under # packet loss and the restart test: it must never act - name: community-bootstrap-quiet @@ -634,4 +648,6 @@ jobs: testnet-run/resources.tsv testnet-run/resources-summary.txt testnet-run/sampler.log - retention-days: 14 + testnet-run/partition-*.txt + # branch runs (test variants) are kept briefly: Actions storage is shared + retention-days: ${{ github.ref_name == 'master' && 14 || 3 }} diff --git a/ci/testnet/partition.py b/ci/testnet/partition.py new file mode 100755 index 000000000..4fcb61f64 --- /dev/null +++ b/ci/testnet/partition.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""The two chains of a partition (run.sh, OFFICIAL_PARTITION_AT) and the chain left after it. + + partition.py dump --from 395 --port 9090 > community.txt # "seq hash transactions" per block + partition.py report --community community.txt --official official.txt --port 9090 --port 9102 + +dump: the blocks of one node from a sequence to its head, taken just before the reconnection. +report: where the two sides forked, and which side's blocks the nodes hold after the reconnection +(each port given, every block of both sides compared); the side whose blocks are gone was unwound, +and the transactions of its blocks above the fork with them (unless written again on the other side). +""" +import argparse +import sys + +from thriftpy2.protocol import TBinaryProtocolFactory +from thriftpy2.rpc import make_client +from thriftpy2.transport import TBufferedTransportFactory + +from fund import api + + +def client_of(port): + return make_client(api.API, host="127.0.0.1", port=int(port), proto_factory=TBinaryProtocolFactory(), + trans_factory=TBufferedTransportFactory(), timeout=60000) + + +def blocks(client, first, last): + """{seq: (hash, transactions)} of the blocks first..last a node holds""" + result = {} + seq = first + while seq <= last: + count = min(100, last - seq + 1) + pools = client.PoolListGetStable(seq + count - 1, count).pools or [] + for pool in pools: + if first <= pool.poolNumber <= last: + result[pool.poolNumber] = (bytes(pool.hash).hex(), pool.transactionsCount) + seq += count + return result + + +def head(client): + return client.SyncStateGet().lastBlock + + +def read(path): + result = {} + with open(path) as f: + for line in f: + seq, h, txs = line.split() + result[int(seq)] = (h, int(txs)) + return result + + +def dump(args): + client = client_of(args.port) + last = head(client) + for seq, (h, txs) in sorted(blocks(client, args.first, last).items()): + print(seq, h, txs) + + +def report(args): + community = read(args.community) + official = read(args.official) + common = sorted(set(community) & set(official)) + fork = next((s for s in common if community[s][0] != official[s][0]), None) + c_head = max(community) if community else 0 + o_head = max(official) if official else 0 + if fork is None: + # one side may only be shorter: the fork is right above the shorter head + fork = min(c_head, o_head) + 1 + c_side = {s: v for s, v in community.items() if s >= fork} + o_side = {s: v for s, v in official.items() if s >= fork} + print(f"PARTITION RESULT: fork at #{fork}; before the reconnection the community side held #{fork}..#{c_head} " + f"({len(c_side)} blocks, {sum(t for _, t in c_side.values())} transactions), the official side #{fork}..#{o_head} " + f"({len(o_side)} blocks, {sum(t for _, t in o_side.values())} transactions)") + failed = False + verdicts = set() + for port in args.port: + client = client_of(port) + last = max(c_head, o_head) + now = blocks(client, fork, last) + kept_c = sum(1 for s, v in c_side.items() if now.get(s, (None,))[0] == v[0]) + kept_o = sum(1 for s, v in o_side.items() if now.get(s, (None,))[0] == v[0]) + lost_c_txs = sum(v[1] for s, v in c_side.items() if now.get(s, (None,))[0] != v[0]) + lost_o_txs = sum(v[1] for s, v in o_side.items() if now.get(s, (None,))[0] != v[0]) + if c_side and kept_c == 0 and kept_o == len(o_side): + verdict = f"the community side was unwound ({len(c_side)} blocks, {lost_c_txs} transactions reverted)" + elif o_side and kept_o == 0 and kept_c == len(c_side): + verdict = f"the official side was unwound ({len(o_side)} blocks, {lost_o_txs} transactions reverted)" + else: + verdict = f"mixed: {kept_c}/{len(c_side)} community and {kept_o}/{len(o_side)} official blocks kept" + failed = True + verdicts.add(verdict) + print(f"PARTITION RESULT: port {port} (head #{head(client)}): {verdict}") + if len(verdicts) > 1: + print("PARTITION RESULT: the nodes compared did not keep the same side") + failed = True + sys.exit(1 if failed else 0) + + +def main(): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + d = sub.add_parser("dump") + d.add_argument("--from", dest="first", type=int, required=True) + d.add_argument("--port", required=True) + r = sub.add_parser("report") + r.add_argument("--community", required=True) + r.add_argument("--official", required=True) + r.add_argument("--port", action="append", required=True) + args = parser.parse_args() + if args.command == "dump": + dump(args) + else: + report(args) + + +if __name__ == "__main__": + main() diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index a901f8e93..4b6d2e608 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -210,7 +210,19 @@ # bootstrap fallback ([community_bootstrap] confidants = every community key, stall_minutes = # COMMUNITY_STALL_MINUTES, default 2), the network must restart without them. COMMUNITY_BOOTSTRAP=config # also runs without OFFICIAL_STOP_AT: the fallback must then never act (no "COMMUNITY BOOTSTRAP:" -# table started or taken), and with OFFICIAL_STOP_AT never before the official nodes stop +# table started or taken), and with OFFICIAL_STOP_AT never before the official nodes stop. +# COMMUNITY_STALL_STARTER (minutes, default 0 = off, with COMMUNITY_BOOTSTRAP=config): the +# community node with the lowest key, which starts the community table, gets that stall_minutes +# and the others COMMUNITY_STALL_MINUTES: its first tables reach nodes whose fallback is still +# closed, so the restart needs a repeat of the table to get through (each community node other +# than the starter must drop one table and take a later one) +# OFFICIAL_PARTITION_AT (default 0 = off, needs BIN_ALT_OFFICIAL=1 and Linux; not with +# OFFICIAL_STOP_AT): at that sequence the community (odd) nodes are cut off from the official +# (even) ones, all of them running (iptables: UDP between their p2p ports, TCP by cgroup), for +# OFFICIAL_PARTITION_MINUTES (default 6), then reconnected. With COMMUNITY_BOOTSTRAP=config the +# community side restarts on its own after stall_minutes: two chains until the reconnection. +# partition.py reports where they forked, which side the nodes kept and how many blocks and +# transactions the other side lost; every node must then end on one chain # # Checks: every node writes blocks up to TARGET_SEQ; a node stopped and restarted (quick start # from its caches) catches up; all nodes report the same STATE DIGEST at every common sequence. @@ -323,6 +335,10 @@ OFFICIAL_FUND=${OFFICIAL_FUND:-0} COMMUNITY_BOOTSTRAP=${COMMUNITY_BOOTSTRAP:-none} case "$COMMUNITY_BOOTSTRAP" in none|trusted|config) ;; *) echo "COMMUNITY_BOOTSTRAP must be none, trusted or config, got '$COMMUNITY_BOOTSTRAP'"; exit 1 ;; esac COMMUNITY_STALL_MINUTES=${COMMUNITY_STALL_MINUTES:-2} +COMMUNITY_STALL_STARTER=${COMMUNITY_STALL_STARTER:-0} +COMMUNITY_STARTER=0 # the node given COMMUNITY_STALL_STARTER, set with the configs +OFFICIAL_PARTITION_AT=${OFFICIAL_PARTITION_AT:-0} +OFFICIAL_PARTITION_MINUTES=${OFFICIAL_PARTITION_MINUTES:-6} JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} @@ -566,6 +582,16 @@ if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then # the community nodes that could start a bootstrap with the extra keys: MinTrustedNodes is 4 [ $(( (NODES + 1) / 2 )) -ge 5 ] || { echo "FAIL: OFFICIAL_STOP_AT needs at least 5 community (odd) nodes, NODES >= 9"; exit 1; } fi +# the community nodes cut off from the running official nodes (see OFFICIAL_PARTITION_AT above) +if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then + [ "$BIN_ALT_OFFICIAL" = 1 ] || { echo "FAIL: OFFICIAL_PARTITION_AT needs official even nodes (BIN_ALT_OFFICIAL=1)"; exit 1; } + [ "$OS_KIND" = linux ] || { echo "FAIL: OFFICIAL_PARTITION_AT needs Linux (iptables, cgroups)"; exit 1; } + [ "$OFFICIAL_STOP_AT" -eq 0 ] && [ "$CRASH_ALL_AT" -eq 0 ] && [ "$DPOS_AT" -eq 0 ] && [ "$JOIN_NODE" -eq 0 ] && [ -z "$NET_FAULT" ] \ + || { echo "FAIL: OFFICIAL_PARTITION_AT runs without OFFICIAL_STOP_AT, CRASH_ALL_AT, DPOS_AT, NET_FAULT and a joining node"; exit 1; } + [ $(( (NODES + 1) / 2 )) -ge 5 ] || { echo "FAIL: OFFICIAL_PARTITION_AT needs at least 5 community (odd) nodes, NODES >= 9"; exit 1; } +fi +[ "$COMMUNITY_STALL_STARTER" -eq 0 ] || [ "$COMMUNITY_BOOTSTRAP" = config ] \ + || { echo "FAIL: COMMUNITY_STALL_STARTER needs COMMUNITY_BOOTSTRAP=config"; exit 1; } mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$ALL_NODES" "$WORK") if [ "$FUND" = "1" ]; then @@ -776,11 +802,20 @@ fi if [ "$COMMUNITY_BOOTSTRAP" = config ]; then listed="" for j in $(seq 1 "$NODES"); do alt_node "$j" || listed="$listed${listed:+,}${KEYS[$((j - 1))]}"; done + # COMMUNITY_STALL_STARTER: the node of this build with the lowest key (bytes, as cs::PublicKey + # compares) starts the community table; it waits less than the others + if [ "$COMMUNITY_STALL_STARTER" -gt 0 ]; then + COMMUNITY_STARTER=$(for j in $(seq 1 "$NODES"); do alt_node "$j" || echo "$j ${KEYS[$((j - 1))]}"; done \ + | python3 -c 'import sys, base58; print(min((base58.b58decode(k), int(j)) for j, k in (l.split() for l in sys.stdin))[1])') + fi for i in $(seq 1 "$NODES"); do alt_node "$i" && continue - printf '\n[community_bootstrap]\nconfidants=%s\nstall_minutes=%s\n' "$listed" "$COMMUNITY_STALL_MINUTES" >> "$WORK/n$i/config.ini" + stall=$COMMUNITY_STALL_MINUTES + [ "$i" -ne "$COMMUNITY_STARTER" ] || stall=$COMMUNITY_STALL_STARTER + printf '\n[community_bootstrap]\nconfidants=%s\nstall_minutes=%s\n' "$listed" "$stall" >> "$WORK/n$i/config.ini" done - echo "community bootstrap: on in every node of this build, $(tr ',' '\n' <<<"$listed" | grep -c .) keys listed, stall_minutes=$COMMUNITY_STALL_MINUTES" + echo "community bootstrap: on in every node of this build, $(tr ',' '\n' <<<"$listed" | grep -c .) keys listed, stall_minutes=$COMMUNITY_STALL_MINUTES" \ + "$([ "$COMMUNITY_STARTER" -eq 0 ] || echo "(n$COMMUNITY_STARTER, the starter: $COMMUNITY_STALL_STARTER)")" fi # test-build hooks of a forged or corrupt snapshot source, and the joiner's list of serving nodes @@ -822,7 +857,7 @@ start_node() { PIDS[$i]=$! # a slow or faulty node runs in its own cgroup: a CPU quota for a slow one, and its packets are # marked for the network fault (its executor, started later, inherits the cgroup) - if [[ ",$SLOW_NODES,$NET_FAULT_NODES," == *",$i,"* ]]; then + if [[ ",$SLOW_NODES,$NET_FAULT_NODES," == *",$i,"* ]] || [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then sudo mkdir -p "/sys/fs/cgroup/cstest-$i" if [[ ",$SLOW_NODES," == *",$i,"* ]]; then echo "$((SLOW_CPU * 1000)) 100000" | sudo tee "/sys/fs/cgroup/cstest-$i/cpu.max" > /dev/null @@ -850,6 +885,31 @@ net_fault_off() { sudo tc qdisc del dev lo root 2>/dev/null || true sudo iptables -t mangle -F OUTPUT 2>/dev/null || true } +# OFFICIAL_PARTITION_AT: nothing passes between a community and an official node, both ways. UDP by +# the p2p ports (a node sends its datagrams from its own port), TCP by the sender's cgroup +partition_on() { + local i j + sudo iptables -N CSPART + sudo iptables -I OUTPUT 1 -o lo -j CSPART + for i in $COMMUNITY_NODES; do + for j in $OFFICIAL_NODES; do + sudo iptables -A CSPART -p udp --sport $((BASE_PORT + i)) --dport $((BASE_PORT + j)) -j DROP + sudo iptables -A CSPART -p udp --sport $((BASE_PORT + j)) --dport $((BASE_PORT + i)) -j DROP + sudo iptables -A CSPART -p tcp -m cgroup --path "cstest-$i" --dport $((BASE_PORT + j)) -j DROP \ + || echo "partition: no cgroup match, TCP from n$i to n$j still passes" + sudo iptables -A CSPART -p tcp -m cgroup --path "cstest-$j" --dport $((BASE_PORT + i)) -j DROP \ + || echo "partition: no cgroup match, TCP from n$j to n$i still passes" + done + done +} +partition_off() { + sudo iptables -D OUTPUT -o lo -j CSPART 2>/dev/null || true + sudo iptables -F CSPART 2>/dev/null || true + sudo iptables -X CSPART 2>/dev/null || true +} +partition_stats() { + sudo iptables -L CSPART -v -x -n 2>/dev/null | awk 'NR > 2 { p += $1 } END { print p + 0 }' +} node_nets_on() { for i in $(seq 1 "$ALL_NODES"); do sudo iptables -t nat -A POSTROUTING -o lo -p udp --sport $((BASE_PORT + i)) -j SNAT --to-source "$(node_ip "$i")" \ @@ -892,6 +952,7 @@ stop_all() { echo "net fault: until the end ($(net_fault_stats))" fi [ -z "$NET_FAULT" ] || net_fault_off + [ "$OFFICIAL_PARTITION_AT" -eq 0 ] || partition_off [ "$NODE_NETS" != 1 ] || node_nets_off [ -z "${GARBAGE_PID:-}" ] || kill "$GARBAGE_PID" 2>/dev/null || true [ ! -f "$WORK/resources-summary.txt" ] || cat "$WORK/resources-summary.txt" @@ -1134,6 +1195,21 @@ official_stop() { echo "OFFICIAL STOP RESULT: restart after $((first_t - t_kill)) s (first block $first_seq), $((c - s0)) blocks in $OFFICIAL_STOP_MINUTES min" fi fi + # COMMUNITY_STALL_STARTER: the others dropped the starter's first tables (fallback still closed) + # and took a repeat: a repeat in the same round is a new message, not a p2p duplicate + if [ "$COMMUNITY_STARTER" -gt 0 ] && [ "$first_t" -gt 0 ]; then + local dropped taken + for i in $COMMUNITY_NODES; do + [ "$i" -ne "$COMMUNITY_STARTER" ] || continue + dropped=$(grep -ac 'community bootstrap table .* has not stalled long enough' "$WORK/n$i/node.log" || true) + taken=$(grep -ac 'COMMUNITY BOOTSTRAP: taking' "$WORK/n$i/node.log" || true) + echo "OFFICIAL STOP RESULT: n$i dropped $dropped community tables while its fallback was closed, then took $taken" + if [ "$dropped" -eq 0 ] || [ "$taken" -eq 0 ]; then + off_fail="${off_fail:+$off_fail; }n$i: $dropped tables dropped and $taken taken, a repeat taken after a drop expected" + fi + done + grep -a 'COMMUNITY BOOTSTRAP: starting' "$WORK/n$COMMUNITY_STARTER/node.log" | head -3 | sed "s/^/official stop: n$COMMUNITY_STARTER (starter): /" + fi # the community nodes agree on every block they hold (all of them, compared over the API) local ports="" head head=$(community_min) @@ -1150,6 +1226,54 @@ official_stop() { echo "official stop: $(printf 'n%s ' $OFFICIAL_NODES)started again, community head $c" } +# OFFICIAL_PARTITION_AT: the community nodes cut off from the running official nodes +part_done=0 # 1 while partitioned, 2 once reconnected +part_fail="" +part_from=0 # the community head at the cut: the chains are dumped from a little below it +official_max() { + local m=0 s i + for i in $OFFICIAL_NODES; do s=$(seq_of "$i"); s=${s:-0}; [ "$s" -le "$m" ] || m=$s; done + echo "$m" +} +official_partition() { + local t_cut s0 o0 first_t=0 c o end i + for i in $COMMUNITY_NODES; do + if grep -aq 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log"; then + part_fail="${part_fail:+$part_fail; }n$i used the community bootstrap before the partition" + fi + done + s0=$(community_max); o0=$(official_max) + part_from=$(( s0 > 5 ? s0 - 5 : 0 )) + partition_on + t_cut=$(date +%s) + echo "partition: $(printf 'n%s ' $COMMUNITY_NODES)cut off from $(printf 'n%s ' $OFFICIAL_NODES)for $OFFICIAL_PARTITION_MINUTES min (community head $s0, official head $o0), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" + end=$(( t_cut + OFFICIAL_PARTITION_MINUTES * 60 )) + while [ "$(date +%s)" -lt "$end" ]; do + sleep 5 + c=$(community_max) + if [ "$first_t" -eq 0 ] && [ "$c" -gt $((s0 + 1)) ]; then + first_t=$(date +%s) + echo "partition: community block $c $((first_t - t_cut)) s after the cut" + fi + done + c=$(community_max); o=$(official_max) + echo "PARTITION RESULT: after $OFFICIAL_PARTITION_MINUTES min the community head is $c ($((c - s0)) blocks, restart after" \ + "$([ "$first_t" -eq 0 ] && echo never || echo "$((first_t - t_cut)) s")), the official head $o ($((o - o0)) blocks); $(partition_stats) packets dropped" + for i in $COMMUNITY_NODES; do + { grep -a 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log" || true; } | tail -1 | sed "s/^/partition: n$i: /" + done + # the two chains just before the reconnection: a community and an official node + local cn on + cn=$(echo $COMMUNITY_NODES | cut -d' ' -f1); on=$(echo $OFFICIAL_NODES | cut -d' ' -f1) + (cd "$HERE" && python3 partition.py dump --from "$part_from" --port "$(api_port "$cn")") > "$WORK/partition-community.txt" \ + || part_fail="${part_fail:+$part_fail; }no chain dump of n$cn" + (cd "$HERE" && python3 partition.py dump --from "$part_from" --port "$(api_port "$on")") > "$WORK/partition-official.txt" \ + || part_fail="${part_fail:+$part_fail; }no chain dump of n$on" + partition_off + part_done=2 + echo "partition: reconnected (community head $c, official head $o)" +} + while :; do sleep 20 line="t=$(( TIMEOUT_MIN * 60 - (deadline - $(date +%s)) ))s" @@ -1304,6 +1428,10 @@ while :; do echo "crash test: network restarted" fi + if [ "$OFFICIAL_PARTITION_AT" -gt 0 ] && [ "$part_done" -eq 0 ] && [ "$min" -ge "$OFFICIAL_PARTITION_AT" ]; then + part_done=1 + official_partition + fi if [ "$OFFICIAL_STOP_AT" -gt 0 ] && [ "$off_done" -eq 0 ] && [ "$min" -ge "$OFFICIAL_STOP_AT" ]; then off_done=1 official_stop @@ -2440,6 +2568,21 @@ if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then if [ -n "$off_fail" ]; then echo "FAIL: official stop: $off_fail"; exit 1; fi fi +if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then + [ "$part_done" -eq 2 ] || part_fail="${part_fail:+$part_fail; }the partition never ended" + if [ "$part_done" -eq 2 ]; then + ports="" + for i in $COMMUNITY_NODES $OFFICIAL_NODES; do ports="$ports --port $(api_port "$i")"; done + (cd "$HERE" && python3 partition.py report --community "$WORK/partition-community.txt" --official "$WORK/partition-official.txt" $ports) \ + || part_fail="${part_fail:+$part_fail; }the nodes did not all keep the blocks of one side" + fi + for i in $COMMUNITY_NODES $OFFICIAL_NODES; do + echo "partition: n$i logged $(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true) bootstraps," \ + "$(grep -ac 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log" || true) community tables started or taken" + done + if [ -n "$part_fail" ]; then echo "FAIL: partition: $part_fail"; exit 1; fi +fi + # official nodes: the same block hashes as n1 (every 100th sequence up to the lowest head) if [ "$BIN_ALT_OFFICIAL" = 1 ]; then ports="" @@ -2451,7 +2594,7 @@ if [ "$BIN_ALT_OFFICIAL" = 1 ]; then done # pruned nodes, and a node that joined from a snapshot, lack the oldest blocks partial=$(for i in $PRUNED_NODES $({ [ "$JOIN_FETCH" = p2p ] || [ "$SNAPSHOT_AT" -gt 0 ]; } && echo "$JOIN_NODE" $EXTRA_JOINERS); do echo "$i"; done | sort -u | paste -sd, -) - (cd "$HERE" && python3 blockhashes.py --head "$head" --partial "$partial" --step "$([ "$OFFICIAL_STOP_AT" -gt 0 ] && echo 1 || echo 100)" $ports) \ + (cd "$HERE" && python3 blockhashes.py --head "$head" --partial "$partial" --step "$([ "$OFFICIAL_STOP_AT" -gt 0 ] || [ "$OFFICIAL_PARTITION_AT" -gt 0 ] && echo 1 || echo 100)" $ports) \ || { echo "FAIL: official and community nodes hold different blocks"; exit 1; } fi @@ -2465,11 +2608,12 @@ if [ -n "$forks" ]; then fi # the community bootstrap fallback in a network that never lost its official nodes: never used -if [ "$COMMUNITY_BOOTSTRAP" = config ] && [ "$OFFICIAL_STOP_AT" -eq 0 ]; then +if [ "$COMMUNITY_BOOTSTRAP" = config ] && [ "$OFFICIAL_STOP_AT" -eq 0 ] && [ "$OFFICIAL_PARTITION_AT" -eq 0 ]; then used=$(grep -alE 'COMMUNITY BOOTSTRAP: (starting|taking)' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) [ -z "$used" ] || { grep -ah 'COMMUNITY BOOTSTRAP' "$WORK"/n*/node.log | head -5; echo "FAIL: the community bootstrap acted on $used"; exit 1; } echo "community bootstrap: never used ($(grep -ah 'NODE> Bootstrap on' "$WORK"/n*/node.log | wc -l) regular bootstrap lines on all nodes)" fi +[ "$OFFICIAL_PARTITION_AT" -eq 0 ] || echo "PARTITION RESULT: one chain: every node holds the same blocks (all compared), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" [ "$OFFICIAL_STOP_AT" -eq 0 ] || echo "OFFICIAL STOP RESULT: one chain: every node holds the same blocks (all compared), COMMUNITY_BOOTSTRAP=$COMMUNITY_BOOTSTRAP" # a network still restarting at the end is stuck, even if it reached the target before (a bootstrap From 897953c174eba5e69ad8739ff02ec122114bdf84 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 07:26:29 +0100 Subject: [PATCH 09/22] docs: differences.md: community bootstrap review fixes and the partition behaviour Co-Authored-By: Claude Opus 5.5 --- docs/differences.md | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/differences.md b/docs/differences.md index acd9e8e3f..4f6e7731b 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -141,29 +141,46 @@ stall_minutes=30 ``` With the list set, and only after `stall_minutes` in which the node saw no new block, no round -package and no regular bootstrap table, and while no neighbour reports a block above its own: +package whose signatures it checked and accepted, and no regular bootstrap table, and while no +neighbour reports a block two or more above its own (one above only holds it for a further +`stall_minutes`, since sync would have fetched that block long before): - a node whose key is an initial confidant or listed may start a bootstrap table of the initial confidants and listed nodes that hold the highest block, as an initial confidant does; the table is signed by its starter, not by the project; - a node with the list takes such a table only if it has more than half of the listed keys, each key is an initial confidant or listed, and the starter's signature checks. The round window and - the stall check of every unsigned table still apply, and a node takes at most one such table per - half `round_elapse_time` unless it is larger. + the stall check of every unsigned table still apply. A node takes one such table per half + `round_elapse_time`; within that time only a larger table whose starter (its lowest key) is not + above the current table's starter replaces it, so a listed node cannot displace a running table + by padding its own with keys of stopped nodes. The starter repeats its table every + `round_elapse_time` with new random bytes after the signature, so that a node whose fallback + opens later gets the repeat (the transport drops a message it has seen before); +- of an even set of nodes the starter drops the last key, as the official code does, unless that + would leave the table without more than half of the listed keys; +- a jump of the clock between two checks (the host suspended and resumed, the clock set) counts + as progress: the node watches again for `stall_minutes` before it acts. In normal operation nothing changes: blocks and round packages keep the fallback closed, a regular bootstrap table is handled exactly as before and always replaces a community one, and blocks, -hashes and validation are unchanged. Official nodes, and community nodes without the list, drop a +hashes and validation are unchanged. A regular table, even one signed by the project, is progress +like a block: if no block follows it for a further `stall_minutes`, a community table may replace it. Official nodes, and community nodes without the list, drop a community table (it holds a key that is not an initial confidant and is not signed by the project); they then take the blocks written meanwhile through ordinary sync once they are back, as the `official-stop-*` scenarios of the robustness workflow check. Every node that should take part needs the same list: it only helps when more than half of the listed nodes are running. The set-up scripts and the Docker image do not offer the option; edit -`config.ini` by hand (in Docker, `~/.credits/config.ini`). The risk it brings: if a network -partition cuts more than half of the listed nodes off from the official nodes for longer than -`stall_minutes`, while the official nodes keep producing blocks, the listed nodes restart on their -own and the two sides write different blocks. Keep `stall_minutes` long. +`config.ini` by hand (in Docker, `~/.credits/config.ini`). + +The risk it brings is a partition. If a network partition cuts more than half of the listed nodes +off from the official nodes for longer than `stall_minutes`, while the official nodes keep +producing blocks, the listed nodes restart on their own and the two sides write different blocks. +There is no rule that makes either side yield: when the two sides reconnect, the longer chain +wins, as for any two chains, and the nodes of the other side unwind their blocks above the fork. +The transactions of the unwound blocks are reverted; those the other side did not also write must +be sent again. The `official-partition` scenario of the robustness workflow measures this. Keep +`stall_minutes` long. ## Start-up behaviour From 67c2b8c451e3cd3ef09f02fcd7333ccd044a5065 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 08:55:43 +0100 Subject: [PATCH 10/22] ci: official-partition: neighbour counts after the reconnection, and a restart variant After a 6 minute cut the two sides did not look each other up again (each node kept only its own side as neighbours). The run now reports the neighbour counts 120 s after the cut is lifted; OFFICIAL_PARTITION_RESTART=1 then restarts the community nodes, which dial every host again (official-partition-restart). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 5 +++++ ci/testnet/run.sh | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 9b323f523..cedb7165a 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -532,6 +532,11 @@ jobs: env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 fixed: true manual: true + # the same, the community nodes restarted after the reconnection (they dial every host again) + - name: official-partition-restart + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 + fixed: true + manual: true # the fallback on (stall_minutes=1) in a network whose official nodes keep running, under # packet loss and the restart test: it must never act - name: community-bootstrap-quiet diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 4b6d2e608..53eddf42c 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -222,7 +222,10 @@ # OFFICIAL_PARTITION_MINUTES (default 6), then reconnected. With COMMUNITY_BOOTSTRAP=config the # community side restarts on its own after stall_minutes: two chains until the reconnection. # partition.py reports where they forked, which side the nodes kept and how many blocks and -# transactions the other side lost; every node must then end on one chain +# transactions the other side lost; every node must then end on one chain. The nodes only +# reconnect if they look each other up again: 120 s after the cut is lifted the run reports +# how many neighbours each node has. OFFICIAL_PARTITION_RESTART=1: then the community nodes are +# restarted (they dial every node of hosts.txt again), as an operator would # # Checks: every node writes blocks up to TARGET_SEQ; a node stopped and restarted (quick start # from its caches) catches up; all nodes report the same STATE DIGEST at every common sequence. @@ -339,6 +342,7 @@ COMMUNITY_STALL_STARTER=${COMMUNITY_STALL_STARTER:-0} COMMUNITY_STARTER=0 # the node given COMMUNITY_STALL_STARTER, set with the configs OFFICIAL_PARTITION_AT=${OFFICIAL_PARTITION_AT:-0} OFFICIAL_PARTITION_MINUTES=${OFFICIAL_PARTITION_MINUTES:-6} +OFFICIAL_PARTITION_RESTART=${OFFICIAL_PARTITION_RESTART:-0} JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} @@ -1272,6 +1276,21 @@ official_partition() { partition_off part_done=2 echo "partition: reconnected (community head $c, official head $o)" + # have the two sides found each other again? (each node logs its neighbour count with its blocks) + sleep 120 + for i in $COMMUNITY_NODES $OFFICIAL_NODES; do + echo "partition: 120 s after the reconnection n$i has $(grep -a 'neighbour nodes = ' "$WORK/n$i/node.log" | tail -1 | sed -E 's/.*neighbour nodes = ([0-9]+).*/\1/') neighbours, head $(seq_of "$i")" + done + if [ "$OFFICIAL_PARTITION_RESTART" = 1 ]; then + echo "partition: restarting $(printf 'n%s ' $COMMUNITY_NODES)(community head $(community_max), official head $(official_max))" + for i in $COMMUNITY_NODES; do kill -TERM "${PIDS[$i]}"; REPORTED_DOWN[$i]=1; done + for i in $COMMUNITY_NODES; do + wait "${PIDS[$i]}" 2>/dev/null || true + pkill -f "cs.testnet.node=n$i " 2>/dev/null || true + done + for i in $COMMUNITY_NODES; do start_node "$i"; REPORTED_DOWN[$i]=""; done + echo "partition: community nodes started again" + fi } while :; do From 24903fe01126db002c218bb30bc4de9d4b921af1 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 11:26:47 +0100 Subject: [PATCH 11/22] ci: official-partition-restart: SIGKILL a community node still running 60 s after SIGTERM Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 53eddf42c..3bab3083e 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -1284,7 +1284,16 @@ official_partition() { if [ "$OFFICIAL_PARTITION_RESTART" = 1 ]; then echo "partition: restarting $(printf 'n%s ' $COMMUNITY_NODES)(community head $(community_max), official head $(official_max))" for i in $COMMUNITY_NODES; do kill -TERM "${PIDS[$i]}"; REPORTED_DOWN[$i]=1; done + # a trusted node waits for its round before it exits, and with most of the round's + # confidants stopping that round never ends: SIGKILL after 60 s + local t_stop + t_stop=$(date +%s) for i in $COMMUNITY_NODES; do + while kill -0 "${PIDS[$i]}" 2>/dev/null && [ $(( $(date +%s) - t_stop )) -lt 60 ]; do sleep 1; done + if kill -0 "${PIDS[$i]}" 2>/dev/null; then + echo "partition: n$i still running 60 s after SIGTERM, SIGKILL" + kill -KILL "${PIDS[$i]}" 2>/dev/null || true + fi wait "${PIDS[$i]}" 2>/dev/null || true pkill -f "cs.testnet.node=n$i " 2>/dev/null || true done From 6d44d6c6c7c3d8b1a4ec949ba9c2dd0d6c3dc67c Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 12:45:35 +0100 Subject: [PATCH 12/22] docs: differences.md: the partition with the bounded-rollback guard (#189), and what CI measured Co-Authored-By: Claude Opus 5.5 --- docs/differences.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/differences.md b/docs/differences.md index 4f6e7731b..8f1ac1fba 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -176,10 +176,25 @@ listed nodes are running. The set-up scripts and the Docker image do not offer t The risk it brings is a partition. If a network partition cuts more than half of the listed nodes off from the official nodes for longer than `stall_minutes`, while the official nodes keep producing blocks, the listed nodes restart on their own and the two sides write different blocks. -There is no rule that makes either side yield: when the two sides reconnect, the longer chain -wins, as for any two chains, and the nodes of the other side unwind their blocks above the fork. -The transactions of the unwound blocks are reverted; those the other side did not also write must -be sent again. The `official-partition` scenario of the robustness workflow measures this. Keep +There is no rule that makes either side yield to the other. What happens at reconnection: + +- official nodes follow the longer chain, as for any two chains, and unwind their blocks above the + fork; the transactions of those blocks are reverted and must be sent again unless the other + side also wrote them; +- community nodes with the bounded-rollback guard (#189, `allow_deep_rollback=false` by default) + do not follow a longer chain they cannot authenticate themselves: a node on the shorter side + keeps its chain and waits until its operator approves the switch (`[params] + allow_deep_rollback` or `CS_ALLOW_DEEP_ROLLBACK`), after which its blocks above the fork, and + their transactions, are reverted. + +Measured in the `official-partition` scenarios of the robustness workflow (before #189, community +side cut off for 6 minutes, `stall_minutes=2`): the two sides did not look each other up again on +their own after the cut was lifted, each node keeping only its own side as neighbours, so both +chains went on side by side until the end of the run. With the community nodes restarted after the +reconnection (`official-partition-restart`), they connected to the official nodes and unwound +their shorter chain block by block from #950 to #544, then stopped two blocks above the fork +(#542) and did not take the official blocks: still no single chain. A +partition with this option on therefore needs operator attention in any case. Keep `stall_minutes` long. ## Start-up behaviour From 6b01fff1c668dc54711d8ab9f859d68e9dede85d Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 20 Nov 2026 09:42:11 +0100 Subject: [PATCH 13/22] ci: official-partition: --resolve-fork after the reconnection (RESOLVE_FORK_AFTER) RESOLVE_FORK_AFTER=1 waits up to RESOLVE_FORK_WAIT minutes for FORK HELD on the community nodes, stops each one holding a fork, runs it once with --resolve-fork and starts it again: the operator choosing the official chain. New scenarios official-partition-resolve and official-partition-restart-resolve. Refusals are expected in the partition scenarios (DEEP_ROLLBACK_EXPECTED=1), and the reorganisation report is printed before the partition verdict. fork-held.json and reorg-episode.json are uploaded with the logs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 19 ++++++- ci/testnet/run.sh | 98 +++++++++++++++++++++++++------- 2 files changed, 94 insertions(+), 23 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index cedb7165a..93903933c 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -528,13 +528,26 @@ jobs: # the community nodes cut off from the RUNNING official nodes for 6 minutes (fallback on, # stall_minutes=2): both sides write blocks; after the reconnection the run reports which # side the nodes kept, how many blocks and transactions the other lost, and needs one chain + # (refusals and FORK HELD on the community side are expected: DEEP_ROLLBACK_EXPECTED=1) - name: official-partition - env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 fixed: true manual: true # the same, the community nodes restarted after the reconnection (they dial every host again) - name: official-partition-restart - env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 + fixed: true + manual: true + # the operator path after a partition: the community nodes hold the fork (FORK HELD, the + # longer chain is not vouched for by their confidants); each one is stopped, run once with + # --resolve-fork and started again, and every node must end on the official chain. Without + # and with the restart after the reconnection + - name: official-partition-resolve + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 RESOLVE_FORK_AFTER=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 + fixed: true + manual: true + - name: official-partition-restart-resolve + env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 RESOLVE_FORK_AFTER=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 fixed: true manual: true # the fallback on (stall_minutes=1) in a network whose official nodes keep running, under @@ -654,5 +667,7 @@ jobs: testnet-run/resources-summary.txt testnet-run/sampler.log testnet-run/partition-*.txt + testnet-run/n*/fork-held.json + testnet-run/n*/reorg-episode.json # branch runs (test variants) are kept briefly: Actions storage is shared retention-days: ${{ github.ref_name == 'master' && 14 || 3 }} diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index e3c1c2169..b30532151 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -225,7 +225,12 @@ # transactions the other side lost; every node must then end on one chain. The nodes only # reconnect if they look each other up again: 120 s after the cut is lifted the run reports # how many neighbours each node has. OFFICIAL_PARTITION_RESTART=1: then the community nodes are -# restarted (they dial every node of hosts.txt again), as an operator would +# restarted (they dial every node of hosts.txt again), as an operator would. +# RESOLVE_FORK_AFTER=1: then, as an operator choosing the official chain, wait up to +# RESOLVE_FORK_WAIT minutes (default 5) for FORK HELD on the community nodes (the bounded-rollback +# guard refuses the longer chain their confidants do not vouch for), stop every community node +# holding a fork, run it once with --resolve-fork (cut at the recorded fork point) and start it +# again: every node must then end on the official chain # # Checks: every node writes blocks up to TARGET_SEQ; a node stopped and restarted (quick start # from its caches) catches up; all nodes report the same STATE DIGEST at every common sequence. @@ -343,6 +348,8 @@ COMMUNITY_STARTER=0 # the node given COMMUNITY_STALL_STARTER, set with the OFFICIAL_PARTITION_AT=${OFFICIAL_PARTITION_AT:-0} OFFICIAL_PARTITION_MINUTES=${OFFICIAL_PARTITION_MINUTES:-6} OFFICIAL_PARTITION_RESTART=${OFFICIAL_PARTITION_RESTART:-0} +RESOLVE_FORK_AFTER=${RESOLVE_FORK_AFTER:-0} +RESOLVE_FORK_WAIT=${RESOLVE_FORK_WAIT:-5} JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} @@ -594,6 +601,7 @@ if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then || { echo "FAIL: OFFICIAL_PARTITION_AT runs without OFFICIAL_STOP_AT, CRASH_ALL_AT, DPOS_AT, NET_FAULT and a joining node"; exit 1; } [ $(( (NODES + 1) / 2 )) -ge 5 ] || { echo "FAIL: OFFICIAL_PARTITION_AT needs at least 5 community (odd) nodes, NODES >= 9"; exit 1; } fi +[ "$RESOLVE_FORK_AFTER" = 0 ] || [ "$OFFICIAL_PARTITION_AT" -gt 0 ] || { echo "FAIL: RESOLVE_FORK_AFTER needs OFFICIAL_PARTITION_AT"; exit 1; } [ "$COMMUNITY_STALL_STARTER" -eq 0 ] || [ "$COMMUNITY_BOOTSTRAP" = config ] \ || { echo "FAIL: COMMUNITY_STALL_STARTER needs COMMUNITY_BOOTSTRAP=config"; exit 1; } mapfile -t KEYS < <(python3 "$HERE/gen_keys.py" "$ALL_NODES" "$WORK") @@ -1283,23 +1291,63 @@ official_partition() { done if [ "$OFFICIAL_PARTITION_RESTART" = 1 ]; then echo "partition: restarting $(printf 'n%s ' $COMMUNITY_NODES)(community head $(community_max), official head $(official_max))" - for i in $COMMUNITY_NODES; do kill -TERM "${PIDS[$i]}"; REPORTED_DOWN[$i]=1; done - # a trusted node waits for its round before it exits, and with most of the round's - # confidants stopping that round never ends: SIGKILL after 60 s - local t_stop - t_stop=$(date +%s) - for i in $COMMUNITY_NODES; do - while kill -0 "${PIDS[$i]}" 2>/dev/null && [ $(( $(date +%s) - t_stop )) -lt 60 ]; do sleep 1; done - if kill -0 "${PIDS[$i]}" 2>/dev/null; then - echo "partition: n$i still running 60 s after SIGTERM, SIGKILL" - kill -KILL "${PIDS[$i]}" 2>/dev/null || true - fi - wait "${PIDS[$i]}" 2>/dev/null || true - pkill -f "cs.testnet.node=n$i " 2>/dev/null || true - done + partition_stop_nodes $COMMUNITY_NODES for i in $COMMUNITY_NODES; do start_node "$i"; REPORTED_DOWN[$i]=""; done echo "partition: community nodes started again" fi + [ "$RESOLVE_FORK_AFTER" != 1 ] || partition_resolve_fork +} + +# stop nodes as an operator would: SIGTERM, then SIGKILL after 60 s (a trusted node waits for its +# round before it exits, and with most of the round's confidants stopping that round never ends) +partition_stop_nodes() { + local i t_stop + for i in "$@"; do kill -TERM "${PIDS[$i]}" 2>/dev/null || true; REPORTED_DOWN[$i]=1; done + t_stop=$(date +%s) + for i in "$@"; do + while kill -0 "${PIDS[$i]}" 2>/dev/null && [ $(( $(date +%s) - t_stop )) -lt 60 ]; do sleep 1; done + if kill -0 "${PIDS[$i]}" 2>/dev/null; then + echo "partition: n$i still running 60 s after SIGTERM, SIGKILL" + kill -KILL "${PIDS[$i]}" 2>/dev/null || true + fi + wait "${PIDS[$i]}" 2>/dev/null || true + pkill -f "cs.testnet.node=n$i " 2>/dev/null || true + done +} + +# RESOLVE_FORK_AFTER: the operator of each community node holding a fork follows the official chain +part_resolved="" # community nodes cut back by --resolve-fork +partition_resolve_fork() { + local t0 held i n_comm rc + n_comm=$(echo $COMMUNITY_NODES | wc -w) + t0=$(date +%s) + while :; do + held="" + for i in $COMMUNITY_NODES; do + [ -f "$WORK/n$i/fork-held.json" ] && grep -aq 'FORK HELD since' "$WORK/n$i/node.log" && held="$held $i" + done + [ "$(echo $held | wc -w)" -lt "$n_comm" ] || break + [ $(( $(date +%s) - t0 )) -lt $(( RESOLVE_FORK_WAIT * 60 )) ] || break + sleep 10 + done + echo "RESOLVE FORK: FORK HELD on $(echo $held | wc -w) of $n_comm community nodes$(printf ' n%s' $held) after $(( $(date +%s) - t0 )) s (community head $(community_max), official head $(official_max))" + for i in $COMMUNITY_NODES; do + { grep -a 'FORK HELD since' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E "s/^.*(FORK HELD since)/partition: n$i: \1/" + done + if [ -z "$held" ]; then + part_fail="${part_fail:+$part_fail; }no community node reported FORK HELD within $RESOLVE_FORK_WAIT min, nothing to resolve" + return 0 + fi + partition_stop_nodes $held + for i in $held; do + rc=0 + (cd "$WORK/n$i" && exec timeout 600 "$(bin_of "$i")" --db-path db --config-file config.ini --resolve-fork > resolve-fork.log 2>&1) || rc=$? + echo "RESOLVE FORK: n$i --resolve-fork exit $rc, head now $(seq_of "$i")" + { grep -a -- '--resolve-fork' "$WORK/n$i/node.log" "$WORK/n$i/resolve-fork.log" 2>/dev/null || true; } | tail -3 | sed "s/^/partition: n$i: /" + if [ "$rc" -eq 0 ]; then part_resolved="$part_resolved $i"; else part_fail="${part_fail:+$part_fail; }--resolve-fork failed on n$i (exit $rc)"; fi + done + for i in $held; do start_node "$i"; REPORTED_DOWN[$i]=""; done + echo "RESOLVE FORK: $(printf 'n%s ' $held)started again after --resolve-fork" } while :; do @@ -2596,7 +2644,20 @@ if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then if [ -n "$off_fail" ]; then echo "FAIL: official stop: $off_fail"; exit 1; fi fi +# chain reorganisations, see below; a partition scenario reports them before its own verdict +reorg_reported=0 +reorg_report() { + reorg_reported=1 + reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) + echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" + refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) + echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" + held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) + [ -z "$held" ] || echo "FORK HELD on: $held" +} + if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then + reorg_report [ "$part_done" -eq 2 ] || part_fail="${part_fail:+$part_fail; }the partition never ended" if [ "$part_done" -eq 2 ]; then ports="" @@ -2639,12 +2700,7 @@ fi # finished rollback episode). In a normal run the guard must never stop a reorg and no competing block # may be refused (a refusal leaves a lagging node waiting for the operator, FORK HELD). # DEEP_ROLLBACK_EXPECTED=1 is for a scenario that provokes either on purpose. -reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) -echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" -refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) -echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" -held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) -[ -z "$held" ] || echo "FORK HELD on: $held" +[ "$reorg_reported" = 1 ] || reorg_report guarded=$(grep -al 'LONGEST-CHAIN GUARD: a competing chain' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) if [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then if [ -n "$guarded" ]; then From bb32d2698dab1048c4af2771123eac9269c0650e Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 20 Nov 2026 11:42:17 +0100 Subject: [PATCH 14/22] ci: official-partition: the operator path end to end; no-fork verdict RESOLVE_FORK_AFTER: after --resolve-fork, a node whose fork point was only an upper bound and that holds the fork again lower is cut with --set-bc-top at the last common block (partition.py fork, from the chains dumped before the reconnection), as docs/quick-start.md tells the operator. A start right after a cut that stops at once is reported and repeated. The no-restart resolve scenario is dropped: without the restart the two sides never meet again, so no node holds a fork. partition.py report: a side that wrote no block while cut off (it stalled) is no fork when every node kept the other side's blocks. REORG DEPTHS and UNWINDS REFUSED are also printed when a partition run times out. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 13 ++-- ci/testnet/partition.py | 29 +++++-- ci/testnet/run.sh | 126 ++++++++++++++++++++++++------- 3 files changed, 127 insertions(+), 41 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 93903933c..dfa90cbf8 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -538,14 +538,11 @@ jobs: env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 fixed: true manual: true - # the operator path after a partition: the community nodes hold the fork (FORK HELD, the - # longer chain is not vouched for by their confidants); each one is stopped, run once with - # --resolve-fork and started again, and every node must end on the official chain. Without - # and with the restart after the reconnection - - name: official-partition-resolve - env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 RESOLVE_FORK_AFTER=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 - fixed: true - manual: true + # the operator path after a partition: the community nodes, restarted after the + # reconnection, hold the fork (FORK HELD: the longer chain is not vouched for by their + # confidants); each one is stopped and run once with --resolve-fork, then, if the fork + # point was only an upper bound and the node holds the fork again lower, with --set-bc-top + # at the last common block; every node must end on the official chain - name: official-partition-restart-resolve env: NODES=10 TRUSTED_NODES=5 INITIAL_NODES=1,2,3,4,6 OFFICIAL_PARTITION_AT=400 OFFICIAL_PARTITION_MINUTES=6 OFFICIAL_PARTITION_RESTART=1 RESOLVE_FORK_AFTER=1 OFFICIAL_FUND=200000 COMMUNITY_BOOTSTRAP=config COMMUNITY_STALL_MINUTES=2 RESTART_AT=0 LOAD_SECONDS=1500 TARGET_SEQ=2000 DEEP_ROLLBACK_EXPECTED=1 fixed: true diff --git a/ci/testnet/partition.py b/ci/testnet/partition.py index 4fcb61f64..dec40b91a 100755 --- a/ci/testnet/partition.py +++ b/ci/testnet/partition.py @@ -3,6 +3,7 @@ partition.py dump --from 395 --port 9090 > community.txt # "seq hash transactions" per block partition.py report --community community.txt --official official.txt --port 9090 --port 9102 + partition.py fork --community community.txt --official official.txt # the first block that differs dump: the blocks of one node from a sequence to its head, taken just before the reconnection. report: where the two sides forked, and which side's blocks the nodes hold after the reconnection @@ -58,16 +59,22 @@ def dump(args): print(seq, h, txs) +def fork_of(community, official): + """the first sequence at which the two sides hold different blocks""" + common = sorted(set(community) & set(official)) + fork = next((s for s in common if community[s][0] != official[s][0]), None) + if fork is None: + # one side may only be shorter: the fork is right above the shorter head + fork = min(max(community, default=0), max(official, default=0)) + 1 + return fork + + def report(args): community = read(args.community) official = read(args.official) - common = sorted(set(community) & set(official)) - fork = next((s for s in common if community[s][0] != official[s][0]), None) c_head = max(community) if community else 0 o_head = max(official) if official else 0 - if fork is None: - # one side may only be shorter: the fork is right above the shorter head - fork = min(c_head, o_head) + 1 + fork = fork_of(community, official) c_side = {s: v for s, v in community.items() if s >= fork} o_side = {s: v for s, v in official.items() if s >= fork} print(f"PARTITION RESULT: fork at #{fork}; before the reconnection the community side held #{fork}..#{c_head} " @@ -83,7 +90,12 @@ def report(args): kept_o = sum(1 for s, v in o_side.items() if now.get(s, (None,))[0] == v[0]) lost_c_txs = sum(v[1] for s, v in c_side.items() if now.get(s, (None,))[0] != v[0]) lost_o_txs = sum(v[1] for s, v in o_side.items() if now.get(s, (None,))[0] != v[0]) - if c_side and kept_c == 0 and kept_o == len(o_side): + if not o_side and c_side and kept_c == len(c_side): + # the official side wrote nothing while cut off (it stalled): no fork, one chain + verdict = f"no fork: the official side wrote no block, the community blocks were kept ({len(c_side)} blocks)" + elif not c_side and o_side and kept_o == len(o_side): + verdict = f"no fork: the community side wrote no block, the official blocks were kept ({len(o_side)} blocks)" + elif c_side and kept_c == 0 and kept_o == len(o_side): verdict = f"the community side was unwound ({len(c_side)} blocks, {lost_c_txs} transactions reverted)" elif o_side and kept_o == 0 and kept_c == len(c_side): verdict = f"the official side was unwound ({len(o_side)} blocks, {lost_o_txs} transactions reverted)" @@ -108,9 +120,14 @@ def main(): r.add_argument("--community", required=True) r.add_argument("--official", required=True) r.add_argument("--port", action="append", required=True) + f = sub.add_parser("fork") + f.add_argument("--community", required=True) + f.add_argument("--official", required=True) args = parser.parse_args() if args.command == "dump": dump(args) + elif args.command == "fork": + print(fork_of(read(args.community), read(args.official))) else: report(args) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index b30532151..34ccfbb11 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -229,8 +229,10 @@ # RESOLVE_FORK_AFTER=1: then, as an operator choosing the official chain, wait up to # RESOLVE_FORK_WAIT minutes (default 5) for FORK HELD on the community nodes (the bounded-rollback # guard refuses the longer chain their confidants do not vouch for), stop every community node -# holding a fork, run it once with --resolve-fork (cut at the recorded fork point) and start it -# again: every node must then end on the official chain +# holding a fork, run it once with --resolve-fork and start it again; a node whose fork point +# was only an upper bound and that holds the fork again lower is cut with --set-bc-top at the +# last common block (docs/quick-start.md, "If your node reports FORK HELD"). A start right after +# a cut that stops at once is reported and repeated. Every node must then end on one chain # # Checks: every node writes blocks up to TARGET_SEQ; a node stopped and restarted (quick start # from its caches) catches up; all nodes report the same STATE DIGEST at every common sequence. @@ -1238,6 +1240,18 @@ official_stop() { echo "official stop: $(printf 'n%s ' $OFFICIAL_NODES)started again, community head $c" } +# chain reorganisations (see "REORG DEPTHS" near the end); a partition scenario reports them before its own verdict +reorg_reported=0 +reorg_report() { + reorg_reported=1 + reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) + echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" + refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) + echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" + held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) + [ -z "$held" ] || echo "FORK HELD on: $held" +} + # OFFICIAL_PARTITION_AT: the community nodes cut off from the running official nodes part_done=0 # 1 while partitioned, 2 once reconnected part_fail="" @@ -1315,39 +1329,107 @@ partition_stop_nodes() { done } -# RESOLVE_FORK_AFTER: the operator of each community node holding a fork follows the official chain -part_resolved="" # community nodes cut back by --resolve-fork +# RESOLVE_FORK_AFTER: the operator of each community node holding a fork follows the official chain, +# as docs/quick-start.md ("If your node reports FORK HELD") says: --resolve-fork; if the node knows +# only an upper bound of the fork point and reports FORK HELD again lower, --set-bc-top at the last +# common block (here from the chains dumped before the reconnection, as an operator would take it +# from the monitor) +part_resolved="" # community nodes cut back to the official chain +part_restart_fail=0 # starts right after a cut that stopped with an error (started again) + +# nodes holding a fork: fork-held.json and a FORK HELD line after line $2 of node.log +partition_held() { + local i from=$1 held="" + for i in $COMMUNITY_NODES; do + [ -f "$WORK/n$i/fork-held.json" ] && tail -n +"$((from + 1))" "$WORK/n$i/node.log" | grep -aq 'FORK HELD since' && held="$held $i" + done + echo $held +} + +# start a node after its chain was cut; if it stops at once (the first start after a cut can fail +# on a quick-start checkpoint above the new head), report it and start it once more +partition_start_after_cut() { + local i t0 status + for i in "$@"; do start_node "$i"; REPORTED_DOWN[$i]=""; done + t0=$(date +%s) + while [ $(( $(date +%s) - t0 )) -lt 60 ]; do sleep 2; done + for i in "$@"; do + kill -0 "${PIDS[$i]}" 2>/dev/null && continue + status=0; wait "${PIDS[$i]}" 2>/dev/null || status=$? + part_restart_fail=$((part_restart_fail + 1)) + echo "RESOLVE FORK: n$i stopped at its first start after the cut (exit $status):" + { grep -a ' error ' "$WORK/n$i/node.log" || true; } | tail -4 | sed "s/^/partition: n$i: /" | cut -c1-260 + start_node "$i"; REPORTED_DOWN[$i]="" + echo "RESOLVE FORK: n$i started again" + done +} + partition_resolve_fork() { - local t0 held i n_comm rc + local t0 held i n_comm rc fork lines upper="" again + local -a lines_i=() n_comm=$(echo $COMMUNITY_NODES | wc -w) t0=$(date +%s) while :; do - held="" - for i in $COMMUNITY_NODES; do - [ -f "$WORK/n$i/fork-held.json" ] && grep -aq 'FORK HELD since' "$WORK/n$i/node.log" && held="$held $i" - done + held=$(partition_held 0) [ "$(echo $held | wc -w)" -lt "$n_comm" ] || break [ $(( $(date +%s) - t0 )) -lt $(( RESOLVE_FORK_WAIT * 60 )) ] || break sleep 10 done - echo "RESOLVE FORK: FORK HELD on $(echo $held | wc -w) of $n_comm community nodes$(printf ' n%s' $held) after $(( $(date +%s) - t0 )) s (community head $(community_max), official head $(official_max))" + echo "RESOLVE FORK: FORK HELD on $(echo $held | wc -w) of $n_comm community nodes${held:+$(printf ' n%s' $held)} after $(( $(date +%s) - t0 )) s (community head $(community_max), official head $(official_max))" for i in $COMMUNITY_NODES; do - { grep -a 'FORK HELD since' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E "s/^.*(FORK HELD since)/partition: n$i: \1/" + { grep -a 'FORK HELD since' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E "s/^.*(FORK HELD since)/partition: n$i: \\1/; s/ What to do:.*//" done if [ -z "$held" ]; then part_fail="${part_fail:+$part_fail; }no community node reported FORK HELD within $RESOLVE_FORK_WAIT min, nothing to resolve" return 0 fi + # 1. --resolve-fork on every node holding the fork partition_stop_nodes $held for i in $held; do + lines=$(wc -l < "$WORK/n$i/node.log") rc=0 (cd "$WORK/n$i" && exec timeout 600 "$(bin_of "$i")" --db-path db --config-file config.ini --resolve-fork > resolve-fork.log 2>&1) || rc=$? - echo "RESOLVE FORK: n$i --resolve-fork exit $rc, head now $(seq_of "$i")" - { grep -a -- '--resolve-fork' "$WORK/n$i/node.log" "$WORK/n$i/resolve-fork.log" 2>/dev/null || true; } | tail -3 | sed "s/^/partition: n$i: /" - if [ "$rc" -eq 0 ]; then part_resolved="$part_resolved $i"; else part_fail="${part_fail:+$part_fail; }--resolve-fork failed on n$i (exit $rc)"; fi + echo "RESOLVE FORK: n$i --resolve-fork exit $rc" + tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | { grep -a -- '--resolve-fork' || true; } | sed -E "s/^.*(--resolve-fork)/partition: n$i: \\1/" | cut -c1-260 + if [ "$rc" -ne 0 ]; then part_fail="${part_fail:+$part_fail; }--resolve-fork failed on n$i (exit $rc)"; continue; fi + tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | grep -aq 'the fork point is an upper bound' && upper="$upper $i" done - for i in $held; do start_node "$i"; REPORTED_DOWN[$i]=""; done + for i in $held; do lines_i[$i]=$(wc -l < "$WORK/n$i/node.log"); done + partition_start_after_cut $held echo "RESOLVE FORK: $(printf 'n%s ' $held)started again after --resolve-fork" + # 2. the fork point was an upper bound: the node holds the fork again, lower; cut at the last + # common block by hand + if [ -n "$upper" ]; then + fork=$(cd "$HERE" && python3 partition.py fork --community "$WORK/partition-community.txt" --official "$WORK/partition-official.txt") + t0=$(date +%s) + while :; do + again="" + for i in $upper; do + tail -n +"$((${lines_i[$i]} + 1))" "$WORK/n$i/node.log" | grep -aq 'FORK HELD since' && again="$again $i" + done + [ "$(echo $again | wc -w)" -lt "$(echo $upper | wc -w)" ] || break + [ $(( $(date +%s) - t0 )) -lt 180 ] || break + sleep 10 + done + echo "RESOLVE FORK: the fork point was an upper bound on$(printf ' n%s' $upper); FORK HELD again on ${again:-none} after $(( $(date +%s) - t0 )) s; the chains differ from #$fork: --set-bc-top $((fork - 1))" + for i in $again; do + tail -n +"$((${lines_i[$i]} + 1))" "$WORK/n$i/node.log" | { grep -a 'FORK HELD since' || true; } | tail -1 | sed -E "s/^.*(FORK HELD since)/partition: n$i: \\1/; s/ What to do:.*//" + done + if [ -n "$again" ]; then + partition_stop_nodes $again + for i in $again; do + lines=$(wc -l < "$WORK/n$i/node.log") + rc=0 + (cd "$WORK/n$i" && exec timeout 600 "$(bin_of "$i")" --db-path db --config-file config.ini --set-bc-top "$((fork - 1))" > resolve-fork.log 2>&1) || rc=$? + echo "RESOLVE FORK: n$i --set-bc-top $((fork - 1)) exit $rc" + tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | { grep -aE 'start remove|set-bc-top|error' || true; } | tail -3 | sed "s/^/partition: n$i: /" | cut -c1-260 + [ "$rc" -eq 0 ] || part_fail="${part_fail:+$part_fail; }--set-bc-top failed on n$i (exit $rc)" + done + partition_start_after_cut $again + echo "RESOLVE FORK: $(printf 'n%s ' $again)started again after --set-bc-top" + fi + fi + part_resolved=$held } while :; do @@ -2034,6 +2116,7 @@ PY && { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/restart-done" ]; } \ && { [ "$SAMPLE_CONTRACTS" != "1" ] || [ -n "$SAMPLES_PID" ]; }; then break; fi if [ "$(date +%s)" -ge "$deadline" ]; then + [ "$OFFICIAL_PARTITION_AT" -eq 0 ] || reorg_report echo "FAIL: not every node reached sequence $TARGET_SEQ within $TIMEOUT_MIN min" exit 1 fi @@ -2644,18 +2727,6 @@ if [ "$OFFICIAL_STOP_AT" -gt 0 ]; then if [ -n "$off_fail" ]; then echo "FAIL: official stop: $off_fail"; exit 1; fi fi -# chain reorganisations, see below; a partition scenario reports them before its own verdict -reorg_reported=0 -reorg_report() { - reorg_reported=1 - reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) - echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" - refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) - echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" - held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) - [ -z "$held" ] || echo "FORK HELD on: $held" -} - if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then reorg_report [ "$part_done" -eq 2 ] || part_fail="${part_fail:+$part_fail; }the partition never ended" @@ -2669,6 +2740,7 @@ if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then echo "partition: n$i logged $(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true) bootstraps," \ "$(grep -ac 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log" || true) community tables started or taken" done + [ "$RESOLVE_FORK_AFTER" != 1 ] || echo "RESOLVE FORK: cut back to the official chain:${part_resolved:- none}; $part_restart_fail start(s) right after a cut stopped with an error and were repeated" if [ -n "$part_fail" ]; then echo "FAIL: partition: $part_fail"; exit 1; fi fi From 947a7a8bdb9d8d819425582532a2ab24a69f4356 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 20 Nov 2026 13:14:20 +0100 Subject: [PATCH 15/22] docs: community bootstrap after a partition with the bounded-rollback guard differences.md: the community side holds the fork (FORK HELD) and its operator chooses --resolve-fork (then --set-bc-top at the last common block when the fork point is an upper bound) or dismiss-fork; official nodes still follow the longer chain. The fallback's bootstrap blocks pass the guard on the community side; the option does not touch block removal. What the partition scenarios measured with the guard. CHANGES row updated. ci: run.sh prints node lists as n1 n3 ... Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 4 +-- docs/CHANGES.md | 2 +- docs/differences.md | 66 ++++++++++++++++++++++++++++++++------------- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 34ccfbb11..67bf0e180 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -1411,7 +1411,7 @@ partition_resolve_fork() { [ $(( $(date +%s) - t0 )) -lt 180 ] || break sleep 10 done - echo "RESOLVE FORK: the fork point was an upper bound on$(printf ' n%s' $upper); FORK HELD again on ${again:-none} after $(( $(date +%s) - t0 )) s; the chains differ from #$fork: --set-bc-top $((fork - 1))" + echo "RESOLVE FORK: the fork point was an upper bound on$(printf ' n%s' $upper); FORK HELD again on$([ -n "$again" ] && printf ' n%s' $again || echo ' none') after $(( $(date +%s) - t0 )) s; the chains differ from #$fork: --set-bc-top $((fork - 1))" for i in $again; do tail -n +"$((${lines_i[$i]} + 1))" "$WORK/n$i/node.log" | { grep -a 'FORK HELD since' || true; } | tail -1 | sed -E "s/^.*(FORK HELD since)/partition: n$i: \\1/; s/ What to do:.*//" done @@ -2740,7 +2740,7 @@ if [ "$OFFICIAL_PARTITION_AT" -gt 0 ]; then echo "partition: n$i logged $(grep -ac 'NODE> Bootstrap on' "$WORK/n$i/node.log" || true) bootstraps," \ "$(grep -ac 'COMMUNITY BOOTSTRAP: \(starting\|taking\)' "$WORK/n$i/node.log" || true) community tables started or taken" done - [ "$RESOLVE_FORK_AFTER" != 1 ] || echo "RESOLVE FORK: cut back to the official chain:${part_resolved:- none}; $part_restart_fail start(s) right after a cut stopped with an error and were repeated" + [ "$RESOLVE_FORK_AFTER" != 1 ] || echo "RESOLVE FORK: cut back to the official chain:$([ -n "$part_resolved" ] && printf ' n%s' $part_resolved || echo ' none'); $part_restart_fail start(s) right after a cut stopped with an error and were repeated" if [ -n "$part_fail" ]; then echo "FAIL: partition: $part_fail"; exit 1; fi fi diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 0ca042bbd..e0e428a94 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,7 @@ lines under the new release. | PR | Change | |---|---| -| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. CI: `official-stop-*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge) | +| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. After a partition the community side reports FORK HELD (#189) and its operator chooses `--resolve-fork` / `--set-bc-top` (official chain) or `dismiss-fork` (community chain). CI: `official-stop-*`, `official-partition*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge) | | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | | #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). A block not vouched for is refused at once but reported only if the head then stays put (no flapping); unreadable guard files fail safe; the files go when the database is replaced. `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | diff --git a/docs/differences.md b/docs/differences.md index cf3817211..3de370c6d 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -181,21 +181,51 @@ There is no rule that makes either side yield to the other. What happens at reco - official nodes follow the longer chain, as for any two chains, and unwind their blocks above the fork; the transactions of those blocks are reverted and must be sent again unless the other side also wrote them; -- community nodes with the bounded-rollback guard (#189, `allow_deep_rollback=false` by default) - do not follow a longer chain they cannot authenticate themselves: a node on the shorter side - keeps its chain and waits until its operator approves the switch (`[params] - allow_deep_rollback` or `CS_ALLOW_DEEP_ROLLBACK`), after which its blocks above the fork, and - their transactions, are reverted. - -Measured in the `official-partition` scenarios of the robustness workflow (before #189, community -side cut off for 6 minutes, `stall_minutes=2`): the two sides did not look each other up again on -their own after the cut was lifted, each node keeping only its own side as neighbours, so both -chains went on side by side until the end of the run. With the community nodes restarted after the -reconnection (`official-partition-restart`), they connected to the official nodes and unwound -their shorter chain block by block from #950 to #544, then stopped two blocks above the fork -(#542) and did not take the official blocks: still no single chain. A -partition with this option on therefore needs operator attention in any case. Keep -`stall_minutes` long. +- community nodes never follow a longer chain on their own unless its blocks are vouched for by + confidants they hold (the bounded-rollback guard, see "Consensus fixes"). The blocks of the other + side are not, so each community node on the shorter side keeps its chain and reports **FORK + HELD** (on every round line, a detail line once a minute, `fork-held.json` next to the + database, an unhealthy Docker healthcheck). Its operator checks the other nodes or the monitor + and chooses (docs/quick-start.md, "If your node reports FORK HELD"): + - **follow the official chain**: stop the node, run it once with `--resolve-fork` (Docker: + `resolve-fork`) and start it again. After a partition the node usually knows only an upper + bound of the fork point (the block it refused does not continue any block it holds), so + `--resolve-fork` removes only the blocks above that bound and the node reports FORK HELD + again a block lower; then cut at the last common block with `--set-bc-top `. The + blocks above the fork, and their transactions, are reverted; + - **keep the community chain**: create `dismiss-fork` next to the database; the node keeps + refusing that branch and no longer reports it. + +The fallback's own blocks pass that guard on the community side: a community bootstrap block is +signed by the confidants of the community table, and a node that took that table holds it as a +round table it accepted, which is what the guard asks of a bootstrap block. Nothing in this option +changes what may unwind a node's chain: it does not touch block removal, the rollback bound or the +fork records. With the option on, the `official-stop-*` scenarios show no refused block and no +reorganisation on any node. + +Measured in the `official-partition` scenarios of the robustness workflow (community side cut off +for 6 minutes, `stall_minutes=2`, the official nodes run the official build), with the guard: + +- the community side restarts by the fallback 120-180 s after its last block; both sides then + write blocks (fork at #533-#563); +- after the cut is lifted the two sides do not look each other up again: every node keeps only + the 4 neighbours of its own side, nothing is refused, and both chains go on until the end of the + run (`official-partition`); +- with the community nodes restarted after the reconnection (they dial every host again, + `official-partition-restart`), all five report FORK HELD within 20 s and refuse every block of + the official chain: the community side is no longer unwound part of the way and left stuck + above the fork, as before the guard; it stays on its own chain until its operator decides. In + one run the official side had stalled during the cut (3 live initial confidants, fewer than 4), + wrote no block of its own, and took the community blocks once the community nodes were + restarted: every node then held all 217 community blocks and the same head; +- with the operator path (`official-partition-restart-resolve`): `--resolve-fork` cut one block + (the fork point was an upper bound), FORK HELD came back a block lower within 20 s, + `--set-bc-top` at the last common block followed, and every node then held the official chain: + one chain, the 179 community blocks and their 42 transactions reverted. The first start after + each cut stopped once ("the QUICK START checkpoint continues after the last block on disk") and + succeeded when started again. + +A partition with this option on therefore needs operator attention. Keep `stall_minutes` long. ## Start-up behaviour @@ -418,9 +448,9 @@ consensus rules. reconnects. The same holds for a node that lags behind with a divergent last block it never replaced (for example, it missed a bootstrap round): when the network's next block is not confirmed by the confidants of that block, the node waits for the operator instead of unwinding - it. This is the deliberate choice of this change; `--resolve-fork` is the remedy. The opt-in - community bootstrap fallback (PR #188, not merged yet) expects the longer chain to win after a - partition; with this change that switch is an operator decision too. + it. This is the deliberate choice of this change; `--resolve-fork` is the remedy. After a + partition with the opt-in community bootstrap fallback, the community side holds the fork in the + same way and its operator decides (see "Community bootstrap fallback"). - **Two blocks of one round.** When a trusted node does not get one stage-2 in time, it fetches it with a request that takes exactly as long as the others wait for its stage-3. Depending on a few From 019269ad6ce465a75415d949a8b33446268e410e Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 20 Nov 2026 13:42:18 +0100 Subject: [PATCH 16/22] csnode: fork guard: count refused blocks instead of a line each A node holding a fork is offered the other chain's blocks again and again. In CI (official-partition-restart, a community node FORK HELD for 75 min while official peers kept offering their chain) it wrote about 2,000 log lines a minute: 136 refusal lines, 680 sync requests, 680 block replies, 136 "blocks remaining". No file was written per refusal. The refusal line now comes once, then at most once a minute with the count since the previous one (cs::forkguard::RepeatedLine); while FORK HELD the sync request, block reply and "blocks remaining" lines go to the debug log. The FORK HELD round marker and the detail line once a minute are unchanged. run.sh sums the counts for UNWINDS REFUSED. Tests: the throttle (first line at once, then one a minute with the count), and refusals while held leave fork-held.json untouched. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 8 +++-- csnode/include/csnode/blockchain.hpp | 3 ++ csnode/include/csnode/forkguardstate.hpp | 29 ++++++++++++++++ csnode/src/blockchain.cpp | 18 ++++++++-- csnode/src/node.cpp | 8 ++++- csnode/src/poolsynchronizer.cpp | 22 +++++++++--- docs/quick-start.md | 3 ++ tests/csnode/blockchain_tests.cpp | 43 ++++++++++++++++++++++++ 8 files changed, 125 insertions(+), 9 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 67bf0e180..a1d6121ab 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -1246,8 +1246,12 @@ reorg_report() { reorg_reported=1 reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" - refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) - echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" + # one line per refusal, or (since the lines are throttled) one a minute with the count since the + # previous one + refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } \ + | sed -E 's/.*\(([0-9]+) refused since.*/\1/; t; s/.*/1/' | awk '{ n += $1 } END { print n + 0 }') + refused_lines=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) + echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants ($refused_lines log lines)" held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) [ -z "$held" ] || echo "FORK HELD on: $held" } diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index f32da4a8a..89250b357 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -351,6 +351,7 @@ class BlockChain { void test_setLastBlock(const csdb::Pool& block) { deferredBlock_ = block; lastSequence_ = block.sequence(); } // let the episode end as if a block above the reference had been recorded void test_onHeadAdvanced() { onHeadAdvanced(); checkApprovedBranch(); } + bool test_recordForkHeld(const char* reason) { return recordForkHeld(reason); } void test_setSelectionUnfinished(const csdb::PoolHash& lastPrev) { selectionFinished_ = false; lastPrevHash_ = lastPrev; antiForkMode_ = true; emittingRequest_ = 1; } bool test_selectionFinished() const { return selectionFinished_; } bool test_antiForkMode() const { return antiForkMode_; } @@ -726,6 +727,8 @@ public slots: mutable std::mutex guardMutex_; std::optional forkRecord_; std::chrono::steady_clock::time_point lastDismissedLog_{}; // dismissed branch offered again (hourly) + // refused competing blocks: one line, then at most one a minute with the count since (guardMutex_) + cs::forkguard::RepeatedLine refusedLine_; // --resolve-fork's approved branch (exact fork point only), pending until the node is past it std::optional approvedBranch_; std::string resolveCompetingHash_; diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp index d3445260d..a5238c341 100644 --- a/csnode/include/csnode/forkguardstate.hpp +++ b/csnode/include/csnode/forkguardstate.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -124,4 +125,32 @@ std::string remedy(const ForkRecord& record); // remedy std::string summary(const ForkRecord& record); +// A line for an event that may repeat many times a minute (a node holding a fork refuses every block +// its peers offer of the other chain, and asks for them again): the first event gets a line at once, +// later ones are counted and reported with the next line, at most one per `period`. onEvent returns +// the number of events the line due now reports (this one included), or 0 when no line is due. +class RepeatedLine { +public: + explicit RepeatedLine(std::chrono::steady_clock::duration period = std::chrono::minutes(1)) + : period_(period) {} + + uint64_t onEvent(std::chrono::steady_clock::time_point now) { + ++pending_; + if (logged_ && now - last_ < period_) { + return 0; + } + const uint64_t count = pending_; + pending_ = 0; + last_ = now; + logged_ = true; + return count; + } + +private: + std::chrono::steady_clock::duration period_; + std::chrono::steady_clock::time_point last_{}; + bool logged_ = false; + uint64_t pending_ = 0; +}; + } // namespace cs::forkguard diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index deb00f63f..409514feb 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -2937,8 +2937,22 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { // last block uncertain (which would let a later replacement remove it). The node // reports FORK HELD until its head moves on or the operator decides. if (recordForkHeld("authority")) { - cslog() << kLogPrefix << "competing block #" << poolSequence - << " is not authenticated by this node's confidants, keep own last block"; + // a node holding a fork is offered the other chain's blocks over and over: one + // line, then one a minute with the count (the FORK HELD detail line says the rest) + uint64_t refused = 0; + { + std::lock_guard l(guardMutex_); + refused = refusedLine_.onEvent(std::chrono::steady_clock::now()); + } + if (refused > 0) { + cslog() << kLogPrefix << "competing block #" << poolSequence + << " is not authenticated by this node's confidants, keep own last block (" << refused + << " refused since the previous such line)"; + } + else { + csdebug() << kLogPrefix << "competing block #" << poolSequence + << " is not authenticated by this node's confidants, keep own last block"; + } } else { csdebug() << kLogPrefix << "competing block #" << poolSequence diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 9ba4bf311..457e02e27 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -2611,7 +2611,13 @@ void Node::getBlockRequest(const uint8_t* data, const size_t size, const cs::Pub } void Node::getBlockReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { - csinfo() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + // while FORK HELD the replies bring the other chain's blocks again and again (see PoolSynchronizer) + if (blockChain_.forkHeld()) { + csdebug() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + } + else { + csinfo() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + } // the blocks of this reply come from `sender`: named in the fork record should the node refuse one blockChain_.setSyncSource(sender, poolSynchronizer_->announcedSequence(sender)); struct ClearSource { diff --git a/csnode/src/poolsynchronizer.cpp b/csnode/src/poolsynchronizer.cpp index cb0a4495f..06bc303fb 100644 --- a/csnode/src/poolsynchronizer.cpp +++ b/csnode/src/poolsynchronizer.cpp @@ -86,7 +86,14 @@ void PoolSynchronizer::sync(cs::RoundNumber roundNum, cs::RoundNumber difference << "\n Last written seq: " << lastWrittenSequence << "\n Cached blocks: " << cachedBlocksSize << "\n Total blocks: " << totalBlocks; - cslog() << "SYNC: Blocks remaining: " << blocksRemaining; + // while FORK HELD the node asks for the other chain's blocks again and again and refuses them: + // the FORK HELD lines say so, these go to the debug log + if (blockChain_->forkHeld()) { + csdebug() << "SYNC: Blocks remaining: " << blocksRemaining; + } + else { + cslog() << "SYNC: Blocks remaining: " << blocksRemaining; + } if (blocksRemaining == 0) { showSyncronizationProgress(lastWrittenSequence); @@ -372,9 +379,16 @@ void PoolSynchronizer::sendBlockRequest() { else { maxRequestedSequence_ = neededSequences.back(); if (checkSynchroLog(neighbour->first)) { - cslog() << "SYNC: requesting for " << neededSequences.size() - << " blocks [" << neededSequences.front() << ", " << neededSequences.back() - << "] from " << cs::Utils::byteStreamToHex(neighbour->first); + if (blockChain_->forkHeld()) { // asked again and again while FORK HELD: debug log + csdebug() << "SYNC: requesting for " << neededSequences.size() + << " blocks [" << neededSequences.front() << ", " << neededSequences.back() + << "] from " << cs::Utils::byteStreamToHex(neighbour->first); + } + else { + cslog() << "SYNC: requesting for " << neededSequences.size() + << " blocks [" << neededSequences.front() << ", " << neededSequences.back() + << "] from " << cs::Utils::byteStreamToHex(neighbour->first); + } emit sendRequest(neighbour->first, neededSequences); pacing_.onRequest(neighbour->first, neededSequences.back(), now); addSynchroLog(neighbour->first, neededSequences, SyncroMessage::NoAnswer); diff --git a/docs/quick-start.md b/docs/quick-start.md index b77e5931d..dac716e41 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -444,6 +444,9 @@ describe that database: if you delete `db/` by hand to start again, delete these A block of another chain that the node refuses while its own chain goes on is not reported: FORK HELD appears only when the node stays at the same block after the refusal (two rounds, or a minute). +While the fork is held, peers keep offering their chain: the node logs the refused blocks once a +minute with a count ("N refused since the previous such line"), and its block requests and replies +go to the debug log only. **1. Check before acting.** Compare the node's head (`Last written sequence`) and the block at the fork point with other nodes: the [monitor](https://monitor.credits.com), the diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index 6b04cf97f..252c4daa5 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -544,6 +544,49 @@ TEST(BlockChain, RollbackReferenceSurvivesRestart) { chain.setGuardStateDir(""); } +// a node holding a fork is offered the other chain's blocks many times a minute: one line at once, +// then at most one a minute with the count since the previous one +TEST(BlockChain, RefusedBlocksAreCountedNotLoggedOneByOne) { + using namespace std::chrono_literals; + cs::forkguard::RepeatedLine line; + const auto t0 = std::chrono::steady_clock::time_point{} + 1h; + EXPECT_EQ(line.onEvent(t0), 1u); // the first refusal: a line at once + for (int i = 1; i < 2000; ++i) { + EXPECT_EQ(line.onEvent(t0 + std::chrono::milliseconds(25 * i)), 0u); // 2000 within 50 s + } + EXPECT_EQ(line.onEvent(t0 + 60s), 2000u); // 1999 counted + this one + EXPECT_EQ(line.onEvent(t0 + 61s), 0u); + EXPECT_EQ(line.onEvent(t0 + 10min), 2u); // a quiet spell: the next one at once, with the one held back + EXPECT_EQ(line.onEvent(t0 + 10min + 1s), 0u); +} + +// once FORK HELD, further refusals change nothing on disk: no file is written per refused block +TEST(BlockChain, RefusalsWhileHeldDoNotTouchTheForkFile) { + installDefaultConfig(); + TempDir dir("refusals"); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(5000); + chain.setGuardStateDir(dir.str()); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); + ASSERT_FALSE(chain.guardAutoRollback()); // beyond the depth bound: FORK HELD + ASSERT_TRUE(chain.forkHeld()); + const auto file = dir.path / cs::forkguard::kForkFile; + ASSERT_TRUE(std::filesystem::exists(file)); + const auto written = std::filesystem::last_write_time(file); + const auto size = std::filesystem::file_size(file); + for (int i = 0; i < 1000; ++i) { + EXPECT_TRUE(chain.test_recordForkHeld("authority")); + } + EXPECT_EQ(std::filesystem::last_write_time(file), written); + EXPECT_EQ(std::filesystem::file_size(file), size); + EXPECT_TRUE(chain.forkHeld()); + chain.setGuardStateDir(""); +} + // the fork record and the episode in their files, and the --resolve-fork plan TEST(BlockChain, ForkGuardFilesRoundTrip) { TempDir dir("files"); From 77c47d42dbd8704b67958c51dae80d8dd935b0ee Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 20 Nov 2026 15:13:21 +0100 Subject: [PATCH 17/22] ci: fund.py: a transfer expired after a chain cut is no fault (RESOLVE_FORK_AFTER) With RESOLVE_FORK_AFTER the chain of node 1, which takes the load, is cut back by --resolve-fork / --set-bc-top: a transfer it took on the cut branch can come back "transaction is expired". run.sh passes --chain-cut and fund.py counts it with the transfers of unknown outcome. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/fund.py | 6 ++++++ ci/testnet/run.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/testnet/fund.py b/ci/testnet/fund.py index bc81ecc55..e9cc22f68 100644 --- a/ci/testnet/fund.py +++ b/ci/testnet/fund.py @@ -214,6 +214,9 @@ def main(): # the even nodes' amount (0 = --amount): with official even nodes, more stake takes them the seats parser.add_argument("--even-amount", type=int, default=0) parser.add_argument("--load-seconds", type=int, default=0) + # the chain of node 1 is cut back during the load (run.sh RESOLVE_FORK_AFTER): a transfer it took on + # the branch that was cut may come back expired, an answer and not a fault + parser.add_argument("--chain-cut", action="store_true") parser.add_argument("--dpos-at", type=int, default=0) parser.add_argument("--reward-round", type=int, default=0) parser.add_argument("--min-stake", type=int, default=0) @@ -324,6 +327,9 @@ def delegate(name, src, dst, amount, value): # the inner id of a transfer whose outcome was unknown: an answer, not a fault pending += 1 print(f"load transfer rejected after an unknown outcome: {message}", flush=True) + elif not ok and args.chain_cut and "transaction is expired" in message: + pending += 1 + print(f"load transfer expired after the chain was cut back: {message}", flush=True) elif not ok and "not syncronized" in message: # the node refuses transfers while it catches up after a restart (power-cut scenario) unsynced += 1 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index a1d6121ab..acb88f599 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2070,7 +2070,7 @@ PY python3 "$HERE/fund.py" "$WORK" "$NODES" "$MASTER_SEED" --load-seconds "$LOAD_SECONDS" \ --dpos-at "$DPOS_AT" --reward-round "$REWARD_ROUND" --delegation-seconds "$DELEGATION_SECONDS" --min-stake "$MIN_STAKE" --syncro-round "$SYNCRO_ROUND" \ --low-stake-nodes "$LOW_STAKE_NODES" --even-amount "$OFFICIAL_FUND" \ - $([ "$DELEGATE" = "1" ] && echo --delegations) > "$WORK/fund.log" 2>&1 & + $([ "$DELEGATE" = "1" ] && echo --delegations) $([ "$RESOLVE_FORK_AFTER" = 1 ] && echo --chain-cut) > "$WORK/fund.log" 2>&1 & FUND_PID=$! fi From da68658b4ecfc772288dd43799549f73c8405f38 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 21 Nov 2026 05:20:19 +0100 Subject: [PATCH 18/22] ci: official-partition-restart-resolve: wait for the located fork point, one cut only Before --resolve-fork, wait (RESOLVE_LOCATE_WAIT, 6 min) until every node holding the fork reports "fork_point_status" in fork-held.json; an unverified located point is cut with --to-located. The run fails when a cut lands on an upper bound or a start right after the cut stops with an error. Needs the fork locator of #192 on master. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index acb88f599..f0061a68b 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -351,6 +351,7 @@ OFFICIAL_PARTITION_AT=${OFFICIAL_PARTITION_AT:-0} OFFICIAL_PARTITION_MINUTES=${OFFICIAL_PARTITION_MINUTES:-6} OFFICIAL_PARTITION_RESTART=${OFFICIAL_PARTITION_RESTART:-0} RESOLVE_FORK_AFTER=${RESOLVE_FORK_AFTER:-0} +RESOLVE_LOCATE_WAIT=${RESOLVE_LOCATE_WAIT:-6} # minutes to wait for the located fork point RESOLVE_FORK_WAIT=${RESOLVE_FORK_WAIT:-5} JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list @@ -1387,13 +1388,38 @@ partition_resolve_fork() { part_fail="${part_fail:+$part_fail; }no community node reported FORK HELD within $RESOLVE_FORK_WAIT min, nothing to resolve" return 0 fi + # 0. while FORK HELD the node looks for the last common block itself: wait until every node holding + # the fork knows it ("fork_point_status": "exact" in fork-held.json; "deeper" is an answer too) + local pending status to_located="" extra + t0=$(date +%s) + while :; do + pending="" + for i in $held; do + grep -aqE '"fork_point_status": "(exact|deeper|located \(unverified\))"' "$WORK/n$i/fork-held.json" 2>/dev/null || pending="$pending $i" + done + [ -n "$pending" ] || break + [ $(( $(date +%s) - t0 )) -lt $(( RESOLVE_LOCATE_WAIT * 60 )) ] || break + sleep 10 + done + echo "RESOLVE FORK: fork point located on$(printf ' n%s' $held) but$([ -n "$pending" ] && printf ' n%s' $pending || echo ' none') after $(( $(date +%s) - t0 )) s" + for i in $held; do + status=$(sed -nE 's/.*"fork_point_status": "([a-z ()]+)".*/\1/p' "$WORK/n$i/fork-held.json" 2>/dev/null) + echo "RESOLVE FORK: n$i fork point #$(sed -nE 's/.*"fork_point": "([0-9]+)".*/\1/p' "$WORK/n$i/fork-held.json" 2>/dev/null) ($status), upper bound #$(sed -nE 's/.*"fork_point_upper_bound": "([0-9]+)".*/\1/p' "$WORK/n$i/fork-held.json" 2>/dev/null)" + { grep -aE 'fork point search|last common block' "$WORK/n$i/node.log" || true; } | tail -3 | sed "s/^/partition: n$i: /" | cut -c1-260 + case "$status" in + exact) ;; + "located (unverified)") to_located="$to_located $i"; echo "RESOLVE FORK: n$i: located but not verified: the operator confirms it with --to-located" ;; + *) part_fail="${part_fail:+$part_fail; }n$i: the fork point was not located ($status) before --resolve-fork" ;; + esac + done # 1. --resolve-fork on every node holding the fork partition_stop_nodes $held for i in $held; do lines=$(wc -l < "$WORK/n$i/node.log") rc=0 - (cd "$WORK/n$i" && exec timeout 600 "$(bin_of "$i")" --db-path db --config-file config.ini --resolve-fork > resolve-fork.log 2>&1) || rc=$? - echo "RESOLVE FORK: n$i --resolve-fork exit $rc" + extra=""; case " $to_located " in *" $i "*) extra=--to-located ;; esac + (cd "$WORK/n$i" && exec timeout 600 "$(bin_of "$i")" --db-path db --config-file config.ini --resolve-fork $extra > resolve-fork.log 2>&1) || rc=$? + echo "RESOLVE FORK: n$i --resolve-fork $extra exit $rc" tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | { grep -a -- '--resolve-fork' || true; } | sed -E "s/^.*(--resolve-fork)/partition: n$i: \\1/" | cut -c1-260 if [ "$rc" -ne 0 ]; then part_fail="${part_fail:+$part_fail; }--resolve-fork failed on n$i (exit $rc)"; continue; fi tail -n +"$((lines + 1))" "$WORK/n$i/node.log" | grep -aq 'the fork point is an upper bound' && upper="$upper $i" @@ -1401,6 +1427,9 @@ partition_resolve_fork() { for i in $held; do lines_i[$i]=$(wc -l < "$WORK/n$i/node.log"); done partition_start_after_cut $held echo "RESOLVE FORK: $(printf 'n%s ' $held)started again after --resolve-fork" + # one --resolve-fork, at the exact fork point, and no start-up failure after the cut + [ -z "$upper" ] || part_fail="${part_fail:+$part_fail; }--resolve-fork cut at an upper bound on$(printf ' n%s' $upper)" + [ "$part_restart_fail" -eq 0 ] || part_fail="${part_fail:+$part_fail; }$part_restart_fail start(s) right after the cut stopped with an error" # 2. the fork point was an upper bound: the node holds the fork again, lower; cut at the last # common block by hand if [ -n "$upper" ]; then From 53333e4d18d5a7e9a1122f171a1c6ec891c6bf90 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 21 Nov 2026 05:33:42 +0100 Subject: [PATCH 19/22] ci: snapshot stress check: median minute rate; stress-loss allows 20 % / 50 % The block rate of a window is the median of its one-minute rates, so one slow minute does not decide; both limits are reported (rate and round p95) and configurable (STRESS_RATE_DROP, STRESS_P95_RISE; defaults 10 and 20 as before). snapshot-p2p-stress-loss failed at -10 % against 10 % allowed: a steady fall of 5-16 % across runs on master and branches, not a regression. Its loss reaches the joiner n9 only after the joins, so the second window carries more lossy traffic than the first, and the first window's round p95 alone varies between identical runs (0.57 s or 0.8-0.93 s). It now allows a 20 % rate fall and a 50 % p95 rise. Checked on the logs of three runs: they pass (-9.5 to -11.4 %); the same logs slowed by 30 % give -37 to -40 % and slowed by 20 % give -26 to -31 %: both still fail. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 10 +++++++-- ci/testnet/run.sh | 37 +++++++++++++++++++++----------- docs/CHANGES.md | 2 +- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index dfa90cbf8..760fe937f 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -392,14 +392,20 @@ jobs: contracts: true # stress on the confidants that serve (n1, n3, n5, n7): four nodes join at once, the first killed # during its fetch and its import; n3 and n9 on a 2 Mbit/s link. The block rate and the round - # times while serving must stay within 10 % / 20 % of the same window before the joins + # times while serving must stay within 10 % / 20 % of the same window before the joins (the rate + # is the median of the one-minute rates) - name: snapshot-p2p-stress official: 'no' # four joining nodes: the even ones would be official env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 TARGET_SEQ=3300 JOIN_EXTRA=3 JOIN_FETCH_RATE=200 JOIN_KILL_DURING_FETCH=1 NET_FAULT=rate NET_FAULT_RATE=2mbit NET_FAULT_NODES=3,9 fixed: true + # 5 % loss on n3 and n9: 20 % / 50 % allowed. The loss reaches the joiner n9 only once it has + # joined, so the second window has more lossy traffic than the first, whatever serving costs; + # and the first window's round p95 alone varies between identical runs (0.57 s or 0.8-0.93 s). + # Measured: a steady fall of 5-16 % (-10 % failed the former 10 % limit); the same logs slowed + # by 30 % (or 20 %) give -37 to -40 % (-26 to -31 %), which still fails - name: snapshot-p2p-stress-loss official: 'no' # four joining nodes: the even ones would be official - env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 TARGET_SEQ=3300 JOIN_EXTRA=3 NET_FAULT=loss NET_FAULT_PCT=5 NET_FAULT_NODES=3,9 + env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 STRESS_RATE_DROP=20 STRESS_P95_RISE=50 TARGET_SEQ=3300 JOIN_EXTRA=3 NET_FAULT=loss NET_FAULT_PCT=5 NET_FAULT_NODES=3,9 fixed: true # with contracts running before and during the joins - name: snapshot-p2p-stress-contracts diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index f0061a68b..f7b3c7564 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -72,8 +72,9 @@ # one and configured like it; each must start from a snapshot (or fall back as JOIN_EXPECT says) # and reach the target. STRESS_WINDOW (seconds, default 0 = off): the block rate and the round # times of n1 in that many seconds before the joins and after they started are compared; the -# run fails when the rate falls by more than 10 % or the 95th percentile of the round time rises -# by more than 20 % +# rate of a window is the median of its one-minute rates, so one slow minute does not decide. +# The run fails when the rate falls by more than STRESS_RATE_DROP % (default 10) or the 95th +# percentile of the round time rises by more than STRESS_P95_RISE % (default 20) # NET_FAULT (default off): loss, corrupt, delay, reorder (with 5% duplicates), duplicate, partition, # or rate (NET_FAULT_RATE, default 2mbit, for the faulty nodes' traffic together), applied with # tc netem to the p2p traffic sent and received by NET_FAULT_NODES (comma separated) from @@ -426,6 +427,8 @@ is_joiner() { [ "$JOIN_NODE" -gt 0 ] && [ "$1" -ge "$JOIN_NODE" ]; } alt_node() { [ -n "$BIN_ALT" ] && [ $(( $1 % 2 )) -eq 0 ] && ! { [ "${JOIN_COMMUNITY:-0}" = 1 ] && [ "$1" -eq "$JOIN_NODE" ]; }; } EXTRA_JOINERS=$( { [ "$JOIN_EXTRA" -gt 0 ] && seq $((JOIN_NODE + 1)) "$ALL_NODES" | tr "\n" " "; } || true) STRESS_WINDOW=${STRESS_WINDOW:-0} +STRESS_RATE_DROP=${STRESS_RATE_DROP:-10} +STRESS_P95_RISE=${STRESS_P95_RISE:-20} BACKFILL_RATE=${BACKFILL_RATE:-} BACKFILL_BATCH=${BACKFILL_BATCH:-} BACKFILL_PER_TURN=${BACKFILL_PER_TURN:-} @@ -2706,9 +2709,10 @@ done # the consensus while the snapshots are served: n1's block rate and round times in the same number of # seconds before the joins and after they started if [ "$STRESS_WINDOW" -gt 0 ] && [ -n "${join_started:-}" ]; then - python3 - "$WORK/n1/node.log" "$join_started" "$STRESS_WINDOW" <<'PY' || { echo "FAIL: the consensus slowed down while snapshots were served"; exit 1; } -import re, sys, datetime + python3 - "$WORK/n1/node.log" "$join_started" "$STRESS_WINDOW" "$STRESS_RATE_DROP" "$STRESS_P95_RISE" <<'PY' || { echo "FAIL: the consensus slowed down while snapshots were served"; exit 1; } +import re, sys, datetime, statistics log, start, window = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) +max_drop, max_rise = float(sys.argv[4]), float(sys.argv[5]) stamp = re.compile(r"^\[(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d+)\]") def t(line): m = stamp.match(line) @@ -2724,25 +2728,34 @@ for line in open(log, errors="replace"): m = re.search(r"Last written sequence = ([0-9']+)", line) if x is not None and m: seqs.append((x, int(m.group(1).replace("'", "")))) +def rate_in(a, b): + s = [q for x, q in seqs if a <= x < b] + return (s[-1] - s[0]) / (b - a) if len(s) > 1 else 0.0 def window_stats(a, b): r = [y - x for x, y in zip(rounds, rounds[1:]) if a <= x < b] - s = [q for x, q in seqs if a <= x < b] - rate = (s[-1] - s[0]) / (b - a) if len(s) > 1 else 0.0 + # the median of the one-minute rates (at least 3 parts): a single slow minute, e.g. a burst of + # lost packets, moves the median far less than the rate over the whole window + parts = max(3, (b - a) // 60) + step = (b - a) / parts + rate = statistics.median(rate_in(a + k * step, a + (k + 1) * step) for k in range(parts)) r.sort() p95 = r[int(0.95 * (len(r) - 1))] if r else 0.0 return rate, p95, len(r) base = window_stats(start - window, start) load = window_stats(start, start + window) -print(f"consensus: before the joins {base[0]:.2f} blocks/s, round p95 {base[1]:.2f} s ({base[2]} rounds); " +print(f"consensus: before the joins {base[0]:.2f} blocks/s (median minute), round p95 {base[1]:.2f} s ({base[2]} rounds); " f"while serving {load[0]:.2f} blocks/s, round p95 {load[1]:.2f} s ({load[2]} rounds)") -ok = base[2] > 10 and load[2] > 10 +ok = base[2] > 10 and load[2] > 10 and base[0] > 0 and base[1] > 0 if not ok: print("FAIL: too few rounds in a window to compare") -elif load[0] < 0.9 * base[0]: - print(f"FAIL: block rate fell by {100 * (1 - load[0] / base[0]):.0f} % (10 % allowed)") + sys.exit(1) +drop, rise = 100 * (1 - load[0] / base[0]), 100 * (load[1] / base[1] - 1) +print(f"consensus: block rate {-drop:+.1f} % ({max_drop:g} % fall allowed), round p95 {rise:+.1f} % ({max_rise:g} % rise allowed)") +if drop > max_drop: + print(f"FAIL: block rate fell by {drop:.0f} % ({max_drop:g} % allowed)") ok = False -elif load[1] > 1.2 * base[1]: - print(f"FAIL: round time p95 rose by {100 * (load[1] / base[1] - 1):.0f} % (20 % allowed)") +if rise > max_rise: + print(f"FAIL: round time p95 rose by {rise:.0f} % ({max_rise:g} % allowed)") ok = False sys.exit(0 if ok else 1) PY diff --git a/docs/CHANGES.md b/docs/CHANGES.md index e0e428a94..717ae86b3 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,7 @@ lines under the new release. | PR | Change | |---|---| -| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. After a partition the community side reports FORK HELD (#189) and its operator chooses `--resolve-fork` / `--set-bc-top` (official chain) or `dismiss-fork` (community chain). CI: `official-stop-*`, `official-partition*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge) | +| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. After a partition the community side reports FORK HELD (#189) and its operator chooses `--resolve-fork` / `--set-bc-top` (official chain) or `dismiss-fork` (community chain). CI: `official-stop-*`, `official-partition*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge). CI: the snapshot stress check takes the median one-minute block rate, and `snapshot-p2p-stress-loss` allows a 20 % rate fall and a 50 % round p95 rise (its loss reaches a joiner only after the joins; 10 % failed on noise, a 30 % slowdown still fails) | | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | | #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). A block not vouched for is refused at once but reported only if the head then stays put (no flapping); unreadable guard files fail safe; the files go when the database is replaced. `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | From 4f83309be3ae065f0a2d353ee9e6562ab7a75226 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 21 Nov 2026 05:34:50 +0100 Subject: [PATCH 20/22] docs: differences.md: community bootstrap, final state Option A after a partition (FORK HELD, the operator follows the official chain with the located fork point of #192, --to-located for an unverified one, --set-bc-top otherwise, or keeps the community chain with dismiss-fork); why the option stays out of the set-up; the gate for H_activate_decentralization of BK's plan. Co-Authored-By: Claude Opus 5.5 --- docs/differences.md | 47 ++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/docs/differences.md b/docs/differences.md index 3de370c6d..c8878a593 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -169,14 +169,19 @@ community table (it holds a key that is not an initial confidant and is not sign project); they then take the blocks written meanwhile through ordinary sync once they are back, as the `official-stop-*` scenarios of the robustness workflow check. -Every node that should take part needs the same list: it only helps when more than half of the -listed nodes are running. The set-up scripts and the Docker image do not offer the option; edit -`config.ini` by hand (in Docker, `~/.credits/config.ini`). +**An expert option, not in the set-up.** Every node that should take part needs the same list: it +only helps when more than half of the listed nodes are running, and mismatched lists weaken that +majority rule. So `setup.sh`, `setup.ps1` and the Docker environment do not offer it until the +option has been reviewed and a coordinated list exists (possibly shipped like `known_hosts.txt`); +edit `config.ini` by hand (in Docker, `~/.credits/config.ini`, which the entrypoint writes only +when it is missing). The risk it brings is a partition. If a network partition cuts more than half of the listed nodes off from the official nodes for longer than `stall_minutes`, while the official nodes keep producing blocks, the listed nodes restart on their own and the two sides write different blocks. -There is no rule that makes either side yield to the other. What happens at reconnection: +There is no rule that makes either side yield to the other, and the threshold stays "more than +half of the list". What happens at reconnection (option A of the bounded-rollback guard, #189: the +shorter side waits for its operator): - official nodes follow the longer chain, as for any two chains, and unwind their blocks above the fork; the transactions of those blocks are reverted and must be sent again unless the other @@ -187,12 +192,15 @@ There is no rule that makes either side yield to the other. What happens at reco HELD** (on every round line, a detail line once a minute, `fork-held.json` next to the database, an unhealthy Docker healthcheck). Its operator checks the other nodes or the monitor and chooses (docs/quick-start.md, "If your node reports FORK HELD"): - - **follow the official chain**: stop the node, run it once with `--resolve-fork` (Docker: - `resolve-fork`) and start it again. After a partition the node usually knows only an upper - bound of the fork point (the block it refused does not continue any block it holds), so - `--resolve-fork` removes only the blocks above that bound and the node reports FORK HELD - again a block lower; then cut at the last common block with `--set-bc-top `. The - blocks above the fork, and their transactions, are reverted; + - **follow the official chain**: after a partition the block the node refused does not continue + any block it holds, so at first it knows only an upper bound of the fork point. While FORK + HELD it looks for the last common block itself (#192). Wait until `fork-held.json` shows + `"fork_point_status": "exact"`, then stop the node, run it once with `--resolve-fork` (Docker: + `resolve-fork`) and start it again: it cuts at that block in one run. If the status is + `"located (unverified)"`, check the point against other nodes or the monitor and add + `--to-located`; if it stays `"upper bound"` or `"deeper"`, cut at the last common block by hand + with `--set-bc-top `. The blocks above the fork, and their transactions, are + reverted; the quick-start checkpoints above the new top go with them; - **keep the community chain**: create `dismiss-fork` next to the database; the node keeps refusing that branch and no longer reports it. @@ -218,15 +226,22 @@ for 6 minutes, `stall_minutes=2`, the official nodes run the official build), wi one run the official side had stalled during the cut (3 live initial confidants, fewer than 4), wrote no block of its own, and took the community blocks once the community nodes were restarted: every node then held all 217 community blocks and the same head; -- with the operator path (`official-partition-restart-resolve`): `--resolve-fork` cut one block - (the fork point was an upper bound), FORK HELD came back a block lower within 20 s, - `--set-bc-top` at the last common block followed, and every node then held the official chain: - one chain, the 179 community blocks and their 42 transactions reverted. The first start after - each cut stopped once ("the QUICK START checkpoint continues after the last block on disk") and - succeeded when started again. +- with the operator path (`official-partition-restart-resolve`) and the fork locator of #192: the + five community nodes held the fork within 20 s and found the exact fork point (#506) 30 s later; + one `--resolve-fork` each cut there, every start after it succeeded, and every node then held + the official chain: one chain, 275 community blocks and their 63 transactions reverted. Without + the locator, `--resolve-fork` cut a single block (the upper bound), FORK HELD came back a block + lower, and `--set-bc-top` at the last common block was needed (179-215 blocks reverted); the + first start after each such cut stopped once on a quick-start checkpoint above the new top, + which #192 removes. The scenario now fails if a cut lands on an upper bound or a start after a + cut fails. A partition with this option on therefore needs operator attention. Keep `stall_minutes` long. +**Decentralisation plan.** When `H_activate_decentralization` of BK's plan lands, the fallback is to +be gated on `lastSeq + 1 < H_activate` (it acts only before the activation height) and later folded +into the plan's BootstrapSignature protocol. + ## Start-up behaviour - **A banner names the release.** At start the console and the log show the CREDITS logo with From 01163d688261f112b21e62a81ed5f3e17eaf023d Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 21 Nov 2026 07:12:17 +0100 Subject: [PATCH 21/22] ci: snapshot stress check: STRESS_ROUND_RISE (no digit in a scenario key) The workflow accepts scenario keys of capitals and underscores only, so STRESS_P95_RISE=50 stopped snapshot-p2p-stress-loss before the network started ("bad scenario entry"). Same meaning: the allowed rise of the round time's 95th percentile, in percent. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 2 +- ci/testnet/run.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 760fe937f..0bf72a7ef 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -405,7 +405,7 @@ jobs: # by 30 % (or 20 %) give -37 to -40 % (-26 to -31 %), which still fails - name: snapshot-p2p-stress-loss official: 'no' # four joining nodes: the even ones would be official - env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 STRESS_RATE_DROP=20 STRESS_P95_RISE=50 TARGET_SEQ=3300 JOIN_EXTRA=3 NET_FAULT=loss NET_FAULT_PCT=5 NET_FAULT_NODES=3,9 + env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=1,3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 STRESS_WINDOW=300 STRESS_RATE_DROP=20 STRESS_ROUND_RISE=50 TARGET_SEQ=3300 JOIN_EXTRA=3 NET_FAULT=loss NET_FAULT_PCT=5 NET_FAULT_NODES=3,9 fixed: true # with contracts running before and during the joins - name: snapshot-p2p-stress-contracts diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index f7b3c7564..29dd1289c 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -74,7 +74,7 @@ # times of n1 in that many seconds before the joins and after they started are compared; the # rate of a window is the median of its one-minute rates, so one slow minute does not decide. # The run fails when the rate falls by more than STRESS_RATE_DROP % (default 10) or the 95th -# percentile of the round time rises by more than STRESS_P95_RISE % (default 20) +# percentile of the round time rises by more than STRESS_ROUND_RISE % (default 20) # NET_FAULT (default off): loss, corrupt, delay, reorder (with 5% duplicates), duplicate, partition, # or rate (NET_FAULT_RATE, default 2mbit, for the faulty nodes' traffic together), applied with # tc netem to the p2p traffic sent and received by NET_FAULT_NODES (comma separated) from @@ -428,7 +428,7 @@ alt_node() { [ -n "$BIN_ALT" ] && [ $(( $1 % 2 )) -eq 0 ] && ! { [ "${JOIN_COMMU EXTRA_JOINERS=$( { [ "$JOIN_EXTRA" -gt 0 ] && seq $((JOIN_NODE + 1)) "$ALL_NODES" | tr "\n" " "; } || true) STRESS_WINDOW=${STRESS_WINDOW:-0} STRESS_RATE_DROP=${STRESS_RATE_DROP:-10} -STRESS_P95_RISE=${STRESS_P95_RISE:-20} +STRESS_ROUND_RISE=${STRESS_ROUND_RISE:-20} BACKFILL_RATE=${BACKFILL_RATE:-} BACKFILL_BATCH=${BACKFILL_BATCH:-} BACKFILL_PER_TURN=${BACKFILL_PER_TURN:-} @@ -2709,7 +2709,7 @@ done # the consensus while the snapshots are served: n1's block rate and round times in the same number of # seconds before the joins and after they started if [ "$STRESS_WINDOW" -gt 0 ] && [ -n "${join_started:-}" ]; then - python3 - "$WORK/n1/node.log" "$join_started" "$STRESS_WINDOW" "$STRESS_RATE_DROP" "$STRESS_P95_RISE" <<'PY' || { echo "FAIL: the consensus slowed down while snapshots were served"; exit 1; } + python3 - "$WORK/n1/node.log" "$join_started" "$STRESS_WINDOW" "$STRESS_RATE_DROP" "$STRESS_ROUND_RISE" <<'PY' || { echo "FAIL: the consensus slowed down while snapshots were served"; exit 1; } import re, sys, datetime, statistics log, start, window = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) max_drop, max_rise = float(sys.argv[4]), float(sys.argv[5]) From fdf123a85f46e94da319c304832496df609ff259 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 21 Nov 2026 09:54:00 +0100 Subject: [PATCH 22/22] csnode: community bootstrap: final review fixes - A set at the highest block with listed keys that hold no majority of the list no longer blocks the regular restart: the initial confidants in it make the regular table (official odd rule), as on a node without the list (cs::bootstrapTableOf). Before, an initial confidant with the fallback open waited, and the official nodes waited for it. - dropForOddTable drops the last key when the rest is a regular table. - FORK HELD: refusedLine_ is reset in holdFork, so a new episode's first refused block is logged at once. - takeCommunityTable: its own log line for an empty table. - ci: partition_on removes a CSPART chain left by an earlier cut. - docs: known limitations (hostile lowest listed key, replay across a chain cut); CHANGES row. Tests: AMixedSetWithoutAMajorityOfTheListFallsBackToTheRegularTable, AnEvenSetKeepsItsMajorityOfTheList (regular rest), ANewForkHeldEpisodeLogsItsFirstRefusalAtOnce. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 1 + csnode/include/csnode/blockchain.hpp | 3 ++ csnode/include/csnode/communitybootstrap.hpp | 39 +++++++++++++++++++- csnode/src/blockchain.cpp | 1 + csnode/src/node.cpp | 28 ++++++++++---- docs/CHANGES.md | 2 +- docs/differences.md | 23 +++++++++++- tests/csnode/blockchain_tests.cpp | 26 +++++++++++++ tests/csnode/communitybootstrap_tests.cpp | 23 ++++++++++++ 9 files changed, 135 insertions(+), 11 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 29dd1289c..d5d9aa93e 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -907,6 +907,7 @@ net_fault_off() { # the p2p ports (a node sends its datagrams from its own port), TCP by the sender's cgroup partition_on() { local i j + partition_off # a chain left by an earlier cut: -N would fail, and its rules would stay sudo iptables -N CSPART sudo iptables -I OUTPUT 1 -o lo -j CSPART for i in $COMMUNITY_NODES; do diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 89250b357..5f6219433 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -352,6 +352,9 @@ class BlockChain { // let the episode end as if a block above the reference had been recorded void test_onHeadAdvanced() { onHeadAdvanced(); checkApprovedBranch(); } bool test_recordForkHeld(const char* reason) { return recordForkHeld(reason); } + uint64_t test_onRefusedBlock(std::chrono::steady_clock::time_point now) { + std::lock_guard l(guardMutex_); return refusedLine_.onEvent(now); + } void test_setSelectionUnfinished(const csdb::PoolHash& lastPrev) { selectionFinished_ = false; lastPrevHash_ = lastPrev; antiForkMode_ = true; emittingRequest_ = 1; } bool test_selectionFinished() const { return selectionFinished_; } bool test_antiForkMode() const { return antiForkMode_; } diff --git a/csnode/include/csnode/communitybootstrap.hpp b/csnode/include/csnode/communitybootstrap.hpp index 3b1366fab..dc250636d 100644 --- a/csnode/include/csnode/communitybootstrap.hpp +++ b/csnode/include/csnode/communitybootstrap.hpp @@ -1,7 +1,9 @@ #ifndef COMMUNITYBOOTSTRAP_HPP #define COMMUNITYBOOTSTRAP_HPP +#include #include +#include #include #include #include @@ -104,7 +106,8 @@ inline bool isCommunityTable(const std::set& initial, const std::set< // The starter of an even set drops its last key (the official rule, for an odd table). Not when that // would leave a community table without more than half of the list: with a list of 6 and 4 of its -// nodes left, the 3 kept would never be enough +// nodes left, the 3 kept would never be enough. A community table whose last key is its only one +// outside the initial confidants drops it: the rest is a regular table, as the official rule gives inline bool dropForOddTable(const std::set& initial, const std::set& community, const std::vector& table) { if (table.empty() || table.size() % 2 != 0) { @@ -113,7 +116,39 @@ inline bool dropForOddTable(const std::set& initial, const std::set

(table.begin(), table.end() - 1)); + const std::vector rest(table.begin(), table.end() - 1); + return isCommunityTable(initial, community, rest) + || std::none_of(rest.cbegin(), rest.cend(), [&initial](const PublicKey& key) { return initial.count(key) == 0; }); +} + +// The table a starter sends from the keys at the highest sequence it sees (its own included), after +// the odd rule. A set with a listed key outside the initial confidants that is not a community table +// (not more than half of the list) cannot start anything as it is; it must not hold back the +// regular restart either, which the initial confidants among it would make without the list: the +// table is then theirs, under the official odd rule, and a node outside them waits as before +inline std::set bootstrapTableOf(const std::set& initial, const std::set& community, + const std::set& seen) { + const auto oddTable = [&initial](const std::set& listed, std::set keys) { + if (dropForOddTable(initial, listed, std::vector(keys.cbegin(), keys.cend()))) { + keys.erase(std::prev(keys.end())); + } + return keys; + }; + std::set table = oddTable(community, seen); + const std::vector asSent(table.cbegin(), table.cend()); + const bool outsideInitial = std::any_of(asSent.cbegin(), asSent.cend(), [&initial](const PublicKey& key) { + return initial.count(key) == 0; + }); + if (!outsideInitial || isCommunityTable(initial, community, asSent)) { + return table; + } + std::set regular; + for (const auto& key : seen) { + if (initial.count(key) > 0) { + regular.insert(key); + } + } + return oddTable({}, regular); } // A neighbour ahead of this node: it syncs from it, the network is not stalled. Two blocks ahead diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 409514feb..50e827cf3 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1061,6 +1061,7 @@ void BlockChain::holdFork(cs::forkguard::ForkRecord r) { { std::lock_guard l(guardMutex_); forkRecord_ = r; + refusedLine_ = cs::forkguard::RepeatedLine{}; // a new episode: its first refusal gets a line at once } cs::forkguard::saveForkRecord(guardStateDir_, r); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 457e02e27..b07291586 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -1223,7 +1223,11 @@ bool Node::takeCommunityTable(const cs::PublicKeys& table, bool own) { const uint64_t elapse = cs::ConfigHolder::instance().config()->roundElapseTime(); std::lock_guard lock(communityMutex_); const uint64_t now = stat_.msSinceStart(); - if (table.empty() || !communityGate_.mayTake(weight, table.front(), now, elapse)) { + if (table.empty()) { + cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "not starting" : "dropping") << " an empty table"; + return false; + } + if (!communityGate_.mayTake(weight, table.front(), now, elapse)) { cslog() << kLogPrefix_ << "COMMUNITY BOOTSTRAP: " << (own ? "not starting" : "dropping") << " a table of " << table.size() << " keys: another was taken less than half a round_elapse_time ago, and this one is not heavier with a starter key not above its starter's"; return false; @@ -5602,12 +5606,21 @@ void Node::onRoundTimeElapsed() { transport_->forEachNeighbour(std::move(callback)); // an odd table: the last key of an even set goes, unless a community table would lose its - // majority of the list with it (cs::dropForOddTable; without the fallback open, always) - if (cs::dropForOddTable(initialConfidants_, listed, cs::PublicKeys(actualConfidants.cbegin(), actualConfidants.cend()))) { - auto it = actualConfidants.end(); - --it; - it = actualConfidants.erase(it); - + // majority of the list with it (cs::dropForOddTable; without the fallback open, always). A set + // with listed keys that is not a community table: the initial confidants in it, as without the + // list, so that it never holds back a regular restart (cs::bootstrapTableOf) + { + const auto isOutsideInitial = [this](const cs::PublicKey& key) { + return initialConfidants_.count(key) == 0; + }; + const size_t seen = actualConfidants.size(); + const bool seenListed = std::any_of(actualConfidants.cbegin(), actualConfidants.cend(), isOutsideInitial); + actualConfidants = cs::bootstrapTableOf(initialConfidants_, listed, actualConfidants); + if (seenListed && std::none_of(actualConfidants.cbegin(), actualConfidants.cend(), isOutsideInitial)) { + cslog() << "COMMUNITY BOOTSTRAP: " << seen << " confidants with max sequence " << maxGlobalBlock + << " do not hold more than half of the " << listed.size() + << " listed keys, regular table of the initial confidants among them"; + } } initBootstrapRP(actualConfidants); @@ -5656,6 +5669,7 @@ void Node::onRoundTimeElapsed() { return initialConfidants_.count(key) == 0; }); if (communityTable && !cs::isCommunityTable(initialConfidants_, listed, confs)) { + // cs::bootstrapTableOf never gives such a set; kept as a guard cslog() << "COMMUNITY BOOTSTRAP: " << confs.size() << " confidants with max sequence " << maxGlobalBlock << " do not hold more than half of the " << listed.size() << " listed keys, wait"; return; diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 717ae86b3..af28d55eb 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,7 @@ lines under the new release. | PR | Change | |---|---| -| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. After a partition the community side reports FORK HELD (#189) and its operator chooses `--resolve-fork` / `--set-bc-top` (official chain) or `dismiss-fork` (community chain). CI: `official-stop-*`, `official-partition*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge). CI: the snapshot stress check takes the median one-minute block rate, and `snapshot-p2p-stress-loss` allows a 20 % rate fall and a 50 % round p95 rise (its loss reaches a joiner only after the joins; 10 % failed on noise, a 30 % slowdown still fails) | +| #188 | csnode: community bootstrap fallback, an expert option off by default (`[community_bootstrap] confidants=`, `stall_minutes=`): after a long stall without blocks, rounds or regular bootstraps, nodes with the list may restart the network with a table of initial and listed keys signed by its starter; official nodes then sync the blocks. After a partition the community side reports FORK HELD (#189) and its operator chooses `--resolve-fork` / `--set-bc-top` (official chain) or `dismiss-fork` (community chain). CI: `official-stop-*`, `official-partition*` and `community-bootstrap-quiet` robustness scenarios (consensus-adjacent, reviewed before merge). A set at the highest block whose listed keys are no majority of the list falls back to the regular table of its initial confidants, so the option never holds back a regular restart. FORK HELD (#189): a new episode's first refused block is logged at once. CI: the snapshot stress check takes the median one-minute block rate, and `snapshot-p2p-stress-loss` allows a 20 % rate fall and a 50 % round p95 rise (its loss reaches a joiner only after the joins; 10 % failed on noise, a 30 % slowdown still fails) | | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | | #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). A block not vouched for is refused at once but reported only if the head then stays put (no flapping); unreadable guard files fail safe; the files go when the database is replaced. `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | diff --git a/docs/differences.md b/docs/differences.md index c8878a593..6f5cfdaee 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -157,7 +157,8 @@ neighbour reports a block two or more above its own (one above only holds it for `round_elapse_time` with new random bytes after the signature, so that a node whose fallback opens later gets the repeat (the transport drops a message it has seen before); - of an even set of nodes the starter drops the last key, as the official code does, unless that - would leave the table without more than half of the listed keys; + would leave the table without more than half of the listed keys (it does drop it when the rest + is a regular table of initial confidants); - a jump of the clock between two checks (the host suspended and resumed, the clock set) counts as progress: the node watches again for `stall_minutes` before it acts. @@ -238,6 +239,26 @@ for 6 minutes, `stall_minutes=2`, the official nodes run the official build), wi A partition with this option on therefore needs operator attention. Keep `stall_minutes` long. +**Known limitations** (accepted; the operators control the list): + +- **A hostile lowest listed key.** The starter of a table is its lowest key, and a node takes a new + community table once half a `round_elapse_time` has passed since the last one. A listed node with + the lowest key that turns hostile can therefore send, every half `round_elapse_time`, a table + padded with keys of stopped nodes that cannot decide a round, and keep the community restart + from deciding while the stall lasts. It cannot write blocks the others would not sign, and a + regular table still replaces its table at once; the remedy is to take its key off the list. +- **Replay across a chain cut.** A community table is signed over the round and the keys only. + After `--set-bc-top` or `--resolve-fork` below the round of a table sent earlier, that table + verifies again for the same round on a node whose fallback is open. It changes nothing unless its + confidants are themselves running and stalled at that height. Binding the last block hash into + the signed bytes would change the format a project-signed table shares; it is left to the + BootstrapSignature protocol of the decentralisation plan. + +A set at the highest block with listed keys that hold no more than half of the list is no +community table; the initial confidants in it then make the regular table they would make without +the list (the official odd rule applies), so a node with the list never holds back a regular +restart, for example when the official nodes come back while a minority of the listed nodes runs. + **Decentralisation plan.** When `H_activate_decentralization` of BK's plan lands, the fallback is to be gated on `lastSeq + 1 < H_activate` (it acts only before the activation height) and later folded into the plan's BootstrapSignature protocol. diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index 252c4daa5..3ca69a82f 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -587,6 +587,32 @@ TEST(BlockChain, RefusalsWhileHeldDoNotTouchTheForkFile) { chain.setGuardStateDir(""); } +// a new FORK HELD episode starts its refusal line afresh: its first refusal is logged at once, even +// within a minute of the previous episode's last line +TEST(BlockChain, ANewForkHeldEpisodeLogsItsFirstRefusalAtOnce) { + using namespace std::chrono_literals; + installDefaultConfig(); + TempDir dir("refusalepisodes"); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.setGuardStateDir(dir.str()); + const auto t0 = std::chrono::steady_clock::now(); + for (int episode = 0; episode < 2; ++episode) { + chain.test_resetRollbackEpisode(); // the previous episode ended + chain.test_setLastSequence(5000); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); + ASSERT_FALSE(chain.guardAutoRollback()); // FORK HELD + ASSERT_TRUE(chain.forkHeld()); + const auto t = t0 + std::chrono::seconds(episode * 5); + EXPECT_EQ(chain.test_onRefusedBlock(t), 1u) << "episode " << episode; + EXPECT_EQ(chain.test_onRefusedBlock(t + 1s), 0u); + EXPECT_EQ(chain.test_onRefusedBlock(t + 2s), 0u); + } + chain.setGuardStateDir(""); +} + // the fork record and the episode in their files, and the --resolve-fork plan TEST(BlockChain, ForkGuardFilesRoundTrip) { TempDir dir("files"); diff --git a/tests/csnode/communitybootstrap_tests.cpp b/tests/csnode/communitybootstrap_tests.cpp index 3a3392a3c..80d7c0865 100644 --- a/tests/csnode/communitybootstrap_tests.cpp +++ b/tests/csnode/communitybootstrap_tests.cpp @@ -282,6 +282,29 @@ TEST(CommunityBootstrap, AnEvenSetKeepsItsMajorityOfTheList) { EXPECT_FALSE(dropForOddTable(initial, six, table({1, 3, 5, 7, 9}))); EXPECT_TRUE(dropForOddTable(initial, {}, table({2, 4, 6, 8}))); EXPECT_TRUE(dropForOddTable(initial, six, table({2, 4, 6, 8}))); + // a community table whose only key outside the initial confidants is its last: the rest is a + // regular table, the official rule applies + EXPECT_TRUE(dropForOddTable(kInitial, kCommunity, table({1, 2, 3, 4, 5, 7}))); + EXPECT_FALSE(isCommunityTable(kInitial, kCommunity, table({1, 2, 3, 4, 5}))); +} + +// a set with a listed key that holds no majority of the list must not hold back the regular restart +// the initial confidants in it would make without the list +TEST(CommunityBootstrap, AMixedSetWithoutAMajorityOfTheListFallsBackToTheRegularTable) { + using cs::bootstrapTableOf; + const std::set initial = keys({1, 2, 3, 4, 6}); + const std::set list = keys({5, 7, 9, 11, 13}); + // the 5 initial confidants and listed 5 alive: 1 of 5 listed is no community table; the 5 initial + // ones make the regular table they make on a node without the list + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 2, 3, 4, 5, 6})), keys({1, 2, 3, 4, 6})); + EXPECT_EQ(bootstrapTableOf(initial, {}, keys({1, 2, 3, 4, 6})), keys({1, 2, 3, 4, 6})); + // 4 initial ones and listed 5: even, the official odd rule leaves 3, below MinTrustedNodes: wait + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 2, 3, 4, 5})), keys({1, 2, 3})); + // a community table stays one + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 5, 7, 9})), keys({1, 5, 7, 9})); + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 3, 5, 7, 9, 11})), keys({1, 3, 5, 7, 9})); + // initial keys only: as before + EXPECT_EQ(bootstrapTableOf(initial, list, keys({1, 2, 3, 4})), keys({1, 2, 3})); } namespace {