From f0d680c22ca9d08a7643071b2cdbd01b08bd0c71 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 02:45:47 +0200 Subject: [PATCH 1/2] csnode, solver: keep the stage-1 hashes of a same-round bootstrap An official stage-1 race (contracts-net-loss stall at block 831, run 39866725719, analysed by Fable): each node sends its hash when it takes a bootstrap table; a hash that reaches a node before that node took the table is dropped (getHash returns when not a confidant, or recv_hash is cleared by onRoundStart(..., false)), so the last node to take the table never gets MinTrustedNodes-1 hashes, and with round_elapse_time below the ~12.5 s fallback chain the bootstrap repeats for ever. - getHash: a node that is not a confidant verifies and hands to the solver a hash of the round after its last block (the solver caches it, NormalState ignores it); other rounds dropped as before - getBootstrapTable: a table of the round the node is in keeps the received hashes (onRoundStart(..., true), as the starter already does), replayed by gotConveyerSync - SolverCore::gotHash: one entry per sender, so kept hashes do not pile up across retries - bootstrap-livelock becomes a regular robustness scenario Every check of a hash runs as for a confidant; the block is built from the same stage messages, so a mixed network does not fork. Consensus-relevant: to be merged only with the maintainer's OK. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 7 +++---- ci/testnet/run.sh | 2 +- csnode/include/csnode/bootstrapgate.hpp | 12 ++++++++++++ csnode/src/node.cpp | 14 ++++++++++++-- docs/CHANGES.md | 1 + docs/differences.md | 7 +++++++ solver/src/solverinterface.cpp | 10 +++++----- tests/csnode/bootstrapgate_tests.cpp | 19 +++++++++++++++++++ 8 files changed, 60 insertions(+), 12 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 5609b6f22..df8726b27 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -286,12 +286,11 @@ jobs: - name: contracts-net-loss env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1 contracts: true - # reproduction of an official stage-1 hash race: with round_elapse_time 10 s a slow round - # under loss triggers a bootstrap of the full table, which never completes; fails until the - # node keeps the hashes of a same-round bootstrap (manual: run by name) + # an official stage-1 hash race: with round_elapse_time 10 s a slow round under loss triggers + # a bootstrap of the full table, which the official node never completes; fixed by keeping the + # hashes of a same-round bootstrap (#77) - name: bootstrap-livelock env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 - manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 37ebb15d5..d0ece122c 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -142,7 +142,7 @@ SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} # a node starts a bootstrap when its round lasts this long. The consensus's own fallback for a # silent trusted node takes about 12.5 s, so below that a slow round is pre-empted by a bootstrap, # and a bootstrap of a full table re-sent every 10 s never completes (official stage-1 hash race): -# 20 s by default (mainnet default 60 s); bootstrap-livelock keeps 10 s to reproduce the race +# 20 s by default (mainnet default 60 s); bootstrap-livelock keeps 10 s for the race fixed in #77 ROUND_ELAPSE_MS=${ROUND_ELAPSE_MS:-20000} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} diff --git a/csnode/include/csnode/bootstrapgate.hpp b/csnode/include/csnode/bootstrapgate.hpp index b912f0cd7..b685b567b 100644 --- a/csnode/include/csnode/bootstrapgate.hpp +++ b/csnode/include/csnode/bootstrapgate.hpp @@ -42,6 +42,18 @@ inline bool isBootstrapStallAccepted(uint64_t msSinceRoundChange, uint64_t msSin return msSinceRoundChange >= bootstrapStallThresholdMs(roundElapseMs); } +// A hash is evidence for the round after this node's last block only (stage 1 compares it with the +// last block's hash): a node that is not a confidant keeps such a hash for a table of that round +inline bool isHashKeptWhileNotConfidant(RoundNumber rNum, Sequence lastSeq) { + return rNum == lastSeq + 1; +} + +// A bootstrap table of the round the node is already in restarts that round: the hashes already +// received stay valid (the official node keeps them across its own same-round table changes) +inline bool bootstrapKeepsRoundHashes(RoundNumber tableRound, RoundNumber currentRound) { + return tableRound == currentRound; +} + } // namespace cs #endif // BOOTSTRAPGATE_HPP diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 917c03e89..1f32b3f28 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -639,7 +639,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R cs::Conveyer::instance().updateRoundTable(rNum, roundTable); stat_.onRoundChanged(); - onRoundStart(roundTable, false); + // a table of the round this node is in restarts that round: the stage-1 hashes already received + // stay (the starter keeps its own, see onRoundTimeElapsed and bootstrap); another round starts afresh + onRoundStart(roundTable, cs::bootstrapKeepsRoundHashes(rNum, currentRound)); reviewConveyerHashes(); if (cs::ConfigHolder::instance().config()->isSyncOn()) { @@ -3566,7 +3568,12 @@ void Node::sendHash(cs::RoundNumber round) { } void Node::getHash(const uint8_t* data, const size_t size, cs::RoundNumber rNum, const cs::PublicKey& sender) { - if (myLevel_ != Level::Confidant) { + // Not a confidant of the current table: the sender's table of the same round lists this node, as + // in a bootstrap this node has not taken yet. The official node drops the hash, so the last node + // to take a bootstrap table never gets enough hashes and the bootstrap can repeat for ever (a + // stage-1 race). The solver caches a verified hash of the round after the last block and replays + // it when the table arrives (gotConveyerSync); a hash of any other round is dropped as before + if (myLevel_ != Level::Confidant && !cs::isHashKeptWhileNotConfidant(rNum, blockChain_.getLastSeq())) { csdebug() << "NODE> ignore hash as no confidant"; return; } @@ -3659,6 +3666,9 @@ void Node::getHash(const uint8_t* data, const size_t size, cs::RoundNumber rNum, myRealTrustedSize = cs::TrustedMask::trustedSize(lastTrusted); } + if (myLevel_ != Level::Confidant) { + csdebug() << "NODE> cache hash of round " << rNum << " until a table of this round lists this node"; + } solver_->gotHash(std::move(sHash), myRealTrustedSize); } diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 81a3c9968..5afebf61e 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -43,6 +43,7 @@ lines under the new release. | #73 | api: the executor version check asks every second up to `executor_check_version_delay` instead of once after it (the node starts as soon as its executor answers); with `executor_multi_instance` a stopping node no longer kills the other nodes' executors | | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | | #76 | ci(testnet): round_elapse_time 20 s by default (10 s pre-empted slow rounds with a bootstrap); a run that ends stuck in a bootstrap round fails; bootstrap-livelock reproduces an official stage-1 hash race (run by name) | +| #77 | csnode: a node keeps the stage-1 hashes of a same-round bootstrap (official race: the last node to take a bootstrap table never got enough hashes, and the bootstrap could repeat for ever) | ## community.9 — 2026-10-22 diff --git a/docs/differences.md b/docs/differences.md index ad6fc1cbf..7edcd250a 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -81,6 +81,13 @@ quick-starts after a conversion. that is several blocks behind and has not learned the round yet, takes the next table instead. A bootstrap through the diag API must name a round within two of what the nodes know (their last block + 1, as the sending node uses when it raises the round). +- **A bootstrap that completes.** Each node sends its stage-1 hash as soon as it takes a bootstrap + table; the official node drops a hash that arrives before it has taken the table itself, so the + last node to take it never gets enough hashes and, with a short `round_elapse_time`, the + bootstrap repeats for ever. The community node keeps those hashes (verified as usual) and uses + them when the table arrives. It never accepts a hash an official confidant would reject, and the + block is built from the same stage messages, so a mixed network does not fork; official nodes + still drop theirs. - **State kept across a quick start.** Pending consensus-settings changes, special orders and deployed smart contracts survive a quick start. The official node can lose them, or apply them differently, after a quick start. diff --git a/solver/src/solverinterface.cpp b/solver/src/solverinterface.cpp index 143e69708..474144a3e 100644 --- a/solver/src/solverinterface.cpp +++ b/solver/src/solverinterface.cpp @@ -117,12 +117,12 @@ void SolverCore::gotHash(const cs::StageHash&& sHash, uint8_t currentTrustedSize if (sHash.realTrustedSize < currentTrustedSize) { csdebug() << "Stake value is lower than that in this node, trow this hash"; } - auto rNum = cs::Conveyer::instance().currentRoundNumber(); - bool isBootstrap = pnode->isBootstrapRound(); + // one entry per sender: a later hash of the same sender replaces the earlier one (realTrustedSize + // has no other use), so hashes kept across bootstrap retries do not pile up auto it = std::find_if( - recv_hash.cbegin(), recv_hash.cend(), - [sHash, rNum, isBootstrap](const cs::StageHash& sh) { - return ((sHash.sender == sh.sender) && (isBootstrap || sHash.realTrustedSize > sh.realTrustedSize) && (sHash.round == rNum)); + recv_hash.cbegin(), recv_hash.cend(), + [&sHash](const cs::StageHash& sh) { + return sHash.sender == sh.sender; } ); if (it != recv_hash.cend()) { diff --git a/tests/csnode/bootstrapgate_tests.cpp b/tests/csnode/bootstrapgate_tests.cpp index dc09e9bec..fc73532cf 100644 --- a/tests/csnode/bootstrapgate_tests.cpp +++ b/tests/csnode/bootstrapgate_tests.cpp @@ -7,6 +7,8 @@ using cs::bootstrapStallThresholdMs; using cs::isBootstrapRoundInWindow; using cs::isBootstrapStallAccepted; +using cs::bootstrapKeepsRoundHashes; +using cs::isHashKeptWhileNotConfidant; // the rounds an honest starter sends to a receiver: its last block + 1, the receiver being level with // it, one block ahead or behind, or still fetching blocks with the network round learned @@ -61,3 +63,20 @@ TEST(BootstrapGate, YoungOrBootstrappingNodesTakeTheTable) { EXPECT_FALSE(isBootstrapStallAccepted(0, 10000, 10000, false)); EXPECT_TRUE(isBootstrapStallAccepted(0, 3600000, 10000, true)); } + +// a node that is not a confidant keeps a hash only for the round after its last block +TEST(BootstrapGate, HashesKeptWhileNotConfidant) { + EXPECT_TRUE(isHashKeptWhileNotConfidant(501, 500)); + EXPECT_TRUE(isHashKeptWhileNotConfidant(1, 0)); + EXPECT_FALSE(isHashKeptWhileNotConfidant(500, 500)); + EXPECT_FALSE(isHashKeptWhileNotConfidant(502, 500)); + EXPECT_FALSE(isHashKeptWhileNotConfidant(0, 0)); +} + +// a bootstrap table of the current round keeps its hashes, any other round starts afresh +TEST(BootstrapGate, SameRoundBootstrapKeepsHashes) { + EXPECT_TRUE(bootstrapKeepsRoundHashes(501, 501)); + EXPECT_FALSE(bootstrapKeepsRoundHashes(501, 0)); + EXPECT_FALSE(bootstrapKeepsRoundHashes(500, 501)); + EXPECT_FALSE(bootstrapKeepsRoundHashes(502, 501)); +} From fb7974f49090bcc455dd10374fc98543524c9ded Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 03:54:35 +0200 Subject: [PATCH 2/2] ci: bootstrap-livelock stays a scenario run by name (intermittent, 3000 blocks); CHANGES.md Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 8 +++++--- docs/CHANGES.md | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index df8726b27..ca3050f50 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -286,11 +286,13 @@ jobs: - name: contracts-net-loss env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1 contracts: true - # an official stage-1 hash race: with round_elapse_time 10 s a slow round under loss triggers - # a bootstrap of the full table, which the official node never completes; fixed by keeping the - # hashes of a same-round bootstrap (#77) + # an official stage-1 hash race: with round_elapse_time 10 s a slow round under loss can + # trigger a bootstrap of the full table, which the official node never completes; fixed by + # keeping the hashes of a same-round bootstrap (#77). Intermittent and long (3000 blocks): + # run by name, repeated - name: bootstrap-livelock env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 + manual: true # a pruned node quick-starting from checkpoints that hold the contract states - name: contracts-pruned-term-kill env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000 diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 3ffd1c553..b88112050 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,6 @@ lines under the new release. | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | | #75 | ci(testnet): the contract load window starts after funding its writers, block rates from the first block (found in the slowdown analysis of the scale runs) | | #76 | ci(testnet): round_elapse_time 20 s by default (10 s pre-empted slow rounds with a bootstrap); a run that ends stuck in a bootstrap round fails; bootstrap-livelock exercises an official stage-1 hash race (intermittent; run by name) | -| #76 | ci(testnet): round_elapse_time 20 s by default (10 s pre-empted slow rounds with a bootstrap); a run that ends stuck in a bootstrap round fails; bootstrap-livelock reproduces an official stage-1 hash race (run by name) | | #77 | csnode: a node keeps the stage-1 hashes of a same-round bootstrap (official race: the last node to take a bootstrap table never got enough hashes, and the bootstrap could repeat for ever) | ## community.9 — 2026-10-22