diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index baed3f6b4..ca3050f50 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -287,8 +287,9 @@ jobs: env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1 contracts: true # an official stage-1 hash race: with round_elapse_time 10 s a slow round under loss can - # trigger a bootstrap of the full table, which then never completes. Intermittent (it - # stalled 1 run of 3 at 10 s, none at 3000 blocks in run 39878938046): run by name, repeated + # trigger a bootstrap of the full table, which the official node never completes; fixed by + # keeping the hashes of a same-round bootstrap (#77). Intermittent and long (3000 blocks): + # run by name, repeated - name: bootstrap-livelock env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 ROUND_ELAPSE_MS=10000 TARGET_SEQ=3000 manual: true diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 37ebb15d5..d0ece122c 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -142,7 +142,7 @@ SAMPLE_SECONDS=${SAMPLE_SECONDS:-60} # a node starts a bootstrap when its round lasts this long. The consensus's own fallback for a # silent trusted node takes about 12.5 s, so below that a slow round is pre-empted by a bootstrap, # and a bootstrap of a full table re-sent every 10 s never completes (official stage-1 hash race): -# 20 s by default (mainnet default 60 s); bootstrap-livelock keeps 10 s to reproduce the race +# 20 s by default (mainnet default 60 s); bootstrap-livelock keeps 10 s for the race fixed in #77 ROUND_ELAPSE_MS=${ROUND_ELAPSE_MS:-20000} TARGET_SEQ=${TARGET_SEQ:-1100} TIMEOUT_MIN=${TIMEOUT_MIN:-45} diff --git a/csnode/include/csnode/bootstrapgate.hpp b/csnode/include/csnode/bootstrapgate.hpp index b912f0cd7..b685b567b 100644 --- a/csnode/include/csnode/bootstrapgate.hpp +++ b/csnode/include/csnode/bootstrapgate.hpp @@ -42,6 +42,18 @@ inline bool isBootstrapStallAccepted(uint64_t msSinceRoundChange, uint64_t msSin return msSinceRoundChange >= bootstrapStallThresholdMs(roundElapseMs); } +// A hash is evidence for the round after this node's last block only (stage 1 compares it with the +// last block's hash): a node that is not a confidant keeps such a hash for a table of that round +inline bool isHashKeptWhileNotConfidant(RoundNumber rNum, Sequence lastSeq) { + return rNum == lastSeq + 1; +} + +// A bootstrap table of the round the node is already in restarts that round: the hashes already +// received stay valid (the official node keeps them across its own same-round table changes) +inline bool bootstrapKeepsRoundHashes(RoundNumber tableRound, RoundNumber currentRound) { + return tableRound == currentRound; +} + } // namespace cs #endif // BOOTSTRAPGATE_HPP diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 917c03e89..1f32b3f28 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -639,7 +639,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R cs::Conveyer::instance().updateRoundTable(rNum, roundTable); stat_.onRoundChanged(); - onRoundStart(roundTable, false); + // a table of the round this node is in restarts that round: the stage-1 hashes already received + // stay (the starter keeps its own, see onRoundTimeElapsed and bootstrap); another round starts afresh + onRoundStart(roundTable, cs::bootstrapKeepsRoundHashes(rNum, currentRound)); reviewConveyerHashes(); if (cs::ConfigHolder::instance().config()->isSyncOn()) { @@ -3566,7 +3568,12 @@ void Node::sendHash(cs::RoundNumber round) { } void Node::getHash(const uint8_t* data, const size_t size, cs::RoundNumber rNum, const cs::PublicKey& sender) { - if (myLevel_ != Level::Confidant) { + // Not a confidant of the current table: the sender's table of the same round lists this node, as + // in a bootstrap this node has not taken yet. The official node drops the hash, so the last node + // to take a bootstrap table never gets enough hashes and the bootstrap can repeat for ever (a + // stage-1 race). The solver caches a verified hash of the round after the last block and replays + // it when the table arrives (gotConveyerSync); a hash of any other round is dropped as before + if (myLevel_ != Level::Confidant && !cs::isHashKeptWhileNotConfidant(rNum, blockChain_.getLastSeq())) { csdebug() << "NODE> ignore hash as no confidant"; return; } @@ -3659,6 +3666,9 @@ void Node::getHash(const uint8_t* data, const size_t size, cs::RoundNumber rNum, myRealTrustedSize = cs::TrustedMask::trustedSize(lastTrusted); } + if (myLevel_ != Level::Confidant) { + csdebug() << "NODE> cache hash of round " << rNum << " until a table of this round lists this node"; + } solver_->gotHash(std::move(sHash), myRealTrustedSize); } diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 8e886809a..b88112050 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,6 +44,7 @@ lines under the new release. | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | | #75 | ci(testnet): the contract load window starts after funding its writers, block rates from the first block (found in the slowdown analysis of the scale runs) | | #76 | ci(testnet): round_elapse_time 20 s by default (10 s pre-empted slow rounds with a bootstrap); a run that ends stuck in a bootstrap round fails; bootstrap-livelock exercises an official stage-1 hash race (intermittent; run by name) | +| #77 | csnode: a node keeps the stage-1 hashes of a same-round bootstrap (official race: the last node to take a bootstrap table never got enough hashes, and the bootstrap could repeat for ever) | ## community.9 — 2026-10-22 diff --git a/docs/differences.md b/docs/differences.md index ad6fc1cbf..7edcd250a 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -81,6 +81,13 @@ quick-starts after a conversion. that is several blocks behind and has not learned the round yet, takes the next table instead. A bootstrap through the diag API must name a round within two of what the nodes know (their last block + 1, as the sending node uses when it raises the round). +- **A bootstrap that completes.** Each node sends its stage-1 hash as soon as it takes a bootstrap + table; the official node drops a hash that arrives before it has taken the table itself, so the + last node to take it never gets enough hashes and, with a short `round_elapse_time`, the + bootstrap repeats for ever. The community node keeps those hashes (verified as usual) and uses + them when the table arrives. It never accepts a hash an official confidant would reject, and the + block is built from the same stage messages, so a mixed network does not fork; official nodes + still drop theirs. - **State kept across a quick start.** Pending consensus-settings changes, special orders and deployed smart contracts survive a quick start. The official node can lose them, or apply them differently, after a quick start. diff --git a/solver/src/solverinterface.cpp b/solver/src/solverinterface.cpp index 143e69708..474144a3e 100644 --- a/solver/src/solverinterface.cpp +++ b/solver/src/solverinterface.cpp @@ -117,12 +117,12 @@ void SolverCore::gotHash(const cs::StageHash&& sHash, uint8_t currentTrustedSize if (sHash.realTrustedSize < currentTrustedSize) { csdebug() << "Stake value is lower than that in this node, trow this hash"; } - auto rNum = cs::Conveyer::instance().currentRoundNumber(); - bool isBootstrap = pnode->isBootstrapRound(); + // one entry per sender: a later hash of the same sender replaces the earlier one (realTrustedSize + // has no other use), so hashes kept across bootstrap retries do not pile up auto it = std::find_if( - recv_hash.cbegin(), recv_hash.cend(), - [sHash, rNum, isBootstrap](const cs::StageHash& sh) { - return ((sHash.sender == sh.sender) && (isBootstrap || sHash.realTrustedSize > sh.realTrustedSize) && (sHash.round == rNum)); + recv_hash.cbegin(), recv_hash.cend(), + [&sHash](const cs::StageHash& sh) { + return sHash.sender == sh.sender; } ); if (it != recv_hash.cend()) { diff --git a/tests/csnode/bootstrapgate_tests.cpp b/tests/csnode/bootstrapgate_tests.cpp index dc09e9bec..fc73532cf 100644 --- a/tests/csnode/bootstrapgate_tests.cpp +++ b/tests/csnode/bootstrapgate_tests.cpp @@ -7,6 +7,8 @@ using cs::bootstrapStallThresholdMs; using cs::isBootstrapRoundInWindow; using cs::isBootstrapStallAccepted; +using cs::bootstrapKeepsRoundHashes; +using cs::isHashKeptWhileNotConfidant; // the rounds an honest starter sends to a receiver: its last block + 1, the receiver being level with // it, one block ahead or behind, or still fetching blocks with the network round learned @@ -61,3 +63,20 @@ TEST(BootstrapGate, YoungOrBootstrappingNodesTakeTheTable) { EXPECT_FALSE(isBootstrapStallAccepted(0, 10000, 10000, false)); EXPECT_TRUE(isBootstrapStallAccepted(0, 3600000, 10000, true)); } + +// a node that is not a confidant keeps a hash only for the round after its last block +TEST(BootstrapGate, HashesKeptWhileNotConfidant) { + EXPECT_TRUE(isHashKeptWhileNotConfidant(501, 500)); + EXPECT_TRUE(isHashKeptWhileNotConfidant(1, 0)); + EXPECT_FALSE(isHashKeptWhileNotConfidant(500, 500)); + EXPECT_FALSE(isHashKeptWhileNotConfidant(502, 500)); + EXPECT_FALSE(isHashKeptWhileNotConfidant(0, 0)); +} + +// a bootstrap table of the current round keeps its hashes, any other round starts afresh +TEST(BootstrapGate, SameRoundBootstrapKeepsHashes) { + EXPECT_TRUE(bootstrapKeepsRoundHashes(501, 501)); + EXPECT_FALSE(bootstrapKeepsRoundHashes(501, 0)); + EXPECT_FALSE(bootstrapKeepsRoundHashes(500, 501)); + EXPECT_FALSE(bootstrapKeepsRoundHashes(502, 501)); +}