diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 5463f9c26..5385b2957 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -21,6 +21,9 @@ on: only: description: 'Comma-separated scenario names to run (empty = all)' default: '' + node_alt_ref: + description: 'Branch or commit of this repository built and run by the even nodes (empty = all the same)' + default: '' permissions: contents: read @@ -102,6 +105,45 @@ jobs: path: build/client/node retention-days: 7 + # mixed nodes, as in testnet.yml: the even nodes run a node built from another ref + - name: Build the node under comparison + if: inputs.node_alt_ref != '' + env: + CCACHE_BASEDIR: ${{ github.workspace }} + NODE_ALT_REF: ${{ inputs.node_alt_ref }} + run: | + git fetch -q --no-recurse-submodules origin -- "$NODE_ALT_REF" + git worktree add -q node-alt FETCH_HEAD + # the submodules are copied from this run's checkout, which must point to the same commits + for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \ + third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do + if [ "$(git ls-tree HEAD -- "$m")" != "$(git -C node-alt ls-tree HEAD -- "$m")" ]; then + echo "$NODE_ALT_REF points $m to another commit than this run"; exit 1 + fi + rm -rf "node-alt/$m" + cp -a "$m" "node-alt/$m" + done + cmake -S node-alt -B build-alt -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTS=OFF \ + -DBoost_ROOT="${BOOST_PREFIX}" \ + -DBoost_NO_SYSTEM_PATHS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \ + -DWITH_HASKELL=OFF \ + -DCREDITS_TESTNET=ON + cmake --build build-alt -j"$(nproc)" --target node + echo "node under comparison: $(git -C node-alt log -1 --format='%h %s')" + + - name: Upload the node under comparison + if: inputs.node_alt_ref != '' + uses: actions/upload-artifact@v4 + with: + name: testnet-node-alt-${{ github.sha }} + path: build-alt/client/node + retention-days: 7 + scenario: needs: build runs-on: ubuntu-24.04 @@ -245,6 +287,13 @@ jobs: name: testnet-node-${{ github.sha }} path: build + - name: Download the node under comparison + if: env.SELECTED == 'true' && inputs.node_alt_ref != '' + uses: actions/download-artifact@v4 + with: + name: testnet-node-alt-${{ github.sha }} + path: build-alt + - name: Run private network if: env.SELECTED == 'true' timeout-minutes: 100 @@ -259,6 +308,10 @@ jobs: WORK: ${{ github.workspace }}/testnet-run run: | chmod +x build/node + if [ -f build-alt/node ]; then + chmod +x build-alt/node + export BIN_ALT="$PWD/build-alt/node" + fi # the scenario's variables come from the matrix above, NAME=value pairs only for kv in $SCENARIO; do echo "$kv" | grep -qE '^[A-Z_]+=[a-z0-9,:]+$' || { echo "bad scenario entry $kv"; exit 1; } diff --git a/csnode/include/csnode/bootstrapgate.hpp b/csnode/include/csnode/bootstrapgate.hpp new file mode 100644 index 000000000..b912f0cd7 --- /dev/null +++ b/csnode/include/csnode/bootstrapgate.hpp @@ -0,0 +1,47 @@ +#ifndef BOOTSTRAPGATE_HPP +#define BOOTSTRAPGATE_HPP + +#include +#include + +#include + +namespace cs { + +// An honest starter (Node::onRoundTimeElapsed, Node::bootstrap, Node::initBootstrapRP) sends +// round = its last block + 1 and holds the highest block among the initial confidants it sees. A +// receiver level with it sees lastSeq + 1; one that holds one block the starter lacks sees lastSeq; +// one that lacks the starter's last block sees lastSeq + 2; one still fetching blocks that learned +// the network round sees currentRound or currentRound + 1. A bootstrap table for any other round is +// not for this node. +constexpr RoundNumber kBootstrapRoundsAhead = 2; + +inline bool isBootstrapRoundInWindow(RoundNumber rNum, Sequence lastSeq, RoundNumber currentRound) { + if (rNum == 0 || rNum < lastSeq) { + return false; + } + const RoundNumber top = std::max(lastSeq, currentRound); + return rNum <= kBootstrapRoundsAhead || rNum - kBootstrapRoundsAhead <= top; +} + +// A node takes an unsigned bootstrap table only when its own round has lasted about +// round_elapse_time, the condition that makes an honest initial confidant start a bootstrap. Half +// of it is margin for the starter's tick (2 s), the propagation of the previous round table and a +// round table fetched late. A node younger than round_elapse_time cannot have observed a stall, and +// a node already in a bootstrap round is restarting with the others: both take the table as the +// official node does. +inline uint64_t bootstrapStallThresholdMs(uint64_t roundElapseMs) { + return roundElapseMs - roundElapseMs / 2; +} + +inline bool isBootstrapStallAccepted(uint64_t msSinceRoundChange, uint64_t msSinceStart, uint64_t roundElapseMs, + bool inBootstrapRound) { + if (inBootstrapRound || msSinceStart < roundElapseMs) { + return true; + } + return msSinceRoundChange >= bootstrapStallThresholdMs(roundElapseMs); +} + +} // namespace cs + +#endif // BOOTSTRAPGATE_HPP diff --git a/csnode/include/csnode/bootstrapweight.hpp b/csnode/include/csnode/bootstrapweight.hpp index b7674c275..ea6f3b56f 100644 --- a/csnode/include/csnode/bootstrapweight.hpp +++ b/csnode/include/csnode/bootstrapweight.hpp @@ -20,8 +20,8 @@ constexpr size_t kMaxBootstrapTable = 25; // Consensus::MaxTrustedNodes // Up to 8 keys, exactly the official node's value on x86-64, so that both pick the same table when a // stalled network bootstraps again: the table is matched against the initial confidants in their // (sorted) order, and the confidant with index k adds 2^(n-1-k) for a table of n keys. In the official -// code an index k >= n shifts by a wrapped size_t, which x86 masks to 31..8 and the uint8_t sum then -// truncates to 0: it adds nothing, as here. From 9 to 25 keys, where the official node returns 0 and +// code an index k >= n shifts by a wrapped size_t, which x86 masks to 31..8 (for fewer than 28 initial +// confidants) and the uint8_t sum then truncates to 0: it adds nothing, as here. From 9 to 25 keys, where the official node returns 0 and // drops the table, 2^n on top, so every larger table beats every smaller one, but only for a table // the starter builds: initial confidants in their order, each once. Any other table of 9 or more keys // (repeated or unordered keys, which an unsigned table could otherwise use to outweigh an honest one) diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index 1c95b20c6..3b885c6d6 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -9,6 +9,7 @@ #include +#include #include #include #include diff --git a/csnode/include/csnode/roundstat.hpp b/csnode/include/csnode/roundstat.hpp index 4c6a3834e..f7b1d0eac 100644 --- a/csnode/include/csnode/roundstat.hpp +++ b/csnode/include/csnode/roundstat.hpp @@ -116,6 +116,12 @@ class RoundStat { void resetLastRoundMs(); bool isCurrentRoundTooLong(size_t longDurationMs = kMaxRoundDelay) const; + + // ms since the current round began on this node (its last round change), on the steady clock; + // unlike the round elapse time point, the elapse check does not reset it + uint64_t msSinceRoundChange(); + // ms since this node started + uint64_t msSinceStart() const; const std::map& getRoundEvaluation() { return minedEvaluation_; } @@ -180,6 +186,8 @@ public signals: std::mutex statsElapseMutex_; std::chrono::steady_clock::time_point roundElapseTimePoint_; std::chrono::steady_clock::time_point storeBlockElapseTimePoint_; + std::chrono::steady_clock::time_point roundChangeTimePoint_; // guarded by statsElapseMutex_ + const std::chrono::steady_clock::time_point startTimePoint_; std::chrono::milliseconds checkPingDelta_{0}; // diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 26433e03b..917c03e89 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -518,6 +518,14 @@ cs::BootstrapWeight Node::calculateBootStrapWeight(const cs::PublicKeys& confida void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::RoundNumber rNum) { cslog() << "NODE> get Boot strap Round Table #" << rNum; + // a table for a round no honest starter would send to this node changes nothing here + const cs::Sequence lastSequence = blockChain_.getLastSeq(); + const cs::RoundNumber currentRound = cs::Conveyer::instance().currentRoundNumber(); + if (!cs::isBootstrapRoundInWindow(rNum, lastSequence, currentRound)) { + cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " does not fit this node (last block " << lastSequence + << ", round " << currentRound << "), drop"; + return; + } solver_->resetGrayList(); cs::IDataStream in(data, size); cs::RoundTable roundTable; @@ -548,14 +556,16 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R } cs::BootstrapWeight currentWeight = calculateBootStrapWeight(confidants); + bool signedTable = false; + const size_t signed_subdata_len = 1 + size_t(confSize) * cscrypto::kPublicKeySize; if (unknown) { cs::Signature sig; stream >> sig; if (!stream.isValid()) { cswarning() << "malformed bootstrap round table, ignore"; + return; } - const size_t signed_subdata_len = 1 + size_t(confSize) * cscrypto::kPublicKeySize; const size_t signed_buf_len = sizeof(cs::RoundNumber) + signed_subdata_len; cs::Bytes buf; buf.reserve(signed_buf_len); @@ -568,7 +578,38 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R return; } currentWeight = cs::kSignedBootstrapWeight; + signedTable = true; } + else if (payload.size() == signed_subdata_len + cscrypto::kSignatureSize) { + // a table of initial confidants signed through the diag API (Node::bootstrap): the official + // node does not read its signature; here it exempts the table from the stall check below. + // An unsigned table ends with 4 random bytes, never with a signature's 64 + cs::Signature sig; + stream >> sig; + cs::Bytes buf; + buf.reserve(sizeof(cs::RoundNumber) + signed_subdata_len); + auto* ptr = reinterpret_cast(&rNum); + std::copy(ptr, ptr + sizeof(cs::RoundNumber), std::back_inserter(buf)); + ptr = payload.data(); + std::copy(ptr, ptr + signed_subdata_len, std::back_inserter(buf)); + signedTable = stream.isValid() && cscrypto::verifySignature(sig, cs::PacketValidator::getBlockChainKey(), buf.data(), buf.size()); + if (!signedTable) { + // e.g. a diag API bootstrap whose round the sender raised to its last block + 1 + csdebug() << kLogPrefix_ << "bootstrap table #" << rNum << " has a signature that does not verify for this round"; + } + } + + // an unsigned table only while this node's own round has stalled, as it has for the starter + if (!signedTable) { + const uint64_t elapse = cs::ConfigHolder::instance().config()->roundElapseTime(); + const uint64_t sinceRound = stat_.msSinceRoundChange(); + if (!cs::isBootstrapStallAccepted(sinceRound, stat_.msSinceStart(), elapse, isBootstrapRound_)) { + cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " while round " << currentRound << " is " << sinceRound + << " ms old (round_elapse_time " << elapse << "), drop"; + return; + } + } + if (currentWeight <= bootStrapWeight_) { cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " with " << confidants.size() << " confidants weighs " << currentWeight << ", not above " << bootStrapWeight_ << ", drop"; @@ -589,12 +630,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R roundTable.confidants = std::move(confidants); roundTable.hashes.clear(); - cs::Sequence lastSequence = blockChain_.getLastSeq(); - - if (lastSequence >= rNum) { - csdebug() << "NODE> remove " << lastSequence - rNum + 1 << " block(s) required (rNum = " << rNum << ", last_seq = " << lastSequence << ")"; - blockChain_.setBlocksToBeRemoved(lastSequence - rNum + 1); - } + // No block-removal request: the official one (lastSequence - rNum + 1) is refused whenever a + // block was loaded or recorded, the permission being set already; the one honest case, + // rNum == lastSequence (one uncertain last block), is resolved by storeBlock. roundTable.round = rNum; confirmationList_.remove(rNum); diff --git a/csnode/src/roundstat.cpp b/csnode/src/roundstat.cpp index 80fe906cc..86cc8def6 100644 --- a/csnode/src/roundstat.cpp +++ b/csnode/src/roundstat.cpp @@ -18,7 +18,9 @@ RoundStat::RoundStat(BlockChain* bch) , startSkipRounds_(2) , lastRoundMs_(0) , roundElapseTimePoint_(std::chrono::steady_clock::now()) -, storeBlockElapseTimePoint_(std::chrono::steady_clock::now()) { +, storeBlockElapseTimePoint_(std::chrono::steady_clock::now()) +, roundChangeTimePoint_(std::chrono::steady_clock::now()) +, startTimePoint_(std::chrono::steady_clock::now()) { //nodes_ = new std::map(); blockChain_ = bch; } @@ -707,6 +709,16 @@ void RoundStat::checkPing(cs::Sequence sequence, const PublicKey& key) { void RoundStat::onRoundChanged() { cs::Lock lock(statsElapseMutex_); roundElapseTimePoint_ = std::chrono::steady_clock::now(); + roundChangeTimePoint_ = roundElapseTimePoint_; +} + +uint64_t RoundStat::msSinceRoundChange() { + cs::Lock lock(statsElapseMutex_); + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - roundChangeTimePoint_).count()); +} + +uint64_t RoundStat::msSinceStart() const { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - startTimePoint_).count()); } void RoundStat::onBlockStored() { diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 4b5513943..b39102762 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -36,6 +36,7 @@ lines under the new release. |---|---| | #67 | csdb, ci, docs: conversion follow-ups from the review of #65 (contract-records message, keep a copy of `db/` before converting from berkeleydb, sturdier db-convert scenario) | | #70 | csnode: bootstrap tables of 9 to 25 confidants are accepted (the official node silently drops them, so a network of 9 or more initial confidants never starts); the same choice as the official node up to 8 | +| #71 | csnode: a bootstrap round table is taken only for a round next to the node's own, and, unless signed with the blockchain key, only once the node's own round has stalled; the block-removal request it set (never effective) is gone | ## community.9 — 2026-10-22 diff --git a/docs/differences.md b/docs/differences.md index c4589a3f4..ad6fc1cbf 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -72,6 +72,15 @@ quick-starts after a conversion. restarts. The community node accepts tables of up to 25 confidants (`MaxTrustedNodes`). Up to 8 it gives exactly the official weights, so both pick the same table when a stalled network starts again; a dropped table is now logged. +- **Bootstrap tables are checked against the node's own state.** A bootstrap round table is + taken only for a round next to the node's own (from its last block to its round + 2), and, + unless it is signed with the blockchain key, only once the node's own round has lasted at + least half of `round_elapse_time` (a node younger than `round_elapse_time`, or already in a + bootstrap round, takes it as before). The tables honest nodes send pass in the cases a mixed + network relies on, so it makes the same choices; a node that joined the stalled round late, or + that is several blocks behind and has not learned the round yet, takes the next table instead. + A bootstrap through the diag API must name a round within two of what the nodes know (their + last block + 1, as the sending node uses when it raises the round). - **State kept across a quick start.** Pending consensus-settings changes, special orders and deployed smart contracts survive a quick start. The official node can lose them, or apply them differently, after a quick start. diff --git a/tests/csnode/bootstrapgate_tests.cpp b/tests/csnode/bootstrapgate_tests.cpp new file mode 100644 index 000000000..dc09e9bec --- /dev/null +++ b/tests/csnode/bootstrapgate_tests.cpp @@ -0,0 +1,63 @@ +#include "gtest/gtest.h" + +#include + +#include + +using cs::bootstrapStallThresholdMs; +using cs::isBootstrapRoundInWindow; +using cs::isBootstrapStallAccepted; + +// the rounds an honest starter sends to a receiver: its last block + 1, the receiver being level with +// it, one block ahead or behind, or still fetching blocks with the network round learned +TEST(BootstrapGate, HonestRoundsAreInTheWindow) { + EXPECT_TRUE(isBootstrapRoundInWindow(1, 0, 0)); // a fresh network + EXPECT_TRUE(isBootstrapRoundInWindow(501, 500, 501)); // level with the starter + EXPECT_TRUE(isBootstrapRoundInWindow(500, 500, 501)); // one block the starter lacks + EXPECT_TRUE(isBootstrapRoundInWindow(502, 500, 500)); // lacks the starter's last block + EXPECT_TRUE(isBootstrapRoundInWindow(9001, 100, 9001)); // fetching blocks, round learned + EXPECT_TRUE(isBootstrapRoundInWindow(9002, 100, 9001)); // and one round table missed + EXPECT_TRUE(isBootstrapRoundInWindow(9003, 100, 9001)); + EXPECT_TRUE(isBootstrapRoundInWindow(500, 499, 501)); // the round moves down by one + EXPECT_TRUE(isBootstrapRoundInWindow(500, 500, 0)); // restarted, round not known yet + EXPECT_TRUE(isBootstrapRoundInWindow(502, 500, 0)); +} + +TEST(BootstrapGate, OtherRoundsAreNot) { + EXPECT_FALSE(isBootstrapRoundInWindow(0, 0, 0)); + EXPECT_FALSE(isBootstrapRoundInWindow(499, 500, 501)); // below the last block + EXPECT_FALSE(isBootstrapRoundInWindow(1, 500, 501)); + EXPECT_FALSE(isBootstrapRoundInWindow(503, 500, 0)); + EXPECT_FALSE(isBootstrapRoundInWindow(9004, 100, 9001)); + EXPECT_FALSE(isBootstrapRoundInWindow(1000000, 500, 501)); +} + +TEST(BootstrapGate, NoOverflowNearTheTop) { + const auto max = std::numeric_limits::max(); + EXPECT_TRUE(isBootstrapRoundInWindow(max, max - 1, max - 1)); + EXPECT_FALSE(isBootstrapRoundInWindow(max, 0, 0)); + EXPECT_TRUE(isBootstrapRoundInWindow(max, max, max)); +} + +TEST(BootstrapGate, StallThresholdIsHalfTheElapseTime) { + EXPECT_EQ(bootstrapStallThresholdMs(10000), 5000u); + EXPECT_EQ(bootstrapStallThresholdMs(60000), 30000u); + EXPECT_EQ(bootstrapStallThresholdMs(0), 0u); +} + +TEST(BootstrapGate, AStalledRoundTakesTheTable) { + const uint64_t up = 3600000; // started an hour ago + EXPECT_FALSE(isBootstrapStallAccepted(0, up, 10000, false)); + EXPECT_FALSE(isBootstrapStallAccepted(4999, up, 10000, false)); + EXPECT_TRUE(isBootstrapStallAccepted(5000, up, 10000, false)); + EXPECT_TRUE(isBootstrapStallAccepted(60000, up, 60000, false)); + EXPECT_TRUE(isBootstrapStallAccepted(0, up, 0, false)); +} + +// a node younger than round_elapse_time, or one already restarting with the others, takes it as the +// official node does +TEST(BootstrapGate, YoungOrBootstrappingNodesTakeTheTable) { + EXPECT_TRUE(isBootstrapStallAccepted(0, 9999, 10000, false)); + EXPECT_FALSE(isBootstrapStallAccepted(0, 10000, 10000, false)); + EXPECT_TRUE(isBootstrapStallAccepted(0, 3600000, 10000, true)); +}