Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ on:
only:
description: 'Comma-separated scenario names to run (empty = all)'
default: ''
node_alt_ref:
description: 'Branch or commit of this repository built and run by the even nodes (empty = all the same)'
default: ''

permissions:
contents: read
Expand Down Expand Up @@ -102,6 +105,45 @@ jobs:
path: build/client/node
retention-days: 7

# mixed nodes, as in testnet.yml: the even nodes run a node built from another ref
- name: Build the node under comparison
if: inputs.node_alt_ref != ''
env:
CCACHE_BASEDIR: ${{ github.workspace }}
NODE_ALT_REF: ${{ inputs.node_alt_ref }}
run: |
git fetch -q --no-recurse-submodules origin -- "$NODE_ALT_REF"
git worktree add -q node-alt FETCH_HEAD
# the submodules are copied from this run's checkout, which must point to the same commits
for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \
third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do
if [ "$(git ls-tree HEAD -- "$m")" != "$(git -C node-alt ls-tree HEAD -- "$m")" ]; then
echo "$NODE_ALT_REF points $m to another commit than this run"; exit 1
fi
rm -rf "node-alt/$m"
cp -a "$m" "node-alt/$m"
done
cmake -S node-alt -B build-alt -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DENABLE_TESTS=OFF \
-DBoost_ROOT="${BOOST_PREFIX}" \
-DBoost_NO_SYSTEM_PATHS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_FLAGS="-include cstdint -include cstddef" \
-DWITH_HASKELL=OFF \
-DCREDITS_TESTNET=ON
cmake --build build-alt -j"$(nproc)" --target node
echo "node under comparison: $(git -C node-alt log -1 --format='%h %s')"

- name: Upload the node under comparison
if: inputs.node_alt_ref != ''
uses: actions/upload-artifact@v4
with:
name: testnet-node-alt-${{ github.sha }}
path: build-alt/client/node
retention-days: 7

scenario:
needs: build
runs-on: ubuntu-24.04
Expand Down Expand Up @@ -245,6 +287,13 @@ jobs:
name: testnet-node-${{ github.sha }}
path: build

- name: Download the node under comparison
if: env.SELECTED == 'true' && inputs.node_alt_ref != ''
uses: actions/download-artifact@v4
with:
name: testnet-node-alt-${{ github.sha }}
path: build-alt

- name: Run private network
if: env.SELECTED == 'true'
timeout-minutes: 100
Expand All @@ -259,6 +308,10 @@ jobs:
WORK: ${{ github.workspace }}/testnet-run
run: |
chmod +x build/node
if [ -f build-alt/node ]; then
chmod +x build-alt/node
export BIN_ALT="$PWD/build-alt/node"
fi
# the scenario's variables come from the matrix above, NAME=value pairs only
for kv in $SCENARIO; do
echo "$kv" | grep -qE '^[A-Z_]+=[a-z0-9,:]+$' || { echo "bad scenario entry $kv"; exit 1; }
Expand Down
47 changes: 47 additions & 0 deletions csnode/include/csnode/bootstrapgate.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#ifndef BOOTSTRAPGATE_HPP
#define BOOTSTRAPGATE_HPP

#include <algorithm>
#include <cstdint>

#include <lib/system/common.hpp>

namespace cs {

// An honest starter (Node::onRoundTimeElapsed, Node::bootstrap, Node::initBootstrapRP) sends
// round = its last block + 1 and holds the highest block among the initial confidants it sees. A
// receiver level with it sees lastSeq + 1; one that holds one block the starter lacks sees lastSeq;
// one that lacks the starter's last block sees lastSeq + 2; one still fetching blocks that learned
// the network round sees currentRound or currentRound + 1. A bootstrap table for any other round is
// not for this node.
constexpr RoundNumber kBootstrapRoundsAhead = 2;

inline bool isBootstrapRoundInWindow(RoundNumber rNum, Sequence lastSeq, RoundNumber currentRound) {
if (rNum == 0 || rNum < lastSeq) {
return false;
}
const RoundNumber top = std::max<RoundNumber>(lastSeq, currentRound);
return rNum <= kBootstrapRoundsAhead || rNum - kBootstrapRoundsAhead <= top;
}

// A node takes an unsigned bootstrap table only when its own round has lasted about
// round_elapse_time, the condition that makes an honest initial confidant start a bootstrap. Half
// of it is margin for the starter's tick (2 s), the propagation of the previous round table and a
// round table fetched late. A node younger than round_elapse_time cannot have observed a stall, and
// a node already in a bootstrap round is restarting with the others: both take the table as the
// official node does.
inline uint64_t bootstrapStallThresholdMs(uint64_t roundElapseMs) {
return roundElapseMs - roundElapseMs / 2;
}

inline bool isBootstrapStallAccepted(uint64_t msSinceRoundChange, uint64_t msSinceStart, uint64_t roundElapseMs,
bool inBootstrapRound) {
if (inBootstrapRound || msSinceStart < roundElapseMs) {
return true;
}
return msSinceRoundChange >= bootstrapStallThresholdMs(roundElapseMs);
}

} // namespace cs

#endif // BOOTSTRAPGATE_HPP
4 changes: 2 additions & 2 deletions csnode/include/csnode/bootstrapweight.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ constexpr size_t kMaxBootstrapTable = 25; // Consensus::MaxTrustedNodes
// Up to 8 keys, exactly the official node's value on x86-64, so that both pick the same table when a
// stalled network bootstraps again: the table is matched against the initial confidants in their
// (sorted) order, and the confidant with index k adds 2^(n-1-k) for a table of n keys. In the official
// code an index k >= n shifts by a wrapped size_t, which x86 masks to 31..8 and the uint8_t sum then
// truncates to 0: it adds nothing, as here. From 9 to 25 keys, where the official node returns 0 and
// code an index k >= n shifts by a wrapped size_t, which x86 masks to 31..8 (for fewer than 28 initial
// confidants) and the uint8_t sum then truncates to 0: it adds nothing, as here. From 9 to 25 keys, where the official node returns 0 and
// drops the table, 2^n on top, so every larger table beats every smaller one, but only for a table
// the starter builds: initial confidants in their order, each once. Any other table of 9 or more keys
// (repeated or unordered keys, which an unsigned table could otherwise use to outweigh an honest one)
Expand Down
1 change: 1 addition & 0 deletions csnode/include/csnode/node.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#include <csconnector/csconnector.hpp>

#include <csnode/bootstrapgate.hpp>
#include <csnode/bootstrapweight.hpp>
#include <csnode/caches_serialization_manager.hpp>
#include <csnode/conveyer.hpp>
Expand Down
8 changes: 8 additions & 0 deletions csnode/include/csnode/roundstat.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,12 @@ class RoundStat {

void resetLastRoundMs();
bool isCurrentRoundTooLong(size_t longDurationMs = kMaxRoundDelay) const;

// ms since the current round began on this node (its last round change), on the steady clock;
// unlike the round elapse time point, the elapse check does not reset it
uint64_t msSinceRoundChange();
// ms since this node started
uint64_t msSinceStart() const;
const std::map<cs::PublicKey, cs::MinedEvaluationDelegator>& getRoundEvaluation() {
return minedEvaluation_;
}
Expand Down Expand Up @@ -180,6 +186,8 @@ public signals:
std::mutex statsElapseMutex_;
std::chrono::steady_clock::time_point roundElapseTimePoint_;
std::chrono::steady_clock::time_point storeBlockElapseTimePoint_;
std::chrono::steady_clock::time_point roundChangeTimePoint_; // guarded by statsElapseMutex_
const std::chrono::steady_clock::time_point startTimePoint_;

std::chrono::milliseconds checkPingDelta_{0};
//<to save>
Expand Down
52 changes: 45 additions & 7 deletions csnode/src/node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,14 @@ cs::BootstrapWeight Node::calculateBootStrapWeight(const cs::PublicKeys& confida

void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::RoundNumber rNum) {
cslog() << "NODE> get Boot strap Round Table #" << rNum;
// a table for a round no honest starter would send to this node changes nothing here
const cs::Sequence lastSequence = blockChain_.getLastSeq();
const cs::RoundNumber currentRound = cs::Conveyer::instance().currentRoundNumber();
if (!cs::isBootstrapRoundInWindow(rNum, lastSequence, currentRound)) {
cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " does not fit this node (last block " << lastSequence
<< ", round " << currentRound << "), drop";
return;
}
solver_->resetGrayList();
cs::IDataStream in(data, size);
cs::RoundTable roundTable;
Expand Down Expand Up @@ -548,14 +556,16 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R
}

cs::BootstrapWeight currentWeight = calculateBootStrapWeight(confidants);
bool signedTable = false;
const size_t signed_subdata_len = 1 + size_t(confSize) * cscrypto::kPublicKeySize;

if (unknown) {
cs::Signature sig;
stream >> sig;
if (!stream.isValid()) {
cswarning() << "malformed bootstrap round table, ignore";
return;
}
const size_t signed_subdata_len = 1 + size_t(confSize) * cscrypto::kPublicKeySize;
const size_t signed_buf_len = sizeof(cs::RoundNumber) + signed_subdata_len;
cs::Bytes buf;
buf.reserve(signed_buf_len);
Expand All @@ -568,7 +578,38 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R
return;
}
currentWeight = cs::kSignedBootstrapWeight;
signedTable = true;
}
else if (payload.size() == signed_subdata_len + cscrypto::kSignatureSize) {
// a table of initial confidants signed through the diag API (Node::bootstrap): the official
// node does not read its signature; here it exempts the table from the stall check below.
// An unsigned table ends with 4 random bytes, never with a signature's 64
cs::Signature sig;
stream >> sig;
cs::Bytes buf;
buf.reserve(sizeof(cs::RoundNumber) + signed_subdata_len);
auto* ptr = reinterpret_cast<const uint8_t*>(&rNum);
std::copy(ptr, ptr + sizeof(cs::RoundNumber), std::back_inserter(buf));
ptr = payload.data();
std::copy(ptr, ptr + signed_subdata_len, std::back_inserter(buf));
signedTable = stream.isValid() && cscrypto::verifySignature(sig, cs::PacketValidator::getBlockChainKey(), buf.data(), buf.size());
if (!signedTable) {
// e.g. a diag API bootstrap whose round the sender raised to its last block + 1
csdebug() << kLogPrefix_ << "bootstrap table #" << rNum << " has a signature that does not verify for this round";
}
}

// an unsigned table only while this node's own round has stalled, as it has for the starter
if (!signedTable) {
const uint64_t elapse = cs::ConfigHolder::instance().config()->roundElapseTime();
const uint64_t sinceRound = stat_.msSinceRoundChange();
if (!cs::isBootstrapStallAccepted(sinceRound, stat_.msSinceStart(), elapse, isBootstrapRound_)) {
cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " while round " << currentRound << " is " << sinceRound
<< " ms old (round_elapse_time " << elapse << "), drop";
return;
}
}

if (currentWeight <= bootStrapWeight_) {
cslog() << kLogPrefix_ << "bootstrap table #" << rNum << " with " << confidants.size() << " confidants weighs "
<< currentWeight << ", not above " << bootStrapWeight_ << ", drop";
Expand All @@ -589,12 +630,9 @@ void Node::getBootstrapTable(const uint8_t* data, const size_t size, const cs::R
roundTable.confidants = std::move(confidants);
roundTable.hashes.clear();

cs::Sequence lastSequence = blockChain_.getLastSeq();

if (lastSequence >= rNum) {
csdebug() << "NODE> remove " << lastSequence - rNum + 1 << " block(s) required (rNum = " << rNum << ", last_seq = " << lastSequence << ")";
blockChain_.setBlocksToBeRemoved(lastSequence - rNum + 1);
}
// No block-removal request: the official one (lastSequence - rNum + 1) is refused whenever a
// block was loaded or recorded, the permission being set already; the one honest case,
// rNum == lastSequence (one uncertain last block), is resolved by storeBlock.

roundTable.round = rNum;
confirmationList_.remove(rNum);
Expand Down
14 changes: 13 additions & 1 deletion csnode/src/roundstat.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ RoundStat::RoundStat(BlockChain* bch)
, startSkipRounds_(2)
, lastRoundMs_(0)
, roundElapseTimePoint_(std::chrono::steady_clock::now())
, storeBlockElapseTimePoint_(std::chrono::steady_clock::now()) {
, storeBlockElapseTimePoint_(std::chrono::steady_clock::now())
, roundChangeTimePoint_(std::chrono::steady_clock::now())
, startTimePoint_(std::chrono::steady_clock::now()) {
//nodes_ = new std::map<cs::PublicKey, cs::NodeStat>();
blockChain_ = bch;
}
Expand Down Expand Up @@ -707,6 +709,16 @@ void RoundStat::checkPing(cs::Sequence sequence, const PublicKey& key) {
void RoundStat::onRoundChanged() {
cs::Lock lock(statsElapseMutex_);
roundElapseTimePoint_ = std::chrono::steady_clock::now();
roundChangeTimePoint_ = roundElapseTimePoint_;
}

uint64_t RoundStat::msSinceRoundChange() {
cs::Lock lock(statsElapseMutex_);
return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - roundChangeTimePoint_).count());
}

uint64_t RoundStat::msSinceStart() const {
return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startTimePoint_).count());
}

void RoundStat::onBlockStored() {
Expand Down
1 change: 1 addition & 0 deletions docs/CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ lines under the new release.
|---|---|
| #67 | csdb, ci, docs: conversion follow-ups from the review of #65 (contract-records message, keep a copy of `db/` before converting from berkeleydb, sturdier db-convert scenario) |
| #70 | csnode: bootstrap tables of 9 to 25 confidants are accepted (the official node silently drops them, so a network of 9 or more initial confidants never starts); the same choice as the official node up to 8 |
| #71 | csnode: a bootstrap round table is taken only for a round next to the node's own, and, unless signed with the blockchain key, only once the node's own round has stalled; the block-removal request it set (never effective) is gone |

## community.9 — 2026-10-22

Expand Down
9 changes: 9 additions & 0 deletions docs/differences.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@ quick-starts after a conversion.
restarts. The community node accepts tables of up to 25 confidants (`MaxTrustedNodes`). Up to 8 it
gives exactly the official weights, so both pick the same table when a stalled network starts
again; a dropped table is now logged.
- **Bootstrap tables are checked against the node's own state.** A bootstrap round table is
taken only for a round next to the node's own (from its last block to its round + 2), and,
unless it is signed with the blockchain key, only once the node's own round has lasted at
least half of `round_elapse_time` (a node younger than `round_elapse_time`, or already in a
bootstrap round, takes it as before). The tables honest nodes send pass in the cases a mixed
network relies on, so it makes the same choices; a node that joined the stalled round late, or
that is several blocks behind and has not learned the round yet, takes the next table instead.
A bootstrap through the diag API must name a round within two of what the nodes know (their
last block + 1, as the sending node uses when it raises the round).
- **State kept across a quick start.** Pending consensus-settings changes, special orders and
deployed smart contracts survive a quick start. The official node can lose them, or apply
them differently, after a quick start.
Expand Down
63 changes: 63 additions & 0 deletions tests/csnode/bootstrapgate_tests.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#include "gtest/gtest.h"

#include <limits>

#include <csnode/bootstrapgate.hpp>

using cs::bootstrapStallThresholdMs;
using cs::isBootstrapRoundInWindow;
using cs::isBootstrapStallAccepted;

// the rounds an honest starter sends to a receiver: its last block + 1, the receiver being level with
// it, one block ahead or behind, or still fetching blocks with the network round learned
TEST(BootstrapGate, HonestRoundsAreInTheWindow) {
EXPECT_TRUE(isBootstrapRoundInWindow(1, 0, 0)); // a fresh network
EXPECT_TRUE(isBootstrapRoundInWindow(501, 500, 501)); // level with the starter
EXPECT_TRUE(isBootstrapRoundInWindow(500, 500, 501)); // one block the starter lacks
EXPECT_TRUE(isBootstrapRoundInWindow(502, 500, 500)); // lacks the starter's last block
EXPECT_TRUE(isBootstrapRoundInWindow(9001, 100, 9001)); // fetching blocks, round learned
EXPECT_TRUE(isBootstrapRoundInWindow(9002, 100, 9001)); // and one round table missed
EXPECT_TRUE(isBootstrapRoundInWindow(9003, 100, 9001));
EXPECT_TRUE(isBootstrapRoundInWindow(500, 499, 501)); // the round moves down by one
EXPECT_TRUE(isBootstrapRoundInWindow(500, 500, 0)); // restarted, round not known yet
EXPECT_TRUE(isBootstrapRoundInWindow(502, 500, 0));
}

TEST(BootstrapGate, OtherRoundsAreNot) {
EXPECT_FALSE(isBootstrapRoundInWindow(0, 0, 0));
EXPECT_FALSE(isBootstrapRoundInWindow(499, 500, 501)); // below the last block
EXPECT_FALSE(isBootstrapRoundInWindow(1, 500, 501));
EXPECT_FALSE(isBootstrapRoundInWindow(503, 500, 0));
EXPECT_FALSE(isBootstrapRoundInWindow(9004, 100, 9001));
EXPECT_FALSE(isBootstrapRoundInWindow(1000000, 500, 501));
}

TEST(BootstrapGate, NoOverflowNearTheTop) {
const auto max = std::numeric_limits<cs::RoundNumber>::max();
EXPECT_TRUE(isBootstrapRoundInWindow(max, max - 1, max - 1));
EXPECT_FALSE(isBootstrapRoundInWindow(max, 0, 0));
EXPECT_TRUE(isBootstrapRoundInWindow(max, max, max));
}

TEST(BootstrapGate, StallThresholdIsHalfTheElapseTime) {
EXPECT_EQ(bootstrapStallThresholdMs(10000), 5000u);
EXPECT_EQ(bootstrapStallThresholdMs(60000), 30000u);
EXPECT_EQ(bootstrapStallThresholdMs(0), 0u);
}

TEST(BootstrapGate, AStalledRoundTakesTheTable) {
const uint64_t up = 3600000; // started an hour ago
EXPECT_FALSE(isBootstrapStallAccepted(0, up, 10000, false));
EXPECT_FALSE(isBootstrapStallAccepted(4999, up, 10000, false));
EXPECT_TRUE(isBootstrapStallAccepted(5000, up, 10000, false));
EXPECT_TRUE(isBootstrapStallAccepted(60000, up, 60000, false));
EXPECT_TRUE(isBootstrapStallAccepted(0, up, 0, false));
}

// a node younger than round_elapse_time, or one already restarting with the others, takes it as the
// official node does
TEST(BootstrapGate, YoungOrBootstrappingNodesTakeTheTable) {
EXPECT_TRUE(isBootstrapStallAccepted(0, 9999, 10000, false));
EXPECT_FALSE(isBootstrapStallAccepted(0, 10000, 10000, false));
EXPECT_TRUE(isBootstrapStallAccepted(0, 3600000, 10000, true));
}
Loading