From c167a290b71d58973df63ec14c919428757a977b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 10:31:23 +0100 Subject: [PATCH 01/14] csnode: bound automatic chain reorganisations A node that is behind and syncs a longer competing chain rolls back its own confirmed blocks down to the fork point before taking the other chain. The official node puts no limit on that depth, so a competing chain can make a syncing node unwind arbitrarily deep. Bound the depth of every automatic, non-operator rollback (this builds on the earlier local draft b8e5cb59, cherry-picked and extended): - The depth check now lives inside removeLastBlock() and compromiseLastBlock(), so every automatic removal path goes through it: the sync fork path, the uncertain-block replacement, the antifork cache and the bad-hash reply. The bound is measured against a fixed reference captured at the first removal of an episode, so forward progress cannot reset it and a node cannot be made to creep past the limit by alternating removals and appends. There is no operator-initiated truncation path, so all callers are automatic. - A competing block that is NOT a bootstrap block triggers a rollback only if its round confirmations verify against the confidants of the node's own last block -- the authority it already trusts -- exactly as BlockVerifier checks a block from the network. A block without confirmations from that authority never makes the node unwind. An honest micro-fork still works, because the honest competing block at the same height was produced by the same deterministically elected trusted group. A bootstrap block carries no such confirmations and cannot be checked locally; for it the depth bound plus the operator override is the only defence. Once the bound would be crossed the node keeps its chain, logs a prominent alert and waits for the operator; it sets no permanent state, so a later legitimate reorg still works, and it does not crash-loop. The bound ([params] max_auto_rollback, default 1000) and a deep-switch override ([params] allow_deep_rollback, or the CS_ALLOW_DEEP_ROLLBACK environment variable) are configurable. A node that adopted a community bootstrap table may unwind back to the sequence it held when it took the table (setCommunityBootstrapFloor), which keeps this coherent with the community bootstrap fallback. Each finished rollback episode logs its depth; ci/testnet/run.sh reports the deepest one and fails a normal run if the bound ever stops a reorg. Local safety bound only: it changes how far this node unwinds on its own and never changes any block content or what the node accepts in normal operation, so it stays byte-compatible and invisible to other nodes, and a mixed official/community network behaves exactly as before. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 12 +++ client/config/config.cpp | 6 ++ client/config/config.hpp | 15 ++++ csnode/include/csnode/blockchain.hpp | 48 ++++++++++ csnode/src/blockchain.cpp | 130 +++++++++++++++++++++++++++ docs/CHANGES.md | 1 + docs/differences.md | 41 +++++++++ packaging/mainnet/config.ini | 8 ++ solver/include/solver/consensus.hpp | 9 ++ tests/csnode/blockchain_tests.cpp | 86 ++++++++++++++++++ 10 files changed, 356 insertions(+) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index b3cb3931e..25dd2565a 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2270,6 +2270,18 @@ if [ -n "$forks" ]; then exit 1 fi +# chain reorganisations: how deep each node unwound on its own (community nodes log one line per +# finished rollback episode), and the bound must never stop a normal run. DEEP_ROLLBACK_EXPECTED=1 +# is for a scenario that provokes a reorg deeper than [params] max_auto_rollback on purpose. +reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) +echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" +guarded=$(grep -al 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) +if [ -n "$guarded" ] && [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then + grep -ah 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log | head -3 + echo "FAIL: the automatic rollback bound stopped a reorg in a normal run on: $guarded" + exit 1 +fi + # a network still restarting at the end is stuck, even if it reached the target before (a bootstrap # round, R-x.1 and up, that never gives way to R-(x+1).0): the last three round headers of n1 stuck=$(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -3 | grep -vc '\.0 ' || true) diff --git a/client/config/config.cpp b/client/config/config.cpp index 1d2050636..e98f92ef7 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -69,6 +69,8 @@ const std::string PARAM_NAME_COMPATIBLE_VERSION = "compatible_version"; const std::string PARAM_NAME_TRAVERSE_NAT = "traverse_nat"; const std::string PARAM_NAME_SYNC_ON = "sync_on"; const std::string PARAM_NAME_MAX_UNCORRECTED_BLOCK = "max_uncorrected_block"; +const std::string PARAM_NAME_MAX_AUTO_ROLLBACK = "max_auto_rollback"; +const std::string PARAM_NAME_ALLOW_DEEP_ROLLBACK = "allow_deep_rollback"; const std::string PARAM_NAME_GENERATE_FORK = "generate_fork"; const std::string PARAM_NAME_IDLE_MODE = "idle_mode"; const std::string PARAM_NAME_DAEMON_MODE = "daemon_mode"; @@ -997,6 +999,10 @@ Config Config::readFromFile(const std::string& fileName) { } result.maxUncorrectedBlock_ = params.count(PARAM_NAME_MAX_UNCORRECTED_BLOCK) ? params.get(PARAM_NAME_MAX_UNCORRECTED_BLOCK) : DEFAULT_MAX_UNCORRECTED_BLOCK; + result.maxAutoRollbackDepth_ = params.count(PARAM_NAME_MAX_AUTO_ROLLBACK) ? params.get(PARAM_NAME_MAX_AUTO_ROLLBACK) : DEFAULT_MAX_AUTO_ROLLBACK; + if (params.count(PARAM_NAME_ALLOW_DEEP_ROLLBACK) > 0) { + result.allowDeepRollback_ = params.get(PARAM_NAME_ALLOW_DEEP_ROLLBACK); + } result.generateFork_ = params.count(PARAM_NAME_GENERATE_FORK) ? params.get(PARAM_NAME_GENERATE_FORK) : false; if (params.count(PARAM_NAME_SYNC_ON)) { diff --git a/client/config/config.hpp b/client/config/config.hpp index 55c762f50..e1ec76ca0 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -31,6 +31,8 @@ const std::string DEFAULT_PATH_TO_PRIVATE_KEY = "NodePrivate.txt"; const uint32_t DEFAULT_MIN_NEIGHBOURS = Neighbourhood::kMinNeighbours; const uint32_t DEFAULT_MAX_NEIGHBOURS = Neighbourhood::kMaxNeighbours; const uint64_t DEFAULT_MAX_UNCORRECTED_BLOCK = 0; +// keep in step with Consensus::MaxAutoRollbackDepth (solver/include/solver/consensus.hpp) +const uint64_t DEFAULT_MAX_AUTO_ROLLBACK = 1000; const uint32_t DEFAULT_CONNECTION_BANDWIDTH = 1 << 19; const uint32_t DEFAULT_OBSERVER_WAIT_TIME = 5 * 60 * 1000; // ms const uint32_t DEFAULT_ROUND_ELAPSE_TIME = 1000 * 60; // ms @@ -327,6 +329,17 @@ class Config { return maxUncorrectedBlock_; } + // longest-chain guard: the most own confirmed blocks the node will roll back automatically when it + // adopts a competing chain during sync, before it stops and waits for the operator. + uint64_t maxAutoRollbackDepth() const { + return maxAutoRollbackDepth_; + } + + // longest-chain guard override: accept a deliberate deep switch without the depth bound. + bool allowDeepRollback() const { + return allowDeepRollback_; + } + bool traverseNAT() const { return traverseNAT_; } @@ -430,6 +443,8 @@ class Config { bool restrictNeighbours_ = false; uint64_t connectionBandwidth_ = DEFAULT_CONNECTION_BANDWIDTH; uint64_t maxUncorrectedBlock_ = DEFAULT_MAX_UNCORRECTED_BLOCK; + uint64_t maxAutoRollbackDepth_ = DEFAULT_MAX_AUTO_ROLLBACK; + bool allowDeepRollback_ = false; bool symmetric_ = false; EndpointData hostAddressEp_; diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index b872e38e5..43702993a 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -242,6 +242,42 @@ class BlockChain { static void setBootstrap(csdb::Pool& block, bool is_bootstrap); static bool isBootstrap(const csdb::Pool& block); + // Longest-chain guard. It is called from inside removeLastBlock() and compromiseLastBlock(), so + // every automatic rollback of the node's own last block passes through it, whatever the caller + // (the sync fork path, the uncertain-replacement path, the antifork cache and the bad-hash reply). + // It returns true while the resulting head would stay at or above rollbackHighWater_ - + // maxAutoRollbackDepth_, where rollbackHighWater_ is the head captured at the start of the current + // rollback episode (the first automatic removal from any path). Once the limit would be crossed it + // logs a prominent alert and returns false (keeping the chain intact and leaving the decision to + // the operator); it does NOT set any permanent state, so a later legitimate reorg still works. + // An operator accepts a deliberate deep switch with the [params] allow_deep_rollback option or the + // CS_ALLOW_DEEP_ROLLBACK environment variable. A node that has adopted a community bootstrap table + // (see setCommunityBootstrapFloor) may unwind freely back to the sequence where it took that table. + // Local only: it changes how far this node unwinds, never block content. + bool guardAutoRollback(); + + // True if the competing block carries round confirmations signed by the confidants the node + // already trusts - the confidants of its own current last block - exactly as BlockVerifier checks + // them. A non-bootstrap competing block that fails this check never triggers a rollback: it is not + // vouched for by any authority this node recognises (a self-signed block vouches only for itself). + // A bootstrap block carries no such confirmations and cannot be checked locally; for it the depth + // floor plus the operator override is the only defence, so callers gate it on isBootstrap() first. + bool hasTrustedConfirmations(const csdb::Pool& competing) const; + + // A node that has just adopted a community bootstrap table (the opt-in liveness fallback) records + // here the sequence it held when it took the table; the guard then lets it unwind back to that + // point without operator action, since taking the table already was the operator's decision. 0 + // (the default) means no such table was taken and the ordinary depth bound applies throughout. + void setCommunityBootstrapFloor(cs::Sequence seq); + +#ifdef TESTING + // test-only accessors for the longest-chain guard (inline, no data members: no layout change) + void test_setLastSequence(cs::Sequence s) { lastSequence_ = s; } + cs::Sequence test_rollbackHighWater() const { return rollbackHighWater_; } + bool test_rollbackEpisode() const { return rollbackEpisode_; } + void test_resetRollbackEpisode() { rollbackEpisode_ = false; rollbackHighWater_ = 0; } +#endif + const cs::MultiWallets& multiWallets() const; /** @@ -595,6 +631,18 @@ public slots: mutable std::atomic uuid_ = 0; std::atomic lastSequence_; cs::Sequence blocksToBeRemoved_ = 0; + // longest-chain guard: a fixed reference for the current automatic-rollback episode. When the + // first automatic rollback of an episode starts, rollbackHighWater_ records the head we had then; + // the node then refuses to unwind automatically below rollbackHighWater_ - MaxAutoRollbackDepth. + // The reference is cleared only once the chain climbs back strictly above rollbackHighWater_, so + // alternating removals and appends cannot creep past the limit. Purely local bookkeeping; it + // never changes any block content. + bool rollbackEpisode_ = false; + cs::Sequence rollbackHighWater_ = 0; + cs::Sequence rollbackLowWater_ = 0; // lowest head reached in the current episode (for the log) + // longest-chain guard runtime state (see setCommunityBootstrapFloor). The depth bound and the + // deep-rollback override are read from config ([params] max_auto_rollback / allow_deep_rollback). + cs::Sequence communityBootstrapFloor_ = 0; std::atomic_bool stop_ = false; // the quick-start checkpoint of the last init() is ahead of the blocks on disk bool checkpointAheadOfBlocks_ = false; diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 149032d7b..28c5247fc 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -829,6 +830,83 @@ csdb::Transaction BlockChain::loadTransaction(const csdb::TransactionID& transId return transaction; } +void BlockChain::setCommunityBootstrapFloor(cs::Sequence seq) { + communityBootstrapFloor_ = seq; +} + +// Longest-chain guard: bound the depth of an automatic rollback while adopting a competing chain. +// A competing chain that asks us to unwind more than the configured bound of own blocks is either a +// deliberate network switch (a human must confirm it) or a chain that should not be followed: +// in both cases the safe local action is to stop unwinding and wait for the operator. The depth is +// measured against a fixed reference (the head when the episode started), not a counter that forward +// progress could reset, so alternating removals and appends cannot creep past it. It is called from +// inside removeLastBlock() and compromiseLastBlock(), so every automatic removal is bounded whatever +// the caller, and rollbackHighWater_ is captured at the first removal from any path. +bool BlockChain::guardAutoRollback() { + // operator override: accept a deliberate deep switch. Config ([params] allow_deep_rollback) or the + // CS_ALLOW_DEEP_ROLLBACK environment variable; either one is enough. + static const bool allowDeepEnv = [] { + const char* v = std::getenv("CS_ALLOW_DEEP_ROLLBACK"); + return v != nullptr && *v != '\0'; + }(); + const auto* config = cs::ConfigHolder::instance().config(); + if (allowDeepEnv || (config != nullptr && config->allowDeepRollback())) { + return true; + } + const uint64_t maxDepth = (config != nullptr) ? config->maxAutoRollbackDepth() + : Consensus::MaxAutoRollbackDepth; + if (maxDepth == 0) { + return true; // max_auto_rollback=0: no bound, the official node's behaviour + } + + const cs::Sequence tip = lastSequence_.load(); + if (!rollbackEpisode_) { + rollbackEpisode_ = true; + rollbackHighWater_ = tip; // the head we had before this rollback episode began + rollbackLowWater_ = tip; + } + + // a node that adopted a community bootstrap table (PR #188) may unwind freely back to the sequence + // it held when it took the table: taking the table was already the operator's explicit choice, and + // after a partition the longer community branch legitimately replaces blocks down to that point. + if (communityBootstrapFloor_ != 0 && tip > communityBootstrapFloor_) { + return true; + } + + // refuse the removal that would take the head below the floor (no wraparound when near genesis) + if (rollbackHighWater_ > maxDepth && tip <= rollbackHighWater_ - maxDepth) { + cserror() << kLogPrefix << "LONGEST-CHAIN GUARD: a competing chain requires rolling back more than " + << maxDepth << " of our own confirmed blocks (reference head #" + << WithDelimiters(rollbackHighWater_) << ", now #" << WithDelimiters(tip) + << "). Keeping our chain, operator review required. This is either a deliberate longer-chain " + << "switch to be confirmed, or a chain not to be followed. Set [params] " + << "allow_deep_rollback=true (or the CS_ALLOW_DEEP_ROLLBACK environment variable) and restart " + << "to accept a deliberate deep switch."; + return false; + } + return true; +} + +// A competing block legitimately asks us to roll back only if it is vouched for by authority this node +// already recognises. We verify its round confirmations against the confidants of our own current last +// block, exactly as BlockVerifier does for a block arriving from the network. Checking the block against +// the confidants it declares itself (as a self-signature check would) proves nothing: any block +// names its own keys and signs for itself. A legitimate micro-fork passes, because the honest competing +// block at the same height was produced by the same deterministically elected trusted group whose keys +// confirm our last block; a block without confirmations from that group fails and never triggers an +// unwind. Bootstrap blocks carry no such confirmations and are handled by the depth floor plus operator, +// so callers must check isBootstrap() before relying on this. +bool BlockChain::hasTrustedConfirmations(const csdb::Pool& competing) const { + std::string reason; + const auto trustedConfidants = getLastBlock().confidants(); + const bool ok = cs::BlockVerifier::verifyConfirmations(competing, trustedConfidants, reason); + if (!ok) { + csdebug() << kLogPrefix << "competing block #" << competing.sequence() + << " not confirmed by our last block's confidants: " << reason; + } + return ok; +} + // - remove the last block from the top of blockchain // - remove pair (hash, sequence) from cache (blockHashes_) // - decrement the last sequence by 1 @@ -838,6 +916,14 @@ void BlockChain::removeLastBlock() { csmeta(csdebug) << kLogPrefix << "There are no blocks, allowed to be removed"; return; } + // longest-chain guard: every caller of removeLastBlock is an automatic reorg/sync removal (there is + // no operator truncation path), so bound the depth here, once, for all of them. When the limit is + // reached the block is kept and the caller sees its own last block unchanged; no permanent state is + // set, so a later legitimate reorg still works. + if (!guardAutoRollback()) { + cswarning() << kLogPrefix << "longest-chain guard kept the last block, automatic rollback stopped"; + return; + } //--blocksToBeRemoved_; cs::Sequence remove_seq = lastSequence_; csdb::PoolHash remove_hash = blockHashes_->find(remove_seq); @@ -909,6 +995,9 @@ void BlockChain::removeLastBlock() { blockHashes_->remove(remove_hash); } --lastSequence_; + if (rollbackEpisode_ && lastSequence_.load() < rollbackLowWater_) { + rollbackLowWater_ = lastSequence_.load(); + } csmeta(csdebug) << kLogPrefix << "done"; } @@ -928,6 +1017,14 @@ bool BlockChain::compromiseLastBlock(const csdb::PoolHash& desired_hash) { return false; } + // longest-chain guard: marking the last block uncertain is the first step of an automatic unwind, + // so it is bounded here too. Beyond the depth limit the block is kept and the caller falls through + // to its keep-own-chain branch. + if (!guardAutoRollback()) { + cswarning() << kLogPrefix << "longest-chain guard kept the last block, not compromising it"; + return false; + } + const auto seq = last_block.sequence(); const auto current = last_block.hash(); const auto desired = desired_hash; @@ -1924,6 +2021,16 @@ std::optional BlockChain::recordBlock(csdb::Pool& pool, bool isTrust lastSequence_ = deferredBlock_.sequence(); } + // forward progress: the rollback episode ends only once the chain climbs back strictly above the + // head it had when the episode began, so the depth cannot creep past the limit by alternating + // removals and appends below that head. + if (rollbackEpisode_ && lastSequence_.load() > rollbackHighWater_) { + rollbackEpisode_ = false; + // one line per finished episode: how deep the node actually unwound (measured in CI) + cslog() << kLogPrefix << "rollback episode ended: unwound " << (rollbackHighWater_ - rollbackLowWater_) + << " block(s) below #" << WithDelimiters(rollbackHighWater_); + } + //csdetails() << kLogPrefix << "Pool #" << deferredBlock_.sequence() << ": " << cs::Utils::byteStreamToHex(deferredBlock_.to_binary().data(), deferredBlock_.to_binary().size()); emit storeBlockEvent(pool); if constexpr (false && (pool.transactions_count() > 0 || pool.sequence() % 10 == 0)) {//log code @@ -2311,10 +2418,20 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { if (compromiseLastBlock(pool.previous_hash())) { csdebug() << kLogPrefix << "compromise own last block and cancel store operation"; } + else if (!BlockChain::isBootstrap(pool) && !hasTrustedConfirmations(pool)) { + // a non-bootstrap competing block not vouched for by authority this node recognises + // (its confirmations do not verify against our last block's confidants): never unwind + // our own chain for it. A block that only vouches for itself cannot make us roll back. + csdebug() << kLogPrefix << "competing block #" << poolSequence + << " is not confirmed by our trusted confidants, keep own last block"; + } else { //if (lastSequence + 5ULL < cs::Conveyer::instance().currentRoundNumber()) { // arrangeBlocksInCache(); //} + // bootstrap block (not locally verifiable), or a competing block confirmed by our + // trusted confidants: unwind, but only within the depth bound that removeLastBlock + // enforces; beyond it the block is kept and the operator decides. csdebug() << kLogPrefix << "remove own last block and cancel store operation"; removeLastBlock(); } @@ -2797,6 +2914,19 @@ void BlockChain::cacheLastBlocks() { cs::Sequence lastSeq; while (!incorrectBlocks_.empty() || !selectionFinished_) { csinfo() << kLogPrefix << "Starting blocks transferring cycle"; + // longest-chain guard: stop unwinding own blocks once the competing chain demands a deeper + // automatic rollback than allowed; leave the chain intact, abandon this antifork episode + // cleanly and wait for the operator. (guardAutoRollback measures depth against the fixed + // reference head via lastSequence_, so this counts real own-block removals only.) + if (!guardAutoRollback()) { + cswarning() << kLogPrefix << "longest-chain guard stopped the antifork rollback; keeping own chain"; + incorrectBlocks_.clear(); + selectionFinished_ = true; + lastPrevHash_ = csdb::PoolHash(); + emittingRequest_ = 0; + antiForkMode_ = false; + return; + } auto lastBlock = getLastBlock(); lastSeq = lastBlock.sequence(); csinfo() << kLogPrefix << "now dealing with " << lastSeq; diff --git a/docs/CHANGES.md b/docs/CHANGES.md index fba892eac..07c3e0c07 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,6 +44,7 @@ lines under the new release. | PR | Change | |---|---| +| #TBD | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A non-bootstrap competing block triggers a rollback only if its confirmations verify against the node's own last confidants. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes") | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | diff --git a/docs/differences.md b/docs/differences.md index 4f74d09c0..264878ad3 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -266,6 +266,47 @@ consensus rules. ## Consensus fixes +- **Bounded automatic rollback.** When a node is behind and syncs a competing chain that forks below + its current head, it rolls back its own blocks down to the fork point before taking the other + chain. The official node puts no limit on that depth. The community node stops the automatic + rollback once it would unwind more than `max_auto_rollback` of its own confirmed blocks (default + 1000) below the head it had when the rollback started: it leaves its chain intact, logs a prominent + alert and waits for the operator. The bound covers every automatic removal path — the sync fork + path, the uncertain-block replacement, the antifork cache and the bad-hash reply — because the + depth check lives inside the two functions they all go through. The depth is measured against a + fixed reference, so forward progress cannot reset it and a node cannot be made to creep past the + limit by alternating removals and appends. A reorg this deep is a rare event — a deliberate switch + to another network, or a rejected one — and the choice is left to a human instead of happening + silently. The node keeps running and does not crash-loop; it simply stays on its chain until an + operator decides. + + Two extra checks sharpen this. First, a competing block that is **not a bootstrap block** triggers a + rollback only if its round confirmations verify against the confidants of the node's own last block — + the authority it already trusts — exactly as the block verifier checks a block from the network. A + block whose confirmations the node does not recognise cannot make it unwind; an honest micro-fork + still works, because the honest competing block at the same height was produced by the same elected trusted group. Second, a **bootstrap + block** carries no such confirmations and cannot be checked locally, so for it the depth bound plus + the operator is the only defence — which is exactly what this feature provides. + + This is a local safety bound only: it changes how far this node unwinds on its own, never what a + block contains or what the node accepts in normal operation, so it is invisible to other nodes and a + mixed official/community network behaves exactly as before. Two settings tune it, both in `[params]`: + `max_auto_rollback=` sets the bound (default 1000; 0 means no bound, the official behaviour), and + `allow_deep_rollback=true` accepts a deliberate deep switch without any bound. The + `CS_ALLOW_DEEP_ROLLBACK` environment variable does the same as the latter, for a one-off run. Either + way the node must be restarted to apply the change. + + **Interplay with the community bootstrap fallback.** The opt-in community bootstrap fallback + (PR #188, not merged yet) relies on the longer chain winning after a partition: the other side + unwinds its blocks down to the fork point. With this bound, a branch that would make a node unwind + more than `max_auto_rollback` blocks stops and waits for the operator instead, which is the safe + default for an unattended node, since locally a wanted and an unwanted longer chain look the same. + To keep the two features coherent, the guard has a hook (`setCommunityBootstrapFloor`): a node + that has itself taken a community bootstrap table may unwind back to the sequence it held when it + took the table without hitting the bound, because taking that table was already the operator's + decision. The fallback sets this hook once both changes are merged; until then, and below that + sequence, the ordinary bound applies and the operator decides. + - **Two blocks of one round.** When a trusted node does not get one stage-2 in time, it fetches it with a request that takes exactly as long as the others wait for its stage-3. Depending on a few milliseconds, some trusted nodes finish the round with every signature while the others retry diff --git a/packaging/mainnet/config.ini b/packaging/mainnet/config.ini index 1df11d87c..b228f6bc9 100644 --- a/packaging/mainnet/config.ini +++ b/packaging/mainnet/config.ini @@ -12,6 +12,14 @@ compatible_version=true ; states that do not match the chain are fetched from the confidants once the node is live. ; false: no re-execution while syncing, every state is fetched (no executor needed to sync) sync_contract_execution=true +; longest-chain guard: the most of the node's own confirmed blocks it will roll back automatically +; when it adopts a competing chain during sync. Beyond this depth it keeps its chain, logs an alert +; and waits for the operator (a reorg this deep is a deliberate network switch or a rejected one). +; Default 1000; 0 disables the bound (the official node's behaviour). Local only, block content is +; unchanged. To accept a deliberate deep switch, set allow_deep_rollback=true (or the +; CS_ALLOW_DEEP_ROLLBACK environment variable) and restart. +; max_auto_rollback=1000 +; allow_deep_rollback=false [host_input] port=9510 diff --git a/solver/include/solver/consensus.hpp b/solver/include/solver/consensus.hpp index 86d8d455b..d22395687 100644 --- a/solver/include/solver/consensus.hpp +++ b/solver/include/solver/consensus.hpp @@ -52,6 +52,15 @@ class Consensus { /** @brief The minimum trusted nodes to start consensus */ const static unsigned int MinTrustedNodes = 4; + /** @brief Default maximum number of own blocks the node will automatically roll back when it + * adopts a competing chain during sync; overridden by [params] max_auto_rollback in config.ini. + * Beyond this depth the automatic rollback is halted and the node waits for the operator: a reorg + * this deep is either a deliberate network switch (to be confirmed by a human) or a competing + * longer chain (to be rejected). Purely local: it changes only how far this node unwinds on its + * own, never the block contents other nodes see. Set high enough not to interfere with ordinary + * micro-forks; raise it (or set allow_deep_rollback) to accept a deeper deliberate switch. */ + constexpr static uint64_t MaxAutoRollbackDepth = 1000; + /** @brief The maximum trusted nodes to take part in consensus */ const static unsigned int MaxTrustedNodes = 25; diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index e9f15a4c9..1999aae2e 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -12,6 +12,8 @@ #include #include +#include +#include #include #include @@ -317,3 +319,87 @@ TEST(BlockChain, ConsensusSettingsDigestCoversEverySetting) { EXPECT_NE(changed.consensusSettingsDigest(), digest) << "change " << i; } } + +namespace { +// a clean default config so the guard reads the default bound (1000) and no deep-rollback override, +// independent of any config a prior test in the binary may have set. +void installDefaultConfig() { + cs::ConfigHolder::instance().setConfig(Config{}); +} +} // namespace + +// The longest-chain guard bounds an automatic rollback at Consensus::MaxAutoRollbackDepth below the +// head captured when the rollback episode began, and that reference is fixed for the episode. +TEST(BlockChain, LongestChainGuardBoundsRollbackDepth) { + installDefaultConfig(); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + + const cs::Sequence head = 5000; + const uint64_t depth = Consensus::MaxAutoRollbackDepth; // default 1000 + chain.test_setLastSequence(head); + + // the first call captures the high-water reference at the current head + ASSERT_TRUE(chain.guardAutoRollback()); + EXPECT_TRUE(chain.test_rollbackEpisode()); + EXPECT_EQ(chain.test_rollbackHighWater(), head); + + // unwinding step by step is allowed right up to the floor (head - depth), and the reference does + // not move as the head drops, so forward progress cannot reset it + chain.test_setLastSequence(head - depth + 1); + EXPECT_TRUE(chain.guardAutoRollback()); + EXPECT_EQ(chain.test_rollbackHighWater(), head); + + // the removal that would take the head to or below the floor is refused + chain.test_setLastSequence(head - depth); + EXPECT_FALSE(chain.guardAutoRollback()); + + // refusing sets no permanent state: a fresh episode from a new head works again + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(head - depth); + EXPECT_TRUE(chain.guardAutoRollback()); + EXPECT_EQ(chain.test_rollbackHighWater(), head - depth); +} + +// Near genesis the bound never underflows: a head below the depth can always be unwound to genesis. +TEST(BlockChain, LongestChainGuardNoUnderflowNearGenesis) { + installDefaultConfig(); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + + chain.test_setLastSequence(10); + for (cs::Sequence s = 10; s > 0; --s) { + chain.test_setLastSequence(s); + EXPECT_TRUE(chain.guardAutoRollback()) << "seq " << s; + } +} + +// A node that adopted a community bootstrap table may unwind back to the sequence where it took the +// table, even past the ordinary depth bound; below that point the bound applies again. +TEST(BlockChain, LongestChainGuardCommunityBootstrapFloor) { + installDefaultConfig(); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + + const cs::Sequence head = 5000; + const cs::Sequence tableSeq = 3500; // deeper than head - 1000 = 4000 + chain.test_setLastSequence(head); + chain.setCommunityBootstrapFloor(tableSeq); + ASSERT_TRUE(chain.guardAutoRollback()); // captures high-water 5000 + + // past the ordinary floor (4000) but still above the table sequence: allowed + chain.test_setLastSequence(4000); + EXPECT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(tableSeq + 1); + EXPECT_TRUE(chain.guardAutoRollback()); + + // at or below the table sequence the ordinary bound takes over again and refuses + chain.test_setLastSequence(tableSeq); + EXPECT_FALSE(chain.guardAutoRollback()); +} From b50503fb97ab6f85c027db7cd7d0986322534edb Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 10:32:06 +0100 Subject: [PATCH 02/14] docs: CHANGES row for #189; ci: keep robustness and scale artifacts 3 days off master Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 6 +++--- .github/workflows/scale.yml | 6 +++--- docs/CHANGES.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index a13751d86..0f1ceb2e6 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -121,7 +121,7 @@ jobs: with: name: testnet-node-${{ github.sha }} path: build/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} # mixed nodes, as in testnet.yml: the even nodes run a node built from another ref - name: Build the node under comparison @@ -182,7 +182,7 @@ jobs: with: name: testnet-node-alt-${{ github.sha }} path: build-alt/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} scenario: needs: build @@ -608,4 +608,4 @@ jobs: testnet-run/resources.tsv testnet-run/resources-summary.txt testnet-run/sampler.log - retention-days: 14 + retention-days: ${{ github.ref_name == 'master' && 14 || 3 }} diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index 13960802c..7b6016335 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -170,7 +170,7 @@ jobs: with: name: testnet-node-${{ github.sha }} path: build/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} # the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI test network # needs (its starter key and genesis recipient), for the even nodes @@ -223,7 +223,7 @@ jobs: with: name: testnet-node-official-${{ github.sha }} path: build-alt/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} network: needs: [plan, build] @@ -356,4 +356,4 @@ jobs: testnet-run/n*/config.ini testnet-run/resources.tsv testnet-run/resources-summary.txt - retention-days: 14 + retention-days: ${{ github.ref_name == 'master' && 14 || 3 }} diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 07c3e0c07..d8129dc3f 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,7 @@ lines under the new release. | PR | Change | |---|---| -| #TBD | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A non-bootstrap competing block triggers a rollback only if its confirmations verify against the node's own last confidants. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes") | +| #189 | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A non-bootstrap competing block triggers a rollback only if its confirmations verify against the node's own last confidants. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes") | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | From 7972d6fc2a1961a76eb21cd030ba701ddeff860a Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 10:55:18 +0100 Subject: [PATCH 03/14] csnode: unwind only for competing blocks this node can authenticate The gate in front of an automatic unwind is now one function, competingBlockMayTriggerUnwind(), checked before the node touches its last block at all, so a competing block that fails it cannot even mark that block uncertain (which would let a later replacement remove it). - A non-bootstrap competing block needs strict round confirmations: a mask of more than one entry, of the same size as the node's last confidants, as many signatures as it marks, and a majority of those confidants confirming; then BlockVerifier::verifyConfirmations. verifyConfirmations itself is unchanged, so normal block validation stays as it was. - A bootstrap flag is a bit any block can carry, so it never opens a weaker gate: a bootstrap-flagged competing block passes only when its own signatures verify and its confidants are exactly those of a round table this node accepted in the last few rounds. - The operator override (allow_deep_rollback / CS_ALLOW_DEEP_ROLLBACK) lifts the gate and the depth bound, for a deliberate switch to a chain confirmed by confidants this node does not hold. Unit tests cover the strict confirmations and the bootstrap gate. run.sh reports how many competing blocks were refused. The docs describe the effect after a partition: a node on the shorter side keeps its chain until the operator decides. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 1 + csnode/include/csnode/blockchain.hpp | 23 ++++- csnode/src/blockchain.cpp | 113 ++++++++++++++++++----- docs/CHANGES.md | 2 +- docs/differences.md | 44 +++++---- packaging/mainnet/config.ini | 7 +- tests/csnode/blockchain_tests.cpp | 133 +++++++++++++++++++++++++++ 7 files changed, 274 insertions(+), 49 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 25dd2565a..36d34fdae 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2275,6 +2275,7 @@ fi # is for a scenario that provokes a reorg deeper than [params] max_auto_rollback on purpose. reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" +echo "UNWINDS REFUSED: $(grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null | wc -l) competing block(s) not vouched for by the node's confidants" guarded=$(grep -al 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) if [ -n "$guarded" ] && [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then grep -ah 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log | head -3 diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 43702993a..42b00ae1e 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -260,10 +260,27 @@ class BlockChain { // already trusts - the confidants of its own current last block - exactly as BlockVerifier checks // them. A non-bootstrap competing block that fails this check never triggers a rollback: it is not // vouched for by any authority this node recognises (a self-signed block vouches only for itself). - // A bootstrap block carries no such confirmations and cannot be checked locally; for it the depth - // floor plus the operator override is the only defence, so callers gate it on isBootstrap() first. + // A bootstrap block carries no such confirmations; competingBlockMayTriggerUnwind() sends it to + // isBootstrapAuthenticatedLocally() instead, so a bootstrap flag never opens a weaker gate. + // Strict for the unwind decision: a confirmation mask of more than one entry, of the same size as our + // last block's confidants, with as many signatures as it marks, and a majority of them confirming. bool hasTrustedConfirmations(const csdb::Pool& competing) const; + // True if a bootstrap-flagged competing block is authenticated locally: its own signatures verify + // against its confidants AND those confidants are exactly the confidants of a round table this node + // itself accepted (the current round or the two before it). A bootstrap flag alone is a bit anyone + // can set, so an unauthenticated bootstrap block never triggers an automatic unwind. + bool isBootstrapAuthenticatedLocally(const csdb::Pool& competing) const; + + // The single gate for an automatic unwind in response to a competing block: the operator override, + // or a non-bootstrap block with trusted confirmations, or a locally authenticated bootstrap block. + // A block that passes is still subject to the depth bound inside removeLastBlock(). + bool competingBlockMayTriggerUnwind(const csdb::Pool& competing) const; + + // The operator has approved a deliberate deep switch: [params] allow_deep_rollback=true or the + // CS_ALLOW_DEEP_ROLLBACK environment variable. It lifts both the authority gate and the depth bound. + static bool deepRollbackApproved(); + // A node that has just adopted a community bootstrap table (the opt-in liveness fallback) records // here the sequence it held when it took the table; the guard then lets it unwind back to that // point without operator action, since taking the table already was the operator's decision. 0 @@ -276,6 +293,8 @@ class BlockChain { cs::Sequence test_rollbackHighWater() const { return rollbackHighWater_; } bool test_rollbackEpisode() const { return rollbackEpisode_; } void test_resetRollbackEpisode() { rollbackEpisode_ = false; rollbackHighWater_ = 0; } + // make `block` the node's last block (the deferred one), as getLastBlock() returns it + void test_setLastBlock(const csdb::Pool& block) { deferredBlock_ = block; lastSequence_ = block.sequence(); } #endif const cs::MultiWallets& multiWallets() const; diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 28c5247fc..8c592d0ee 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -845,18 +845,14 @@ void BlockChain::setCommunityBootstrapFloor(cs::Sequence seq) { bool BlockChain::guardAutoRollback() { // operator override: accept a deliberate deep switch. Config ([params] allow_deep_rollback) or the // CS_ALLOW_DEEP_ROLLBACK environment variable; either one is enough. - static const bool allowDeepEnv = [] { - const char* v = std::getenv("CS_ALLOW_DEEP_ROLLBACK"); - return v != nullptr && *v != '\0'; - }(); - const auto* config = cs::ConfigHolder::instance().config(); - if (allowDeepEnv || (config != nullptr && config->allowDeepRollback())) { + if (deepRollbackApproved()) { return true; } + const auto* config = cs::ConfigHolder::instance().config(); const uint64_t maxDepth = (config != nullptr) ? config->maxAutoRollbackDepth() : Consensus::MaxAutoRollbackDepth; if (maxDepth == 0) { - return true; // max_auto_rollback=0: no bound, the official node's behaviour + return true; // max_auto_rollback=0: no depth bound (the authority gate still applies) } const cs::Sequence tip = lastSequence_.load(); @@ -896,15 +892,84 @@ bool BlockChain::guardAutoRollback() { // confirm our last block; a block without confirmations from that group fails and never triggers an // unwind. Bootstrap blocks carry no such confirmations and are handled by the depth floor plus operator, // so callers must check isBootstrap() before relying on this. +bool BlockChain::deepRollbackApproved() { + static const bool allowDeepEnv = [] { + const char* v = std::getenv("CS_ALLOW_DEEP_ROLLBACK"); + return v != nullptr && *v != '\0'; + }(); + const auto* config = cs::ConfigHolder::instance().config(); + return allowDeepEnv || (config != nullptr && config->allowDeepRollback()); +} + bool BlockChain::hasTrustedConfirmations(const csdb::Pool& competing) const { - std::string reason; - const auto trustedConfidants = getLastBlock().confidants(); - const bool ok = cs::BlockVerifier::verifyConfirmations(competing, trustedConfidants, reason); - if (!ok) { + // Strict pre-checks for the unwind decision only; BlockVerifier::verifyConfirmations keeps the + // exceptions it has for normal block validation (sequence < 2, one-entry masks), which must not be + // enough to make this node unwind its own chain. + const auto fail = [&competing](const char* why) { csdebug() << kLogPrefix << "competing block #" << competing.sequence() - << " not confirmed by our last block's confidants: " << reason; + << " not confirmed by our last block's confidants: " << why; + return false; + }; + if (!competing.is_valid() || competing.sequence() < 2) { + return fail("no confirmations to check"); } - return ok; + const auto trustedConfidants = getLastBlock().confidants(); + if (competing.numberConfirmations() == 0 || competing.roundConfirmations().empty()) { + return fail("no round confirmations"); + } + const auto confMask = cs::Utils::bitsToMask(competing.numberConfirmations(), competing.roundConfirmationMask()); + if (confMask.size() <= 1 || confMask.size() != trustedConfidants.size()) { + return fail("confirmation mask does not match our last block's confidants"); + } + const size_t confirming = NodeUtils::realTrustedValue(confMask); + if (competing.roundConfirmations().size() != confirming) { + return fail("confirmation count does not match its mask"); + } + if (confirming * 2 <= trustedConfidants.size()) { + return fail("fewer than a majority of our confidants confirm it"); + } + std::string reason; + if (!cs::BlockVerifier::verifyConfirmations(competing, trustedConfidants, reason)) { + return fail(reason.c_str()); + } + return true; +} + +bool BlockChain::isBootstrapAuthenticatedLocally(const csdb::Pool& competing) const { + std::string reason; + if (!cs::BlockVerifier::verifySignatures(competing, reason)) { + csdebug() << kLogPrefix << "bootstrap block #" << competing.sequence() << " signatures: " << reason; + return false; + } + const auto& confidants = competing.confidants(); + if (confidants.empty()) { + return false; + } + // a round table this node accepted itself (its bootstrap table was checked when it was taken) + const auto& conveyer = cs::Conveyer::instance(); + if (conveyer.currentRoundTable().confidants == confidants) { + return true; + } + const cs::RoundNumber current = conveyer.currentRoundNumber(); + for (cs::RoundNumber back = 0; back < 3 && back <= current; ++back) { + const cs::RoundTable* table = conveyer.roundTable(current - back); + if (table != nullptr && table->confidants == confidants) { + return true; + } + } + csdebug() << kLogPrefix << "bootstrap block #" << competing.sequence() + << " names confidants of no round table this node accepted"; + return false; +} + +bool BlockChain::competingBlockMayTriggerUnwind(const csdb::Pool& competing) const { + if (deepRollbackApproved()) { + return true; // operator path: a deliberate switch to a chain confirmed by other confidants + } + if (BlockChain::isBootstrap(competing)) { + return isBootstrapAuthenticatedLocally(competing); + } + return hasTrustedConfirmations(competing); } // - remove the last block from the top of blockchain @@ -2415,23 +2480,23 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { cserror() << "BLOCKCHAIN> new pool\'s prev. hash is empty, don\'t write it, do not any harm to our blockchain"; return false; } - if (compromiseLastBlock(pool.previous_hash())) { - csdebug() << kLogPrefix << "compromise own last block and cancel store operation"; + if (!competingBlockMayTriggerUnwind(pool)) { + // a competing block not vouched for by authority this node recognises (no strict + // confirmations by our last block's confidants, or a bootstrap flag that is not + // authenticated locally): never unwind our own chain for it, not even by marking our + // last block uncertain (which would let a later replacement remove it). + cslog() << kLogPrefix << "competing block #" << poolSequence + << " is not authenticated by this node's confidants, keep own last block"; } - else if (!BlockChain::isBootstrap(pool) && !hasTrustedConfirmations(pool)) { - // a non-bootstrap competing block not vouched for by authority this node recognises - // (its confirmations do not verify against our last block's confidants): never unwind - // our own chain for it. A block that only vouches for itself cannot make us roll back. - csdebug() << kLogPrefix << "competing block #" << poolSequence - << " is not confirmed by our trusted confidants, keep own last block"; + else if (compromiseLastBlock(pool.previous_hash())) { + csdebug() << kLogPrefix << "compromise own last block and cancel store operation"; } else { //if (lastSequence + 5ULL < cs::Conveyer::instance().currentRoundNumber()) { // arrangeBlocksInCache(); //} - // bootstrap block (not locally verifiable), or a competing block confirmed by our - // trusted confidants: unwind, but only within the depth bound that removeLastBlock - // enforces; beyond it the block is kept and the operator decides. + // authenticated competing block: unwind, but only within the depth bound that + // removeLastBlock enforces; beyond it the block is kept and the operator decides. csdebug() << kLogPrefix << "remove own last block and cancel store operation"; removeLastBlock(); } diff --git a/docs/CHANGES.md b/docs/CHANGES.md index d8129dc3f..cf53c1935 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,7 @@ lines under the new release. | PR | Change | |---|---| -| #189 | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A non-bootstrap competing block triggers a rollback only if its confirmations verify against the node's own last confidants. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes") | +| #189 | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes") | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | diff --git a/docs/differences.md b/docs/differences.md index 264878ad3..3f8bce1fa 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -280,32 +280,38 @@ consensus rules. silently. The node keeps running and does not crash-loop; it simply stays on its chain until an operator decides. - Two extra checks sharpen this. First, a competing block that is **not a bootstrap block** triggers a - rollback only if its round confirmations verify against the confidants of the node's own last block — - the authority it already trusts — exactly as the block verifier checks a block from the network. A - block whose confirmations the node does not recognise cannot make it unwind; an honest micro-fork - still works, because the honest competing block at the same height was produced by the same elected trusted group. Second, a **bootstrap - block** carries no such confirmations and cannot be checked locally, so for it the depth bound plus - the operator is the only defence — which is exactly what this feature provides. + Before any of this, the competing block itself must be vouched for by authority the node already + recognises, or the node does not unwind at all, not even by one block. A competing block that is + **not a bootstrap block** needs round confirmations from a majority of the confidants of the node's + own last block, checked as the block verifier checks a block from the network, but more strictly + (no one-entry confirmation masks). An honest micro-fork passes, because the honest competing block + at the same height was produced by the same elected trusted group. A **bootstrap block** carries no + such confirmations, and its bootstrap flag alone proves nothing, so it passes only when it is + authenticated locally: its own signatures verify and its confidants are exactly those of a round + table this node itself accepted in the last few rounds. Otherwise the node keeps its chain and the + operator decides. Everything that passes is still subject to the depth bound. This is a local safety bound only: it changes how far this node unwinds on its own, never what a block contains or what the node accepts in normal operation, so it is invisible to other nodes and a mixed official/community network behaves exactly as before. Two settings tune it, both in `[params]`: - `max_auto_rollback=` sets the bound (default 1000; 0 means no bound, the official behaviour), and - `allow_deep_rollback=true` accepts a deliberate deep switch without any bound. The + `max_auto_rollback=` sets the bound (default 1000; 0 removes the depth bound, the authority check + stays), and + `allow_deep_rollback=true` accepts a deliberate switch: it lifts both the depth bound and the + check above, since the other chain's blocks are confirmed by confidants this node does not hold. The `CS_ALLOW_DEEP_ROLLBACK` environment variable does the same as the latter, for a one-off run. Either way the node must be restarted to apply the change. - **Interplay with the community bootstrap fallback.** The opt-in community bootstrap fallback - (PR #188, not merged yet) relies on the longer chain winning after a partition: the other side - unwinds its blocks down to the fork point. With this bound, a branch that would make a node unwind - more than `max_auto_rollback` blocks stops and waits for the operator instead, which is the safe - default for an unattended node, since locally a wanted and an unwanted longer chain look the same. - To keep the two features coherent, the guard has a hook (`setCommunityBootstrapFloor`): a node - that has itself taken a community bootstrap table may unwind back to the sequence it held when it - took the table without hitting the bound, because taking that table was already the operator's - decision. The fallback sets this hook once both changes are merged; until then, and below that - sequence, the ordinary bound applies and the operator decides. + **After a partition, and the community bootstrap fallback.** When both sides of a split keep + producing blocks, each side's blocks are confirmed by its own confidants. On reconnection a node on + the shorter side does not follow the longer chain on its own, whatever the depth, because that + chain's blocks are not confirmed by any confidants it holds: it keeps its chain and logs it, and the + operator sets `allow_deep_rollback` to switch. The official node unwinds to the longer chain without + asking. A single node or a minority that cannot reach consensus while cut off builds no branch and + simply syncs forward when it reconnects. The opt-in community bootstrap fallback (PR #188, not + merged yet) expects the longer chain to win after a partition; with this change that switch is an + operator decision. The guard keeps a hook for it (`setCommunityBootstrapFloor`): a node that took a + community bootstrap table may unwind back to the sequence where it took it without hitting the depth + bound, but only for blocks that pass the authority check above. - **Two blocks of one round.** When a trusted node does not get one stage-2 in time, it fetches it with a request that takes exactly as long as the others wait for its stage-3. Depending on a few diff --git a/packaging/mainnet/config.ini b/packaging/mainnet/config.ini index b228f6bc9..8618d2918 100644 --- a/packaging/mainnet/config.ini +++ b/packaging/mainnet/config.ini @@ -15,9 +15,10 @@ sync_contract_execution=true ; longest-chain guard: the most of the node's own confirmed blocks it will roll back automatically ; when it adopts a competing chain during sync. Beyond this depth it keeps its chain, logs an alert ; and waits for the operator (a reorg this deep is a deliberate network switch or a rejected one). -; Default 1000; 0 disables the bound (the official node's behaviour). Local only, block content is -; unchanged. To accept a deliberate deep switch, set allow_deep_rollback=true (or the -; CS_ALLOW_DEEP_ROLLBACK environment variable) and restart. +; A competing block must also be confirmed by confidants this node holds. Default 1000; 0 removes +; the depth bound only. Local only, block content is unchanged. To accept a deliberate switch to +; another chain, set allow_deep_rollback=true (or the CS_ALLOW_DEEP_ROLLBACK environment variable) +; and restart; remove it again afterwards. ; max_auto_rollback=1000 ; allow_deep_rollback=false diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index 1999aae2e..9a8ee35de 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include @@ -403,3 +406,133 @@ TEST(BlockChain, LongestChainGuardCommunityBootstrapFloor) { chain.test_setLastSequence(tableSeq); EXPECT_FALSE(chain.guardAutoRollback()); } + +namespace { +struct GuardKeys { + std::vector pub; + std::vector priv; +}; + +GuardKeys guardKeys(size_t n) { + EXPECT_TRUE(cscrypto::cryptoInit()); + GuardKeys k; + for (size_t i = 0; i < n; ++i) { + cs::PublicKey p; + k.priv.push_back(cscrypto::generateKeyPair(p)); + k.pub.push_back(p); + } + return k; +} + +// a block of `seq` whose confidants are `keys`, all of them signing; when `prev` is given it carries +// round confirmations by the confidants of `prev` selected in `confirmMask` (bit i = confidant i) +csdb::Pool guardBlock(cs::Sequence seq, const GuardKeys& keys, const GuardKeys* prev = nullptr, + uint64_t confirmMask = 0, bool bootstrap = false, bool corruptSignature = false) { + csdb::Pool p(csdb::PoolHash{}, seq); + p.set_confidants(keys.pub); + p.add_number_trusted(static_cast(keys.pub.size())); + p.add_real_trusted((1ull << keys.pub.size()) - 1); + p.add_user_field(200, std::string("payload")); // an id no block field uses + BlockChain::setBootstrap(p, bootstrap); + if (prev) { + cs::Bytes trustedToHash; + cs::ODataStream stream(trustedToHash); + stream << seq; + stream << keys.pub; + const cs::Hash h = cscrypto::calculateHash(trustedToHash.data(), trustedToHash.size()); + std::vector confirmations; + for (size_t i = 0; i < prev->priv.size(); ++i) { + if (confirmMask >> i & 1) { + confirmations.push_back(cscrypto::generateSignature(prev->priv[i], h.data(), h.size())); + } + } + p.add_number_confirmations(static_cast(prev->pub.size())); + p.add_confirmation_mask(confirmMask); + p.add_round_confirmations(confirmations); + } + auto probe = p.clone(); + EXPECT_TRUE(probe.compose()); + cs::Hash hash{}; + const auto hb = probe.hash().to_binary(); + std::copy(hb.begin(), hb.end(), hash.begin()); + std::vector sigs; + for (const auto& k : keys.priv) { + sigs.push_back(cscrypto::generateSignature(k, hash.data(), hash.size())); + } + if (corruptSignature) { + sigs.front()[0] ^= 0xff; + } + p.set_signatures(sigs); + EXPECT_TRUE(p.compose()); + return csdb::Pool::from_binary(p.to_binary()); +} + +BlockChain& guardChain() { + static const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + static const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + static BlockChain chain(genesis, start); + return chain; +} +} // namespace + +// A non-bootstrap competing block may make the node unwind only with confirmations by a majority of +// the confidants of the node's own last block; stricter than block validation (no one-entry masks). +TEST(BlockChain, CompetingBlockNeedsStrictTrustedConfirmations) { + installDefaultConfig(); + auto& chain = guardChain(); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + chain.test_setLastBlock(guardBlock(20, ours)); + + EXPECT_TRUE(chain.hasTrustedConfirmations(guardBlock(21, next, &ours, 0b111))); + EXPECT_TRUE(chain.hasTrustedConfirmations(guardBlock(21, next, &ours, 0b011))); // 2 of 3 + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next, &ours, 0b001))); // a minority + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next))); // none + const auto strangers = guardKeys(3); + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next, &strangers, 0b111))); // keys we do not trust + const auto four = guardKeys(4); + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next, &four, 0b1111))); // mask of another size + EXPECT_TRUE(chain.competingBlockMayTriggerUnwind(guardBlock(21, next, &ours, 0b111))); + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(guardBlock(21, next, &strangers, 0b111))); +} + +// The bootstrap flag is a bit anyone can set: a bootstrap-flagged competing block passes the gate +// only when it is authenticated locally (valid own signatures AND the confidants of a round table this +// node accepted), never on the flag alone, and never more easily than a normal block. +TEST(BlockChain, BootstrapFlagAloneNeverTriggersUnwind) { + installDefaultConfig(); + auto& chain = guardChain(); + const auto ours = guardKeys(3); + const auto others = guardKeys(3); + chain.test_setLastBlock(guardBlock(30, ours)); + + auto& conveyer = cs::Conveyer::instance(); + cs::RoundTable unrelated; + unrelated.round = 1000; + unrelated.confidants = guardKeys(3).pub; + conveyer.setRound(unrelated.round); + conveyer.setTable(unrelated); + + const auto flagged = guardBlock(31, others, nullptr, 0, true); + ASSERT_TRUE(BlockChain::isBootstrap(flagged)); + EXPECT_FALSE(chain.isBootstrapAuthenticatedLocally(flagged)); + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(flagged)); + // the same block without the flag fails too: the flag never opens a gate a normal block cannot pass + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(guardBlock(31, others))); + + // the node took a round table naming exactly these confidants: the bootstrap block is authenticated + cs::RoundTable taken; + taken.round = 1001; + taken.confidants = others.pub; + conveyer.setRound(taken.round); + conveyer.setTable(taken); + EXPECT_TRUE(chain.isBootstrapAuthenticatedLocally(flagged)); + EXPECT_TRUE(chain.competingBlockMayTriggerUnwind(flagged)); + // ... but not with a bad signature + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(guardBlock(31, others, nullptr, 0, true, true))); + + // leave the conveyer on a table that matches nothing here + unrelated.round = 1002; + conveyer.setRound(unrelated.round); + conveyer.setTable(unrelated); +} From d8f6433aab193b4db9afa5fa29e195be39882831 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 14:21:17 +0100 Subject: [PATCH 04/14] csdb: forget removed blocks in the storage read cache Storage keeps a small read cache of pools indexed uniquely by sequence and by hash. pool_remove_last() did not drop the removed pool from it, and pools_cache_insert() silently refused a new pool at a sequence still cached. After a node unwound blocks already written to storage and saved the competing blocks at the same sequences, pool_load(sequence) kept returning the removed block. The next recordBlock() then failed its consistency check ("block hash in pool #N doesn't correspond to the last one") on every attempt, and the node stopped a few blocks above the fork point. Erase the pool from the cache on every removal path, and let an insert replace any cached pool with the same sequence or hash. Read-side only: block content and what the node accepts are unchanged. Co-Authored-By: Claude Opus 5.5 --- csdb/src/storage.cpp | 18 +++++++++- docs/CHANGES.md | 2 +- tests/csnode/storage_writer_tests.cpp | 52 +++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index bd049743d..3a63b6be8 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -219,14 +219,26 @@ class Storage::priv { PoolCache pools_cache; static const size_t cacheSize = 50; + // the cache keeps one pool per sequence and per hash: a pool saved at a sequence whose previous + // pool was removed (a chain reorganisation) replaces the cached one, instead of being refused by + // the unique index and leaving the removed pool to be returned by pool_load(sequence) void pools_cache_insert(const cs::Sequence& seq, const PoolHash &hash, const Pool &pool) { - if (pools_cache.size() == cacheSize) { + pools_cache_erase(seq, hash); + if (pools_cache.size() >= cacheSize) { auto random = pools_cache.begin(); pools_cache.erase(random); } pools_cache.insert({seq, hash, pool}); } + // forget a pool removed from the top of the chain (by sequence and by hash) + void pools_cache_erase(const cs::Sequence& seq, const PoolHash &hash) { + pools_cache.get().erase(seq); + if (!hash.is_empty()) { + pools_cache.get().erase(hash); + } + } + friend class ::csdb::Storage; private signals: @@ -1068,6 +1080,7 @@ Pool Storage::pool_remove_last() { bool found = write_queue_pop(res); if (found) { + d->pools_cache_erase(res.sequence(), res.hash()); std::unique_lock lock(d->data_lock); --d->count_pool; d->last_hash = res.previous_hash(); @@ -1099,6 +1112,7 @@ Pool Storage::pool_remove_last() { // error nearly impossible /*bool ok =*/ d->db->remove(last_hash().to_binary()); + d->pools_cache_erase(res.sequence(), res.hash()); --d->count_pool; d->last_hash = res.previous_hash(); @@ -1134,6 +1148,7 @@ bool Storage::pool_remove_last_repair(cs::Sequence test_sequence, const csdb::Po if (!write_queue_pop_last(test_hash, queued)) { return false; } + d->pools_cache_erase(test_sequence, test_hash); if (queued) { return true; } @@ -1198,6 +1213,7 @@ bool Storage::pool_remove_(cs::Sequence testSequence) { if (!write_queue_pop_last(testHash, queued)) { return false; } + d->pools_cache_erase(testSequence, testHash); if (queued) { return true; } diff --git a/docs/CHANGES.md b/docs/CHANGES.md index cf53c1935..23f68b8f4 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,7 @@ lines under the new release. | PR | Change | |---|---| -| #189 | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes") | +| #189 | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes"). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | diff --git a/tests/csnode/storage_writer_tests.cpp b/tests/csnode/storage_writer_tests.cpp index 8c54e5809..6965599cf 100644 --- a/tests/csnode/storage_writer_tests.cpp +++ b/tests/csnode/storage_writer_tests.cpp @@ -218,3 +218,55 @@ TEST(StorageWriter, RemovingTheLastQueuedBlockEndsTheFailure) { EXPECT_TRUE(db->has(pools[0].hash())); storage.close(); } + +// A chain reorganisation below the deferred block: the node removes stored blocks and saves the +// competing ones at the same sequences. pool_load(sequence) must then return the new blocks, not the +// removed ones still held by the storage's read cache (the stale block made the next consistency +// check fail for ever, and the node stopped two blocks above the fork point). +namespace { + +void checkReplacedBlocksAreLoaded(bool asyncWrites) { + auto db = std::make_shared(); + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.writeBatchSize = 100; + opt.asyncWrites = asyncWrites; + ASSERT_TRUE(storage.open(opt)); + + const auto own = chain(5); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + for (cs::Sequence s = 0; s < 5; ++s) ASSERT_EQ(storage.pool_load(s).hash(), own[s].hash()); + + // unwind to #2, then the competing #3 and #4 + ASSERT_EQ(storage.pool_remove_last().hash(), own[4].hash()); + ASSERT_EQ(storage.pool_remove_last().hash(), own[3].hash()); + std::vector other; + csdb::PoolHash prev = own[2].hash(); + for (cs::Sequence s = 3; s < 5; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(100, static_cast('z' - s))); + ASSERT_TRUE(p.compose()); + ASSERT_NE(p.hash(), own[s].hash()); + prev = p.hash(); + other.push_back(p); + ASSERT_TRUE(storage.pool_save(p)); + } + + EXPECT_EQ(storage.pool_load(3).hash(), other[0].hash()); + EXPECT_EQ(storage.pool_load(4).hash(), other[1].hash()); + EXPECT_EQ(storage.pool_load(4).previous_hash(), storage.pool_load(3).hash()); + EXPECT_FALSE(storage.pool_load(own[3].hash()).is_valid()); + EXPECT_FALSE(storage.pool_load(own[4].hash()).is_valid()); + storage.close(); +} + +} // namespace + +TEST(StorageWriter, BlocksSavedAfterAnUnwindReplaceTheCachedOnes) { + checkReplacedBlocksAreLoaded(false); +} + +TEST(StorageWriter, BlocksSavedAfterAnUnwindReplaceTheCachedOnesAsync) { + checkReplacedBlocksAreLoaded(true); +} From edcb8ffe56a4ba228286d2cff9775826a4f3e247 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Wed, 18 Nov 2026 22:36:16 +0100 Subject: [PATCH 05/14] tests: storage read cache on the repair path and after a restart BlockChain::removeLastBlock() falls back to pool_remove_last_repair() when the removed block cannot be read back. Check that this path also forgets the block in the storage read cache, that a block saved afterwards at the same sequence is the one read back, and that after a restart (the storage rescan fills the cache again) the node reads the replacement, not the removed block. Both with synchronous and asynchronous writes. Tests only. Co-Authored-By: Claude Opus 5.5 --- tests/csnode/storage_writer_tests.cpp | 64 +++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/csnode/storage_writer_tests.cpp b/tests/csnode/storage_writer_tests.cpp index 6965599cf..ea7abc8ad 100644 --- a/tests/csnode/storage_writer_tests.cpp +++ b/tests/csnode/storage_writer_tests.cpp @@ -270,3 +270,67 @@ TEST(StorageWriter, BlocksSavedAfterAnUnwindReplaceTheCachedOnes) { TEST(StorageWriter, BlocksSavedAfterAnUnwindReplaceTheCachedOnesAsync) { checkReplacedBlocksAreLoaded(true); } + +// The recovery path of BlockChain::removeLastBlock(): when the removed block cannot be read back, the +// node removes it with pool_remove_last_repair(). That path must forget the block in the read cache +// too, and a node restarted afterwards (the storage rescan fills the cache again) must read the blocks +// saved after the removal, not the removed ones. +namespace { + +csdb::Pool competing(const csdb::PoolHash& prev, cs::Sequence seq) { + csdb::Pool p(prev, seq); + p.add_user_field(1, std::string(100, static_cast('z' - seq))); + EXPECT_TRUE(p.compose()); + return p; +} + +void checkRepairedBlockIsForgotten(bool asyncWrites) { + auto db = std::make_shared(); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.writeBatchSize = 100; + opt.asyncWrites = asyncWrites; + + const auto own = chain(4); + csdb::Pool other; + { + csdb::Storage storage; + ASSERT_TRUE(storage.open(opt)); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + ASSERT_TRUE(storage.flush()); + ASSERT_EQ(storage.pool_load(3).hash(), own[3].hash()); // #3 now in the read cache + + ASSERT_TRUE(storage.pool_remove_last_repair(3, own[3].hash())); + EXPECT_EQ(storage.size(), 3u); + EXPECT_EQ(storage.last_hash(), own[2].hash()); + EXPECT_FALSE(storage.pool_load(3).is_valid()) << "the repaired block is still read from the cache"; + EXPECT_FALSE(storage.pool_load(own[3].hash()).is_valid()); + + other = competing(own[2].hash(), 3); + ASSERT_NE(other.hash(), own[3].hash()); + ASSERT_TRUE(storage.pool_save(other)); + EXPECT_EQ(storage.pool_load(3).hash(), other.hash()); + ASSERT_TRUE(storage.flush()); + storage.close(); + } + + // restart: the rescan reads every stored block and fills the cache from the database + csdb::Storage storage; + ASSERT_TRUE(storage.open(opt)); + EXPECT_EQ(storage.size(), 4u); + EXPECT_EQ(storage.last_hash(), other.hash()); + EXPECT_EQ(storage.pool_load(3).hash(), other.hash()); + EXPECT_EQ(storage.pool_load(other.hash()).sequence(), 3u); + EXPECT_FALSE(storage.pool_load(own[3].hash()).is_valid()); + storage.close(); +} + +} // namespace + +TEST(StorageWriter, RepairedLastBlockIsForgottenAndRestartReadsItsReplacement) { + checkRepairedBlockIsForgotten(false); +} + +TEST(StorageWriter, RepairedLastBlockIsForgottenAndRestartReadsItsReplacementAsync) { + checkRepairedBlockIsForgotten(true); +} From 51f7a1aa42c4b923e03aa0f081e58873d0fe9ae5 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 07:12:02 +0100 Subject: [PATCH 06/14] csnode: FORK HELD state, persisted rollback reference, operator remedies The bound on automatic chain reorganisations now defaults to 100 blocks (Consensus::MaxAutoRollbackDepth and DEFAULT_MAX_AUTO_ROLLBACK, kept in step). - The episode reference is kept in reorg-episode.json next to the database and loaded at start-up, so a restart does not give a fresh full depth. - When the node refuses to unwind (beyond the bound, or a competing block not vouched for by its confidants) it records the fork in fork-held.json (fork point and hash, our head, the refused block, peer key, address and height, time, reference) and reports FORK HELD: a marker on every round line and a detail line with both remedies once a minute. The state clears when the head is back above the reference or the operator truncates. - Keep this chain: a dismiss-fork file next to the database, picked up by the running node within a round (also at start-up). The record keeps that branch as dismissed: still refused, logged at most hourly, not reported again; a different branch is reported again. The dismissal survives restarts. - Follow the other chain: --resolve-fork cuts the chain at the recorded fork point through the --set-bc-top path, only if the stored block there still has the recorded hash (otherwise nothing is removed), clears the record and approves following the other chain for the one episode that follows. - allow_deep_rollback / CS_ALLOW_DEEP_ROLLBACK turns itself off after one completed episode, loudly. - The start-up integrity repair (incorrectBlocks_) is not bounded by the guard. - hasTrustedConfirmations refuses a bootstrap block, which BlockVerifier accepts without confirmations. - The unused community-bootstrap floor hook is removed; no message advises a restart any more. Tests: storeBlock refusals leave the head and the uncertain flag untouched, the antifork abandonment state, the integrity repair, config and environment parsing, the reference and FORK HELD across a simulated restart, dismissal (cleared, still refused, same branch quiet, other branch raised, trigger consumed, kept across restart), the resolve plan, and the storage cut landing exactly at the fork point or refusing a mismatched record. Co-Authored-By: Claude Opus 5.5 --- client/config/config.cpp | 12 +- client/config/config.hpp | 14 +- client/include/client/cmdlineargs.hpp | 1 + client/src/main.cpp | 1 + client/src/peer.cpp | 2 +- csdb/include/csdb/storage.hpp | 7 + csdb/src/storage.cpp | 22 + csnode/CMakeLists.txt | 2 + csnode/include/csnode/blockchain.hpp | 120 ++++-- csnode/include/csnode/forkguardstate.hpp | 110 +++++ csnode/include/csnode/node.hpp | 5 + csnode/include/csnode/poolsynchronizer.hpp | 2 + csnode/src/blockchain.cpp | 452 +++++++++++++++++--- csnode/src/forkguardstate.cpp | 289 +++++++++++++ csnode/src/node.cpp | 89 +++- csnode/src/poolsynchronizer.cpp | 5 + solver/include/solver/consensus.hpp | 2 +- tests/csnode/blockchain_tests.cpp | 460 +++++++++++++++++++-- tests/csnode/storage_writer_tests.cpp | 63 +++ 19 files changed, 1537 insertions(+), 121 deletions(-) create mode 100644 csnode/include/csnode/forkguardstate.hpp create mode 100644 csnode/src/forkguardstate.cpp diff --git a/client/config/config.cpp b/client/config/config.cpp index e98f92ef7..78a804e5a 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -135,6 +135,7 @@ const std::string ARG_NAME_PRIVATE_KEY_FILE = "private-key-file"; const std::string ARG_NAME_ENCRYPT_KEY_FILE = "encryptkey"; const std::string ARG_NAME_RECREATE_INDEX = "recreate-index"; const std::string ARG_NAME_NEW_BC_TOP = "set-bc-top"; +const std::string ARG_NAME_RESOLVE_FORK = "resolve-fork"; const std::string ARG_NAME_BAL_CHANGE = "bal-change"; const std::string ARG_NAME_DISABLE_AUTO_SHUTDOWN = "disable-auto-shutdown"; @@ -279,6 +280,7 @@ Config Config::read(po::variables_map& vm, bool atStart) { result.pathToDb_ = getArgFromCmdLine(vm, ARG_NAME_DB_PATH, DEFAULT_PATH_TO_DB); if (atStart) { + result.resolveFork_ = vm.count(ARG_NAME_RESOLVE_FORK) > 0; if (vm.count(ARG_NAME_NEW_BC_TOP)) { result.newBlockchainTop_ = true; result.newBlockchainTopSeq_ = @@ -999,10 +1001,7 @@ Config Config::readFromFile(const std::string& fileName) { } result.maxUncorrectedBlock_ = params.count(PARAM_NAME_MAX_UNCORRECTED_BLOCK) ? params.get(PARAM_NAME_MAX_UNCORRECTED_BLOCK) : DEFAULT_MAX_UNCORRECTED_BLOCK; - result.maxAutoRollbackDepth_ = params.count(PARAM_NAME_MAX_AUTO_ROLLBACK) ? params.get(PARAM_NAME_MAX_AUTO_ROLLBACK) : DEFAULT_MAX_AUTO_ROLLBACK; - if (params.count(PARAM_NAME_ALLOW_DEEP_ROLLBACK) > 0) { - result.allowDeepRollback_ = params.get(PARAM_NAME_ALLOW_DEEP_ROLLBACK); - } + readRollbackGuardSettings(params, result.maxAutoRollbackDepth_, result.allowDeepRollback_); result.generateFork_ = params.count(PARAM_NAME_GENERATE_FORK) ? params.get(PARAM_NAME_GENERATE_FORK) : false; if (params.count(PARAM_NAME_SYNC_ON)) { @@ -1207,6 +1206,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) { } } +void Config::readRollbackGuardSettings(const boost::property_tree::ptree& params, uint64_t& maxDepth, bool& allowDeep) { + maxDepth = params.count(PARAM_NAME_MAX_AUTO_ROLLBACK) ? params.get(PARAM_NAME_MAX_AUTO_ROLLBACK) : DEFAULT_MAX_AUTO_ROLLBACK; + allowDeep = params.count(PARAM_NAME_ALLOW_DEEP_ROLLBACK) ? params.get(PARAM_NAME_ALLOW_DEEP_ROLLBACK) : false; +} + void Config::readBackfillSettings(const boost::property_tree::ptree& data, StorageData& storage) { if (data.count("backfill")) { const auto text = data.get("backfill"); diff --git a/client/config/config.hpp b/client/config/config.hpp index e1ec76ca0..ed7a69afd 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -32,7 +32,7 @@ const uint32_t DEFAULT_MIN_NEIGHBOURS = Neighbourhood::kMinNeighbours; const uint32_t DEFAULT_MAX_NEIGHBOURS = Neighbourhood::kMaxNeighbours; const uint64_t DEFAULT_MAX_UNCORRECTED_BLOCK = 0; // keep in step with Consensus::MaxAutoRollbackDepth (solver/include/solver/consensus.hpp) -const uint64_t DEFAULT_MAX_AUTO_ROLLBACK = 1000; +const uint64_t DEFAULT_MAX_AUTO_ROLLBACK = 100; const uint32_t DEFAULT_CONNECTION_BANDWIDTH = 1 << 19; const uint32_t DEFAULT_OBSERVER_WAIT_TIME = 5 * 60 * 1000; // ms const uint32_t DEFAULT_ROUND_ELAPSE_TIME = 1000 * 60; // ms @@ -335,11 +335,17 @@ class Config { return maxAutoRollbackDepth_; } - // longest-chain guard override: accept a deliberate deep switch without the depth bound. + // longest-chain guard override: accept a deliberate deep switch without the depth bound (for one + // completed rollback episode; then it is off until the node restarts). bool allowDeepRollback() const { return allowDeepRollback_; } + // --resolve-fork: truncate the chain to the fork point recorded in fork-held.json, then stop + bool resolveFork() const { + return resolveFork_; + } + bool traverseNAT() const { return traverseNAT_; } @@ -408,6 +414,9 @@ class Config { static SnapshotData parseSnapshotData(const boost::property_tree::ptree& config); // the backfill keys of a [storage] section (backfill, backfill_batch, backfill_rate) static void readBackfillSettings(const boost::property_tree::ptree& storage, StorageData& data); + // the [params] keys of the bound on automatic chain reorganisations: max_auto_rollback (default + // DEFAULT_MAX_AUTO_ROLLBACK, 0 = no depth bound) and allow_deep_rollback (default false) + static void readRollbackGuardSettings(const boost::property_tree::ptree& params, uint64_t& maxDepth, bool& allowDeep); private: void setLoggerSettings(const boost::property_tree::ptree& config); @@ -445,6 +454,7 @@ class Config { uint64_t maxUncorrectedBlock_ = DEFAULT_MAX_UNCORRECTED_BLOCK; uint64_t maxAutoRollbackDepth_ = DEFAULT_MAX_AUTO_ROLLBACK; bool allowDeepRollback_ = false; + bool resolveFork_ = false; bool symmetric_ = false; EndpointData hostAddressEp_; diff --git a/client/include/client/cmdlineargs.hpp b/client/include/client/cmdlineargs.hpp index bfadfda9d..349a62b88 100644 --- a/client/include/client/cmdlineargs.hpp +++ b/client/include/client/cmdlineargs.hpp @@ -7,6 +7,7 @@ struct cmdline { static constexpr const char* argSeed = "seed"; static constexpr const char* argDumpKeys = "dumpkeys"; static constexpr const char* argSetBCTop = "set-bc-top"; + static constexpr const char* argResolveFork = "resolve-fork"; static constexpr const char* argBalChange = "bal-change"; #ifdef _WIN32 static constexpr const char* argInstall = "install"; diff --git a/client/src/main.cpp b/client/src/main.cpp index 6989d85f5..19acf500b 100644 --- a/client/src/main.cpp +++ b/client/src/main.cpp @@ -56,6 +56,7 @@ int main(int argc, char* argv[]) { ("recreate-index", "recreate index.db") (cmdline::argSeed, "enter with seed instead of keys") (cmdline::argSetBCTop, po::value(), "all blocks in blockchain with higher sequence will be removed") + (cmdline::argResolveFork, "a node that reports FORK HELD: remove the blocks above the fork point recorded in fork-held.json (only if the block there is still the recorded one), then exit; at the next start the node follows the other chain. Check with other nodes first (the node must be stopped)") (cmdline::argBalChange, po::value(), "prints all changes in account balance in log file") ("disable-auto-shutdown", "node will be prohibited to shutdown in case of fatal errors") ("version", "show node version") diff --git a/client/src/peer.cpp b/client/src/peer.cpp index 33ae2e656..10e0c2eae 100644 --- a/client/src/peer.cpp +++ b/client/src/peer.cpp @@ -144,7 +144,7 @@ bool Peer::onInit(const char*) { } bool Peer::onRun(const char*) { - if (vm_.count(cmdline::argSetBCTop) == 0) { + if (vm_.count(cmdline::argSetBCTop) == 0 && vm_.count(cmdline::argResolveFork) == 0) { cslog() << "Running Node"; node_->run(); } diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 109033fa7..dfbe0153e 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -81,6 +81,8 @@ class Storage final { // a quick start asked to read from a block after the last one stored: its checkpoint is // ahead of the blocks on disk StartAfterLastBlock = 7, + // --resolve-fork: the block at the requested top is not the one recorded (set_truncate_check) + TruncateMismatch = 8, UnknownError = 255, }; @@ -152,6 +154,11 @@ class Storage final { * In case of fail the error info can be received with methods: \ref last_error, * \ref last_error_message, \ref db_last_error() and \ref db_last_error_message() */ + // The next open() with a new blockchain top (--set-bc-top) cuts the chain only if the stored block + // at that top has this hash; otherwise it removes nothing and fails with TruncateMismatch. An + // empty hash (the default) checks nothing. Used by --resolve-fork. + void set_truncate_check(const PoolHash& expectedTopHash); + // dbBackend: "berkeleydb" (default), "rocksdb", or "lmdb" in /lmdb, whose removed blocks // LMDB reuses the space of (pruned storage) bool open(const ::std::string& path_to_base = ::std::string{}, diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index 3a63b6be8..b8d1765bd 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -218,6 +218,8 @@ class Storage::priv { > PoolCache; PoolCache pools_cache; static const size_t cacheSize = 50; + // set_truncate_check(): the hash the block at the new blockchain top must have + PoolHash truncate_expected_hash; // the cache keeps one pool per sequence and per hash: a pool saved at a sequence whose previous // pool was removed (a chain reorganisation) replaces the cached one, instead of being refused by @@ -477,6 +479,10 @@ Storage::WeakPtr Storage::weak_ptr() const noexcept { return d; } +void Storage::set_truncate_check(const PoolHash& expectedTopHash) { + d->truncate_expected_hash = expectedTopHash; +} + Storage::Error Storage::last_error() const { return last_error_map(d.get()).last_error_; } @@ -549,6 +555,22 @@ bool Storage::open(const OpenOptions& opt, OpenCallback callback) { } if (opt.newBlockchainTop != cs::kWrongSequence) { + if (!d->truncate_expected_hash.is_empty()) { + const PoolHash expected = d->truncate_expected_hash; + d->truncate_expected_hash = PoolHash{}; + cs::Bytes topBinary; + PoolHash stored; + if (d->db->get(static_cast(opt.newBlockchainTop), &topBinary)) { + stored = csdb::Pool::hash_from_binary(std::move(topBinary)); + } + if (stored != expected) { + d->set_last_error(TruncateMismatch, "block %llu is %s, not the recorded %s: nothing removed", + static_cast(opt.newBlockchainTop), + stored.is_empty() ? "not stored" : stored.to_string().c_str(), + expected.to_string().c_str()); + return false; + } + } auto seqToRemove = static_cast(opt.newBlockchainTop + 1); auto seqLast = seqToRemove; { diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index a2819cc27..d6d81a1cb 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -80,6 +80,7 @@ add_library(csnode include/csnode/snapshot_bootstrap.hpp include/csnode/snapshot_fetcher.hpp include/csnode/blockverifier.hpp + include/csnode/forkguardstate.hpp include/csnode/backwardsync.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -134,6 +135,7 @@ add_library(csnode src/snapshot_bootstrap.cpp src/snapshot_fetcher.cpp src/blockverifier.cpp + src/forkguardstate.cpp src/backwardsync.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 42b00ae1e..10f730aed 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -3,7 +3,9 @@ #include #include +#include #include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include #include +#include #include @@ -246,22 +249,23 @@ class BlockChain { // every automatic rollback of the node's own last block passes through it, whatever the caller // (the sync fork path, the uncertain-replacement path, the antifork cache and the bad-hash reply). // It returns true while the resulting head would stay at or above rollbackHighWater_ - - // maxAutoRollbackDepth_, where rollbackHighWater_ is the head captured at the start of the current - // rollback episode (the first automatic removal from any path). Once the limit would be crossed it - // logs a prominent alert and returns false (keeping the chain intact and leaving the decision to - // the operator); it does NOT set any permanent state, so a later legitimate reorg still works. - // An operator accepts a deliberate deep switch with the [params] allow_deep_rollback option or the - // CS_ALLOW_DEEP_ROLLBACK environment variable. A node that has adopted a community bootstrap table - // (see setCommunityBootstrapFloor) may unwind freely back to the sequence where it took that table. - // Local only: it changes how far this node unwinds, never block content. + // max_auto_rollback, where rollbackHighWater_ is the head captured at the start of the current + // rollback episode (the first automatic removal from any path) and kept in reorg-episode.json next + // to the database, so a restart does not give a fresh full depth. Once the limit would be crossed + // it records the fork (fork-held.json), logs a prominent alert and returns false: the chain stays + // intact, the node reports FORK HELD and the operator decides (--resolve-fork, --set-bc-top). The + // state clears when the episode ends (the head climbs above the reference) or the operator + // truncates. An operator accepts a deliberate deep switch with [params] allow_deep_rollback or the + // CS_ALLOW_DEEP_ROLLBACK environment variable, for one episode. Local only: it changes how far + // this node unwinds, never block content. bool guardAutoRollback(); // True if the competing block carries round confirmations signed by the confidants the node // already trusts - the confidants of its own current last block - exactly as BlockVerifier checks // them. A non-bootstrap competing block that fails this check never triggers a rollback: it is not // vouched for by any authority this node recognises (a self-signed block vouches only for itself). - // A bootstrap block carries no such confirmations; competingBlockMayTriggerUnwind() sends it to - // isBootstrapAuthenticatedLocally() instead, so a bootstrap flag never opens a weaker gate. + // A bootstrap block carries no such confirmations and always fails here (BlockVerifier accepts it + // without them); competingBlockMayTriggerUnwind() sends it to isBootstrapAuthenticatedLocally(). // Strict for the unwind decision: a confirmation mask of more than one entry, of the same size as our // last block's confidants, with as many signatures as it marks, and a majority of them confirming. bool hasTrustedConfirmations(const csdb::Pool& competing) const; @@ -278,23 +282,66 @@ class BlockChain { bool competingBlockMayTriggerUnwind(const csdb::Pool& competing) const; // The operator has approved a deliberate deep switch: [params] allow_deep_rollback=true or the - // CS_ALLOW_DEEP_ROLLBACK environment variable. It lifts both the authority gate and the depth bound. - static bool deepRollbackApproved(); - - // A node that has just adopted a community bootstrap table (the opt-in liveness fallback) records - // here the sequence it held when it took the table; the guard then lets it unwind back to that - // point without operator action, since taking the table already was the operator's decision. 0 - // (the default) means no such table was taken and the ordinary depth bound applies throughout. - void setCommunityBootstrapFloor(cs::Sequence seq); + // CS_ALLOW_DEEP_ROLLBACK environment variable (until one episode has completed with it: then it is + // off for the rest of the run), or --resolve-fork for the episode it started. It lifts both the + // authority gate and the depth bound. + bool deepRollbackApproved() const; + // CS_ALLOW_DEEP_ROLLBACK is set to a non-empty value + static bool deepRollbackEnvSet(); + + // The local files of the guard live in `dir` (the folder next to the database). Loads the + // persisted episode reference and the fork record; an episode the chain has already passed is + // dropped. An empty dir keeps everything in memory only. Called by init(). + void setGuardStateDir(const std::string& dir); + + // FORK HELD: the node refused to unwind for a competing chain, the episode has not ended and the + // operator has not dismissed it. + bool forkHeld() const; + // the record while FORK HELD, else nothing + std::optional forkHeldRecord() const; + // the record, held or keeping a dismissal + std::optional forkRecord() const; + + // The operator keeps this node's chain: FORK HELD ends, the record keeps the competing branch as + // dismissed (still refused, never reported again; another branch is). False if no fork is held. + bool dismissFork(); + // the dismiss-fork file next to the database: dismissFork() and remove the file. Called every + // round and at start-up; true if a fork was dismissed. + bool checkDismissTrigger(); + + // the sync reply being stored comes from this peer, which announced this height (for the fork + // record); cleared with clearSyncSource() + void setSyncSource(const cs::PublicKey& key, cs::Sequence announcedHeight); + void clearSyncSource(); + // resolves a peer key to "ip:port" for the fork record (set by the node; may return "") + void setPeerAddressResolver(std::function resolver); + + // --resolve-fork: init() truncates to `seq` only if the stored block there has `hash`, then clears + // the fork record and approves following the competing chain for the one episode that follows + void setResolveFork(cs::Sequence seq, const csdb::PoolHash& hash); + // init() refused to truncate: the stored block at the requested sequence has another hash + bool truncationRefused() const { + return truncationRefused_; + } #ifdef TESTING - // test-only accessors for the longest-chain guard (inline, no data members: no layout change) + // test-only accessors for the longest-chain guard void test_setLastSequence(cs::Sequence s) { lastSequence_ = s; } cs::Sequence test_rollbackHighWater() const { return rollbackHighWater_; } bool test_rollbackEpisode() const { return rollbackEpisode_; } - void test_resetRollbackEpisode() { rollbackEpisode_ = false; rollbackHighWater_ = 0; } + void test_resetRollbackEpisode() { + rollbackEpisode_ = false; rollbackHighWater_ = 0; episodeApprovedSwitch_ = false; deepOverrideSpent_ = false; + std::lock_guard l(guardMutex_); forkRecord_.reset(); + } // make `block` the node's last block (the deferred one), as getLastBlock() returns it void test_setLastBlock(const csdb::Pool& block) { deferredBlock_ = block; lastSequence_ = block.sequence(); } + // let the episode end as if a block above the reference had been recorded + void test_onHeadAdvanced() { onHeadAdvanced(); } + void test_setSelectionUnfinished(const csdb::PoolHash& lastPrev) { selectionFinished_ = false; lastPrevHash_ = lastPrev; antiForkMode_ = true; emittingRequest_ = 1; } + bool test_selectionFinished() const { return selectionFinished_; } + bool test_antiForkMode() const { return antiForkMode_; } + uint8_t test_emittingRequest() const { return emittingRequest_; } + csdb::PoolHash test_lastPrevHash() const { return lastPrevHash_; } #endif const cs::MultiWallets& multiWallets() const; @@ -659,9 +706,36 @@ public slots: bool rollbackEpisode_ = false; cs::Sequence rollbackHighWater_ = 0; cs::Sequence rollbackLowWater_ = 0; // lowest head reached in the current episode (for the log) - // longest-chain guard runtime state (see setCommunityBootstrapFloor). The depth bound and the - // deep-rollback override are read from config ([params] max_auto_rollback / allow_deep_rollback). - cs::Sequence communityBootstrapFloor_ = 0; + // longest-chain guard: where its files live (empty: memory only), the fork it refused (FORK + // HELD), the --resolve-fork / allow_deep_rollback approvals, the integrity repair in progress + std::string guardStateDir_; + mutable std::mutex guardMutex_; + std::optional forkRecord_; + std::chrono::steady_clock::time_point lastDismissedLog_{}; // dismissed branch offered again (hourly) + bool episodeApprovedSwitch_ = false; // --resolve-fork approved this episode + bool deepOverrideSpent_ = false; // allow_deep_rollback served one episode: off until restart + bool deepOverrideUsed_ = false; // the current episode ran with allow_deep_rollback + bool integrityRepair_ = false; // cacheLastBlocks() is removing blocks of incorrectBlocks_ + struct CompetingBlock { + cs::Sequence seq = 0; + csdb::PoolHash hash; + csdb::PoolHash previous; + }; + std::optional competing_; // the competing block being handled (for the record) + std::optional> syncSource_; + std::function peerAddressResolver_; + cs::Sequence resolveForkSeq_ = cs::kWrongSequence; + csdb::PoolHash resolveForkHash_; + bool truncationRefused_ = false; + // starts the episode at the current head if none is running (persisting its reference) + void beginRollbackEpisode(); + // the chain grew: ends the episode once the head is above its reference + void onHeadAdvanced(); + // records the refused fork (once per episode) and logs the alert; reason "depth" or "authority". + // False (and nothing raised) when the competing block belongs to a branch the operator dismissed. + bool recordForkHeld(const char* reason); + // removeLastBlock()/compromiseLastBlock() may go on: an integrity repair, or the guard allows it + bool mayUnwindLastBlock(); std::atomic_bool stop_ = false; // the quick-start checkpoint of the last init() is ahead of the blocks on disk bool checkpointAheadOfBlocks_ = false; diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp new file mode 100644 index 000000000..f691fd5b6 --- /dev/null +++ b/csnode/include/csnode/forkguardstate.hpp @@ -0,0 +1,110 @@ +#pragma once + +#include +#include +#include +#include + +#include + +// Local files of the bound on automatic chain reorganisations, next to the block database (the +// parent folder of --db-path; the data folder in Docker). Purely local bookkeeping: nothing here is +// sent to other nodes or changes what a block contains. +// +// - reorg-episode.json: the fixed reference of the current automatic-rollback episode (the head the +// node had when the episode began). Kept across restarts, so a restart does not give a fresh full +// depth. Removed when the episode ends (the head is back above the reference) or the operator +// truncates the chain. +// - fork-held.json: written when the node refuses to unwind for a competing chain (beyond the depth +// bound, or a competing block not vouched for by the node's confidants). While its "held" is true +// the node reports FORK HELD. The operator either keeps this chain (the dismiss-fork file: the +// record then keeps the dismissed branch, still refused, and "held" turns false) or follows the +// other one (--resolve-fork / --set-bc-top, which remove the file). When the episode ends on its +// own, the file goes unless it holds a dismissal. +namespace cs::forkguard { + +inline constexpr const char* kEpisodeFile = "reorg-episode.json"; +inline constexpr const char* kForkFile = "fork-held.json"; +// created by the operator (any content) to dismiss the held fork and keep this node's chain; the +// running node picks it up within a round, acts, logs and deletes it +inline constexpr const char* kDismissTrigger = "dismiss-fork"; +// at most this many block hashes of dismissed branches are remembered +inline constexpr size_t kMaxBranchHashes = 256; + +// the folder of these files for a database path: its parent folder ("." for a relative "db") +std::string stateDirFor(const std::string& dbPath); + +struct Episode { + cs::Sequence highWater = 0; + // set by --resolve-fork: the operator approved following the competing chain for this one + // episode (no depth bound, no authority check); cleared when the episode ends + bool approvedSwitch = false; +}; + +struct ForkRecord { + // FORK HELD: the node refuses a competing chain and the operator has not decided yet. False once + // the operator dismissed it (the record then keeps the dismissal) or the episode ended. + bool held = true; + std::string reason; // "depth" (the bound) or "authority" (not vouched for) + int64_t sinceUnix = 0; // first refusal of the episode, UTC seconds + cs::Sequence highWater = 0; // the episode's reference head + cs::Sequence ourHead = 0; // our head when the node refused + std::string ourHeadHash; + // the highest of our blocks the competing chain can share: exact when the competing block names + // it as its previous block, otherwise the competing chain diverges at or below it + cs::Sequence forkPoint = 0; + std::string forkPointHash; + bool forkPointExact = false; + cs::Sequence competingSeq = 0; // the refused block (0: not known, e.g. a bad-hash reply) + std::string competingHash; + std::string peerKey; // base58, empty when not known + std::string peerAddress; // ip:port, empty when not known + cs::Sequence peerHeight = 0; // the height the peer announced, 0 when not known + // the operator kept this node's chain (dismiss-fork): the first known block of the dismissed + // branch after the fork point, and the hashes of that branch seen since. A competing block that is + // one of them, or continues one of them, is still refused but raises FORK HELD no more. + int64_t dismissedUnix = 0; + cs::Sequence dismissedSeq = 0; + std::string dismissedHash; + std::vector branchHashes; + + bool hasDismissal() const { + return !branchHashes.empty(); + } + // the competing block (hash, previous hash) belongs to a dismissed branch + bool inDismissedBranch(const std::string& hash, const std::string& previous) const; + // remember one more block hash of a dismissed branch (the oldest go beyond kMaxBranchHashes) + void addBranchHash(const std::string& hash); +}; + +// An empty dir turns persistence off (the functions do nothing and load returns nothing). +bool saveEpisode(const std::string& dir, const Episode& episode); +std::optional loadEpisode(const std::string& dir); +void clearEpisode(const std::string& dir); + +bool saveForkRecord(const std::string& dir, const ForkRecord& record); +std::optional loadForkRecord(const std::string& dir); +void clearForkRecord(const std::string& dir); + +// --resolve-fork: the fork point to cut the chain at and the hash the stored block there must have, +// from the record in dir; nothing (and *error says why) when there is no record or it cannot be +// checked against the database +struct ResolvePlan { + cs::Sequence top = 0; + std::string hash; + ForkRecord record; +}; +std::optional planResolve(const std::string& dir, std::string* error); + +// "2026-11-19 10:04:05 UTC" +std::string utcTime(int64_t unixSeconds); + +// the operator's two choices for a held fork: keep this chain (the dismiss-fork file) or follow the +// other one (--resolve-fork, or --set-bc-top by hand), native and Docker +std::string remedy(const ForkRecord& record); + +// the periodic FORK HELD detail line: since when, where the chains diverge, the refused peer and the +// remedy +std::string summary(const ForkRecord& record); + +} // namespace cs::forkguard diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index ceb1fd570..c22cfed17 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -471,6 +471,11 @@ public slots: }; bool initCommon(); ChainInit initChain(); + // --resolve-fork: the fork point recorded in fork-held.json, checked and announced; none (logged) + // when there is nothing to resolve + std::optional resolveForkTop(); + // FORK HELD: the time of the last detail line (one a minute at most) + std::chrono::steady_clock::time_point lastForkHeldDetail_{}; // snapshot.json of a restored folder against the digest of the loaded caches (design 6.8); // false with exit code 4 on a mismatch bool verifyAtStart(); diff --git a/csnode/include/csnode/poolsynchronizer.hpp b/csnode/include/csnode/poolsynchronizer.hpp index 7f5675c7b..12a28277f 100644 --- a/csnode/include/csnode/poolsynchronizer.hpp +++ b/csnode/include/csnode/poolsynchronizer.hpp @@ -30,6 +30,8 @@ class PoolSynchronizer { void stop(); cs::Sequence getMaxNeighbourSequence(); + // the last sequence this neighbour announced above ours, 0 if none + cs::Sequence announcedSequence(const cs::PublicKey& key) const; static const RoundNumber kRoundDifferentForSync = values::kDefaultMetaStorageMaxSize; void getSyncroMessage(const cs::PublicKey& sender, SyncroMessage msg); diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 8c592d0ee..87f1072f6 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -241,12 +242,21 @@ bool BlockChain::init( // LMDB reuses the space of pruned blocks; BerkeleyDB's Recno never does ([storage] db_backend) cslog() << kLogPrefix << "block database: " << storageCfg.dbBackend << (storageCfg.dbBackend == "lmdb" ? " in " + path + "/lmdb" : std::string{}); + // --resolve-fork: the storage cuts the chain only if the block at the fork point is the recorded one + if (newBlockchainTop != cs::kWrongSequence && resolveForkSeq_ == newBlockchainTop) { + storage_.set_truncate_check(resolveForkHash_); + } + truncationRefused_ = false; if (!storage_.open(path, progress, newBlockchainTop, firstBlockToReadInDatabase, storageCfg.asyncWriteQueueSize, storageCfg.writeBatchSize, static_cast(storageCfg.rocksdbBlockCacheMb) << 20, static_cast(storageCfg.rocksdbMemtableMb) << 20, storageCfg.dbBackend)) { cserror() << kLogPrefix << "Couldn't open database at " << path; + truncationRefused_ = storage_.last_error() == csdb::Storage::TruncateMismatch; + if (truncationRefused_) { + cserror() << kLogPrefix << storage_.last_error_message(); + } checkpointAheadOfBlocks_ = successfulQuickStart && storage_.last_error() == csdb::Storage::StartAfterLastBlock; if (checkpointAheadOfBlocks_) { cserror() << kLogPrefix << "the QUICK START checkpoint continues after the last block on disk (" @@ -263,6 +273,16 @@ bool BlockChain::init( if (newBlockchainTop != cs::kWrongSequence) { if (trxIndex_) trxIndex_->trimToFloor(newBlockchainTop); + // the operator truncated: the rollback episode and FORK HELD end here + const std::string dir = cs::forkguard::stateDirFor(path); + cs::forkguard::clearForkRecord(dir); + cs::forkguard::clearEpisode(dir); + if (resolveForkSeq_ == newBlockchainTop) { + // --resolve-fork: the operator approved following the competing chain. If it diverges + // below the recorded fork point (not known exactly), the node unwinds further on its own, + // for this one episode only: it ends once the head is above the fork point again. + cs::forkguard::saveEpisode(dir, {newBlockchainTop, true}); + } return true; } @@ -324,6 +344,10 @@ bool BlockChain::init( serializationManPtr_->setCompletedFromGenesis(); } + // the longest-chain guard's files next to the database: a rollback episode (and FORK HELD) that + // was running when the node stopped goes on with the same reference + setGuardStateDir(cs::forkguard::stateDirFor(path)); + good_ = true; blocksToBeRemoved_ = totalLoaded - 1; // any amount to remave after start return true; @@ -830,22 +854,24 @@ csdb::Transaction BlockChain::loadTransaction(const csdb::TransactionID& transId return transaction; } -void BlockChain::setCommunityBootstrapFloor(cs::Sequence seq) { - communityBootstrapFloor_ = seq; -} - // Longest-chain guard: bound the depth of an automatic rollback while adopting a competing chain. // A competing chain that asks us to unwind more than the configured bound of own blocks is either a // deliberate network switch (a human must confirm it) or a chain that should not be followed: // in both cases the safe local action is to stop unwinding and wait for the operator. The depth is // measured against a fixed reference (the head when the episode started), not a counter that forward -// progress could reset, so alternating removals and appends cannot creep past it. It is called from -// inside removeLastBlock() and compromiseLastBlock(), so every automatic removal is bounded whatever -// the caller, and rollbackHighWater_ is captured at the first removal from any path. +// progress could reset, so alternating removals and appends cannot creep past it; the reference is +// kept in reorg-episode.json, so a restart does not reset it either. It is called from inside +// removeLastBlock() and compromiseLastBlock(), so every automatic removal is bounded whatever the +// caller, and rollbackHighWater_ is captured at the first removal from any path. bool BlockChain::guardAutoRollback() { - // operator override: accept a deliberate deep switch. Config ([params] allow_deep_rollback) or the - // CS_ALLOW_DEEP_ROLLBACK environment variable; either one is enough. + beginRollbackEpisode(); + + // operator override: accept a deliberate deep switch ([params] allow_deep_rollback, the + // CS_ALLOW_DEEP_ROLLBACK environment variable, or --resolve-fork for this episode) if (deepRollbackApproved()) { + if (!episodeApprovedSwitch_) { + deepOverrideUsed_ = true; // spent when this episode completes + } return true; } const auto* config = cs::ConfigHolder::instance().config(); @@ -855,34 +881,326 @@ bool BlockChain::guardAutoRollback() { return true; // max_auto_rollback=0: no depth bound (the authority gate still applies) } + // refuse the removal that would take the head below the floor (no wraparound when near genesis) const cs::Sequence tip = lastSequence_.load(); - if (!rollbackEpisode_) { - rollbackEpisode_ = true; - rollbackHighWater_ = tip; // the head we had before this rollback episode began - rollbackLowWater_ = tip; + if (rollbackHighWater_ > maxDepth && tip <= rollbackHighWater_ - maxDepth) { + recordForkHeld("depth"); + return false; } + return true; +} - // a node that adopted a community bootstrap table (PR #188) may unwind freely back to the sequence - // it held when it took the table: taking the table was already the operator's explicit choice, and - // after a partition the longer community branch legitimately replaces blocks down to that point. - if (communityBootstrapFloor_ != 0 && tip > communityBootstrapFloor_) { - return true; +void BlockChain::beginRollbackEpisode() { + if (rollbackEpisode_) { + return; + } + const cs::Sequence tip = lastSequence_.load(); + rollbackEpisode_ = true; + rollbackHighWater_ = tip; // the head we had before this rollback episode began + rollbackLowWater_ = tip; + deepOverrideUsed_ = false; + cs::forkguard::saveEpisode(guardStateDir_, {rollbackHighWater_, episodeApprovedSwitch_}); +} + +void BlockChain::onHeadAdvanced() { + // the rollback episode ends only once the chain climbs back strictly above the head it had when + // the episode began, so the depth cannot creep past the limit by alternating removals and appends + // below that head + if (!rollbackEpisode_ || lastSequence_.load() <= rollbackHighWater_) { + return; + } + rollbackEpisode_ = false; + // one line per finished episode: how deep the node actually unwound (measured in CI) + cslog() << kLogPrefix << "rollback episode ended: unwound " << (rollbackHighWater_ - rollbackLowWater_) + << " block(s) below #" << WithDelimiters(rollbackHighWater_); + if (episodeApprovedSwitch_) { + episodeApprovedSwitch_ = false; + cswarning() << kLogPrefix << "the switch approved with --resolve-fork is complete: the automatic rollback " + << "bound applies again"; + } + if (deepOverrideUsed_ && !deepOverrideSpent_) { + deepOverrideSpent_ = true; + cserror() << kLogPrefix << "LONGEST-CHAIN GUARD: allow_deep_rollback (or CS_ALLOW_DEEP_ROLLBACK) has served " + << "one completed rollback episode and is now OFF until the node restarts. Remove it from " + << "config.ini (or the environment) now: left on, it would lift the bound again at the next start."; + } + deepOverrideUsed_ = false; + cs::forkguard::clearEpisode(guardStateDir_); + std::optional keep; + bool wasHeld = false; + { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value()) { + wasHeld = forkRecord_->held; + forkRecord_->held = false; + if (forkRecord_->hasDismissal()) { + keep = forkRecord_; // the dismissed branch stays refused and quiet + } + else { + forkRecord_.reset(); + } + } } + if (keep.has_value()) { + cs::forkguard::saveForkRecord(guardStateDir_, *keep); + } + else { + cs::forkguard::clearForkRecord(guardStateDir_); + } + if (wasHeld) { + cswarning() << kLogPrefix << "FORK HELD cleared: the chain is back above #" << WithDelimiters(rollbackHighWater_); + } +} - // refuse the removal that would take the head below the floor (no wraparound when near genesis) - if (rollbackHighWater_ > maxDepth && tip <= rollbackHighWater_ - maxDepth) { +bool BlockChain::recordForkHeld(const char* reason) { + const std::string competingHash = competing_.has_value() ? competing_->hash.to_string() : std::string{}; + const std::string competingPrev = competing_.has_value() ? competing_->previous.to_string() : std::string{}; + { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value() && forkRecord_->hasDismissal() && competing_.has_value() && + forkRecord_->inDismissedBranch(competingHash, competingPrev)) { + // a branch the operator dismissed: still refused, never FORK HELD again; follow it by its hashes + const size_t before = forkRecord_->branchHashes.size(); + forkRecord_->addBranchHash(competingHash); + const bool grew = forkRecord_->branchHashes.size() != before; + const auto now = std::chrono::steady_clock::now(); + if (lastDismissedLog_ == std::chrono::steady_clock::time_point{} || now - lastDismissedLog_ >= std::chrono::hours(1)) { + lastDismissedLog_ = now; + csinfo() << kLogPrefix << "competing branch at #" << WithDelimiters(forkRecord_->dismissedSeq) << " (" + << forkRecord_->dismissedHash << ") dismissed by the operator on " + << cs::forkguard::utcTime(forkRecord_->dismissedUnix) << "; still refused (block #" + << WithDelimiters(competing_->seq) << ")"; + } + if (grew) { + cs::forkguard::saveForkRecord(guardStateDir_, *forkRecord_); + } + return false; + } + if (forkRecord_.has_value() && forkRecord_->held) { + return true; // one record per episode: the first refusal + } + } + beginRollbackEpisode(); // FORK HELD ends with the episode + cs::forkguard::ForkRecord r; + { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value()) { + r = *forkRecord_; // keeps an earlier dismissal: that branch stays dismissed + } + } + r.held = true; + r.reason = reason; + r.sinceUnix = static_cast(std::time(nullptr)); + r.highWater = rollbackHighWater_; + const cs::Sequence tip = lastSequence_.load(); + r.ourHead = tip; + r.ourHeadHash = getHashBySequence(tip).to_string(); + // the fork point: the highest of our blocks the competing chain can share + cs::Sequence candidate = tip; + bool candidateExcluded = true; // without a competing block, our head is the one disputed + r.forkPointExact = false; + r.competingSeq = 0; + r.competingHash.clear(); + if (competing_.has_value() && competing_->seq > 0) { + r.competingSeq = competing_->seq; + r.competingHash = competingHash; + candidate = std::min(competing_->seq - 1, tip); + if (candidate + 1 == competing_->seq) { + // the competing block names its previous block: ours at that height is shared or not + r.forkPointExact = !competing_->previous.is_empty() && getHashBySequence(candidate) == competing_->previous; + candidateExcluded = !r.forkPointExact; + } + else { + candidateExcluded = false; // nothing known about our block at that height + } + } + r.forkPoint = (candidateExcluded && candidate > 0) ? candidate - 1 : candidate; + r.forkPointHash = getHashBySequence(r.forkPoint).to_string(); + r.peerKey.clear(); + r.peerAddress.clear(); + r.peerHeight = 0; + if (syncSource_.has_value()) { + const auto& key = syncSource_->first; + r.peerKey = EncodeBase58(key.data(), key.data() + key.size()); + r.peerHeight = syncSource_->second; + if (peerAddressResolver_) { + r.peerAddress = peerAddressResolver_(key); + } + } + { + std::lock_guard l(guardMutex_); + forkRecord_ = r; + } + cs::forkguard::saveForkRecord(guardStateDir_, r); + + const auto* config = cs::ConfigHolder::instance().config(); + const uint64_t maxDepth = (config != nullptr) ? config->maxAutoRollbackDepth() : Consensus::MaxAutoRollbackDepth; + if (r.reason == "depth") { cserror() << kLogPrefix << "LONGEST-CHAIN GUARD: a competing chain requires rolling back more than " - << maxDepth << " of our own confirmed blocks (reference head #" - << WithDelimiters(rollbackHighWater_) << ", now #" << WithDelimiters(tip) - << "). Keeping our chain, operator review required. This is either a deliberate longer-chain " - << "switch to be confirmed, or a chain not to be followed. Set [params] " - << "allow_deep_rollback=true (or the CS_ALLOW_DEEP_ROLLBACK environment variable) and restart " - << "to accept a deliberate deep switch."; - return false; + << maxDepth << " of our own confirmed blocks (reference head #" << WithDelimiters(rollbackHighWater_) + << ", now #" << WithDelimiters(tip) << "). Keeping our chain: FORK HELD, operator review required."; + } + else { + cserror() << kLogPrefix << "a competing block #" << WithDelimiters(r.competingSeq) + << " is not vouched for by this node's confidants. Keeping our chain: FORK HELD, operator review required."; + } + cserror() << kLogPrefix << cs::forkguard::summary(r); + return true; +} + +bool BlockChain::dismissFork() { + cs::forkguard::ForkRecord r; + { + std::lock_guard l(guardMutex_); + if (!forkRecord_.has_value() || !forkRecord_->held) { + return false; + } + forkRecord_->held = false; + forkRecord_->dismissedUnix = static_cast(std::time(nullptr)); + // the first block of that branch after the fork point, as far as this node has seen it + forkRecord_->dismissedSeq = forkRecord_->competingSeq; + forkRecord_->dismissedHash = forkRecord_->competingHash; + forkRecord_->addBranchHash(forkRecord_->competingHash); + if (!forkRecord_->hasDismissal()) { + // no competing block known (a bad-hash reply): nothing to recognise it by, just clear it + forkRecord_.reset(); + r.held = false; + } + else { + r = *forkRecord_; + } + } + if (r.hasDismissal()) { + cs::forkguard::saveForkRecord(guardStateDir_, r); + cswarning() << kLogPrefix << "FORK HELD dismissed by the operator: keeping this chain; the competing branch at #" + << WithDelimiters(r.dismissedSeq) << " (" << r.dismissedHash << ") stays refused"; + } + else { + cs::forkguard::clearForkRecord(guardStateDir_); + cswarning() << kLogPrefix << "FORK HELD dismissed by the operator: keeping this chain"; } return true; } +bool BlockChain::checkDismissTrigger() { + if (guardStateDir_.empty()) { + return false; + } + namespace fs = std::filesystem; + std::error_code ec; + const fs::path trigger = fs::path(guardStateDir_) / cs::forkguard::kDismissTrigger; + if (!fs::exists(trigger, ec)) { + return false; + } + const bool dismissed = dismissFork(); + if (!dismissed) { + cslog() << kLogPrefix << "the " << cs::forkguard::kDismissTrigger << " file is there but no fork is held: nothing to dismiss"; + } + fs::remove(trigger, ec); + if (ec) { + cserror() << kLogPrefix << "cannot remove " << trigger.string() << ": " << ec.message(); + } + return dismissed; +} + +bool BlockChain::mayUnwindLastBlock() { + // the start-up integrity repair removes blocks of this node's own database that failed the hash + // check (cacheLastBlocks, incorrectBlocks_): a local repair, not a network reorg, so not bounded + return integrityRepair_ || guardAutoRollback(); +} + +bool BlockChain::forkHeld() const { + std::lock_guard l(guardMutex_); + return forkRecord_.has_value() && forkRecord_->held; +} + +std::optional BlockChain::forkHeldRecord() const { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value() && forkRecord_->held) { + return forkRecord_; + } + return std::nullopt; +} + +std::optional BlockChain::forkRecord() const { + std::lock_guard l(guardMutex_); + return forkRecord_; +} + +void BlockChain::setSyncSource(const cs::PublicKey& key, cs::Sequence announcedHeight) { + syncSource_ = std::make_pair(key, announcedHeight); +} + +void BlockChain::clearSyncSource() { + syncSource_.reset(); +} + +void BlockChain::setPeerAddressResolver(std::function resolver) { + peerAddressResolver_ = std::move(resolver); +} + +void BlockChain::setResolveFork(cs::Sequence seq, const csdb::PoolHash& hash) { + resolveForkSeq_ = seq; + resolveForkHash_ = hash; +} + +void BlockChain::setGuardStateDir(const std::string& dir) { + guardStateDir_ = dir; + const cs::Sequence tip = lastSequence_.load(); + if (const auto e = cs::forkguard::loadEpisode(dir)) { + if (tip <= e->highWater) { + rollbackEpisode_ = true; + rollbackHighWater_ = e->highWater; + rollbackLowWater_ = tip; + episodeApprovedSwitch_ = e->approvedSwitch; + cswarning() << kLogPrefix << "rollback episode resumed: reference head #" << WithDelimiters(e->highWater) + << ", head #" << WithDelimiters(tip) + << (e->approvedSwitch ? ", following the competing chain as approved with --resolve-fork" : ""); + } + else { + cs::forkguard::clearEpisode(dir); + } + } + if (auto r = cs::forkguard::loadForkRecord(dir)) { + if (!rollbackEpisode_) { + r->held = false; // the chain is past that episode + } + if (r->held || r->hasDismissal()) { + { + std::lock_guard l(guardMutex_); + forkRecord_ = *r; + } + if (r->held) { + cserror() << kLogPrefix << cs::forkguard::summary(*r); + } + else { + cs::forkguard::saveForkRecord(dir, *r); + } + } + else { + cs::forkguard::clearForkRecord(dir); + } + } + // a dismiss-fork file left while the node was stopped + checkDismissTrigger(); +} + +bool BlockChain::deepRollbackEnvSet() { + const char* v = std::getenv("CS_ALLOW_DEEP_ROLLBACK"); + return v != nullptr && *v != '\0'; +} + +bool BlockChain::deepRollbackApproved() const { + if (episodeApprovedSwitch_) { + return true; + } + if (deepOverrideSpent_) { + return false; + } + const auto* config = cs::ConfigHolder::instance().config(); + return deepRollbackEnvSet() || (config != nullptr && config->allowDeepRollback()); +} + // A competing block legitimately asks us to roll back only if it is vouched for by authority this node // already recognises. We verify its round confirmations against the confidants of our own current last // block, exactly as BlockVerifier does for a block arriving from the network. Checking the block against @@ -890,17 +1208,8 @@ bool BlockChain::guardAutoRollback() { // names its own keys and signs for itself. A legitimate micro-fork passes, because the honest competing // block at the same height was produced by the same deterministically elected trusted group whose keys // confirm our last block; a block without confirmations from that group fails and never triggers an -// unwind. Bootstrap blocks carry no such confirmations and are handled by the depth floor plus operator, -// so callers must check isBootstrap() before relying on this. -bool BlockChain::deepRollbackApproved() { - static const bool allowDeepEnv = [] { - const char* v = std::getenv("CS_ALLOW_DEEP_ROLLBACK"); - return v != nullptr && *v != '\0'; - }(); - const auto* config = cs::ConfigHolder::instance().config(); - return allowDeepEnv || (config != nullptr && config->allowDeepRollback()); -} - +// unwind. Bootstrap blocks carry no such confirmations and fail here; they go through +// isBootstrapAuthenticatedLocally() instead. bool BlockChain::hasTrustedConfirmations(const csdb::Pool& competing) const { // Strict pre-checks for the unwind decision only; BlockVerifier::verifyConfirmations keeps the // exceptions it has for normal block validation (sequence < 2, one-entry masks), which must not be @@ -913,6 +1222,10 @@ bool BlockChain::hasTrustedConfirmations(const csdb::Pool& competing) const { if (!competing.is_valid() || competing.sequence() < 2) { return fail("no confirmations to check"); } + // BlockVerifier::verifyConfirmations accepts a bootstrap block without checking any confirmation + if (BlockChain::isBootstrap(competing)) { + return fail("bootstrap block carries no confirmations"); + } const auto trustedConfidants = getLastBlock().confidants(); if (competing.numberConfirmations() == 0 || competing.roundConfirmations().empty()) { return fail("no round confirmations"); @@ -981,11 +1294,12 @@ void BlockChain::removeLastBlock() { csmeta(csdebug) << kLogPrefix << "There are no blocks, allowed to be removed"; return; } - // longest-chain guard: every caller of removeLastBlock is an automatic reorg/sync removal (there is - // no operator truncation path), so bound the depth here, once, for all of them. When the limit is - // reached the block is kept and the caller sees its own last block unchanged; no permanent state is - // set, so a later legitimate reorg still works. - if (!guardAutoRollback()) { + // longest-chain guard: every caller of removeLastBlock is an automatic reorg/sync removal or the + // start-up integrity repair (the operator truncates with --set-bc-top / --resolve-fork, at start-up, + // in the storage, not here), so bound the depth here, once, for all of the automatic ones. When the + // limit is reached the block is kept and the caller sees its own last block unchanged; the node + // reports FORK HELD until the episode ends or the operator truncates. + if (!mayUnwindLastBlock()) { cswarning() << kLogPrefix << "longest-chain guard kept the last block, automatic rollback stopped"; return; } @@ -1085,7 +1399,7 @@ bool BlockChain::compromiseLastBlock(const csdb::PoolHash& desired_hash) { // longest-chain guard: marking the last block uncertain is the first step of an automatic unwind, // so it is bounded here too. Beyond the depth limit the block is kept and the caller falls through // to its keep-own-chain branch. - if (!guardAutoRollback()) { + if (!mayUnwindLastBlock()) { cswarning() << kLogPrefix << "longest-chain guard kept the last block, not compromising it"; return false; } @@ -2086,15 +2400,8 @@ std::optional BlockChain::recordBlock(csdb::Pool& pool, bool isTrust lastSequence_ = deferredBlock_.sequence(); } - // forward progress: the rollback episode ends only once the chain climbs back strictly above the - // head it had when the episode began, so the depth cannot creep past the limit by alternating - // removals and appends below that head. - if (rollbackEpisode_ && lastSequence_.load() > rollbackHighWater_) { - rollbackEpisode_ = false; - // one line per finished episode: how deep the node actually unwound (measured in CI) - cslog() << kLogPrefix << "rollback episode ended: unwound " << (rollbackHighWater_ - rollbackLowWater_) - << " block(s) below #" << WithDelimiters(rollbackHighWater_); - } + // forward progress: ends the rollback episode (and FORK HELD) once the head is above its reference + onHeadAdvanced(); //csdetails() << kLogPrefix << "Pool #" << deferredBlock_.sequence() << ": " << cs::Utils::byteStreamToHex(deferredBlock_.to_binary().data(), deferredBlock_.to_binary().size()); emit storeBlockEvent(pool); @@ -2453,7 +2760,9 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { } else { csdebug() << kLogPrefix << "the desired last block has the different content, drop it and remove own last block"; + competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash()}; removeLastBlock(); + competing_.reset(); return false; } } @@ -2480,13 +2789,26 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { cserror() << "BLOCKCHAIN> new pool\'s prev. hash is empty, don\'t write it, do not any harm to our blockchain"; return false; } + // the competing block being handled, for the fork record if the node refuses to unwind + competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash()}; + struct ClearCompeting { + std::optional& c; + ~ClearCompeting() { c.reset(); } + } clearCompeting{competing_}; if (!competingBlockMayTriggerUnwind(pool)) { // a competing block not vouched for by authority this node recognises (no strict // confirmations by our last block's confidants, or a bootstrap flag that is not // authenticated locally): never unwind our own chain for it, not even by marking our - // last block uncertain (which would let a later replacement remove it). - cslog() << kLogPrefix << "competing block #" << poolSequence - << " is not authenticated by this node's confidants, keep own last block"; + // last block uncertain (which would let a later replacement remove it). The node + // reports FORK HELD until its head moves on or the operator decides. + if (recordForkHeld("authority")) { + cslog() << kLogPrefix << "competing block #" << poolSequence + << " is not authenticated by this node's confidants, keep own last block"; + } + else { + csdebug() << kLogPrefix << "competing block #" << poolSequence + << " of a branch dismissed by the operator, keep own last block"; + } } else if (compromiseLastBlock(pool.previous_hash())) { csdebug() << kLogPrefix << "compromise own last block and cancel store operation"; @@ -2979,11 +3301,19 @@ void BlockChain::cacheLastBlocks() { cs::Sequence lastSeq; while (!incorrectBlocks_.empty() || !selectionFinished_) { csinfo() << kLogPrefix << "Starting blocks transferring cycle"; - // longest-chain guard: stop unwinding own blocks once the competing chain demands a deeper - // automatic rollback than allowed; leave the chain intact, abandon this antifork episode - // cleanly and wait for the operator. (guardAutoRollback measures depth against the fixed - // reference head via lastSequence_, so this counts real own-block removals only.) - if (!guardAutoRollback()) { + // removals while incorrectBlocks_ is not empty repair this node's own database (blocks that + // failed the hash check at start-up): a local repair, not a network reorg, so the guard does + // not bound them. The flag lives for this iteration only. + integrityRepair_ = !incorrectBlocks_.empty(); + struct EndRepair { + bool& f; + ~EndRepair() { f = false; } + } endRepair{integrityRepair_}; + // longest-chain guard, for the rest of the selection: stop unwinding own blocks once a deeper + // automatic rollback than allowed would be needed; leave the chain intact, abandon this + // antifork episode cleanly and wait for the operator. (guardAutoRollback measures depth + // against the fixed reference head via lastSequence_, so this counts real own-block removals.) + if (!integrityRepair_ && !guardAutoRollback()) { cswarning() << kLogPrefix << "longest-chain guard stopped the antifork rollback; keeping own chain"; incorrectBlocks_.clear(); selectionFinished_ = true; diff --git a/csnode/src/forkguardstate.cpp b/csnode/src/forkguardstate.cpp new file mode 100644 index 000000000..501a5eaf7 --- /dev/null +++ b/csnode/src/forkguardstate.cpp @@ -0,0 +1,289 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace cs::forkguard { + +namespace fs = std::filesystem; +using boost::property_tree::ptree; + +namespace { + +// a small file written whole: to a temporary name first, then renamed over the old one +bool writeTree(const std::string& dir, const char* name, const ptree& tree) { + if (dir.empty()) { + return false; + } + std::error_code ec; + const fs::path target = fs::path(dir) / name; + const fs::path tmp = fs::path(dir) / (std::string(name) + ".tmp"); + try { + std::ofstream out(tmp, std::ios::trunc); + if (!out) { + return false; + } + boost::property_tree::write_json(out, tree); + out.close(); + if (!out) { + fs::remove(tmp, ec); + return false; + } + } + catch (const std::exception&) { + fs::remove(tmp, ec); + return false; + } + fs::rename(tmp, target, ec); + if (ec) { + fs::remove(tmp, ec); + return false; + } + return true; +} + +std::optional readTree(const std::string& dir, const char* name) { + if (dir.empty()) { + return std::nullopt; + } + const fs::path path = fs::path(dir) / name; + std::error_code ec; + if (!fs::is_regular_file(path, ec) || fs::file_size(path, ec) > 64 * 1024) { + return std::nullopt; + } + try { + std::ifstream in(path); + ptree tree; + boost::property_tree::read_json(in, tree); + return tree; + } + catch (const std::exception&) { + return std::nullopt; + } +} + +void removeFile(const std::string& dir, const char* name) { + if (dir.empty()) { + return; + } + std::error_code ec; + fs::remove(fs::path(dir) / name, ec); +} + +} // namespace + +std::string stateDirFor(const std::string& dbPath) { + fs::path p(dbPath); + // "db/" and "db" name the same folder + while (!p.empty() && !p.has_filename() && p.has_parent_path() && p != p.parent_path()) { + p = p.parent_path(); + } + const fs::path parent = p.parent_path(); + return parent.empty() ? std::string(".") : parent.string(); +} + +bool saveEpisode(const std::string& dir, const Episode& episode) { + ptree tree; + tree.put("high_water", episode.highWater); + tree.put("approved_switch", episode.approvedSwitch); + return writeTree(dir, kEpisodeFile, tree); +} + +std::optional loadEpisode(const std::string& dir) { + const auto tree = readTree(dir, kEpisodeFile); + if (!tree) { + return std::nullopt; + } + try { + Episode e; + e.highWater = tree->get("high_water"); + e.approvedSwitch = tree->get("approved_switch", false); + return e; + } + catch (const std::exception&) { + return std::nullopt; + } +} + +void clearEpisode(const std::string& dir) { + removeFile(dir, kEpisodeFile); +} + +bool ForkRecord::inDismissedBranch(const std::string& hash, const std::string& previous) const { + for (const auto& h : branchHashes) { + if ((!hash.empty() && h == hash) || (!previous.empty() && h == previous)) { + return true; + } + } + return false; +} + +void ForkRecord::addBranchHash(const std::string& hash) { + if (hash.empty()) { + return; + } + for (const auto& h : branchHashes) { + if (h == hash) { + return; + } + } + branchHashes.push_back(hash); + if (branchHashes.size() > kMaxBranchHashes) { + branchHashes.erase(branchHashes.begin(), branchHashes.begin() + (branchHashes.size() - kMaxBranchHashes)); + } +} + +bool saveForkRecord(const std::string& dir, const ForkRecord& r) { + ptree tree; + tree.put("held", r.held); + tree.put("reason", r.reason); + tree.put("since", utcTime(r.sinceUnix)); + tree.put("since_unix", r.sinceUnix); + tree.put("high_water", r.highWater); + tree.put("our_head", r.ourHead); + tree.put("our_head_hash", r.ourHeadHash); + tree.put("fork_point", r.forkPoint); + tree.put("fork_point_hash", r.forkPointHash); + tree.put("fork_point_exact", r.forkPointExact); + tree.put("competing_block", r.competingSeq); + tree.put("competing_hash", r.competingHash); + tree.put("peer_key", r.peerKey); + tree.put("peer_address", r.peerAddress); + tree.put("peer_height", r.peerHeight); + if (r.hasDismissal()) { + tree.put("dismissed", utcTime(r.dismissedUnix)); + tree.put("dismissed_unix", r.dismissedUnix); + tree.put("dismissed_block", r.dismissedSeq); + tree.put("dismissed_hash", r.dismissedHash); + ptree hashes; + for (const auto& h : r.branchHashes) { + ptree item; + item.put("", h); + hashes.push_back(std::make_pair("", item)); + } + tree.add_child("dismissed_branch_hashes", hashes); + } + return writeTree(dir, kForkFile, tree); +} + +std::optional loadForkRecord(const std::string& dir) { + const auto tree = readTree(dir, kForkFile); + if (!tree) { + return std::nullopt; + } + try { + ForkRecord r; + r.held = tree->get("held", true); + r.reason = tree->get("reason", ""); + r.sinceUnix = tree->get("since_unix", 0); + r.highWater = tree->get("high_water"); + r.ourHead = tree->get("our_head"); + r.ourHeadHash = tree->get("our_head_hash", ""); + r.forkPoint = tree->get("fork_point"); + r.forkPointHash = tree->get("fork_point_hash", ""); + r.forkPointExact = tree->get("fork_point_exact", false); + r.competingSeq = tree->get("competing_block", 0); + r.competingHash = tree->get("competing_hash", ""); + r.peerKey = tree->get("peer_key", ""); + r.peerAddress = tree->get("peer_address", ""); + r.peerHeight = tree->get("peer_height", 0); + r.dismissedUnix = tree->get("dismissed_unix", 0); + r.dismissedSeq = tree->get("dismissed_block", 0); + r.dismissedHash = tree->get("dismissed_hash", ""); + if (const auto hashes = tree->get_child_optional("dismissed_branch_hashes")) { + for (const auto& item : *hashes) { + r.addBranchHash(item.second.get_value()); + } + } + return r; + } + catch (const std::exception&) { + return std::nullopt; + } +} + +void clearForkRecord(const std::string& dir) { + removeFile(dir, kForkFile); +} + +std::optional planResolve(const std::string& dir, std::string* error) { + const auto record = loadForkRecord(dir); + if (!record.has_value()) { + if (error) { + *error = std::string("no fork record (") + kForkFile + ") in " + dir + + ": this node holds no refused fork, nothing to do. To cut the chain at a known block, use --set-bc-top "; + } + return std::nullopt; + } + if (record->forkPointHash.empty()) { + if (error) { + *error = "the fork record has no hash for block #" + std::to_string(record->forkPoint) + + " and cannot be checked against the database: nothing removed. Use --set-bc-top "; + } + return std::nullopt; + } + return ResolvePlan{record->forkPoint, record->forkPointHash, *record}; +} + +std::string utcTime(int64_t unixSeconds) { + const std::time_t t = static_cast(unixSeconds); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &t); +#else + gmtime_r(&t, &tm); +#endif + char buf[32] = {}; + std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S UTC", &tm); + return buf; +} + +std::string remedy(const ForkRecord& r) { + std::ostringstream os; + os << "first check the head and the block at #" << r.forkPoint << " with other nodes or the monitor. If this node's " + << "chain is the right one: create the file " << kDismissTrigger << " next to the db folder (native: touch " + << kDismissTrigger << " in the node folder; Docker: docker compose exec node touch /data/" << kDismissTrigger + << "); the node keeps refusing that branch and stops reporting it. If the other chain is the right one: stop " + << "the node and run once node --config-file config.ini --resolve-fork (Docker: docker compose stop && docker " + << "compose run --rm node resolve-fork && docker compose up -d), then start it as usual; by hand: --set-bc-top " + << r.forkPoint << " (Docker: docker compose run --rm node set-bc-top " << r.forkPoint << ")."; + return os.str(); +} + +std::string summary(const ForkRecord& r) { + std::ostringstream os; + os << "FORK HELD since " << utcTime(r.sinceUnix) << ": "; + if (r.reason == "authority") { + os << "a competing block is not vouched for by this node's confidants"; + } + else { + os << "a competing chain asks to unwind beyond the automatic bound (reference head #" << r.highWater << ")"; + } + os << ". Fork point: block #" << r.forkPoint << (r.forkPointExact ? "" : " or lower"); + if (!r.forkPointHash.empty()) { + os << " (" << r.forkPointHash << ")"; + } + os << "; our head #" << r.ourHead; + if (r.competingSeq != 0) { + os << ", refused block #" << r.competingSeq; + } + if (!r.peerKey.empty() || !r.peerAddress.empty()) { + os << " from " << (r.peerKey.empty() ? std::string("?") : r.peerKey); + if (!r.peerAddress.empty()) { + os << " (" << r.peerAddress << ")"; + } + } + if (r.peerHeight != 0) { + os << ", peer height #" << r.peerHeight; + } + os << ". The node keeps its chain. What to do: " << remedy(r); + return os.str(); +} + +} // namespace cs::forkguard diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index f903688e5..4e26ec6dc 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -123,6 +123,20 @@ Node::Node(cs::config::Observer& observer) cs::Connector::connect(&blockChain_.lastBlockReplacedEvent, this, static_cast(&Node::logStateDigest)); cs::Connector::connect(&blockChain_.uncertainBlock, this, &Node::sendBlockRequestToConfidants); cs::Connector::connect(&blockChain_.orderNecessaryBlock, this, &Node::sendNecessaryBlockRequest); + // the fork record names the address of the refused peer when the routing table knows it + blockChain_.setPeerAddressResolver([this](const cs::PublicKey& key) -> std::string { + if (!transport_) { + return {}; + } + std::vector peers; + transport_->getKnownPeers(peers); + for (const auto& p : peers) { + if (p.key == key && !p.ip.empty()) { + return p.ip + ":" + std::to_string(p.port); + } + } + return {}; + }); cs::Connector::connect(&stat_.accountInitiationRequest, this, &Node::accountInitiationRequest); cs::Connector::connect(&blockChain_.successfullQuickStartEvent, this, &Node::onSuccessQS); initPoolSynchronizer(); @@ -236,14 +250,52 @@ bool Node::initCommon() { return true; } +std::optional Node::resolveForkTop() { + const auto* config = cs::ConfigHolder::instance().config(); + if (config->newBlockchainTop()) { + cserror() << "--resolve-fork and --set-bc-top together: use one of them"; + return std::nullopt; + } + std::string error; + const auto plan = cs::forkguard::planResolve(cs::forkguard::stateDirFor(config->getPathToDB()), &error); + if (!plan.has_value()) { + cserror() << "--resolve-fork: " << error; + return std::nullopt; + } + cslog() << cs::forkguard::summary(plan->record); + cslog() << "--resolve-fork: removing every block above #" << WithDelimiters(plan->top) << " (" << plan->hash + << "), our head was #" << WithDelimiters(plan->record.ourHead) << ", if the block at #" + << WithDelimiters(plan->top) << " is still the recorded one"; + blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash)); + return plan->top; +} + Node::ChainInit Node::initChain() { - if (cs::ConfigHolder::instance().config()->newBlockchainTop()) { + const auto* initConfig = cs::ConfigHolder::instance().config(); + if (initConfig->newBlockchainTop() || initConfig->resolveFork()) { + cs::Sequence newTop = initConfig->newBlockchainTopSeq(); + if (initConfig->resolveFork()) { + const auto resolved = resolveForkTop(); + if (!resolved.has_value()) { + exitCode_ = 3; + return ChainInit::Failed; + } + newTop = *resolved; + } if (!blockChain_.init( - cs::ConfigHolder::instance().config()->getPathToDB(), + initConfig->getPathToDB(), &cachesSerializationManager_, initialConfidants_, - cs::ConfigHolder::instance().config()->newBlockchainTopSeq()) + newTop) ) { + if (blockChain_.truncationRefused()) { + // nothing was removed: the database is as it was, the quick-start data stays + cserror() << "--resolve-fork: the block at the recorded fork point is not the recorded one (the " + << "chain changed since the fork was recorded): nothing removed. Check the node's head " + << "with other nodes or the monitor; if it must still be cut, use --set-bc-top ."; + exitCode_ = 3; + return ChainInit::Failed; + } if (isStopRequested()) { cslog() << "Init aborted by user; preserving QUICK START state."; cs::CheckpointHead headInfo; @@ -284,6 +336,11 @@ Node::ChainInit Node::initChain() { solver_->smart_contracts().rehydrateContractDbCache(blockChain_); solver_->smart_contracts().validateRestoredStatesAgainstChain(blockChain_); } + if (initConfig->resolveFork()) { + cslog() << "--resolve-fork: the chain now ends at block #" << WithDelimiters(newTop) + << " and the fork record is cleared. Start the node as usual: it syncs from there and " + << "follows the other chain (for this one switch, without the automatic rollback bound)."; + } return ChainInit::StopAfterInit; } if (!blockChain_.init( @@ -530,7 +587,7 @@ bool Node::startBootstrap() { const std::string dbPath = config.getPathToDB(); const std::string backend = config.getStorageSettings().dbBackend; bool removedTmp = false; - if (const auto skip = sb::notNeeded(cfg, ".", dbPath, backend, config.newBlockchainTop(), &removedTmp)) { + if (const auto skip = sb::notNeeded(cfg, ".", dbPath, backend, config.newBlockchainTop() || config.resolveFork(), &removedTmp)) { if (skip->why == sb::Skip::Backend) { cswarning() << "SNAPSHOT: " << skip->text << ": syncing from block 0"; } @@ -2440,6 +2497,12 @@ void Node::getBlockRequest(const uint8_t* data, const size_t size, const cs::Pub void Node::getBlockReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { csinfo() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + // the blocks of this reply come from `sender`: named in the fork record should the node refuse one + blockChain_.setSyncSource(sender, poolSynchronizer_->announcedSequence(sender)); + struct ClearSource { + BlockChain& chain; + ~ClearSource() { chain.clearSyncSource(); } + } clearSource{blockChain_}; bool isSyncOn = poolSynchronizer_->isSyncroStarted(); bool isBlockchainUncertain = blockChain_.isLastBlockUncertain(); @@ -4716,6 +4779,14 @@ void Node::onRoundStart(const cs::RoundTable& roundTable, bool updateRound) { line1 << "TRUSTED [" << cs::numeric_cast(myConfidantIndex_) << "]"; status_ = cs::NodeStatus::Trusted; } + // the operator keeps this chain: a dismiss-fork file next to the database, picked up every round + blockChain_.checkDismissTrigger(); + // the node refused to unwind for a competing chain and holds its own: on every round line, so it + // cannot be missed (the detail, with the two remedies, once a minute below) + const auto forkHeld = blockChain_.forkHeldRecord(); + if (forkHeld.has_value()) { + line1 << " | FORK HELD"; + } line1 << ' '; @@ -4730,6 +4801,16 @@ void Node::onRoundStart(const cs::RoundTable& roundTable, bool updateRound) { csdebug() << " Node key " << KeyToBase58(nodeIdKey_); cslog() << " Last written sequence = " << WithDelimiters(blockChain_.getLastSeq()) << ", neighbour nodes = " << transport_->getNeighboursCount() << (backwardSync_ && backwardSync_->stats().started ? ", oldest stored = " + std::to_string(blockChain_.getFirstStoredSequence()) : std::string{}); + if (forkHeld.has_value()) { + const auto now = std::chrono::steady_clock::now(); + if (lastForkHeldDetail_ == std::chrono::steady_clock::time_point{} || now - lastForkHeldDetail_ >= std::chrono::minutes(1)) { + lastForkHeldDetail_ = now; + cswarning() << " " << cs::forkguard::summary(*forkHeld); + } + } + else { + lastForkHeldDetail_ = {}; + } std::ostringstream line2; diff --git a/csnode/src/poolsynchronizer.cpp b/csnode/src/poolsynchronizer.cpp index 87dbc7298..cb0a4495f 100644 --- a/csnode/src/poolsynchronizer.cpp +++ b/csnode/src/poolsynchronizer.cpp @@ -276,6 +276,11 @@ void PoolSynchronizer::onStoreBlockTimeElapsed() { // //} +cs::Sequence PoolSynchronizer::announcedSequence(const cs::PublicKey& key) const { + const auto it = neighbours_.find(key); + return it == neighbours_.end() ? 0 : it->second; +} + cs::Sequence PoolSynchronizer::getMaxNeighbourSequence() { cs::Sequence maxSequence = 0; for (auto& it : neighbours_) { diff --git a/solver/include/solver/consensus.hpp b/solver/include/solver/consensus.hpp index d22395687..9f1043514 100644 --- a/solver/include/solver/consensus.hpp +++ b/solver/include/solver/consensus.hpp @@ -59,7 +59,7 @@ class Consensus { * longer chain (to be rejected). Purely local: it changes only how far this node unwinds on its * own, never the block contents other nodes see. Set high enough not to interfere with ordinary * micro-forks; raise it (or set allow_deep_rollback) to accept a deeper deliberate switch. */ - constexpr static uint64_t MaxAutoRollbackDepth = 1000; + constexpr static uint64_t MaxAutoRollbackDepth = 100; /** @brief The maximum trusted nodes to take part in consensus */ const static unsigned int MaxTrustedNodes = 25; diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index 9a8ee35de..b203de7cb 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -17,6 +17,16 @@ #include #include #include +#include +#include + +#include +#include + +#include +#include +#include +#include #include #include @@ -324,13 +334,48 @@ TEST(BlockChain, ConsensusSettingsDigestCoversEverySetting) { } namespace { -// a clean default config so the guard reads the default bound (1000) and no deep-rollback override, +// a clean default config so the guard reads the default bound and no deep-rollback override, // independent of any config a prior test in the binary may have set. void installDefaultConfig() { cs::ConfigHolder::instance().setConfig(Config{}); } + +void setEnv(const char* name, const char* value) { +#ifdef _WIN32 + _putenv_s(name, value); // an empty value removes the variable on Windows +#else + ::setenv(name, value, 1); +#endif +} + +void unsetEnv(const char* name) { +#ifdef _WIN32 + _putenv_s(name, ""); +#else + ::unsetenv(name); +#endif +} + +// an empty folder of its own under the system temporary folder, removed with the object +struct TempDir { + std::filesystem::path path; + explicit TempDir(const std::string& name) { + path = std::filesystem::temp_directory_path() / ("csnode-guard-" + name + "-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + std::filesystem::remove_all(path); + std::filesystem::create_directories(path); + } + ~TempDir() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + std::string str() const { return path.string(); } +}; } // namespace +static_assert(DEFAULT_MAX_AUTO_ROLLBACK == Consensus::MaxAutoRollbackDepth, + "the config default and the consensus constant of the rollback bound must stay in step"); +static_assert(Consensus::MaxAutoRollbackDepth == 100, "owner decision: default max_auto_rollback is 100"); + // The longest-chain guard bounds an automatic rollback at Consensus::MaxAutoRollbackDepth below the // head captured when the rollback episode began, and that reference is fixed for the episode. TEST(BlockChain, LongestChainGuardBoundsRollbackDepth) { @@ -341,7 +386,7 @@ TEST(BlockChain, LongestChainGuardBoundsRollbackDepth) { chain.test_resetRollbackEpisode(); const cs::Sequence head = 5000; - const uint64_t depth = Consensus::MaxAutoRollbackDepth; // default 1000 + const uint64_t depth = Consensus::MaxAutoRollbackDepth; chain.test_setLastSequence(head); // the first call captures the high-water reference at the current head @@ -354,16 +399,19 @@ TEST(BlockChain, LongestChainGuardBoundsRollbackDepth) { chain.test_setLastSequence(head - depth + 1); EXPECT_TRUE(chain.guardAutoRollback()); EXPECT_EQ(chain.test_rollbackHighWater(), head); + EXPECT_FALSE(chain.forkHeld()); - // the removal that would take the head to or below the floor is refused + // the removal that would take the head to or below the floor is refused: FORK HELD chain.test_setLastSequence(head - depth); EXPECT_FALSE(chain.guardAutoRollback()); + EXPECT_TRUE(chain.forkHeld()); // refusing sets no permanent state: a fresh episode from a new head works again chain.test_resetRollbackEpisode(); chain.test_setLastSequence(head - depth); EXPECT_TRUE(chain.guardAutoRollback()); EXPECT_EQ(chain.test_rollbackHighWater(), head - depth); + EXPECT_FALSE(chain.forkHeld()); } // Near genesis the bound never underflows: a head below the depth can always be unwound to genesis. @@ -381,30 +429,194 @@ TEST(BlockChain, LongestChainGuardNoUnderflowNearGenesis) { } } -// A node that adopted a community bootstrap table may unwind back to the sequence where it took the -// table, even past the ordinary depth bound; below that point the bound applies again. -TEST(BlockChain, LongestChainGuardCommunityBootstrapFloor) { +// [params] max_auto_rollback and allow_deep_rollback, and the CS_ALLOW_DEEP_ROLLBACK variable +TEST(BlockChain, RollbackGuardConfigParsing) { + const auto parse = [](const std::string& params, uint64_t& depth, bool& allow) { + std::istringstream in("[params]\n" + params); + boost::property_tree::ptree tree; + boost::property_tree::read_ini(in, tree); + Config::readRollbackGuardSettings(tree.get_child("params"), depth, allow); + }; + uint64_t depth = 7; + bool allow = true; + parse("node_type=client\n", depth, allow); + EXPECT_EQ(depth, 100u); + EXPECT_FALSE(allow); + parse("max_auto_rollback=250\nallow_deep_rollback=true\n", depth, allow); + EXPECT_EQ(depth, 250u); + EXPECT_TRUE(allow); + parse("max_auto_rollback=0\nallow_deep_rollback=false\n", depth, allow); + EXPECT_EQ(depth, 0u); + EXPECT_FALSE(allow); + EXPECT_EQ(Config{}.maxAutoRollbackDepth(), 100u); + EXPECT_FALSE(Config{}.allowDeepRollback()); + + unsetEnv("CS_ALLOW_DEEP_ROLLBACK"); + EXPECT_FALSE(BlockChain::deepRollbackEnvSet()); + setEnv("CS_ALLOW_DEEP_ROLLBACK", ""); + EXPECT_FALSE(BlockChain::deepRollbackEnvSet()); // empty: not set + setEnv("CS_ALLOW_DEEP_ROLLBACK", "1"); + EXPECT_TRUE(BlockChain::deepRollbackEnvSet()); + unsetEnv("CS_ALLOW_DEEP_ROLLBACK"); +} + +// allow_deep_rollback (here the environment variable) lifts the bound for one completed episode, +// then it is off for the rest of the run +TEST(BlockChain, DeepRollbackOverrideServesOneEpisode) { installDefaultConfig(); const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); BlockChain chain(genesis, start); chain.test_resetRollbackEpisode(); + setEnv("CS_ALLOW_DEEP_ROLLBACK", "1"); - const cs::Sequence head = 5000; - const cs::Sequence tableSeq = 3500; // deeper than head - 1000 = 4000 - chain.test_setLastSequence(head); - chain.setCommunityBootstrapFloor(tableSeq); - ASSERT_TRUE(chain.guardAutoRollback()); // captures high-water 5000 - - // past the ordinary floor (4000) but still above the table sequence: allowed - chain.test_setLastSequence(4000); - EXPECT_TRUE(chain.guardAutoRollback()); - chain.test_setLastSequence(tableSeq + 1); + chain.test_setLastSequence(5000); + EXPECT_TRUE(chain.deepRollbackApproved()); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4000); // far below the bound: allowed by the override EXPECT_TRUE(chain.guardAutoRollback()); + EXPECT_FALSE(chain.forkHeld()); + + // the episode completes: the override is spent + chain.test_setLastSequence(5001); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.test_rollbackEpisode()); + EXPECT_FALSE(chain.deepRollbackApproved()); + chain.test_setLastSequence(5001); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(5001 - Consensus::MaxAutoRollbackDepth); + EXPECT_FALSE(chain.guardAutoRollback()); + unsetEnv("CS_ALLOW_DEEP_ROLLBACK"); + chain.test_resetRollbackEpisode(); +} - // at or below the table sequence the ordinary bound takes over again and refuses - chain.test_setLastSequence(tableSeq); +// The episode reference is kept next to the database: a restart neither gives a fresh full depth nor +// forgets FORK HELD +TEST(BlockChain, RollbackReferenceSurvivesRestart) { + installDefaultConfig(); + TempDir dir("restart"); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(5000); + chain.setGuardStateDir(dir.str()); + ASSERT_TRUE(chain.guardAutoRollback()); + ASSERT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + chain.test_setLastSequence(4950); + ASSERT_TRUE(chain.guardAutoRollback()); + + // "restart" at head 4950: the reference is still 5000, not 4950 + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(4950); + chain.setGuardStateDir(dir.str()); + EXPECT_TRUE(chain.test_rollbackEpisode()); + EXPECT_EQ(chain.test_rollbackHighWater(), 5000u); + chain.test_setLastSequence(4901); + EXPECT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); EXPECT_FALSE(chain.guardAutoRollback()); + ASSERT_TRUE(chain.forkHeld()); + ASSERT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + + // restart again: still FORK HELD, same record + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(4900); + chain.setGuardStateDir(dir.str()); + EXPECT_TRUE(chain.forkHeld()); + ASSERT_TRUE(chain.forkHeldRecord().has_value()); + EXPECT_EQ(chain.forkHeldRecord()->highWater, 5000u); + EXPECT_EQ(chain.forkHeldRecord()->ourHead, 4900u); + + // the chain climbs above the reference: the episode and FORK HELD end, both files go + chain.test_setLastSequence(5001); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.forkHeld()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + + // a restart above an old reference drops it + ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {4000, false})); + chain.test_resetRollbackEpisode(); + chain.setGuardStateDir(dir.str()); + EXPECT_FALSE(chain.test_rollbackEpisode()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + chain.setGuardStateDir(""); +} + +// the fork record and the episode in their files, and the --resolve-fork plan +TEST(BlockChain, ForkGuardFilesRoundTrip) { + TempDir dir("files"); + EXPECT_EQ(cs::forkguard::stateDirFor("db"), "."); + EXPECT_EQ(cs::forkguard::stateDirFor("db/"), "."); + EXPECT_EQ(cs::forkguard::stateDirFor("/data/db"), "/data"); + + std::string error; + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("no fork record"), std::string::npos); + + cs::forkguard::ForkRecord r; + r.reason = "depth"; + r.sinceUnix = 1795000000; + r.highWater = 5000; + r.ourHead = 4900; + r.forkPoint = 4899; + r.forkPointExact = false; + r.competingSeq = 4901; + r.competingHash = "AB"; + r.peerKey = "key"; + r.peerAddress = "1.2.3.4:9510"; + r.peerHeight = 6000; + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), r)); + // no hash for the fork point: cannot be checked, refused + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("cannot be checked"), std::string::npos); + + r.forkPointHash = "CD"; + r.dismissedUnix = 1795000100; + r.dismissedSeq = 4901; + r.dismissedHash = "AB"; + r.addBranchHash("AB"); + r.addBranchHash("EF"); + r.addBranchHash("AB"); // once only + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), r)); + { + // the Docker healthcheck reads these lines with grep and sed + std::ifstream in(dir.path / cs::forkguard::kForkFile); + const std::string text((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + EXPECT_NE(text.find("\"held\": \"true\""), std::string::npos) << text; + EXPECT_NE(text.find("\"fork_point\": \"4899\""), std::string::npos) << text; + EXPECT_NE(text.find("\"since\": \"2026-"), std::string::npos) << text; + } + const auto back = cs::forkguard::loadForkRecord(dir.str()); + ASSERT_TRUE(back.has_value()); + EXPECT_TRUE(back->held); + EXPECT_EQ(back->reason, "depth"); + EXPECT_EQ(back->highWater, 5000u); + EXPECT_EQ(back->forkPoint, 4899u); + EXPECT_EQ(back->forkPointHash, "CD"); + EXPECT_EQ(back->competingSeq, 4901u); + EXPECT_EQ(back->peerAddress, "1.2.3.4:9510"); + EXPECT_EQ(back->peerHeight, 6000u); + EXPECT_EQ(back->dismissedSeq, 4901u); + EXPECT_EQ(back->branchHashes, (std::vector{"AB", "EF"})); + EXPECT_TRUE(back->inDismissedBranch("XX", "EF")); + EXPECT_FALSE(back->inDismissedBranch("XX", "YY")); + const auto plan = cs::forkguard::planResolve(dir.str(), &error); + ASSERT_TRUE(plan.has_value()); + EXPECT_EQ(plan->top, 4899u); + EXPECT_EQ(plan->hash, "CD"); + + // the detail line names both choices + const auto line = cs::forkguard::summary(r); + EXPECT_NE(line.find("FORK HELD since 2026-"), std::string::npos); + EXPECT_NE(line.find("--resolve-fork"), std::string::npos); + EXPECT_NE(line.find("--set-bc-top 4899"), std::string::npos); + EXPECT_NE(line.find(cs::forkguard::kDismissTrigger), std::string::npos); + EXPECT_NE(line.find("peer height #6000"), std::string::npos); + + cs::forkguard::clearForkRecord(dir.str()); + EXPECT_FALSE(cs::forkguard::loadForkRecord(dir.str()).has_value()); } namespace { @@ -427,12 +639,13 @@ GuardKeys guardKeys(size_t n) { // a block of `seq` whose confidants are `keys`, all of them signing; when `prev` is given it carries // round confirmations by the confidants of `prev` selected in `confirmMask` (bit i = confidant i) csdb::Pool guardBlock(cs::Sequence seq, const GuardKeys& keys, const GuardKeys* prev = nullptr, - uint64_t confirmMask = 0, bool bootstrap = false, bool corruptSignature = false) { - csdb::Pool p(csdb::PoolHash{}, seq); + uint64_t confirmMask = 0, bool bootstrap = false, bool corruptSignature = false, + const csdb::PoolHash& previousHash = csdb::PoolHash{}, const std::string& payload = "payload") { + csdb::Pool p(previousHash, seq); p.set_confidants(keys.pub); p.add_number_trusted(static_cast(keys.pub.size())); p.add_real_trusted((1ull << keys.pub.size()) - 1); - p.add_user_field(200, std::string("payload")); // an id no block field uses + p.add_user_field(200, payload); // an id no block field uses BlockChain::setBootstrap(p, bootstrap); if (prev) { cs::Bytes trustedToHash; @@ -473,13 +686,29 @@ BlockChain& guardChain() { static BlockChain chain(genesis, start); return chain; } + +// the guard chain in a clean state: no episode, no record, no files +BlockChain& freshGuardChain(const std::string& dir = std::string{}) { + installDefaultConfig(); + auto& chain = guardChain(); + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); + chain.setGuardStateDir(dir); + return chain; +} + +csdb::PoolHash someHash(char c) { + csdb::Pool p(csdb::PoolHash{}, 1); + p.add_user_field(1, std::string(10, c)); + EXPECT_TRUE(p.compose()); + return p.hash(); +} } // namespace // A non-bootstrap competing block may make the node unwind only with confirmations by a majority of // the confidants of the node's own last block; stricter than block validation (no one-entry masks). TEST(BlockChain, CompetingBlockNeedsStrictTrustedConfirmations) { - installDefaultConfig(); - auto& chain = guardChain(); + auto& chain = freshGuardChain(); const auto ours = guardKeys(3); const auto next = guardKeys(3); chain.test_setLastBlock(guardBlock(20, ours)); @@ -496,12 +725,24 @@ TEST(BlockChain, CompetingBlockNeedsStrictTrustedConfirmations) { EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(guardBlock(21, next, &strangers, 0b111))); } +// BlockVerifier accepts a bootstrap block without looking at confirmations: hasTrustedConfirmations +// must not, even when the block carries valid confirmations by our confidants +TEST(BlockChain, BootstrapBlockHasNoTrustedConfirmations) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + chain.test_setLastBlock(guardBlock(40, ours)); + ASSERT_TRUE(chain.hasTrustedConfirmations(guardBlock(41, next, &ours, 0b111))); + const auto flagged = guardBlock(41, next, &ours, 0b111, true); + ASSERT_TRUE(BlockChain::isBootstrap(flagged)); + EXPECT_FALSE(chain.hasTrustedConfirmations(flagged)); +} + // The bootstrap flag is a bit anyone can set: a bootstrap-flagged competing block passes the gate // only when it is authenticated locally (valid own signatures AND the confidants of a round table this // node accepted), never on the flag alone, and never more easily than a normal block. TEST(BlockChain, BootstrapFlagAloneNeverTriggersUnwind) { - installDefaultConfig(); - auto& chain = guardChain(); + auto& chain = freshGuardChain(); const auto ours = guardKeys(3); const auto others = guardKeys(3); chain.test_setLastBlock(guardBlock(30, ours)); @@ -536,3 +777,172 @@ TEST(BlockChain, BootstrapFlagAloneNeverTriggersUnwind) { conveyer.setRound(unrelated.round); conveyer.setTable(unrelated); } + +// storeBlock: a competing block the node refuses (not vouched for, or beyond the depth bound) leaves +// the head and the uncertain flag untouched, and raises FORK HELD with a fork record +TEST(BlockChain, RefusedCompetingBlockLeavesTheChainUntouched) { + TempDir dir("store"); + auto& chain = freshGuardChain(dir.str()); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + const auto strangers = guardKeys(3); + + // not vouched for by our confidants + const auto own = guardBlock(20, ours); + chain.test_setLastBlock(own); + auto foreign = guardBlock(21, next, &strangers, 0b111, false, false, someHash('x')); + EXPECT_FALSE(chain.storeBlock(foreign, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 20u); + EXPECT_EQ(chain.getLastHash(), own.hash()); + EXPECT_FALSE(chain.isLastBlockUncertain()); + ASSERT_TRUE(chain.forkHeld()); + auto rec = chain.forkHeldRecord(); + EXPECT_EQ(rec->reason, "authority"); + EXPECT_EQ(rec->competingSeq, 21u); + EXPECT_EQ(rec->competingHash, foreign.hash().to_string()); + EXPECT_EQ(rec->ourHead, 20u); + EXPECT_EQ(rec->forkPoint, 19u); // our #20 is not the competing block's previous one + EXPECT_FALSE(rec->forkPointExact); + EXPECT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + + // vouched for, but beyond the depth bound + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(200); + ASSERT_TRUE(chain.guardAutoRollback()); // reference head #200 + const auto deep = guardBlock(100, ours); + chain.test_setLastBlock(deep); + chain.setBlocksToBeRemoved(1); + auto vouched = guardBlock(101, next, &ours, 0b111, false, false, someHash('y')); + ASSERT_TRUE(chain.competingBlockMayTriggerUnwind(vouched)); + EXPECT_FALSE(chain.storeBlock(vouched, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 100u); + EXPECT_EQ(chain.getLastHash(), deep.hash()); + EXPECT_FALSE(chain.isLastBlockUncertain()); + ASSERT_TRUE(chain.forkHeld()); + EXPECT_EQ(chain.forkHeldRecord()->reason, "depth"); + EXPECT_EQ(chain.forkHeldRecord()->highWater, 200u); + chain.test_resetRollbackEpisode(); +} + +// The operator keeps this chain (the dismiss-fork file): FORK HELD ends, the branch stays refused and +// is not reported again, even after a restart; another branch is reported again +TEST(BlockChain, DismissedForkStaysRefusedAndQuiet) { + TempDir dir("dismiss"); + auto& chain = freshGuardChain(dir.str()); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + const auto strangers = guardKeys(3); + const auto own = guardBlock(50, ours); + chain.test_setLastBlock(own); + + auto branch = guardBlock(51, next, &strangers, 0b111, false, false, someHash('a')); + EXPECT_FALSE(chain.storeBlock(branch, cs::PoolStoreType::Synced)); + ASSERT_TRUE(chain.forkHeld()); + + // no trigger: nothing happens + EXPECT_FALSE(chain.checkDismissTrigger()); + EXPECT_TRUE(chain.forkHeld()); + // the trigger file: dismissed and consumed + { std::ofstream(dir.path / cs::forkguard::kDismissTrigger) << "\n"; } + EXPECT_TRUE(chain.checkDismissTrigger()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kDismissTrigger)); + EXPECT_FALSE(chain.forkHeld()); + const auto rec = cs::forkguard::loadForkRecord(dir.str()); + ASSERT_TRUE(rec.has_value()); + EXPECT_FALSE(rec->held); + EXPECT_EQ(rec->dismissedSeq, 51u); + EXPECT_EQ(rec->dismissedHash, branch.hash().to_string()); + + // the same branch again: still refused, the chain untouched, no FORK HELD + EXPECT_FALSE(chain.storeBlock(branch, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 50u); + EXPECT_EQ(chain.getLastHash(), own.hash()); + EXPECT_FALSE(chain.forkHeld()); + // a trigger with nothing held is consumed and does nothing + { std::ofstream(dir.path / cs::forkguard::kDismissTrigger) << "\n"; } + EXPECT_FALSE(chain.checkDismissTrigger()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kDismissTrigger)); + + // restart: the dismissal is in the record; the branch, continued by its next block, stays quiet + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); + chain.test_setLastBlock(guardBlock(51, ours, nullptr, 0, false, false, own.hash())); + chain.setGuardStateDir(dir.str()); + EXPECT_FALSE(chain.forkHeld()); + ASSERT_TRUE(chain.forkRecord().has_value()); + EXPECT_TRUE(chain.forkRecord()->hasDismissal()); + auto continued = guardBlock(52, next, &strangers, 0b111, false, false, branch.hash()); + EXPECT_FALSE(chain.storeBlock(continued, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 51u); + EXPECT_FALSE(chain.forkHeld()); + + // a different branch is reported again, and the earlier dismissal is kept + auto other = guardBlock(52, next, &strangers, 0b111, false, false, someHash('b'), "other"); + EXPECT_FALSE(chain.storeBlock(other, cs::PoolStoreType::Synced)); + ASSERT_TRUE(chain.forkHeld()); + EXPECT_EQ(chain.forkHeldRecord()->competingHash, other.hash().to_string()); + EXPECT_TRUE(chain.forkHeldRecord()->inDismissedBranch(continued.hash().to_string(), "")); + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); +} + +// The antifork selection stopped by the guard leaves a clean state: no pending request, selection +// finished, antifork mode off, the chain as it was, FORK HELD +TEST(BlockChain, AntiforkSelectionAbandonedByTheGuard) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + chain.test_setLastSequence(300); + ASSERT_TRUE(chain.guardAutoRollback()); // reference head #300 + const auto low = guardBlock(200, ours); + chain.test_setLastBlock(low); + chain.test_setSelectionUnfinished(someHash('s')); + ASSERT_TRUE(chain.getIncorrectBlockNumbers()->empty()); + + chain.cacheLastBlocks(); + EXPECT_TRUE(chain.test_selectionFinished()); + EXPECT_FALSE(chain.test_antiForkMode()); + EXPECT_EQ(chain.test_emittingRequest(), 0); + EXPECT_TRUE(chain.test_lastPrevHash().is_empty()); + EXPECT_TRUE(chain.getIncorrectBlockNumbers()->empty()); + EXPECT_EQ(chain.getLastSeq(), 200u); + EXPECT_EQ(chain.getLastHash(), low.hash()); + EXPECT_TRUE(chain.forkHeld()); + chain.test_resetRollbackEpisode(); +} + +// The start-up integrity repair (blocks of incorrectBlocks_) is a local repair, not a network reorg: +// the guard does not bound it and raises nothing, even deep below an episode's reference +TEST(BlockChain, IntegrityRepairIsNotBoundedByTheGuard) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + chain.test_setLastSequence(300); + ASSERT_TRUE(chain.guardAutoRollback()); // reference head #300: #200 would be refused + chain.test_setLastBlock(guardBlock(200, ours)); + chain.addIncorrectBlockNumber(200); + chain.setBlocksToBeRemoved(1); + + chain.cacheLastBlocks(); + EXPECT_EQ(chain.getLastSeq(), 199u); // the incorrect block was removed + EXPECT_TRUE(chain.getIncorrectBlockNumbers()->empty()); + EXPECT_FALSE(chain.forkHeld()); + chain.test_resetRollbackEpisode(); +} + +// --resolve-fork's approval: the episode it starts lifts the gate and the bound, and ends with it +TEST(BlockChain, ResolveForkApprovalServesOneEpisode) { + TempDir dir("approve"); + auto& chain = freshGuardChain(); + ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {4899, true})); + chain.test_setLastSequence(4899); + chain.setGuardStateDir(dir.str()); + EXPECT_TRUE(chain.deepRollbackApproved()); + chain.test_setLastSequence(4000); + EXPECT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.deepRollbackApproved()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); +} diff --git a/tests/csnode/storage_writer_tests.cpp b/tests/csnode/storage_writer_tests.cpp index ea7abc8ad..a8f9dce75 100644 --- a/tests/csnode/storage_writer_tests.cpp +++ b/tests/csnode/storage_writer_tests.cpp @@ -334,3 +334,66 @@ TEST(StorageWriter, RepairedLastBlockIsForgottenAndRestartReadsItsReplacement) { TEST(StorageWriter, RepairedLastBlockIsForgottenAndRestartReadsItsReplacementAsync) { checkRepairedBlockIsForgotten(true); } + +// --resolve-fork cuts the chain through the --set-bc-top path, but only when the stored block at the +// fork point is the recorded one: then every block above it goes and the fork point stays the last +// block; otherwise nothing is removed and the open fails with TruncateMismatch. +TEST(StorageWriter, ResolveForkTruncatesExactlyAtTheRecordedForkPoint) { + auto db = std::make_shared(); + const auto own = chain(10); + { + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + ASSERT_TRUE(storage.open(opt)); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + ASSERT_TRUE(storage.flush()); + storage.close(); + } + ASSERT_EQ(db->stored(), 10u); + + // a record that no longer matches (another hash at #6): refused, nothing removed + { + csdb::Storage storage; + storage.set_truncate_check(own[5].hash()); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = 6; + EXPECT_FALSE(storage.open(opt)); + EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); + EXPECT_EQ(db->stored(), 10u); + } + // a fork point that is not stored at all: refused too + { + csdb::Storage storage; + storage.set_truncate_check(own[6].hash()); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = 12; + EXPECT_FALSE(storage.open(opt)); + EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); + EXPECT_EQ(db->stored(), 10u); + } + // the recorded block: everything above #6 goes, #6 stays + { + csdb::Storage storage; + storage.set_truncate_check(own[6].hash()); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = 6; + ASSERT_TRUE(storage.open(opt)); + storage.close(); + } + EXPECT_EQ(db->stored(), 7u); + for (cs::Sequence s = 0; s <= 6; ++s) EXPECT_TRUE(db->has(own[s].hash())) << s; + for (cs::Sequence s = 7; s < 10; ++s) EXPECT_FALSE(db->has(own[s].hash())) << s; + + // and the storage opens normally afterwards, ending at the fork point + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + ASSERT_TRUE(storage.open(opt)); + EXPECT_EQ(storage.size(), 7u); + EXPECT_EQ(storage.last_hash(), own[6].hash()); + storage.close(); +} From 2b0bb4dc5a393d546dbed782e3a784b632963bc8 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 07:13:38 +0100 Subject: [PATCH 07/14] packaging, ci: FORK HELD commands and healthcheck; refused unwinds fail a normal run - Docker: resolve-fork, set-bc-top and dismiss-fork commands (docker compose run --rm node ); the healthcheck reports the container unhealthy while fork-held.json holds a fork, with both choices. Entrypoint test covers the commands and the healthcheck. - config.ini: max_auto_rollback default 100, --resolve-fork / --set-bc-top as the remedy, allow_deep_rollback as the last resort. - ci/testnet/run.sh: any refused competing block (UNWINDS REFUSED > 0) fails a normal run, as a stopped reorg already did, unless DEEP_ROLLBACK_EXPECTED=1; FORK HELD nodes are listed. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 27 ++++++++++++++++------- packaging/docker/README.md | 20 +++++++++++++++++ packaging/docker/entrypoint.sh | 28 +++++++++++++++++++++++- packaging/docker/healthcheck.sh | 11 ++++++++++ packaging/docker/test/test-entrypoint.sh | 28 +++++++++++++++++++++++- packaging/mainnet/config.ini | 16 ++++++++------ 6 files changed, 113 insertions(+), 17 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 36d34fdae..a6d982b8f 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2271,16 +2271,27 @@ if [ -n "$forks" ]; then fi # chain reorganisations: how deep each node unwound on its own (community nodes log one line per -# finished rollback episode), and the bound must never stop a normal run. DEEP_ROLLBACK_EXPECTED=1 -# is for a scenario that provokes a reorg deeper than [params] max_auto_rollback on purpose. +# finished rollback episode). In a normal run the guard must never stop a reorg and no competing block +# may be refused (a refusal leaves a lagging node waiting for the operator, FORK HELD). +# DEEP_ROLLBACK_EXPECTED=1 is for a scenario that provokes either on purpose. reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" -echo "UNWINDS REFUSED: $(grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null | wc -l) competing block(s) not vouched for by the node's confidants" -guarded=$(grep -al 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) -if [ -n "$guarded" ] && [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then - grep -ah 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log | head -3 - echo "FAIL: the automatic rollback bound stopped a reorg in a normal run on: $guarded" - exit 1 +refused=$(grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null | wc -l) +echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" +held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) +[ -z "$held" ] || echo "FORK HELD on: $held" +guarded=$(grep -al 'LONGEST-CHAIN GUARD: a competing chain' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) +if [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then + if [ -n "$guarded" ]; then + grep -ah 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log | head -3 + echo "FAIL: the automatic rollback bound stopped a reorg in a normal run on: $guarded" + exit 1 + fi + if [ "$refused" -gt 0 ]; then + grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log | head -3 + echo "FAIL: $refused competing block(s) refused in a normal run (set DEEP_ROLLBACK_EXPECTED=1 for a scenario that provokes it)" + exit 1 + fi fi # a network still restarting at the end is stuck, even if it reached the target before (a bootstrap diff --git a/packaging/docker/README.md b/packaging/docker/README.md index b59686cd3..6a56d0e06 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -84,6 +84,26 @@ password asks again. The password can also be piped: 1 GB of free memory for a few seconds: `CREDITS_MEMORY` must be at least about `1.2g` (default `4g`). A plain key is as exposed as the machine it is on. +## FORK HELD + +When the node refuses a competing chain (a rollback deeper than `max_auto_rollback`, 100 blocks by +default, or blocks not confirmed by the confidants it trusts), it keeps its chain, writes +`fork-held.json` to the data directory, shows `| FORK HELD` on every round line with a detail line +once a minute, and the healthcheck reports the container unhealthy with the two choices. Check with +other nodes or the monitor first, then: + +```sh +# this node's chain is the right one: it keeps refusing that branch and stops reporting it +docker compose exec node touch /data/dismiss-fork # stopped container: docker compose run --rm node dismiss-fork +# the other chain is the right one: cut at the recorded fork point, then follow the other chain +docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d +# or cut by hand at a block of your choice (the last one to keep) +docker compose stop && docker compose run --rm node set-bc-top && docker compose up -d +``` + +`resolve-fork` removes nothing when there is no record or the block at the recorded fork point is no +longer the recorded one. See "If your node reports FORK HELD" in `docs/quick-start.md`. + ## Changing the storage | Change | Result | diff --git a/packaging/docker/entrypoint.sh b/packaging/docker/entrypoint.sh index 48ccb8f88..3004c7dcf 100755 --- a/packaging/docker/entrypoint.sh +++ b/packaging/docker/entrypoint.sh @@ -58,7 +58,33 @@ case "${1:-}" in # configuration or database [ -f keys/NodePrivate.txt ] || { log "no $DATA/keys/NodePrivate.txt: copy NodePrivate.txt and NodePublic.txt into the keys folder (~/.credits/keys on the host), owned by uid 1000, then run decrypt-key again"; exit 2; } exec "$OPT/node" --decryptkey --db-path db --public-key-file keys/NodePublic.txt --private-key-file keys/NodePrivate.txt ;; - *) log "unknown command '$1': the only command is decrypt-key (docker compose run --rm node decrypt-key)"; exit 2 ;; + resolve-fork|set-bc-top) + # a node that reports FORK HELD, the other chain being the right one (check with other nodes or + # the monitor first): resolve-fork cuts the chain at the fork point the node recorded in + # fork-held.json (only if the block there is still the recorded one), set-bc-top at a + # block given by hand; the node then exits and syncs from there at the next start. Stop the + # container first: docker compose stop && docker compose run --rm node resolve-fork + [ -f config.ini ] && [ -d db ] || { log "no config.ini or db in $DATA: nothing to cut"; exit 2; } + if [ "$1" = resolve-fork ]; then + node_args=(--resolve-fork) + else + [[ "${2:-}" =~ ^[0-9]+$ ]] || { log "set-bc-top needs the sequence of the last block to keep: docker compose run --rm node set-bc-top "; exit 2; } + node_args=(--set-bc-top "$2") + fi + mkdir -p log + rc=0 + "$OPT/node" --db-path db --config-file config.ini --public-key-file keys/NodePublic.txt \ + --private-key-file keys/NodePrivate.txt "${node_args[@]}" || rc=$? + [ "$rc" = 0 ] && log "done: start the node again with docker compose up -d" || log "$1 failed (exit $rc): nothing to do or nothing removed, see the lines above" + exit "$rc" ;; + dismiss-fork) + # a node that reports FORK HELD, this node's chain being the right one: the node keeps refusing + # that branch and stops reporting it. A running node picks the file up within a round + # (docker compose exec node touch /data/dismiss-fork does the same); a stopped one at its start + touch dismiss-fork + log "dismiss-fork left in $DATA: the node dismisses the held fork within a round, or at its next start" + exit 0 ;; + *) log "unknown command '$1': the commands are decrypt-key, resolve-fork, set-bc-top and dismiss-fork (docker compose run --rm node )"; exit 2 ;; esac # a data directory with a database but no config.ini (removed by hand): without CREDITS_DB_BACKEND, # config.ini is written for the backend already there, not for the default diff --git a/packaging/docker/healthcheck.sh b/packaging/docker/healthcheck.sh index 9745b71d5..1b3554d91 100755 --- a/packaging/docker/healthcheck.sh +++ b/packaging/docker/healthcheck.sh @@ -1,5 +1,16 @@ #!/bin/sh # Healthy while the last written block advances: fails when it has not changed for 10 minutes. +# Unhealthy while the node reports FORK HELD (fork-held.json with "held": "true"): it refused a +# competing chain and waits for the operator. +fork=/data/fork-held.json +if [ -f "$fork" ] && grep -q '"held": "true"' "$fork"; then + at=$(sed -nE 's/.*"fork_point": "([0-9]+)".*/\1/p' "$fork") + since=$(sed -nE 's/.*"since": "([^"]+)".*/\1/p' "$fork") + echo "FORK HELD since $since: the node refused a competing chain (fork point #$at) and keeps its own." + echo "Check with other nodes or the monitor first. Our chain is right: docker compose exec node touch /data/dismiss-fork." + echo "The other chain is right: docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d" + exit 1 +fi # the two newest log files, the older first: with the newest first, a rotated log made the last line # come from the old file, a block of hours ago (unhealthy while the node was at the head) seq=$(cat $(ls -tr /data/log/log_*.txt 2>/dev/null | tail -2) 2>/dev/null \ diff --git a/packaging/docker/test/test-entrypoint.sh b/packaging/docker/test/test-entrypoint.sh index 51a2bcd97..94cf7975c 100755 --- a/packaging/docker/test/test-entrypoint.sh +++ b/packaging/docker/test/test-entrypoint.sh @@ -3,7 +3,8 @@ # wallet on the PATH with its data on the volume, the token cache on request (CREDITS_TOKENS_CACHE), # the snapshot download on request (--download-snapshot) and the [snapshot] settings, for pruned and # full nodes, the refusal after a snapshot mismatch, storage changes, and a key encrypted by the node -# (the start stops, the decrypt-key command). +# (the start stops, the decrypt-key command), the fork commands (resolve-fork, set-bc-top, +# dismiss-fork) and the FORK HELD healthcheck. # Usage: test-entrypoint.sh set -euo pipefail here=$(cd "$(dirname "$0")/.." && pwd) @@ -297,4 +298,29 @@ set +e; out=$(command_run "$d" frobnicate); rc=$?; set -e [ "$rc" = 2 ] && grep -q "unknown command 'frobnicate'" <<<"$out" || { echo "$out"; fail "unknown command: exit $rc"; } pass "an encrypted key stops the start; decrypt-key makes it plain" +# 18. FORK HELD: resolve-fork and set-bc-top run the node once with the cut and exit, dismiss-fork +# leaves the trigger file for the node; the healthcheck is unhealthy while fork-held.json holds a fork +d=$(newdata 18) +run "$d" -e CREDITS_SNAPSHOT=off >/dev/null +mkdir -p "$d/db" +fork_run() { local data=$1; shift; docker run --rm --label credits-entrypoint-test --user "$(id -u)" -e STUB_SECONDS=0 -v "$data:/data" "$image" "$@" 2>&1; } +rm -f "$d/stub-args" +out=$(fork_run "$d" resolve-fork) || { echo "$out"; fail "resolve-fork"; } +grep -q -- '--resolve-fork' "$d/stub-args" && grep -q -- '--config-file config.ini' "$d/stub-args" || fail "resolve-fork arguments: $(cat "$d/stub-args")" +rm -f "$d/stub-args" +out=$(fork_run "$d" set-bc-top 1234) || { echo "$out"; fail "set-bc-top"; } +grep -q -- '--set-bc-top 1234' "$d/stub-args" || fail "set-bc-top arguments: $(cat "$d/stub-args")" +set +e; out=$(fork_run "$d" set-bc-top); rc=$?; set -e +[ "$rc" = 2 ] && grep -q 'set-bc-top needs the sequence' <<<"$out" || { echo "$out"; fail "set-bc-top without a sequence: exit $rc"; } +out=$(fork_run "$d" dismiss-fork) || { echo "$out"; fail "dismiss-fork"; } +[ -e "$d/dismiss-fork" ] || fail "dismiss-fork left no trigger file" +health() { docker run --rm --label credits-entrypoint-test --user "$(id -u)" --entrypoint /opt/credits/healthcheck.sh -v "$1:/data" "$image" 2>&1; } +printf '{\n "held": "true",\n "since": "2026-11-19 10:00:00 UTC",\n "fork_point": "4899"\n}\n' > "$d/fork-held.json" +set +e; out=$(health "$d"); rc=$?; set -e +[ "$rc" = 1 ] && grep -q 'FORK HELD since 2026-11-19 10:00:00 UTC' <<<"$out" && grep -q 'fork point #4899' <<<"$out" \ + && grep -q 'resolve-fork' <<<"$out" && grep -q 'dismiss-fork' <<<"$out" || { echo "$out"; fail "healthcheck with a fork held: exit $rc"; } +printf '{\n "held": "false",\n "fork_point": "4899"\n}\n' > "$d/fork-held.json" +out=$(health "$d") || { echo "$out"; fail "healthcheck with a dismissed fork"; } +pass "resolve-fork, set-bc-top and dismiss-fork; the healthcheck reports FORK HELD" + echo "PASS: entrypoint" diff --git a/packaging/mainnet/config.ini b/packaging/mainnet/config.ini index 8618d2918..ece621622 100644 --- a/packaging/mainnet/config.ini +++ b/packaging/mainnet/config.ini @@ -13,13 +13,15 @@ compatible_version=true ; false: no re-execution while syncing, every state is fetched (no executor needed to sync) sync_contract_execution=true ; longest-chain guard: the most of the node's own confirmed blocks it will roll back automatically -; when it adopts a competing chain during sync. Beyond this depth it keeps its chain, logs an alert -; and waits for the operator (a reorg this deep is a deliberate network switch or a rejected one). -; A competing block must also be confirmed by confidants this node holds. Default 1000; 0 removes -; the depth bound only. Local only, block content is unchanged. To accept a deliberate switch to -; another chain, set allow_deep_rollback=true (or the CS_ALLOW_DEEP_ROLLBACK environment variable) -; and restart; remove it again afterwards. -; max_auto_rollback=1000 +; when it adopts a competing chain during sync. Beyond this depth it keeps its chain, logs an alert, +; reports FORK HELD and waits for the operator (a reorg this deep is a deliberate network switch or a +; rejected one); see "If your node reports FORK HELD" in docs/quick-start.md. A competing block must +; also be confirmed by confidants this node holds. Default 100; 0 removes the depth bound only. Local +; only, block content is unchanged. To follow the other chain prefer node --resolve-fork (or +; --set-bc-top ); allow_deep_rollback=true (or the CS_ALLOW_DEEP_ROLLBACK environment +; variable) is the last resort: it lifts the bound for one completed episode, then the node turns it +; off until the next start; remove it again afterwards. +; max_auto_rollback=100 ; allow_deep_rollback=false [host_input] From f0f28ae03f44e7a00f89ed6fe49bc156ac195869 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 07:13:38 +0100 Subject: [PATCH 08/14] docs: FORK HELD, its two remedies and the default of 100 - quick-start.md: "If your node reports FORK HELD": what it means, check with other nodes or the monitor first, keep this chain (dismiss-fork) or follow the other one (--resolve-fork, --set-bc-top), native and Docker, the override as the last resort. - differences.md: the FORK HELD state, the persisted reference, the remedies, the default of 100, the lagging node that waits for the operator, the integrity repair; the community-bootstrap floor sentence is gone. - CHANGES.md: the #189 row. Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 2 +- docs/differences.md | 76 +++++++++++++++++++++++++++++---------------- docs/quick-start.md | 45 ++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 29 deletions(-) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 23f68b8f4..aefac4f73 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,7 +44,7 @@ lines under the new release. | PR | Change | |---|---| -| #189 | csnode: bound the automatic rollback depth when adopting a competing chain during sync. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 1000), on every automatic removal path, leaves its chain intact and waits for the operator instead of unwinding without limit. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. Configurable depth and an `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` operator override. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes"). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | +| #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | diff --git a/docs/differences.md b/docs/differences.md index 3f8bce1fa..942c0ba11 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -270,15 +270,17 @@ consensus rules. its current head, it rolls back its own blocks down to the fork point before taking the other chain. The official node puts no limit on that depth. The community node stops the automatic rollback once it would unwind more than `max_auto_rollback` of its own confirmed blocks (default - 1000) below the head it had when the rollback started: it leaves its chain intact, logs a prominent + 100) below the head it had when the rollback started: it leaves its chain intact, logs a prominent alert and waits for the operator. The bound covers every automatic removal path — the sync fork path, the uncertain-block replacement, the antifork cache and the bad-hash reply — because the depth check lives inside the two functions they all go through. The depth is measured against a fixed reference, so forward progress cannot reset it and a node cannot be made to creep past the - limit by alternating removals and appends. A reorg this deep is a rare event — a deliberate switch - to another network, or a rejected one — and the choice is left to a human instead of happening - silently. The node keeps running and does not crash-loop; it simply stays on its chain until an - operator decides. + limit by alternating removals and appends; the reference is kept in `reorg-episode.json` next to + the database, so a restart does not reset it either. A reorg this deep is a rare event — a + deliberate switch to another network, or a rejected one — and the choice is left to a human + instead of happening silently. The node keeps running and does not crash-loop. The start-up + integrity repair (blocks of the node's own database that fail the hash check, removed when the + operator chooses to resolve them) is a local repair, not a reorg, and is not bounded. Before any of this, the competing block itself must be vouched for by authority the node already recognises, or the node does not unwind at all, not even by one block. A competing block that is @@ -286,32 +288,52 @@ consensus rules. own last block, checked as the block verifier checks a block from the network, but more strictly (no one-entry confirmation masks). An honest micro-fork passes, because the honest competing block at the same height was produced by the same elected trusted group. A **bootstrap block** carries no - such confirmations, and its bootstrap flag alone proves nothing, so it passes only when it is - authenticated locally: its own signatures verify and its confidants are exactly those of a round - table this node itself accepted in the last few rounds. Otherwise the node keeps its chain and the - operator decides. Everything that passes is still subject to the depth bound. + such confirmations (the verifier accepts it without them, so this check refuses it), and its + bootstrap flag alone proves nothing, so it passes only when it is authenticated locally: its own + signatures verify and its confidants are exactly those of a round table this node itself accepted + in the last few rounds. Otherwise the node keeps its chain and the operator decides. Everything + that passes is still subject to the depth bound. + + **FORK HELD.** Whenever the node refuses to unwind (beyond the bound, or a competing block not + vouched for), it records the fork in `fork-held.json` next to the database — the fork point (the + highest of its blocks the other chain can share, with its hash, exact when the competing block + names it as its previous block), its head, the refused block, the peer and its announced height, + the time and the episode's reference — and reports **FORK HELD** until the episode ends or the + operator decides: `| FORK HELD` on every round line (`R- NORMAL | FORK HELD`), a detail line + with the two remedies once a minute, and an unhealthy Docker healthcheck. The state clears by + itself when the head climbs back above the reference. The operator, after checking with other + nodes or the monitor, either keeps this chain — a `dismiss-fork` file next to the database, which + the running node picks up within a round: the record keeps that branch (its first known block and + the hashes seen since) as dismissed, the node still refuses it but no longer reports it, and a + different branch is reported again — or follows the other one: `--resolve-fork` (no argument) + cuts the chain at the recorded fork point, only if the block there is still the recorded one, and + lets the next start follow the other chain for that one episode, also below the recorded point + when the fork turns out deeper; `--set-bc-top ` does the cut by hand. Both are one-shot + and exit; neither advice is ever "restart", since a restart alone changes nothing. See "If your + node reports FORK HELD" in `docs/quick-start.md`. This is a local safety bound only: it changes how far this node unwinds on its own, never what a block contains or what the node accepts in normal operation, so it is invisible to other nodes and a mixed official/community network behaves exactly as before. Two settings tune it, both in `[params]`: - `max_auto_rollback=` sets the bound (default 1000; 0 removes the depth bound, the authority check - stays), and - `allow_deep_rollback=true` accepts a deliberate switch: it lifts both the depth bound and the - check above, since the other chain's blocks are confirmed by confidants this node does not hold. The - `CS_ALLOW_DEEP_ROLLBACK` environment variable does the same as the latter, for a one-off run. Either - way the node must be restarted to apply the change. - - **After a partition, and the community bootstrap fallback.** When both sides of a split keep - producing blocks, each side's blocks are confirmed by its own confidants. On reconnection a node on - the shorter side does not follow the longer chain on its own, whatever the depth, because that - chain's blocks are not confirmed by any confidants it holds: it keeps its chain and logs it, and the - operator sets `allow_deep_rollback` to switch. The official node unwinds to the longer chain without - asking. A single node or a minority that cannot reach consensus while cut off builds no branch and - simply syncs forward when it reconnects. The opt-in community bootstrap fallback (PR #188, not - merged yet) expects the longer chain to win after a partition; with this change that switch is an - operator decision. The guard keeps a hook for it (`setCommunityBootstrapFloor`): a node that took a - community bootstrap table may unwind back to the sequence where it took it without hitting the depth - bound, but only for blocks that pass the authority check above. + `max_auto_rollback=` sets the bound (default 100; 0 removes the depth bound, the authority check + stays), and `allow_deep_rollback=true` accepts a deliberate switch: it lifts both the depth bound + and the check above, since the other chain's blocks are confirmed by confidants this node does not + hold. The `CS_ALLOW_DEEP_ROLLBACK` environment variable does the same. It is the last resort, after + `--resolve-fork` and `--set-bc-top`: once one rollback episode has completed with it, the node + turns it off for the rest of the run and says so loudly; remove it from the configuration then. + + **After a partition, and a lagging node.** When both sides of a split keep producing blocks, each + side's blocks are confirmed by its own confidants. On reconnection a node on the shorter side does + not follow the longer chain on its own, whatever the depth, because that chain's blocks are not + confirmed by any confidants it holds: it keeps its chain, reports FORK HELD and the operator + decides. The official node unwinds to the longer chain without asking. A single node or a minority + that cannot reach consensus while cut off builds no branch and simply syncs forward when it + reconnects. The same holds for a node that lags behind with a divergent last block it never + replaced (for example, it missed a bootstrap round): when the network's next block is not + confirmed by the confidants of that block, the node waits for the operator instead of unwinding + it. This is the deliberate choice of this change; `--resolve-fork` is the remedy. The opt-in + community bootstrap fallback (PR #188, not merged yet) expects the longer chain to win after a + partition; with this change that switch is an operator decision too. - **Two blocks of one round.** When a trusted node does not get one stage-2 in time, it fetches it with a request that takes exactly as long as the others wait for its stage-3. Depending on a few diff --git a/docs/quick-start.md b/docs/quick-start.md index 8d2b4a6dc..d631b5d04 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -421,7 +421,49 @@ On Windows, type the password in a console window (`node.exe` reads it from the a pipe). The plain key and the `.encrypted-...` copy keep the folder's permissions (Windows has no mode 600): keep the `csnode` folder readable by your user only. -## 11. Troubleshooting +## 11. If your node reports FORK HELD + +``` +=========== R-12'345'678.0 NORMAL | FORK HELD =========== + Last written sequence = 12'345'577, neighbour nodes = 14 + FORK HELD since 2026-11-19 10:04:05 UTC: a competing block is not vouched for by this node's confidants. Fork point: block #12345576 or lower; our head #12345577, refused block #12345578 from 7Q4e... (203.0.113.7:9510), peer height #12349000. The node keeps its chain. What to do: ... +``` + +**What it means.** Another chain asked this node to roll back its own confirmed blocks, and the node +refused: either the rollback would go deeper than `max_auto_rollback` blocks (100 by default), or +the other chain's blocks are not confirmed by the confidants this node trusts. The node keeps its +chain and waits for you. Nothing is lost and nothing was changed. The state shows on every round +line (`| FORK HELD`), as a detail line once a minute, in `fork-held.json` next to the `db` folder +(the `csnode` folder; `~/.credits` in Docker), and as an unhealthy container in `docker ps`. It +clears by itself if the node's chain grows past the point where the rollback started. + +A restart changes nothing: the node remembers the fork and the rollback limit across restarts. + +**1. Check before acting.** Compare the node's head (`Last written sequence`) and the block at the +fork point with other nodes: the [monitor](https://monitor.credits.com), the +[technical Telegram](https://t.me/creditstechnical), or a node you trust +(`credits-wallet status` on it). The question is which chain the network is on. + +**2. Then choose one.** + +| | Your chain is the right one | The other chain is the right one | +|---|---|---| +| What it does | The node keeps its chain. That branch stays refused but is no longer reported; a different branch would be reported again. The node keeps running. | The node removes its blocks above the recorded fork point (only if the block there is still the one it recorded), exits, and at the next start follows the other chain. | +| Native (node running) | in the `csnode` folder: `touch dismiss-fork` (Windows: `type nul > dismiss-fork`); the node picks it up within a round and deletes it | stop the node (`Ctrl+C` once, or `sudo systemctl stop credits-node`), then `./node --config-file config.ini --resolve-fork` (Windows: `node.exe ...`), then start it as usual | +| Docker | `docker compose exec node touch /data/dismiss-fork` | `docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d` | + +`--resolve-fork` refuses, and removes nothing, when there is no fork record or when the block at +the recorded fork point is no longer the recorded one. To cut the chain at a block of your choice +instead, use `--set-bc-top ` (the last block to keep) with the node stopped; Docker: +`docker compose run --rm node set-bc-top `. If the other chain forked deeper than the +block you chose, the node reports FORK HELD again with a lower fork point. + +**Last resort.** `allow_deep_rollback=true` in `[params]` of `config.ini` (or the +`CS_ALLOW_DEEP_ROLLBACK` environment variable) lets the node follow any competing chain without the +limit or the confidants check. It turns itself off after one completed rollback and logs that you +must remove it; remove it then. + +## 12. Troubleshooting | Problem | What to do | |---|---| @@ -431,6 +473,7 @@ mode 600): keep the `csnode` folder readable by your user only. | `the block database is X, config.ini asks for Y` (exit code 3) | `config.ini` names another database than the one in `db/`. Set `db_backend` back to X, or convert on purpose with `./node --convert-db` (Docker: `CREDITS_CONVERT_DB=yes`). See [storage.md](storage.md#conversion). | | Snapshot download stopped | Run it again: `./node --download-snapshot` (node stopped) resumes, or run the set-up again. | | The node stops with exit code 4, `snapshot.mismatch` exists | The restored state did not match the snapshot. Start over with an empty `db/`, `caches/` and `qs/` (Docker: `CREDITS_RESET=1` once). | +| `FORK HELD` on the round lines; the container is unhealthy | See [If your node reports FORK HELD](#11-if-your-node-reports-fork-held). | | Disk full | A full node needs 250 to 300 GB with RocksDB. Free space or move to a bigger disk, or turn the backfill off (`backfill=off` in `[storage]`, `CREDITS_BACKFILL=off` in Docker before the first start). | | Linux: `GLIBC_2.38 not found` | Use the `linux-x64-glibc2.31` package. | | macOS: "cannot be opened" | `xattr -dr com.apple.quarantine csnode` | From a3e382acaa145890a466ecd17db77ef62ad3281c Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 07:26:29 +0100 Subject: [PATCH 09/14] csnode: --resolve-fork approves only the recorded branch --resolve-fork no longer lifts the authority check and the depth bound for the episode after the cut. The cut alone brings the node to the fork point, from which it syncs forward. When the recorded fork point is exact, the recorded competing block right after it, and the block built on it, may replace a block the node stored above the fork point again (resolve-approved.json). Nothing is ever unwound to or below the fork point, and any other competing block meets the usual checks. The approval ends once the node follows that block, goes past that height with another block, or is offered another block at that height. When the fork point is only an upper bound, nothing is approved and the operator is told how to cut by hand. Tests: another branch is still refused after a resolve, the recorded branch replaces the re-stored block, nothing goes below the fork point, and the approval clears. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/blockchain.hpp | 26 +++++-- csnode/include/csnode/forkguardstate.hpp | 18 ++++- csnode/src/blockchain.cpp | 93 ++++++++++++++++------- csnode/src/forkguardstate.cpp | 34 ++++++++- csnode/src/node.cpp | 14 +++- docs/differences.md | 11 ++- docs/quick-start.md | 8 +- tests/csnode/blockchain_tests.cpp | 95 ++++++++++++++++++++---- 8 files changed, 236 insertions(+), 63 deletions(-) diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 10f730aed..53e79d096 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -282,9 +282,9 @@ class BlockChain { bool competingBlockMayTriggerUnwind(const csdb::Pool& competing) const; // The operator has approved a deliberate deep switch: [params] allow_deep_rollback=true or the - // CS_ALLOW_DEEP_ROLLBACK environment variable (until one episode has completed with it: then it is - // off for the rest of the run), or --resolve-fork for the episode it started. It lifts both the - // authority gate and the depth bound. + // CS_ALLOW_DEEP_ROLLBACK environment variable, until one episode has completed with it (then it is + // off for the rest of the run). It lifts both the authority gate and the depth bound. (--resolve-fork + // approves nothing as wide: see approvedBranch_.) bool deepRollbackApproved() const; // CS_ALLOW_DEEP_ROLLBACK is set to a non-empty value static bool deepRollbackEnvSet(); @@ -317,8 +317,13 @@ class BlockChain { void setPeerAddressResolver(std::function resolver); // --resolve-fork: init() truncates to `seq` only if the stored block there has `hash`, then clears - // the fork record and approves following the competing chain for the one episode that follows - void setResolveFork(cs::Sequence seq, const csdb::PoolHash& hash); + // the fork record. When the record's fork point was exact, it also records the competing block + // right after it (`competingHash` at seq + 1) as the one approved branch (resolve-approved.json). + void setResolveFork(cs::Sequence seq, const csdb::PoolHash& hash, const std::string& competingHash = std::string{}); + // the branch approved with --resolve-fork, while it is pending + std::optional approvedBranch() const { + return approvedBranch_; + } // init() refused to truncate: the stored block at the requested sequence has another hash bool truncationRefused() const { return truncationRefused_; @@ -330,13 +335,13 @@ class BlockChain { cs::Sequence test_rollbackHighWater() const { return rollbackHighWater_; } bool test_rollbackEpisode() const { return rollbackEpisode_; } void test_resetRollbackEpisode() { - rollbackEpisode_ = false; rollbackHighWater_ = 0; episodeApprovedSwitch_ = false; deepOverrideSpent_ = false; + rollbackEpisode_ = false; rollbackHighWater_ = 0; approvedBranch_.reset(); deepOverrideSpent_ = false; std::lock_guard l(guardMutex_); forkRecord_.reset(); } // make `block` the node's last block (the deferred one), as getLastBlock() returns it void test_setLastBlock(const csdb::Pool& block) { deferredBlock_ = block; lastSequence_ = block.sequence(); } // let the episode end as if a block above the reference had been recorded - void test_onHeadAdvanced() { onHeadAdvanced(); } + void test_onHeadAdvanced() { onHeadAdvanced(); checkApprovedBranch(); } void test_setSelectionUnfinished(const csdb::PoolHash& lastPrev) { selectionFinished_ = false; lastPrevHash_ = lastPrev; antiForkMode_ = true; emittingRequest_ = 1; } bool test_selectionFinished() const { return selectionFinished_; } bool test_antiForkMode() const { return antiForkMode_; } @@ -712,7 +717,12 @@ public slots: mutable std::mutex guardMutex_; std::optional forkRecord_; std::chrono::steady_clock::time_point lastDismissedLog_{}; // dismissed branch offered again (hourly) - bool episodeApprovedSwitch_ = false; // --resolve-fork approved this episode + // --resolve-fork's approved branch (exact fork point only), pending until the node is past it + std::optional approvedBranch_; + std::string resolveCompetingHash_; + // the approved branch is settled: the node is past its block, or another block took that height + void checkApprovedBranch(); + void dropApprovedBranch(const char* why); bool deepOverrideSpent_ = false; // allow_deep_rollback served one episode: off until restart bool deepOverrideUsed_ = false; // the current episode ran with allow_deep_rollback bool integrityRepair_ = false; // cacheLastBlocks() is removing blocks of incorrectBlocks_ diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp index f691fd5b6..575587724 100644 --- a/csnode/include/csnode/forkguardstate.hpp +++ b/csnode/include/csnode/forkguardstate.hpp @@ -36,11 +36,23 @@ std::string stateDirFor(const std::string& dbPath); struct Episode { cs::Sequence highWater = 0; - // set by --resolve-fork: the operator approved following the competing chain for this one - // episode (no depth bound, no authority check); cleared when the episode ends - bool approvedSwitch = false; }; +// resolve-approved.json, written by --resolve-fork when the fork point was exact: the one branch the +// operator chose, the competing block right after the fork point. Until the node is past it (or at +// the first other block offered at that height), a block of exactly that branch may replace a block +// this node stored above the fork point again; nothing is unwound to or below the fork point, and +// any other competing block meets the usual checks. +inline constexpr const char* kApprovedFile = "resolve-approved.json"; +struct ApprovedBranch { + cs::Sequence forkPoint = 0; + cs::Sequence seq = 0; // forkPoint + 1 + std::string hash; // the recorded competing block +}; +bool saveApprovedBranch(const std::string& dir, const ApprovedBranch& branch); +std::optional loadApprovedBranch(const std::string& dir); +void clearApprovedBranch(const std::string& dir); + struct ForkRecord { // FORK HELD: the node refuses a competing chain and the operator has not decided yet. False once // the operator dismissed it (the record then keeps the dismissal) or the episode ended. diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 87f1072f6..d7460d096 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -277,11 +277,13 @@ bool BlockChain::init( const std::string dir = cs::forkguard::stateDirFor(path); cs::forkguard::clearForkRecord(dir); cs::forkguard::clearEpisode(dir); - if (resolveForkSeq_ == newBlockchainTop) { - // --resolve-fork: the operator approved following the competing chain. If it diverges - // below the recorded fork point (not known exactly), the node unwinds further on its own, - // for this one episode only: it ends once the head is above the fork point again. - cs::forkguard::saveEpisode(dir, {newBlockchainTop, true}); + cs::forkguard::clearApprovedBranch(dir); + if (resolveForkSeq_ == newBlockchainTop && !resolveCompetingHash_.empty()) { + // --resolve-fork with an exact fork point: the recorded competing block right after it is + // the one approved branch. Normally the node simply syncs it forward from the fork point; + // the approval only lets that very branch replace a block stored above the fork point + // again before it arrives. Nothing else is relaxed. + cs::forkguard::saveApprovedBranch(dir, {newBlockchainTop, newBlockchainTop + 1, resolveCompetingHash_}); } return true; } @@ -866,12 +868,16 @@ csdb::Transaction BlockChain::loadTransaction(const csdb::TransactionID& transId bool BlockChain::guardAutoRollback() { beginRollbackEpisode(); - // operator override: accept a deliberate deep switch ([params] allow_deep_rollback, the - // CS_ALLOW_DEEP_ROLLBACK environment variable, or --resolve-fork for this episode) + // after --resolve-fork, never unwind to or below the fork point the operator cut the chain at + if (approvedBranch_.has_value() && lastSequence_.load() <= approvedBranch_->forkPoint) { + recordForkHeld("depth"); + return false; + } + + // operator override: accept a deliberate deep switch ([params] allow_deep_rollback or the + // CS_ALLOW_DEEP_ROLLBACK environment variable), for one completed episode if (deepRollbackApproved()) { - if (!episodeApprovedSwitch_) { - deepOverrideUsed_ = true; // spent when this episode completes - } + deepOverrideUsed_ = true; // spent when this episode completes return true; } const auto* config = cs::ConfigHolder::instance().config(); @@ -899,7 +905,7 @@ void BlockChain::beginRollbackEpisode() { rollbackHighWater_ = tip; // the head we had before this rollback episode began rollbackLowWater_ = tip; deepOverrideUsed_ = false; - cs::forkguard::saveEpisode(guardStateDir_, {rollbackHighWater_, episodeApprovedSwitch_}); + cs::forkguard::saveEpisode(guardStateDir_, {rollbackHighWater_}); } void BlockChain::onHeadAdvanced() { @@ -913,11 +919,6 @@ void BlockChain::onHeadAdvanced() { // one line per finished episode: how deep the node actually unwound (measured in CI) cslog() << kLogPrefix << "rollback episode ended: unwound " << (rollbackHighWater_ - rollbackLowWater_) << " block(s) below #" << WithDelimiters(rollbackHighWater_); - if (episodeApprovedSwitch_) { - episodeApprovedSwitch_ = false; - cswarning() << kLogPrefix << "the switch approved with --resolve-fork is complete: the automatic rollback " - << "bound applies again"; - } if (deepOverrideUsed_ && !deepOverrideSpent_) { deepOverrideSpent_ = true; cserror() << kLogPrefix << "LONGEST-CHAIN GUARD: allow_deep_rollback (or CS_ALLOW_DEEP_ROLLBACK) has served " @@ -1139,9 +1140,10 @@ void BlockChain::setPeerAddressResolver(std::functionhighWater; rollbackLowWater_ = tip; - episodeApprovedSwitch_ = e->approvedSwitch; cswarning() << kLogPrefix << "rollback episode resumed: reference head #" << WithDelimiters(e->highWater) - << ", head #" << WithDelimiters(tip) - << (e->approvedSwitch ? ", following the competing chain as approved with --resolve-fork" : ""); + << ", head #" << WithDelimiters(tip); } else { cs::forkguard::clearEpisode(dir); @@ -1181,19 +1181,48 @@ void BlockChain::setGuardStateDir(const std::string& dir) { cs::forkguard::clearForkRecord(dir); } } + approvedBranch_ = cs::forkguard::loadApprovedBranch(dir); + if (approvedBranch_.has_value()) { + cslog() << kLogPrefix << "branch approved with --resolve-fork: block #" << WithDelimiters(approvedBranch_->seq) + << " (" << approvedBranch_->hash << ") after the fork point #" << WithDelimiters(approvedBranch_->forkPoint); + checkApprovedBranch(); + } // a dismiss-fork file left while the node was stopped checkDismissTrigger(); } +void BlockChain::dropApprovedBranch(const char* why) { + if (!approvedBranch_.has_value()) { + return; + } + cslog() << kLogPrefix << "branch approved with --resolve-fork (block #" << WithDelimiters(approvedBranch_->seq) + << "): " << why << "; the approval ends"; + approvedBranch_.reset(); + cs::forkguard::clearApprovedBranch(guardStateDir_); +} + +void BlockChain::checkApprovedBranch() { + if (!approvedBranch_.has_value()) { + return; + } + const cs::Sequence tip = lastSequence_.load(); + if (tip < approvedBranch_->seq) { + return; + } + if (getHashBySequence(approvedBranch_->seq).to_string() == approvedBranch_->hash) { + dropApprovedBranch("the node follows it"); + } + else if (tip > approvedBranch_->seq) { + dropApprovedBranch("the node went on with another block at that height"); + } +} + bool BlockChain::deepRollbackEnvSet() { const char* v = std::getenv("CS_ALLOW_DEEP_ROLLBACK"); return v != nullptr && *v != '\0'; } bool BlockChain::deepRollbackApproved() const { - if (episodeApprovedSwitch_) { - return true; - } if (deepOverrideSpent_) { return false; } @@ -2400,8 +2429,10 @@ std::optional BlockChain::recordBlock(csdb::Pool& pool, bool isTrust lastSequence_ = deferredBlock_.sequence(); } - // forward progress: ends the rollback episode (and FORK HELD) once the head is above its reference + // forward progress: ends the rollback episode (and FORK HELD) once the head is above its reference, + // and settles the branch approved with --resolve-fork onHeadAdvanced(); + checkApprovedBranch(); //csdetails() << kLogPrefix << "Pool #" << deferredBlock_.sequence() << ": " << cs::Utils::byteStreamToHex(deferredBlock_.to_binary().data(), deferredBlock_.to_binary().size()); emit storeBlockEvent(pool); @@ -2795,7 +2826,19 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { std::optional& c; ~ClearCompeting() { c.reset(); } } clearCompeting{competing_}; - if (!competingBlockMayTriggerUnwind(pool)) { + // after --resolve-fork: a block of exactly the approved branch (its block right after the + // fork point, or the next one built on it) may replace a block stored above the fork point + // again; any other block offered at those heights ends the approval + bool approvedBranchBlock = false; + if (approvedBranch_.has_value()) { + const auto& ab = *approvedBranch_; + approvedBranchBlock = (poolSequence == ab.seq && pool.hash().to_string() == ab.hash) || + (poolSequence == ab.seq + 1 && pool.previous_hash().to_string() == ab.hash); + if (!approvedBranchBlock && (poolSequence == ab.seq || poolSequence == ab.seq + 1)) { + dropApprovedBranch("another competing block was offered at that height"); + } + } + if (!approvedBranchBlock && !competingBlockMayTriggerUnwind(pool)) { // a competing block not vouched for by authority this node recognises (no strict // confirmations by our last block's confidants, or a bootstrap flag that is not // authenticated locally): never unwind our own chain for it, not even by marking our diff --git a/csnode/src/forkguardstate.cpp b/csnode/src/forkguardstate.cpp index 501a5eaf7..dd657b04a 100644 --- a/csnode/src/forkguardstate.cpp +++ b/csnode/src/forkguardstate.cpp @@ -91,7 +91,6 @@ std::string stateDirFor(const std::string& dbPath) { bool saveEpisode(const std::string& dir, const Episode& episode) { ptree tree; tree.put("high_water", episode.highWater); - tree.put("approved_switch", episode.approvedSwitch); return writeTree(dir, kEpisodeFile, tree); } @@ -103,7 +102,6 @@ std::optional loadEpisode(const std::string& dir) { try { Episode e; e.highWater = tree->get("high_water"); - e.approvedSwitch = tree->get("approved_switch", false); return e; } catch (const std::exception&) { @@ -115,6 +113,38 @@ void clearEpisode(const std::string& dir) { removeFile(dir, kEpisodeFile); } +bool saveApprovedBranch(const std::string& dir, const ApprovedBranch& b) { + ptree tree; + tree.put("fork_point", b.forkPoint); + tree.put("block", b.seq); + tree.put("hash", b.hash); + return writeTree(dir, kApprovedFile, tree); +} + +std::optional loadApprovedBranch(const std::string& dir) { + const auto tree = readTree(dir, kApprovedFile); + if (!tree) { + return std::nullopt; + } + try { + ApprovedBranch b; + b.forkPoint = tree->get("fork_point"); + b.seq = tree->get("block"); + b.hash = tree->get("hash"); + if (b.hash.empty() || b.seq != b.forkPoint + 1) { + return std::nullopt; + } + return b; + } + catch (const std::exception&) { + return std::nullopt; + } +} + +void clearApprovedBranch(const std::string& dir) { + removeFile(dir, kApprovedFile); +} + bool ForkRecord::inDismissedBranch(const std::string& hash, const std::string& previous) const { for (const auto& h : branchHashes) { if ((!hash.empty() && h == hash) || (!previous.empty() && h == previous)) { diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 4e26ec6dc..cd89fe71a 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -266,7 +266,16 @@ std::optional Node::resolveForkTop() { cslog() << "--resolve-fork: removing every block above #" << WithDelimiters(plan->top) << " (" << plan->hash << "), our head was #" << WithDelimiters(plan->record.ourHead) << ", if the block at #" << WithDelimiters(plan->top) << " is still the recorded one"; - blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash)); + if (plan->record.forkPointExact && plan->record.competingSeq == plan->top + 1) { + blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash), plan->record.competingHash); + } + else { + cswarning() << "--resolve-fork: the fork point is an upper bound (the other chain diverges at #" + << WithDelimiters(plan->top) << " or lower). If it diverges lower, the node reports FORK HELD " + << "again with a lower fork point: then find the last common block with the monitor or other " + << "nodes and cut there with --set-bc-top ."; + blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash)); + } return plan->top; } @@ -338,8 +347,7 @@ Node::ChainInit Node::initChain() { } if (initConfig->resolveFork()) { cslog() << "--resolve-fork: the chain now ends at block #" << WithDelimiters(newTop) - << " and the fork record is cleared. Start the node as usual: it syncs from there and " - << "follows the other chain (for this one switch, without the automatic rollback bound)."; + << " and the fork record is cleared. Start the node as usual: it syncs from there."; } return ChainInit::StopAfterInit; } diff --git a/docs/differences.md b/docs/differences.md index 942c0ba11..057258467 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -306,9 +306,14 @@ consensus rules. the running node picks up within a round: the record keeps that branch (its first known block and the hashes seen since) as dismissed, the node still refuses it but no longer reports it, and a different branch is reported again — or follows the other one: `--resolve-fork` (no argument) - cuts the chain at the recorded fork point, only if the block there is still the recorded one, and - lets the next start follow the other chain for that one episode, also below the recorded point - when the fork turns out deeper; `--set-bc-top ` does the cut by hand. Both are one-shot + cuts the chain at the recorded fork point, only if the block there is still the recorded one; the + node then syncs forward from there. It relaxes nothing else: when the fork point is exact, only + the recorded competing block right after it (and the block built on it) may replace a block the + node stored above the fork point again, until the node is past that height or another block is + offered there, and nothing is ever unwound to or below the fork point. When the fork point is + only an upper bound and the other chain diverges lower, the node reports FORK HELD again with a + lower fork point; `--set-bc-top ` then cuts by hand at the last common block found with + the monitor or other nodes. Both are one-shot and exit; neither advice is ever "restart", since a restart alone changes nothing. See "If your node reports FORK HELD" in `docs/quick-start.md`. diff --git a/docs/quick-start.md b/docs/quick-start.md index d631b5d04..24be75c16 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -448,15 +448,17 @@ fork point with other nodes: the [monitor](https://monitor.credits.com), the | | Your chain is the right one | The other chain is the right one | |---|---|---| -| What it does | The node keeps its chain. That branch stays refused but is no longer reported; a different branch would be reported again. The node keeps running. | The node removes its blocks above the recorded fork point (only if the block there is still the one it recorded), exits, and at the next start follows the other chain. | +| What it does | The node keeps its chain. That branch stays refused but is no longer reported; a different branch would be reported again. The node keeps running. | The node removes its blocks above the recorded fork point (only if the block there is still the one it recorded) and exits; at the next start it syncs the other chain from there. | | Native (node running) | in the `csnode` folder: `touch dismiss-fork` (Windows: `type nul > dismiss-fork`); the node picks it up within a round and deletes it | stop the node (`Ctrl+C` once, or `sudo systemctl stop credits-node`), then `./node --config-file config.ini --resolve-fork` (Windows: `node.exe ...`), then start it as usual | | Docker | `docker compose exec node touch /data/dismiss-fork` | `docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d` | `--resolve-fork` refuses, and removes nothing, when there is no fork record or when the block at the recorded fork point is no longer the recorded one. To cut the chain at a block of your choice instead, use `--set-bc-top ` (the last block to keep) with the node stopped; Docker: -`docker compose run --rm node set-bc-top `. If the other chain forked deeper than the -block you chose, the node reports FORK HELD again with a lower fork point. +`docker compose run --rm node set-bc-top `. If the detail line says +`Fork point: block #N or lower`, the node knows only an upper bound: should the other chain fork +lower, the node reports FORK HELD again with a lower fork point. Then find the last common block +with the monitor or other nodes and cut there with `--set-bc-top`. **Last resort.** `allow_deep_rollback=true` in `[params]` of `config.ini` (or the `CS_ALLOW_DEEP_ROLLBACK` environment variable) lets the node follow any competing chain without the diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index b203de7cb..e330ec9d0 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -536,7 +536,7 @@ TEST(BlockChain, RollbackReferenceSurvivesRestart) { EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); // a restart above an old reference drops it - ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {4000, false})); + ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {4000})); chain.test_resetRollbackEpisode(); chain.setGuardStateDir(dir.str()); EXPECT_FALSE(chain.test_rollbackEpisode()); @@ -929,20 +929,83 @@ TEST(BlockChain, IntegrityRepairIsNotBoundedByTheGuard) { chain.test_resetRollbackEpisode(); } -// --resolve-fork's approval: the episode it starts lifts the gate and the bound, and ends with it -TEST(BlockChain, ResolveForkApprovalServesOneEpisode) { +// --resolve-fork approves only the recorded branch, and only until it is settled: another competing +// branch is still refused, the recorded one may replace a block stored above the fork point again, +// nothing is ever unwound to or below the fork point, and the approval clears +TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBranch) { TempDir dir("approve"); - auto& chain = freshGuardChain(); - ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {4899, true})); - chain.test_setLastSequence(4899); - chain.setGuardStateDir(dir.str()); - EXPECT_TRUE(chain.deepRollbackApproved()); - chain.test_setLastSequence(4000); - EXPECT_TRUE(chain.guardAutoRollback()); - chain.test_setLastSequence(4900); - chain.test_onHeadAdvanced(); - EXPECT_FALSE(chain.deepRollbackApproved()); - EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); - chain.setGuardStateDir(""); - chain.test_resetRollbackEpisode(); + const auto ours = guardKeys(3); + const auto theirs = guardKeys(3); + const auto strangers = guardKeys(3); + const auto forkBlock = guardBlock(60, ours); // #60, the fork point + const auto recorded = guardBlock(61, theirs, &strangers, 0b111, false, false, forkBlock.hash()); // their #61 + const cs::forkguard::ApprovedBranch approval{60, 61, recorded.hash().to_string()}; + + // a different branch after the resolve: refused as usual (and the approval ends) + { + auto& chain = freshGuardChain(); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(forkBlock); + chain.setGuardStateDir(dir.str()); + ASSERT_TRUE(chain.approvedBranch().has_value()); + EXPECT_FALSE(chain.deepRollbackApproved()); // nothing as wide as allow_deep_rollback + auto other = guardBlock(61, strangers, &theirs, 0b111, false, false, someHash('o')); + EXPECT_FALSE(chain.storeBlock(other, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 60u); + EXPECT_EQ(chain.getLastHash(), forkBlock.hash()); + EXPECT_TRUE(chain.forkHeld()); + EXPECT_FALSE(chain.approvedBranch().has_value()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + } + // nothing is unwound to or below the fork point while the approval is pending, even for the branch + { + auto& chain = freshGuardChain(); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(forkBlock); + chain.setGuardStateDir(dir.str()); + chain.setBlocksToBeRemoved(1); + EXPECT_FALSE(chain.guardAutoRollback()); + chain.removeLastBlock(); + EXPECT_EQ(chain.getLastSeq(), 60u); + EXPECT_EQ(chain.getLastHash(), forkBlock.hash()); + EXPECT_TRUE(chain.approvedBranch().has_value()); + } + // the recorded branch: the node stored its own #61 again; the block built on the recorded #61 + // makes it give that block up (uncertain, then replaced), down to the fork point and no further + { + auto& chain = freshGuardChain(); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + const auto ownAgain = guardBlock(61, ours, &ours, 0b111, false, false, forkBlock.hash()); + chain.test_setLastBlock(ownAgain); + chain.setGuardStateDir(dir.str()); + ASSERT_TRUE(chain.approvedBranch().has_value()); + auto next = guardBlock(62, strangers, &theirs, 0b111, false, false, recorded.hash()); + ASSERT_FALSE(chain.competingBlockMayTriggerUnwind(next)); // the usual gate alone would refuse it + EXPECT_FALSE(chain.storeBlock(next, cs::PoolStoreType::Synced)); + EXPECT_FALSE(chain.forkHeld()); + EXPECT_TRUE(chain.isLastBlockUncertain()); // waits for the recorded #61 + auto replacement = recorded; + EXPECT_FALSE(chain.storeBlock(replacement, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 60u); // own #61 removed, the fork point kept + EXPECT_FALSE(chain.forkHeld()); + EXPECT_TRUE(chain.approvedBranch().has_value()); + EXPECT_FALSE(chain.guardAutoRollback()); // and never below it + + // the recorded #61 becomes the head: the approval clears + chain.test_setLastBlock(recorded); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.approvedBranch().has_value()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + } + // the node goes past that height with another block: the approval clears too + { + auto& chain = freshGuardChain(); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(forkBlock); + chain.setGuardStateDir(dir.str()); + chain.test_setLastBlock(guardBlock(62, ours)); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.approvedBranch().has_value()); + } + freshGuardChain(); } From 51b7f241277d8d20d4854c7470eb22c74be7750b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 07:27:51 +0100 Subject: [PATCH 10/14] docs: CHANGES rows in PR order Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 732e72e27..8fc1410ce 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,8 +44,8 @@ lines under the new release. | PR | Change | |---|---| -| #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | +| #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | From e6a6a9729fd87d35b7d08825833619306beb2519 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 07:40:44 +0100 Subject: [PATCH 11/14] csnode: only the recorded block itself is approved by --resolve-fork The approval no longer lets a block at fork+2 that merely points at the recorded hash skip the authority check. Only the recorded competing block, that very hash, can make the node give up its own block, and only when that block was stored again right after the fork point. The recorded block must build on the same block as the own one. Every other competing block meets the usual checks. An unauthenticated offer no longer cancels the approval or changes any persisted guard state. The approval ends when the node goes on past that height with a block of its own, or when a competing block that passes the usual checks is offered there. Tests: - a forged fork+2 block pointing at the recorded hash is refused, and the own block and the approval are kept; - unauthenticated offers do not cancel the approval; - the recorded block replaces the own one, down to the fork point and never below; - a vouched offer ends the approval. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/forkguardstate.hpp | 11 ++-- csnode/src/blockchain.cpp | 32 ++++++---- docs/differences.md | 8 ++- tests/csnode/blockchain_tests.cpp | 74 ++++++++++++++++-------- 4 files changed, 80 insertions(+), 45 deletions(-) diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp index 575587724..b71914c7b 100644 --- a/csnode/include/csnode/forkguardstate.hpp +++ b/csnode/include/csnode/forkguardstate.hpp @@ -38,11 +38,12 @@ struct Episode { cs::Sequence highWater = 0; }; -// resolve-approved.json, written by --resolve-fork when the fork point was exact: the one branch the -// operator chose, the competing block right after the fork point. Until the node is past it (or at -// the first other block offered at that height), a block of exactly that branch may replace a block -// this node stored above the fork point again; nothing is unwound to or below the fork point, and -// any other competing block meets the usual checks. +// resolve-approved.json, written by --resolve-fork when the fork point was exact: the one block the +// operator chose, the competing block right after the fork point. If the node stores its own block at +// that height again before the recorded one arrives, the recorded block itself (that very hash, built +// on the fork point) makes the node give its own block up, never going to or below the fork point. +// Nothing else bypasses the usual checks. The approval ends once the node stores a block at that +// height and goes on, or a competing block that passes the usual checks is offered there. inline constexpr const char* kApprovedFile = "resolve-approved.json"; struct ApprovedBranch { cs::Sequence forkPoint = 0; diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index d7460d096..506ea3463 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -2775,6 +2775,20 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { if (poolSequence == lastSequence) { csdebug() << kLogPrefix << "poolSequence == lastSequence"; + // after --resolve-fork: the node stored its own block right after the fork point again, and the + // recorded competing block itself (the very hash the operator approved, built on the same + // fork point block as the own one) is offered: give up the own block, down to the fork point and never below, so the + // recorded block is appended next. No other block bypasses the usual checks. + if (approvedBranch_.has_value() && poolSequence == approvedBranch_->seq && + pool.hash().to_string() == approvedBranch_->hash && getLastHash().to_string() != approvedBranch_->hash && + !pool.previous_hash().is_empty() && pool.previous_hash() == getLastBlock().previous_hash()) { + cslog() << kLogPrefix << "the block #" << WithDelimiters(poolSequence) << " approved with --resolve-fork is offered: " + << "giving up the own block at that height"; + competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash()}; + removeLastBlock(); + competing_.reset(); + return false; + } if (isLastBlockUncertain() && pool.sequence() == uncertainSequence_) { cslog() << kLogPrefix << "review replacement for uncertain block " << WithDelimiters(poolSequence); if (pool.hash() == desiredHash_) { @@ -2826,19 +2840,13 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { std::optional& c; ~ClearCompeting() { c.reset(); } } clearCompeting{competing_}; - // after --resolve-fork: a block of exactly the approved branch (its block right after the - // fork point, or the next one built on it) may replace a block stored above the fork point - // again; any other block offered at those heights ends the approval - bool approvedBranchBlock = false; - if (approvedBranch_.has_value()) { - const auto& ab = *approvedBranch_; - approvedBranchBlock = (poolSequence == ab.seq && pool.hash().to_string() == ab.hash) || - (poolSequence == ab.seq + 1 && pool.previous_hash().to_string() == ab.hash); - if (!approvedBranchBlock && (poolSequence == ab.seq || poolSequence == ab.seq + 1)) { - dropApprovedBranch("another competing block was offered at that height"); - } + // after --resolve-fork, only a competing block that passes the usual checks ends the approval + // of the recorded branch; an unauthenticated offer changes nothing + if (approvedBranch_.has_value() && (poolSequence == approvedBranch_->seq || poolSequence == approvedBranch_->seq + 1) && + pool.hash().to_string() != approvedBranch_->hash && competingBlockMayTriggerUnwind(pool)) { + dropApprovedBranch("another block vouched for by this node's confidants was offered at that height"); } - if (!approvedBranchBlock && !competingBlockMayTriggerUnwind(pool)) { + if (!competingBlockMayTriggerUnwind(pool)) { // a competing block not vouched for by authority this node recognises (no strict // confirmations by our last block's confidants, or a bootstrap flag that is not // authenticated locally): never unwind our own chain for it, not even by marking our diff --git a/docs/differences.md b/docs/differences.md index 057258467..2fd5f56a5 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -308,9 +308,11 @@ consensus rules. different branch is reported again — or follows the other one: `--resolve-fork` (no argument) cuts the chain at the recorded fork point, only if the block there is still the recorded one; the node then syncs forward from there. It relaxes nothing else: when the fork point is exact, only - the recorded competing block right after it (and the block built on it) may replace a block the - node stored above the fork point again, until the node is past that height or another block is - offered there, and nothing is ever unwound to or below the fork point. When the fork point is + the recorded competing block right after it — that very hash — may replace a block the node + stored again right after the fork point; nothing is ever unwound to or below the fork point, every + other block meets the usual checks, and an unauthenticated offer changes nothing. The approval + ends once the node goes on past that height, or a competing block that passes the usual checks + is offered there. When the fork point is only an upper bound and the other chain diverges lower, the node reports FORK HELD again with a lower fork point; `--set-bc-top ` then cuts by hand at the last common block found with the monitor or other nodes. Both are one-shot diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index e330ec9d0..04f6bcfc8 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -693,6 +693,9 @@ BlockChain& freshGuardChain(const std::string& dir = std::string{}) { auto& chain = guardChain(); chain.setGuardStateDir(""); chain.test_resetRollbackEpisode(); + cs::forkguard::clearForkRecord(dir); + cs::forkguard::clearEpisode(dir); + cs::forkguard::clearApprovedBranch(dir); chain.setGuardStateDir(dir); return chain; } @@ -929,10 +932,11 @@ TEST(BlockChain, IntegrityRepairIsNotBoundedByTheGuard) { chain.test_resetRollbackEpisode(); } -// --resolve-fork approves only the recorded branch, and only until it is settled: another competing -// branch is still refused, the recorded one may replace a block stored above the fork point again, -// nothing is ever unwound to or below the fork point, and the approval clears -TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBranch) { +// --resolve-fork approves only the recorded block: it alone (that very hash, built on the fork point) +// may replace a block the node stored again right after the fork point; a forged block pointing at it, +// or any other unauthenticated offer, is refused and does not cancel the approval; nothing is ever +// unwound to or below the fork point; the approval clears once the node is past that height +TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBlock) { TempDir dir("approve"); const auto ours = guardKeys(3); const auto theirs = guardKeys(3); @@ -940,10 +944,11 @@ TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBranch) { const auto forkBlock = guardBlock(60, ours); // #60, the fork point const auto recorded = guardBlock(61, theirs, &strangers, 0b111, false, false, forkBlock.hash()); // their #61 const cs::forkguard::ApprovedBranch approval{60, 61, recorded.hash().to_string()}; + const auto ownAgain = guardBlock(61, ours, &ours, 0b111, false, false, forkBlock.hash()); // our #61, stored again - // a different branch after the resolve: refused as usual (and the approval ends) + // unauthenticated offers at #61 and #62: refused as usual, and the approval stays { - auto& chain = freshGuardChain(); + auto& chain = freshGuardChain(dir.str()); ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); chain.test_setLastBlock(forkBlock); chain.setGuardStateDir(dir.str()); @@ -954,12 +959,27 @@ TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBranch) { EXPECT_EQ(chain.getLastSeq(), 60u); EXPECT_EQ(chain.getLastHash(), forkBlock.hash()); EXPECT_TRUE(chain.forkHeld()); - EXPECT_FALSE(chain.approvedBranch().has_value()); - EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + EXPECT_TRUE(chain.approvedBranch().has_value()); + EXPECT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + } + // a forged #62 pointing at the recorded hash, while the node holds its own #61 again: refused, the + // own block kept, the approval kept + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(ownAgain); + chain.setGuardStateDir(dir.str()); + auto forged = guardBlock(62, strangers, &theirs, 0b111, false, false, recorded.hash()); + EXPECT_FALSE(chain.storeBlock(forged, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 61u); + EXPECT_EQ(chain.getLastHash(), ownAgain.hash()); + EXPECT_FALSE(chain.isLastBlockUncertain()); + EXPECT_TRUE(chain.forkHeld()); + EXPECT_TRUE(chain.approvedBranch().has_value()); } - // nothing is unwound to or below the fork point while the approval is pending, even for the branch + // nothing is unwound to or below the fork point while the approval is pending { - auto& chain = freshGuardChain(); + auto& chain = freshGuardChain(dir.str()); ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); chain.test_setLastBlock(forkBlock); chain.setGuardStateDir(dir.str()); @@ -970,36 +990,40 @@ TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBranch) { EXPECT_EQ(chain.getLastHash(), forkBlock.hash()); EXPECT_TRUE(chain.approvedBranch().has_value()); } - // the recorded branch: the node stored its own #61 again; the block built on the recorded #61 - // makes it give that block up (uncertain, then replaced), down to the fork point and no further + // the recorded block: it replaces the own #61, down to the fork point and no further, then the + // node appends it and the approval clears { - auto& chain = freshGuardChain(); + auto& chain = freshGuardChain(dir.str()); ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); - const auto ownAgain = guardBlock(61, ours, &ours, 0b111, false, false, forkBlock.hash()); chain.test_setLastBlock(ownAgain); chain.setGuardStateDir(dir.str()); - ASSERT_TRUE(chain.approvedBranch().has_value()); - auto next = guardBlock(62, strangers, &theirs, 0b111, false, false, recorded.hash()); - ASSERT_FALSE(chain.competingBlockMayTriggerUnwind(next)); // the usual gate alone would refuse it - EXPECT_FALSE(chain.storeBlock(next, cs::PoolStoreType::Synced)); - EXPECT_FALSE(chain.forkHeld()); - EXPECT_TRUE(chain.isLastBlockUncertain()); // waits for the recorded #61 - auto replacement = recorded; - EXPECT_FALSE(chain.storeBlock(replacement, cs::PoolStoreType::Synced)); + chain.setBlocksToBeRemoved(1); + auto offered = recorded; + EXPECT_FALSE(chain.storeBlock(offered, cs::PoolStoreType::Synced)); EXPECT_EQ(chain.getLastSeq(), 60u); // own #61 removed, the fork point kept EXPECT_FALSE(chain.forkHeld()); EXPECT_TRUE(chain.approvedBranch().has_value()); EXPECT_FALSE(chain.guardAutoRollback()); // and never below it - // the recorded #61 becomes the head: the approval clears - chain.test_setLastBlock(recorded); + chain.test_setLastBlock(recorded); // appended as the next block chain.test_onHeadAdvanced(); EXPECT_FALSE(chain.approvedBranch().has_value()); EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); } + // a competing block vouched for by this node's confidants at that height ends the approval + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(ownAgain); + chain.setGuardStateDir(dir.str()); + auto vouched = guardBlock(62, theirs, &ours, 0b111, false, false, someHash('v')); + ASSERT_TRUE(chain.competingBlockMayTriggerUnwind(vouched)); + chain.storeBlock(vouched, cs::PoolStoreType::Synced); + EXPECT_FALSE(chain.approvedBranch().has_value()); + } // the node goes past that height with another block: the approval clears too { - auto& chain = freshGuardChain(); + auto& chain = freshGuardChain(dir.str()); ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); chain.test_setLastBlock(forkBlock); chain.setGuardStateDir(dir.str()); From 3e6aa106d986582d40dae08dc708f1534b4d095b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 09:28:40 +0100 Subject: [PATCH 12/14] ci: count refused unwinds without ending run.sh when there are none With pipefail, a grep that finds nothing ended the script right after REORG DEPTHS, so every run failed. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index a6d982b8f..554ae8700 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2276,7 +2276,7 @@ fi # DEEP_ROLLBACK_EXPECTED=1 is for a scenario that provokes either on purpose. reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" -refused=$(grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null | wc -l) +refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) [ -z "$held" ] || echo "FORK HELD on: $held" From 9be2ac591ad6966fe972576c8816aa5074329f6f Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 20:44:44 +0100 Subject: [PATCH 13/14] csnode: fork guard review fixes (stale files, dismissed resolve, debounce, corrupt files) - M1: the guard files (reorg-episode.json, fork-held.json, resolve-approved.json) are removed when the database is replaced: a new empty database at init, and a successful snapshot import. A restored node no longer resumes an old episode or FORK HELD. - M2: --resolve-fork refuses a dismissed record ("dismissed on : nothing held"), and also an unreadable one. - M3: a refusal for a block not vouched for by our confidants is still immediate, but becomes FORK HELD (files, banner, healthcheck) only if the head stays put, at the second round start after it or a minute later. A head that moves drops it, so a peer on a minority branch cannot make FORK HELD flap. Depth refusals are reported at once. - M4: an existing guard file that cannot be read is logged and fails safe: - reorg-episode.json starts an episode at the head; - an unreadable fork-held.json stays FORK HELD with an "unreadable" record that --resolve-fork cannot act on; - an unreadable resolve-approved.json approves nothing. The files are written to a temporary file, flushed and fsynced, then renamed, and the folder is fsynced. - L1: the dismissed block's hash is never evicted, and the dismissed branch grows only through blocks that pass the authority check. - L2: a 5 s timer on the processor thread polls dismiss-fork and pending holds and logs the FORK HELD detail line, also on a node with no rounds. Tests: - a dismissed record refused by planResolve; - empty and garbage files of all three kinds; - a restart below the floor, and clearAll; - an authority refusal followed by the next block: no files, no flap; - dismissed-hash eviction; - a pruned fork point refused with TruncateMismatch. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/blockchain.hpp | 25 +++- csnode/include/csnode/forkguardstate.hpp | 12 +- csnode/include/csnode/node.hpp | 4 + csnode/src/blockchain.cpp | 160 +++++++++++++++++------ csnode/src/forkguardstate.cpp | 139 ++++++++++++++++---- csnode/src/node.cpp | 41 ++++-- csnode/src/snapshot_importer.cpp | 3 + tests/csnode/blockchain_tests.cpp | 154 +++++++++++++++++++++- tests/csnode/storage_writer_tests.cpp | 22 ++++ 9 files changed, 483 insertions(+), 77 deletions(-) diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 53e79d096..f32da4a8a 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -306,9 +306,18 @@ class BlockChain { // dismissed (still refused, never reported again; another branch is). False if no fork is held. bool dismissFork(); // the dismiss-fork file next to the database: dismissFork() and remove the file. Called every - // round and at start-up; true if a fork was dismissed. + // round, every few seconds from a timer and at start-up; true if a fork was dismissed. bool checkDismissTrigger(); + // A refusal for a competing block not vouched for by our confidants becomes FORK HELD only if the + // head has not moved since, at the second round start after it or a minute later (whichever is + // first); a head that moves drops it. Called at every round start (roundStarted) and from a timer. + // True if FORK HELD was raised now. + bool promotePendingHold(bool roundStarted); + bool forkHoldPending() const { + return pendingHold_.has_value(); + } + // the sync reply being stored comes from this peer, which announced this height (for the fork // record); cleared with clearSyncSource() void setSyncSource(const cs::PublicKey& key, cs::Sequence announcedHeight); @@ -335,7 +344,7 @@ class BlockChain { cs::Sequence test_rollbackHighWater() const { return rollbackHighWater_; } bool test_rollbackEpisode() const { return rollbackEpisode_; } void test_resetRollbackEpisode() { - rollbackEpisode_ = false; rollbackHighWater_ = 0; approvedBranch_.reset(); deepOverrideSpent_ = false; + rollbackEpisode_ = false; rollbackHighWater_ = 0; approvedBranch_.reset(); deepOverrideSpent_ = false; pendingHold_.reset(); std::lock_guard l(guardMutex_); forkRecord_.reset(); } // make `block` the node's last block (the deferred one), as getLastBlock() returns it @@ -730,6 +739,7 @@ public slots: cs::Sequence seq = 0; csdb::PoolHash hash; csdb::PoolHash previous; + bool vouched = false; // passed competingBlockMayTriggerUnwind() }; std::optional competing_; // the competing block being handled (for the record) std::optional> syncSource_; @@ -743,7 +753,18 @@ public slots: void onHeadAdvanced(); // records the refused fork (once per episode) and logs the alert; reason "depth" or "authority". // False (and nothing raised) when the competing block belongs to a branch the operator dismissed. + // An "authority" refusal is only kept pending until promotePendingHold() sees the head stuck. bool recordForkHeld(const char* reason); + cs::forkguard::ForkRecord buildForkRecord(const char* reason); + // FORK HELD now: episode, record, file, alert + void holdFork(cs::forkguard::ForkRecord record); + struct PendingHold { + cs::forkguard::ForkRecord record; + cs::Sequence head = 0; + std::chrono::steady_clock::time_point since; + unsigned rounds = 0; + }; + std::optional pendingHold_; // removeLastBlock()/compromiseLastBlock() may go on: an integrity repair, or the guard allows it bool mayUnwindLastBlock(); std::atomic_bool stop_ = false; diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp index b71914c7b..d3445260d 100644 --- a/csnode/include/csnode/forkguardstate.hpp +++ b/csnode/include/csnode/forkguardstate.hpp @@ -51,7 +51,7 @@ struct ApprovedBranch { std::string hash; // the recorded competing block }; bool saveApprovedBranch(const std::string& dir, const ApprovedBranch& branch); -std::optional loadApprovedBranch(const std::string& dir); +std::optional loadApprovedBranch(const std::string& dir, bool* corrupt = nullptr); void clearApprovedBranch(const std::string& dir); struct ForkRecord { @@ -86,18 +86,22 @@ struct ForkRecord { } // the competing block (hash, previous hash) belongs to a dismissed branch bool inDismissedBranch(const std::string& hash, const std::string& previous) const; - // remember one more block hash of a dismissed branch (the oldest go beyond kMaxBranchHashes) + // remember one more block hash of a dismissed branch (the oldest go beyond kMaxBranchHashes, never + // dismissedHash) void addBranchHash(const std::string& hash); }; // An empty dir turns persistence off (the functions do nothing and load returns nothing). bool saveEpisode(const std::string& dir, const Episode& episode); -std::optional loadEpisode(const std::string& dir); +// *corrupt (if given) is set when the file exists but cannot be read +std::optional loadEpisode(const std::string& dir, bool* corrupt = nullptr); void clearEpisode(const std::string& dir); bool saveForkRecord(const std::string& dir, const ForkRecord& record); -std::optional loadForkRecord(const std::string& dir); +std::optional loadForkRecord(const std::string& dir, bool* corrupt = nullptr); void clearForkRecord(const std::string& dir); +// all three files: the database they describe was replaced (wiped, restored from a snapshot) +void clearAll(const std::string& dir); // --resolve-fork: the fork point to cut the chain at and the hash the stored block there must have, // from the record in dir; nothing (and *error says why) when there is no record or it cannot be diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index c22cfed17..6fe51844c 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -476,6 +476,10 @@ public slots: std::optional resolveForkTop(); // FORK HELD: the time of the last detail line (one a minute at most) std::chrono::steady_clock::time_point lastForkHeldDetail_{}; + void logForkHeldDetail(const std::optional& forkHeld); + // the fork guard's timer (every 5 s, queued to the processor thread): dismiss-fork, pending holds + cs::Timer forkGuardTimer_; + void onForkGuardTick(); // snapshot.json of a restored folder against the digest of the loaded caches (design 6.8); // false with exit code 4 on a mismatch bool verifyAtStart(); diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 506ea3463..deb00f63f 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -299,6 +299,8 @@ bool BlockChain::init( if (successfulQuickStart) { serializationManPtr_->clear(); } + // a new database: the rollback-bound files of a previous one (removed db/) do not apply + cs::forkguard::clearAll(cs::forkguard::stateDirFor(path)); writeGenesisBlock(); } else { @@ -909,6 +911,9 @@ void BlockChain::beginRollbackEpisode() { } void BlockChain::onHeadAdvanced() { + if (pendingHold_.has_value() && lastSequence_.load() != pendingHold_->head) { + pendingHold_.reset(); // a refused block that did not stop the chain: no FORK HELD + } // the rollback episode ends only once the chain climbs back strictly above the head it had when // the episode began, so the depth cannot creep past the limit by alternating removals and appends // below that head @@ -953,35 +958,7 @@ void BlockChain::onHeadAdvanced() { } } -bool BlockChain::recordForkHeld(const char* reason) { - const std::string competingHash = competing_.has_value() ? competing_->hash.to_string() : std::string{}; - const std::string competingPrev = competing_.has_value() ? competing_->previous.to_string() : std::string{}; - { - std::lock_guard l(guardMutex_); - if (forkRecord_.has_value() && forkRecord_->hasDismissal() && competing_.has_value() && - forkRecord_->inDismissedBranch(competingHash, competingPrev)) { - // a branch the operator dismissed: still refused, never FORK HELD again; follow it by its hashes - const size_t before = forkRecord_->branchHashes.size(); - forkRecord_->addBranchHash(competingHash); - const bool grew = forkRecord_->branchHashes.size() != before; - const auto now = std::chrono::steady_clock::now(); - if (lastDismissedLog_ == std::chrono::steady_clock::time_point{} || now - lastDismissedLog_ >= std::chrono::hours(1)) { - lastDismissedLog_ = now; - csinfo() << kLogPrefix << "competing branch at #" << WithDelimiters(forkRecord_->dismissedSeq) << " (" - << forkRecord_->dismissedHash << ") dismissed by the operator on " - << cs::forkguard::utcTime(forkRecord_->dismissedUnix) << "; still refused (block #" - << WithDelimiters(competing_->seq) << ")"; - } - if (grew) { - cs::forkguard::saveForkRecord(guardStateDir_, *forkRecord_); - } - return false; - } - if (forkRecord_.has_value() && forkRecord_->held) { - return true; // one record per episode: the first refusal - } - } - beginRollbackEpisode(); // FORK HELD ends with the episode +cs::forkguard::ForkRecord BlockChain::buildForkRecord(const char* reason) { cs::forkguard::ForkRecord r; { std::lock_guard l(guardMutex_); @@ -992,7 +969,7 @@ bool BlockChain::recordForkHeld(const char* reason) { r.held = true; r.reason = reason; r.sinceUnix = static_cast(std::time(nullptr)); - r.highWater = rollbackHighWater_; + r.highWater = rollbackEpisode_ ? rollbackHighWater_ : lastSequence_.load(); const cs::Sequence tip = lastSequence_.load(); r.ourHead = tip; r.ourHeadHash = getHashBySequence(tip).to_string(); @@ -1004,7 +981,7 @@ bool BlockChain::recordForkHeld(const char* reason) { r.competingHash.clear(); if (competing_.has_value() && competing_->seq > 0) { r.competingSeq = competing_->seq; - r.competingHash = competingHash; + r.competingHash = competing_->hash.to_string(); candidate = std::min(competing_->seq - 1, tip); if (candidate + 1 == competing_->seq) { // the competing block names its previous block: ours at that height is shared or not @@ -1028,6 +1005,59 @@ bool BlockChain::recordForkHeld(const char* reason) { r.peerAddress = peerAddressResolver_(key); } } + return r; +} + +bool BlockChain::recordForkHeld(const char* reason) { + const std::string competingHash = competing_.has_value() ? competing_->hash.to_string() : std::string{}; + const std::string competingPrev = competing_.has_value() ? competing_->previous.to_string() : std::string{}; + { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value() && forkRecord_->hasDismissal() && competing_.has_value() && + forkRecord_->inDismissedBranch(competingHash, competingPrev)) { + // a branch the operator dismissed: still refused, never FORK HELD again. The branch is + // followed by its hashes only through blocks vouched for by our confidants, so blocks + // anyone can forge never grow (or evict from) that list. + bool grew = false; + if (competing_->vouched) { + const size_t before = forkRecord_->branchHashes.size(); + forkRecord_->addBranchHash(competingHash); + grew = forkRecord_->branchHashes.size() != before; + } + const auto now = std::chrono::steady_clock::now(); + if (lastDismissedLog_ == std::chrono::steady_clock::time_point{} || now - lastDismissedLog_ >= std::chrono::hours(1)) { + lastDismissedLog_ = now; + csinfo() << kLogPrefix << "competing branch at #" << WithDelimiters(forkRecord_->dismissedSeq) << " (" + << forkRecord_->dismissedHash << ") dismissed by the operator on " + << cs::forkguard::utcTime(forkRecord_->dismissedUnix) << "; still refused (block #" + << WithDelimiters(competing_->seq) << ")"; + } + if (grew) { + cs::forkguard::saveForkRecord(guardStateDir_, *forkRecord_); + } + return false; + } + if (forkRecord_.has_value() && forkRecord_->held) { + return true; // one record per episode: the first refusal + } + } + if (std::string(reason) == "authority") { + // any sync reply can carry a block not vouched for by our confidants (a peer on a minority + // branch): the block is refused at once, but FORK HELD (files, banner, healthcheck) is + // reported only if the head stays where it was (promotePendingHold), so that it does not flap + if (!pendingHold_.has_value()) { + pendingHold_ = PendingHold{buildForkRecord(reason), lastSequence_.load(), std::chrono::steady_clock::now(), 0}; + } + return true; + } + holdFork(buildForkRecord(reason)); + return true; +} + +void BlockChain::holdFork(cs::forkguard::ForkRecord r) { + pendingHold_.reset(); + beginRollbackEpisode(); // FORK HELD ends with the episode + r.highWater = rollbackHighWater_; { std::lock_guard l(guardMutex_); forkRecord_ = r; @@ -1039,13 +1069,37 @@ bool BlockChain::recordForkHeld(const char* reason) { if (r.reason == "depth") { cserror() << kLogPrefix << "LONGEST-CHAIN GUARD: a competing chain requires rolling back more than " << maxDepth << " of our own confirmed blocks (reference head #" << WithDelimiters(rollbackHighWater_) - << ", now #" << WithDelimiters(tip) << "). Keeping our chain: FORK HELD, operator review required."; + << ", now #" << WithDelimiters(r.ourHead) << "). Keeping our chain: FORK HELD, operator review required."; } - else { + else if (r.reason == "authority") { cserror() << kLogPrefix << "a competing block #" << WithDelimiters(r.competingSeq) - << " is not vouched for by this node's confidants. Keeping our chain: FORK HELD, operator review required."; + << " is not vouched for by this node's confidants and the head has not moved since. Keeping our chain: " + << "FORK HELD, operator review required."; + } + else { + cserror() << kLogPrefix << "the fork record " << cs::forkguard::kForkFile << " could not be read: keeping our " + << "chain, FORK HELD, operator review required."; } cserror() << kLogPrefix << cs::forkguard::summary(r); +} + +bool BlockChain::promotePendingHold(bool roundStarted) { + if (!pendingHold_.has_value()) { + return false; + } + if (lastSequence_.load() != pendingHold_->head) { + pendingHold_.reset(); // the head moved: the refused block was no fork this node is stuck on + return false; + } + if (roundStarted) { + ++pendingHold_->rounds; + } + const bool stuck = pendingHold_->rounds >= 2 || + std::chrono::steady_clock::now() - pendingHold_->since >= std::chrono::minutes(1); + if (!stuck) { + return false; + } + holdFork(pendingHold_->record); return true; } @@ -1149,7 +1203,8 @@ void BlockChain::setResolveFork(cs::Sequence seq, const csdb::PoolHash& hash, co void BlockChain::setGuardStateDir(const std::string& dir) { guardStateDir_ = dir; const cs::Sequence tip = lastSequence_.load(); - if (const auto e = cs::forkguard::loadEpisode(dir)) { + bool corrupt = false; + if (const auto e = cs::forkguard::loadEpisode(dir, &corrupt)) { if (tip <= e->highWater) { rollbackEpisode_ = true; rollbackHighWater_ = e->highWater; @@ -1161,7 +1216,15 @@ void BlockChain::setGuardStateDir(const std::string& dir) { cs::forkguard::clearEpisode(dir); } } - if (auto r = cs::forkguard::loadForkRecord(dir)) { + else if (corrupt) { + // fail safe: an unreadable reference must not lift the bound, so a new episode starts here + cserror() << kLogPrefix << cs::forkguard::kEpisodeFile << " in " << dir << " cannot be read: a rollback " + << "episode starts at the current head #" << WithDelimiters(tip) << " instead"; + rollbackEpisode_ = false; + beginRollbackEpisode(); + } + corrupt = false; + if (auto r = cs::forkguard::loadForkRecord(dir, &corrupt)) { if (!rollbackEpisode_) { r->held = false; // the chain is past that episode } @@ -1181,7 +1244,28 @@ void BlockChain::setGuardStateDir(const std::string& dir) { cs::forkguard::clearForkRecord(dir); } } - approvedBranch_ = cs::forkguard::loadApprovedBranch(dir); + else if (corrupt) { + // the record is lost: report FORK HELD with what is known and keep refusing through the episode + cserror() << kLogPrefix << cs::forkguard::kForkFile << " in " << dir << " cannot be read"; + { + std::lock_guard l(guardMutex_); + forkRecord_.reset(); + } + cs::forkguard::ForkRecord r; + r.reason = "unreadable"; + r.sinceUnix = static_cast(std::time(nullptr)); + r.ourHead = tip; + r.ourHeadHash = getHashBySequence(tip).to_string(); + r.forkPoint = tip; // unknown; no hash, so --resolve-fork refuses and the operator cuts by hand + holdFork(r); + } + corrupt = false; + approvedBranch_ = cs::forkguard::loadApprovedBranch(dir, &corrupt); + if (corrupt) { + cserror() << kLogPrefix << cs::forkguard::kApprovedFile << " in " << dir << " cannot be read: no branch is " + << "approved; removing it"; + cs::forkguard::clearApprovedBranch(dir); + } if (approvedBranch_.has_value()) { cslog() << kLogPrefix << "branch approved with --resolve-fork: block #" << WithDelimiters(approvedBranch_->seq) << " (" << approvedBranch_->hash << ") after the fork point #" << WithDelimiters(approvedBranch_->forkPoint); @@ -2835,7 +2919,7 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { return false; } // the competing block being handled, for the fork record if the node refuses to unwind - competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash()}; + competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash(), competingBlockMayTriggerUnwind(pool)}; struct ClearCompeting { std::optional& c; ~ClearCompeting() { c.reset(); } diff --git a/csnode/src/forkguardstate.cpp b/csnode/src/forkguardstate.cpp index dd657b04a..96398fa1f 100644 --- a/csnode/src/forkguardstate.cpp +++ b/csnode/src/forkguardstate.cpp @@ -6,6 +6,14 @@ #include #include +#include +#ifdef _WIN32 +#include +#else +#include +#include +#endif + #include #include @@ -16,7 +24,32 @@ using boost::property_tree::ptree; namespace { -// a small file written whole: to a temporary name first, then renamed over the old one +// flushes a written file (and, on POSIX, the folder entry of a rename) to the disk +bool syncFile(std::FILE* f) { + if (std::fflush(f) != 0) { + return false; + } +#ifdef _WIN32 + return _commit(_fileno(f)) == 0; +#else + return ::fsync(::fileno(f)) == 0; +#endif +} + +void syncDir(const fs::path& dir) { +#ifndef _WIN32 + const int fd = ::open(dir.c_str(), O_RDONLY); + if (fd >= 0) { + ::fsync(fd); + ::close(fd); + } +#else + (void)dir; +#endif +} + +// a small file written whole: to a temporary name first, flushed to the disk, then renamed over the +// old one, so a crash leaves either the old file or the new one bool writeTree(const std::string& dir, const char* name, const ptree& tree) { if (dir.empty()) { return false; @@ -24,19 +57,22 @@ bool writeTree(const std::string& dir, const char* name, const ptree& tree) { std::error_code ec; const fs::path target = fs::path(dir) / name; const fs::path tmp = fs::path(dir) / (std::string(name) + ".tmp"); + std::string text; try { - std::ofstream out(tmp, std::ios::trunc); - if (!out) { - return false; - } + std::ostringstream out; boost::property_tree::write_json(out, tree); - out.close(); - if (!out) { - fs::remove(tmp, ec); - return false; - } + text = out.str(); } catch (const std::exception&) { + return false; + } + std::FILE* f = std::fopen(tmp.string().c_str(), "wb"); + if (f == nullptr) { + return false; + } + const bool written = std::fwrite(text.data(), 1, text.size(), f) == text.size() && syncFile(f); + const bool closed = std::fclose(f) == 0; + if (!written || !closed) { fs::remove(tmp, ec); return false; } @@ -45,16 +81,28 @@ bool writeTree(const std::string& dir, const char* name, const ptree& tree) { fs::remove(tmp, ec); return false; } + syncDir(fs::path(dir)); return true; } -std::optional readTree(const std::string& dir, const char* name) { +// the file's tree; nothing when it does not exist. A file that exists but cannot be read sets +// *corrupt (an empty file, bad JSON, or larger than any record) +std::optional readTree(const std::string& dir, const char* name, bool* corrupt) { + if (corrupt) { + *corrupt = false; + } if (dir.empty()) { return std::nullopt; } const fs::path path = fs::path(dir) / name; std::error_code ec; - if (!fs::is_regular_file(path, ec) || fs::file_size(path, ec) > 64 * 1024) { + if (!fs::exists(path, ec)) { + return std::nullopt; + } + if (!fs::is_regular_file(path, ec) || fs::file_size(path, ec) > 64 * 1024 || fs::file_size(path, ec) == 0) { + if (corrupt) { + *corrupt = true; + } return std::nullopt; } try { @@ -64,6 +112,9 @@ std::optional readTree(const std::string& dir, const char* name) { return tree; } catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } return std::nullopt; } } @@ -94,8 +145,8 @@ bool saveEpisode(const std::string& dir, const Episode& episode) { return writeTree(dir, kEpisodeFile, tree); } -std::optional loadEpisode(const std::string& dir) { - const auto tree = readTree(dir, kEpisodeFile); +std::optional loadEpisode(const std::string& dir, bool* corrupt) { + const auto tree = readTree(dir, kEpisodeFile, corrupt); if (!tree) { return std::nullopt; } @@ -105,6 +156,9 @@ std::optional loadEpisode(const std::string& dir) { return e; } catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } return std::nullopt; } } @@ -121,8 +175,8 @@ bool saveApprovedBranch(const std::string& dir, const ApprovedBranch& b) { return writeTree(dir, kApprovedFile, tree); } -std::optional loadApprovedBranch(const std::string& dir) { - const auto tree = readTree(dir, kApprovedFile); +std::optional loadApprovedBranch(const std::string& dir, bool* corrupt) { + const auto tree = readTree(dir, kApprovedFile, corrupt); if (!tree) { return std::nullopt; } @@ -132,11 +186,17 @@ std::optional loadApprovedBranch(const std::string& dir) { b.seq = tree->get("block"); b.hash = tree->get("hash"); if (b.hash.empty() || b.seq != b.forkPoint + 1) { + if (corrupt) { + *corrupt = true; + } return std::nullopt; } return b; } catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } return std::nullopt; } } @@ -164,8 +224,13 @@ void ForkRecord::addBranchHash(const std::string& hash) { } } branchHashes.push_back(hash); - if (branchHashes.size() > kMaxBranchHashes) { - branchHashes.erase(branchHashes.begin(), branchHashes.begin() + (branchHashes.size() - kMaxBranchHashes)); + // the oldest go first, but never the dismissed block itself: forged blocks cannot push it out + while (branchHashes.size() > kMaxBranchHashes) { + auto victim = branchHashes.begin(); + if (*victim == dismissedHash && branchHashes.size() > 1) { + ++victim; + } + branchHashes.erase(victim); } } @@ -202,8 +267,8 @@ bool saveForkRecord(const std::string& dir, const ForkRecord& r) { return writeTree(dir, kForkFile, tree); } -std::optional loadForkRecord(const std::string& dir) { - const auto tree = readTree(dir, kForkFile); +std::optional loadForkRecord(const std::string& dir, bool* corrupt) { + const auto tree = readTree(dir, kForkFile, corrupt); if (!tree) { return std::nullopt; } @@ -234,6 +299,9 @@ std::optional loadForkRecord(const std::string& dir) { return r; } catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } return std::nullopt; } } @@ -242,8 +310,22 @@ void clearForkRecord(const std::string& dir) { removeFile(dir, kForkFile); } +void clearAll(const std::string& dir) { + removeFile(dir, kEpisodeFile); + removeFile(dir, kForkFile); + removeFile(dir, kApprovedFile); +} + std::optional planResolve(const std::string& dir, std::string* error) { - const auto record = loadForkRecord(dir); + bool corrupt = false; + const auto record = loadForkRecord(dir, &corrupt); + if (corrupt) { + if (error) { + *error = std::string("the fork record (") + kForkFile + ") in " + dir + + " cannot be read: nothing removed. Use --set-bc-top after checking with other nodes"; + } + return std::nullopt; + } if (!record.has_value()) { if (error) { *error = std::string("no fork record (") + kForkFile + ") in " + dir + @@ -251,6 +333,15 @@ std::optional planResolve(const std::string& dir, std::string* erro } return std::nullopt; } + if (!record->held) { + if (error) { + *error = record->hasDismissal() + ? "the fork was dismissed on " + utcTime(record->dismissedUnix) + + ": nothing held, nothing removed. To cut the chain at a known block, use --set-bc-top " + : std::string("nothing held, nothing removed. To cut the chain at a known block, use --set-bc-top "); + } + return std::nullopt; + } if (record->forkPointHash.empty()) { if (error) { *error = "the fork record has no hash for block #" + std::to_string(record->forkPoint) + @@ -279,7 +370,7 @@ std::string remedy(const ForkRecord& r) { os << "first check the head and the block at #" << r.forkPoint << " with other nodes or the monitor. If this node's " << "chain is the right one: create the file " << kDismissTrigger << " next to the db folder (native: touch " << kDismissTrigger << " in the node folder; Docker: docker compose exec node touch /data/" << kDismissTrigger - << "); the node keeps refusing that branch and stops reporting it. If the other chain is the right one: stop " + << "); the running node picks it up within a few seconds, keeps refusing that branch and stops reporting it. If the other chain is the right one: stop " << "the node and run once node --config-file config.ini --resolve-fork (Docker: docker compose stop && docker " << "compose run --rm node resolve-fork && docker compose up -d), then start it as usual; by hand: --set-bc-top " << r.forkPoint << " (Docker: docker compose run --rm node set-bc-top " << r.forkPoint << ")."; @@ -292,6 +383,10 @@ std::string summary(const ForkRecord& r) { if (r.reason == "authority") { os << "a competing block is not vouched for by this node's confidants"; } + else if (r.reason == "unreadable") { + os << "the fork record " << kForkFile << " could not be read; the fork point is not known (check the chain " + << "with other nodes)"; + } else { os << "a competing chain asks to unwind beyond the automatic bound (reference head #" << r.highWater << ")"; } diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index cd89fe71a..6a11ba977 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -156,6 +156,7 @@ Node::Node(cs::config::Observer& observer) cs::Connector::connect(&poolSynchronizer_->sendRequest, this, &Node::sendBlockRequest); cs::Connector::connect(&snapshotTickTimer_.timeOut, this, &Node::onSnapshotTick); cs::Connector::connect(&backfillTimer_.timeOut, this, &Node::onBackfillTick); + cs::Connector::connect(&forkGuardTimer_.timeOut, this, &Node::onForkGuardTick); globalPublicKey_.fill(0); globalPublicKey_.at(31) = 7U; @@ -250,6 +251,30 @@ bool Node::initCommon() { return true; } +void Node::logForkHeldDetail(const std::optional& forkHeld) { + if (forkHeld.has_value()) { + const auto now = std::chrono::steady_clock::now(); + if (lastForkHeldDetail_ == std::chrono::steady_clock::time_point{} || now - lastForkHeldDetail_ >= std::chrono::minutes(1)) { + lastForkHeldDetail_ = now; + cswarning() << " " << cs::forkguard::summary(*forkHeld); + } + } + else { + lastForkHeldDetail_ = {}; + } +} + +// every few seconds on the processor thread, rounds or not (an isolated node has none): the +// dismiss-fork file, a pending refusal the head did not move past, the FORK HELD detail line +void Node::onForkGuardTick() { + if (!good_ || bootstrapping_) { + return; + } + blockChain_.checkDismissTrigger(); + blockChain_.promotePendingHold(false); + logForkHeldDetail(blockChain_.forkHeldRecord()); +} + std::optional Node::resolveForkTop() { const auto* config = cs::ConfigHolder::instance().config(); if (config->newBlockchainTop()) { @@ -475,6 +500,8 @@ bool Node::initRunning() { initBootstrapRP(initialConfidants_); EventReport::sendRunningStatus(*this, Running::Status::Run); startBackfill(); + // the fork guard: dismiss-fork and pending holds also without rounds (an isolated node) + forkGuardTimer_.start(5000, cs::Timer::Type::Standard, cs::RunPolicy::CallQueuePolicy); return true; } @@ -932,6 +959,7 @@ void Node::stop() { { std::lock_guard lock(backfillTimerMutex_); backfillTimer_.stop(); + forkGuardTimer_.stop(); if (backwardSync_) { backwardSync_->stop(); } @@ -4788,7 +4816,9 @@ void Node::onRoundStart(const cs::RoundTable& roundTable, bool updateRound) { status_ = cs::NodeStatus::Trusted; } // the operator keeps this chain: a dismiss-fork file next to the database, picked up every round + // (and by the fork guard timer); a refusal the head did not move past becomes FORK HELD blockChain_.checkDismissTrigger(); + blockChain_.promotePendingHold(true); // the node refused to unwind for a competing chain and holds its own: on every round line, so it // cannot be missed (the detail, with the two remedies, once a minute below) const auto forkHeld = blockChain_.forkHeldRecord(); @@ -4809,16 +4839,7 @@ void Node::onRoundStart(const cs::RoundTable& roundTable, bool updateRound) { csdebug() << " Node key " << KeyToBase58(nodeIdKey_); cslog() << " Last written sequence = " << WithDelimiters(blockChain_.getLastSeq()) << ", neighbour nodes = " << transport_->getNeighboursCount() << (backwardSync_ && backwardSync_->stats().started ? ", oldest stored = " + std::to_string(blockChain_.getFirstStoredSequence()) : std::string{}); - if (forkHeld.has_value()) { - const auto now = std::chrono::steady_clock::now(); - if (lastForkHeldDetail_ == std::chrono::steady_clock::time_point{} || now - lastForkHeldDetail_ >= std::chrono::minutes(1)) { - lastForkHeldDetail_ = now; - cswarning() << " " << cs::forkguard::summary(*forkHeld); - } - } - else { - lastForkHeldDetail_ = {}; - } + logForkHeldDetail(forkHeld); std::ostringstream line2; diff --git a/csnode/src/snapshot_importer.cpp b/csnode/src/snapshot_importer.cpp index 83849699f..a33ba76eb 100644 --- a/csnode/src/snapshot_importer.cpp +++ b/csnode/src/snapshot_importer.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -334,6 +335,8 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife } syncPath(nodeDir / "snapshot.json", false); syncPath(nodeDir, true); + // the bound on automatic reorganisations kept its files for the database this one replaces + cs::forkguard::clearAll(cs::forkguard::stateDirFor(dbPath.string())); result.ok = true; return result; } diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index 04f6bcfc8..6b04cf97f 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -687,6 +687,12 @@ BlockChain& guardChain() { return chain; } +// an authority refusal becomes FORK HELD when the head stays put over two round starts +void stayStuck(BlockChain& chain) { + EXPECT_FALSE(chain.promotePendingHold(true)); + EXPECT_TRUE(chain.promotePendingHold(true)); +} + // the guard chain in a clean state: no episode, no record, no files BlockChain& freshGuardChain(const std::string& dir = std::string{}) { installDefaultConfig(); @@ -798,6 +804,9 @@ TEST(BlockChain, RefusedCompetingBlockLeavesTheChainUntouched) { EXPECT_EQ(chain.getLastSeq(), 20u); EXPECT_EQ(chain.getLastHash(), own.hash()); EXPECT_FALSE(chain.isLastBlockUncertain()); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); ASSERT_TRUE(chain.forkHeld()); auto rec = chain.forkHeldRecord(); EXPECT_EQ(rec->reason, "authority"); @@ -841,6 +850,9 @@ TEST(BlockChain, DismissedForkStaysRefusedAndQuiet) { auto branch = guardBlock(51, next, &strangers, 0b111, false, false, someHash('a')); EXPECT_FALSE(chain.storeBlock(branch, cs::PoolStoreType::Synced)); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); ASSERT_TRUE(chain.forkHeld()); // no trigger: nothing happens @@ -883,9 +895,13 @@ TEST(BlockChain, DismissedForkStaysRefusedAndQuiet) { // a different branch is reported again, and the earlier dismissal is kept auto other = guardBlock(52, next, &strangers, 0b111, false, false, someHash('b'), "other"); EXPECT_FALSE(chain.storeBlock(other, cs::PoolStoreType::Synced)); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); ASSERT_TRUE(chain.forkHeld()); EXPECT_EQ(chain.forkHeldRecord()->competingHash, other.hash().to_string()); - EXPECT_TRUE(chain.forkHeldRecord()->inDismissedBranch(continued.hash().to_string(), "")); + EXPECT_TRUE(chain.forkHeldRecord()->inDismissedBranch(branch.hash().to_string(), "")); + EXPECT_FALSE(chain.forkHeldRecord()->inDismissedBranch(continued.hash().to_string(), "")); // not vouched for: not added chain.setGuardStateDir(""); chain.test_resetRollbackEpisode(); } @@ -958,6 +974,9 @@ TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBlock) { EXPECT_FALSE(chain.storeBlock(other, cs::PoolStoreType::Synced)); EXPECT_EQ(chain.getLastSeq(), 60u); EXPECT_EQ(chain.getLastHash(), forkBlock.hash()); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); EXPECT_TRUE(chain.forkHeld()); EXPECT_TRUE(chain.approvedBranch().has_value()); EXPECT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); @@ -974,6 +993,9 @@ TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBlock) { EXPECT_EQ(chain.getLastSeq(), 61u); EXPECT_EQ(chain.getLastHash(), ownAgain.hash()); EXPECT_FALSE(chain.isLastBlockUncertain()); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); EXPECT_TRUE(chain.forkHeld()); EXPECT_TRUE(chain.approvedBranch().has_value()); } @@ -1033,3 +1055,133 @@ TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBlock) { } freshGuardChain(); } + +// --resolve-fork acts only on a held fork: a dismissed record is refused, with the remedy +TEST(BlockChain, ResolvePlanRefusesADismissedRecord) { + TempDir dir("plan-dismissed"); + cs::forkguard::ForkRecord r; + r.held = false; + r.reason = "authority"; + r.forkPoint = 99; + r.forkPointHash = "AB"; + r.dismissedUnix = 1795000000; + r.dismissedSeq = 100; + r.dismissedHash = "CD"; + r.addBranchHash("CD"); + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), r)); + std::string error; + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("dismissed on 2026-"), std::string::npos) << error; + EXPECT_NE(error.find("--set-bc-top "), std::string::npos) << error; +} + +// an unreadable (or empty) file of each kind is reported, never read as "nothing there": the episode +// fails safe (a new one at the head), the fork record stays FORK HELD, an approval is dropped +TEST(BlockChain, CorruptGuardFilesFailSafe) { + for (const std::string content : {std::string("{ not json"), std::string()}) { + SCOPED_TRACE(content.empty() ? "empty" : "garbage"); + TempDir dir("corrupt"); + const auto write = [&](const char* name) { std::ofstream(dir.path / name, std::ios::binary) << content; }; + bool corrupt = false; + + write(cs::forkguard::kEpisodeFile); + EXPECT_FALSE(cs::forkguard::loadEpisode(dir.str(), &corrupt).has_value()); + EXPECT_TRUE(corrupt); + write(cs::forkguard::kForkFile); + EXPECT_FALSE(cs::forkguard::loadForkRecord(dir.str(), &corrupt).has_value()); + EXPECT_TRUE(corrupt); + std::string error; + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("cannot be read"), std::string::npos) << error; + write(cs::forkguard::kApprovedFile); + EXPECT_FALSE(cs::forkguard::loadApprovedBranch(dir.str(), &corrupt).has_value()); + EXPECT_TRUE(corrupt); + // a missing file is not corrupt + EXPECT_FALSE(cs::forkguard::loadEpisode((dir.path / "none").string(), &corrupt).has_value()); + EXPECT_FALSE(corrupt); + + auto& chain = freshGuardChain(); + chain.test_setLastSequence(700); + write(cs::forkguard::kEpisodeFile); + write(cs::forkguard::kForkFile); + write(cs::forkguard::kApprovedFile); + chain.setGuardStateDir(dir.str()); + EXPECT_TRUE(chain.test_rollbackEpisode()); // a new episode at the head + EXPECT_EQ(chain.test_rollbackHighWater(), 700u); + const auto episode = cs::forkguard::loadEpisode(dir.str(), &corrupt); + ASSERT_TRUE(episode.has_value()); + EXPECT_EQ(episode->highWater, 700u); + ASSERT_TRUE(chain.forkHeld()); // the lost record: still FORK HELD + EXPECT_EQ(chain.forkHeldRecord()->reason, "unreadable"); + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); // no hash to check + EXPECT_FALSE(chain.approvedBranch().has_value()); // no approval + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + freshGuardChain(); + } +} + +// a restart below the floor of the persisted reference refuses at once; and the files go together +// when the database they describe is replaced +TEST(BlockChain, RestartBelowTheFloorAndCleanup) { + TempDir dir("floor"); + auto& chain = freshGuardChain(); + ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {5000})); + chain.test_setLastSequence(4800); // already below 5000 - 100 + chain.setGuardStateDir(dir.str()); + EXPECT_EQ(chain.test_rollbackHighWater(), 5000u); + EXPECT_FALSE(chain.guardAutoRollback()); + EXPECT_TRUE(chain.forkHeld()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), {10, 11, "AB"})); + for (const char* f : {cs::forkguard::kEpisodeFile, cs::forkguard::kForkFile, cs::forkguard::kApprovedFile}) { + EXPECT_TRUE(std::filesystem::exists(dir.path / f)) << f; + } + cs::forkguard::clearAll(dir.str()); + for (const char* f : {cs::forkguard::kEpisodeFile, cs::forkguard::kForkFile, cs::forkguard::kApprovedFile}) { + EXPECT_FALSE(std::filesystem::exists(dir.path / f)) << f; + } + freshGuardChain(); +} + +// a block not vouched for, refused while the chain goes on: no FORK HELD, no file, no flap +TEST(BlockChain, AuthorityRefusalDoesNotFlap) { + TempDir dir("flap"); + auto& chain = freshGuardChain(dir.str()); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + const auto strangers = guardKeys(3); + const auto own = guardBlock(70, ours); + chain.test_setLastBlock(own); + auto stray = guardBlock(71, next, &strangers, 0b111, false, false, someHash('f')); + EXPECT_FALSE(chain.storeBlock(stray, cs::PoolStoreType::Synced)); // refused at once + EXPECT_TRUE(chain.forkHoldPending()); + EXPECT_FALSE(chain.forkHeld()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + EXPECT_FALSE(chain.promotePendingHold(true)); // one round start: not yet + + // the next block of our chain: the refusal is forgotten + chain.test_setLastBlock(guardBlock(71, ours, &ours, 0b111, false, false, own.hash())); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.forkHoldPending()); + EXPECT_FALSE(chain.promotePendingHold(true)); + EXPECT_FALSE(chain.promotePendingHold(true)); + EXPECT_FALSE(chain.forkHeld()); + for (const char* f : {cs::forkguard::kEpisodeFile, cs::forkguard::kForkFile, cs::forkguard::kApprovedFile}) { + EXPECT_FALSE(std::filesystem::exists(dir.path / f)) << f; + } + freshGuardChain(); +} + +// forged blocks cannot push the dismissed block out of the remembered hashes +TEST(BlockChain, DismissedHashIsNeverEvicted) { + cs::forkguard::ForkRecord r; + r.dismissedHash = "D"; + r.addBranchHash("D"); + for (size_t i = 0; i < cs::forkguard::kMaxBranchHashes * 2; ++i) { + r.addBranchHash("F" + std::to_string(i)); + } + EXPECT_EQ(r.branchHashes.size(), cs::forkguard::kMaxBranchHashes); + EXPECT_TRUE(r.inDismissedBranch("D", "")); + EXPECT_TRUE(r.inDismissedBranch("x", "D")); + EXPECT_FALSE(r.inDismissedBranch("F0", "")); // the oldest others did go +} diff --git a/tests/csnode/storage_writer_tests.cpp b/tests/csnode/storage_writer_tests.cpp index a8f9dce75..b74b52acf 100644 --- a/tests/csnode/storage_writer_tests.cpp +++ b/tests/csnode/storage_writer_tests.cpp @@ -374,6 +374,28 @@ TEST(StorageWriter, ResolveForkTruncatesExactlyAtTheRecordedForkPoint) { EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); EXPECT_EQ(db->stored(), 10u); } + // a fork point below the oldest stored block (pruned): refused too + { + auto pruned = std::make_shared(); + { + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = pruned; + ASSERT_TRUE(storage.open(opt)); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + ASSERT_TRUE(storage.flush()); + storage.close(); + } + ASSERT_TRUE(pruned->remove(own[1].hash().to_binary())); + csdb::Storage storage; + storage.set_truncate_check(own[1].hash()); + csdb::Storage::OpenOptions opt; + opt.db = pruned; + opt.newBlockchainTop = 1; + EXPECT_FALSE(storage.open(opt)); + EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); + EXPECT_EQ(pruned->stored(), 9u); + } // the recorded block: everything above #6 goes, #6 stays { csdb::Storage storage; From 8c767a5138201c56c2221a66c5f7104f1328a217 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 19 Nov 2026 20:47:58 +0100 Subject: [PATCH 14/14] packaging, docs: guard files go with the database; FORK HELD notes Packaging: - snapshot-restore.sh, CREDITS_RESET=1 and the cleanup of an interrupted restore remove reorg-episode.json, fork-held.json and resolve-approved.json with the database. Docs: - quick-start and the Docker README say to delete these files with db/. - FORK HELD is reported only when the head stays put; the trigger file is picked up within a few seconds. - With an encrypted key, resolve-fork and set-bc-top ask the password. - Swarm, Kubernetes and autoheal would restart-loop on the unhealthy check; plain restart: unless-stopped does not. - differences.md and CHANGES.md are updated. Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 2 +- docs/differences.md | 9 +++++++-- docs/quick-start.md | 10 ++++++++-- packaging/docker/README.md | 15 +++++++++++++-- packaging/docker/entrypoint.sh | 9 +++++---- packaging/docker/snapshot-restore.sh | 2 ++ 6 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 8fc1410ce..a8369230a 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -45,7 +45,7 @@ lines under the new release. | PR | Change | |---|---| | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | -| #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | +| #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). A block not vouched for is refused at once but reported only if the head then stays put (no flapping); unreadable guard files fail safe; the files go when the database is replaced. `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | diff --git a/docs/differences.md b/docs/differences.md index 2fd5f56a5..b8e22d884 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -301,9 +301,14 @@ consensus rules. the time and the episode's reference — and reports **FORK HELD** until the episode ends or the operator decides: `| FORK HELD` on every round line (`R- NORMAL | FORK HELD`), a detail line with the two remedies once a minute, and an unhealthy Docker healthcheck. The state clears by - itself when the head climbs back above the reference. The operator, after checking with other + itself when the head climbs back above the reference. A competing block not vouched for is + refused at once, but reported as FORK HELD only when the head stays at the same block after it + (two round starts, or a minute), so a peer on a minority branch cannot make it flap. An unreadable + guard file is reported and fails safe (a new rollback episode at the head; FORK HELD kept), the + files are written with a flush to the disk, and they are removed when the database is replaced (a + new empty database, a snapshot restore). The operator, after checking with other nodes or the monitor, either keeps this chain — a `dismiss-fork` file next to the database, which - the running node picks up within a round: the record keeps that branch (its first known block and + the running node picks up within a few seconds: the record keeps that branch (its first known block and the hashes seen since) as dismissed, the node still refuses it but no longer reports it, and a different branch is reported again — or follows the other one: `--resolve-fork` (no argument) cuts the chain at the recorded fork point, only if the block there is still the recorded one; the diff --git a/docs/quick-start.md b/docs/quick-start.md index 24be75c16..b77e5931d 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -437,7 +437,13 @@ line (`| FORK HELD`), as a detail line once a minute, in `fork-held.json` next t (the `csnode` folder; `~/.credits` in Docker), and as an unhealthy container in `docker ps`. It clears by itself if the node's chain grows past the point where the rollback started. -A restart changes nothing: the node remembers the fork and the rollback limit across restarts. +A restart changes nothing: the node remembers the fork and the rollback limit across restarts, in +`reorg-episode.json`, `fork-held.json` and `resolve-approved.json` next to the `db` folder. They +describe that database: if you delete `db/` by hand to start again, delete these three files too +(a snapshot restore and Docker's `CREDITS_RESET=1` remove them for you). + +A block of another chain that the node refuses while its own chain goes on is not reported: FORK +HELD appears only when the node stays at the same block after the refusal (two rounds, or a minute). **1. Check before acting.** Compare the node's head (`Last written sequence`) and the block at the fork point with other nodes: the [monitor](https://monitor.credits.com), the @@ -449,7 +455,7 @@ fork point with other nodes: the [monitor](https://monitor.credits.com), the | | Your chain is the right one | The other chain is the right one | |---|---|---| | What it does | The node keeps its chain. That branch stays refused but is no longer reported; a different branch would be reported again. The node keeps running. | The node removes its blocks above the recorded fork point (only if the block there is still the one it recorded) and exits; at the next start it syncs the other chain from there. | -| Native (node running) | in the `csnode` folder: `touch dismiss-fork` (Windows: `type nul > dismiss-fork`); the node picks it up within a round and deletes it | stop the node (`Ctrl+C` once, or `sudo systemctl stop credits-node`), then `./node --config-file config.ini --resolve-fork` (Windows: `node.exe ...`), then start it as usual | +| Native (node running) | in the `csnode` folder: `touch dismiss-fork` (Windows: `type nul > dismiss-fork`); the node picks it up within a few seconds and deletes it | stop the node (`Ctrl+C` once, or `sudo systemctl stop credits-node`), then `./node --config-file config.ini --resolve-fork` (Windows: `node.exe ...`), then start it as usual | | Docker | `docker compose exec node touch /data/dismiss-fork` | `docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d` | `--resolve-fork` refuses, and removes nothing, when there is no fork record or when the block at diff --git a/packaging/docker/README.md b/packaging/docker/README.md index 6a56d0e06..0bab1a5cb 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -101,8 +101,19 @@ docker compose stop && docker compose run --rm node resolve-fork && docker compo docker compose stop && docker compose run --rm node set-bc-top && docker compose up -d ``` -`resolve-fork` removes nothing when there is no record or the block at the recorded fork point is no -longer the recorded one. See "If your node reports FORK HELD" in `docs/quick-start.md`. +`resolve-fork` removes nothing when there is no record, the fork was dismissed, or the block at the +recorded fork point is no longer the recorded one. With a key encrypted by the node, `resolve-fork` +and `set-bc-top` start the node, which asks the key's password in the terminal (`docker compose run` +is interactive). See "If your node reports FORK HELD" in `docs/quick-start.md`. + +The files `reorg-episode.json`, `fork-held.json` and `resolve-approved.json` in `~/.credits` describe +the database in `db/`: `CREDITS_RESET=1` and a snapshot restore remove them; if you delete `db/` by +hand, delete them too. + +While a fork is held the container is reported unhealthy. Plain Docker Compose with `restart: +unless-stopped` (this file) does not restart it for that, which is what you want: a restart changes +nothing. Docker Swarm, Kubernetes liveness probes or an autoheal container would restart it in a +loop: there, do not act on this healthcheck, or watch `fork-held.json` instead. ## Changing the storage diff --git a/packaging/docker/entrypoint.sh b/packaging/docker/entrypoint.sh index 3004c7dcf..008979a0b 100755 --- a/packaging/docker/entrypoint.sh +++ b/packaging/docker/entrypoint.sh @@ -79,10 +79,10 @@ case "${1:-}" in exit "$rc" ;; dismiss-fork) # a node that reports FORK HELD, this node's chain being the right one: the node keeps refusing - # that branch and stops reporting it. A running node picks the file up within a round + # that branch and stops reporting it. A running node picks the file up within a few seconds # (docker compose exec node touch /data/dismiss-fork does the same); a stopped one at its start touch dismiss-fork - log "dismiss-fork left in $DATA: the node dismisses the held fork within a round, or at its next start" + log "dismiss-fork left in $DATA: the node dismisses the held fork within a few seconds, or at its next start" exit 0 ;; *) log "unknown command '$1': the commands are decrypt-key, resolve-fork, set-bc-top and dismiss-fork (docker compose run --rm node )"; exit 2 ;; esac @@ -100,7 +100,8 @@ fi rm -rf .restore-tmp .digest-before if [ -e .restore-incomplete ]; then log "an interrupted snapshot restore of an earlier release left part of a snapshot: removing it" - rm -rf db caches qs roundstat.dat smartcontracts.dat snapshot.json snapshot.verified + rm -rf db caches qs roundstat.dat smartcontracts.dat snapshot.json snapshot.verified \ + reorg-episode.json fork-held.json resolve-approved.json rm -f .restore-incomplete fi @@ -175,7 +176,7 @@ if [ "$RESET" = 1 ] && { has_data || [ -e caches ] || [ -e qs ]; }; then old=old-$(date -u +%Y%m%d-%H%M%S) mkdir "$old" for p in db caches qs roundstat.dat smartcontracts.dat snapshot.json snapshot.verified snapshot.mismatch snapshot.sources.json \ - snapshot-tmp snapshot-download snapshot .storage; do + snapshot-tmp snapshot-download snapshot .storage reorg-episode.json fork-held.json resolve-approved.json; do [ ! -e "$p" ] || mv "$p" "$old/" done log "CREDITS_RESET=1: the previous state is in $DATA/$old; remove CREDITS_RESET from .env" diff --git a/packaging/docker/snapshot-restore.sh b/packaging/docker/snapshot-restore.sh index c98825f12..1c6d50787 100755 --- a/packaging/docker/snapshot-restore.sh +++ b/packaging/docker/snapshot-restore.sh @@ -55,6 +55,8 @@ for p in db caches qs roundstat.dat smartcontracts.dat; do [ ! -e "$tmp/x/$p" ] || { rm -rf "${data:?}/$p"; mv "$tmp/x/$p" "$data/$p"; } done mv "$tmp/latest.json" "$data/snapshot.json" +# the node's files about the database this one replaces (rollback bound, FORK HELD) +rm -f "$data/reorg-episode.json" "$data/fork-held.json" "$data/resolve-approved.json" rm -f "$data/.restore-incomplete" rm -rf "$tmp" echo "snapshot restored: block $seq from $file"