diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index a13751d86..0f1ceb2e6 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -121,7 +121,7 @@ jobs: with: name: testnet-node-${{ github.sha }} path: build/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} # mixed nodes, as in testnet.yml: the even nodes run a node built from another ref - name: Build the node under comparison @@ -182,7 +182,7 @@ jobs: with: name: testnet-node-alt-${{ github.sha }} path: build-alt/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} scenario: needs: build @@ -608,4 +608,4 @@ jobs: testnet-run/resources.tsv testnet-run/resources-summary.txt testnet-run/sampler.log - retention-days: 14 + retention-days: ${{ github.ref_name == 'master' && 14 || 3 }} diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index 13960802c..7b6016335 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -170,7 +170,7 @@ jobs: with: name: testnet-node-${{ github.sha }} path: build/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} # the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI test network # needs (its starter key and genesis recipient), for the even nodes @@ -223,7 +223,7 @@ jobs: with: name: testnet-node-official-${{ github.sha }} path: build-alt/client/node - retention-days: 7 + retention-days: ${{ github.ref_name == 'master' && 7 || 3 }} network: needs: [plan, build] @@ -356,4 +356,4 @@ jobs: testnet-run/n*/config.ini testnet-run/resources.tsv testnet-run/resources-summary.txt - retention-days: 14 + retention-days: ${{ github.ref_name == 'master' && 14 || 3 }} diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index b3cb3931e..554ae8700 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -2270,6 +2270,30 @@ if [ -n "$forks" ]; then exit 1 fi +# chain reorganisations: how deep each node unwound on its own (community nodes log one line per +# finished rollback episode). In a normal run the guard must never stop a reorg and no competing block +# may be refused (a refusal leaves a lagging node waiting for the operator, FORK HELD). +# DEEP_ROLLBACK_EXPECTED=1 is for a scenario that provokes either on purpose. +reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true) +echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)" +refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l) +echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants" +held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) +[ -z "$held" ] || echo "FORK HELD on: $held" +guarded=$(grep -al 'LONGEST-CHAIN GUARD: a competing chain' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true) +if [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then + if [ -n "$guarded" ]; then + grep -ah 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log | head -3 + echo "FAIL: the automatic rollback bound stopped a reorg in a normal run on: $guarded" + exit 1 + fi + if [ "$refused" -gt 0 ]; then + grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log | head -3 + echo "FAIL: $refused competing block(s) refused in a normal run (set DEEP_ROLLBACK_EXPECTED=1 for a scenario that provokes it)" + exit 1 + fi +fi + # a network still restarting at the end is stuck, even if it reached the target before (a bootstrap # round, R-x.1 and up, that never gives way to R-(x+1).0): the last three round headers of n1 stuck=$(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -3 | grep -vc '\.0 ' || true) diff --git a/client/config/config.cpp b/client/config/config.cpp index 1d2050636..78a804e5a 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -69,6 +69,8 @@ const std::string PARAM_NAME_COMPATIBLE_VERSION = "compatible_version"; const std::string PARAM_NAME_TRAVERSE_NAT = "traverse_nat"; const std::string PARAM_NAME_SYNC_ON = "sync_on"; const std::string PARAM_NAME_MAX_UNCORRECTED_BLOCK = "max_uncorrected_block"; +const std::string PARAM_NAME_MAX_AUTO_ROLLBACK = "max_auto_rollback"; +const std::string PARAM_NAME_ALLOW_DEEP_ROLLBACK = "allow_deep_rollback"; const std::string PARAM_NAME_GENERATE_FORK = "generate_fork"; const std::string PARAM_NAME_IDLE_MODE = "idle_mode"; const std::string PARAM_NAME_DAEMON_MODE = "daemon_mode"; @@ -133,6 +135,7 @@ const std::string ARG_NAME_PRIVATE_KEY_FILE = "private-key-file"; const std::string ARG_NAME_ENCRYPT_KEY_FILE = "encryptkey"; const std::string ARG_NAME_RECREATE_INDEX = "recreate-index"; const std::string ARG_NAME_NEW_BC_TOP = "set-bc-top"; +const std::string ARG_NAME_RESOLVE_FORK = "resolve-fork"; const std::string ARG_NAME_BAL_CHANGE = "bal-change"; const std::string ARG_NAME_DISABLE_AUTO_SHUTDOWN = "disable-auto-shutdown"; @@ -277,6 +280,7 @@ Config Config::read(po::variables_map& vm, bool atStart) { result.pathToDb_ = getArgFromCmdLine(vm, ARG_NAME_DB_PATH, DEFAULT_PATH_TO_DB); if (atStart) { + result.resolveFork_ = vm.count(ARG_NAME_RESOLVE_FORK) > 0; if (vm.count(ARG_NAME_NEW_BC_TOP)) { result.newBlockchainTop_ = true; result.newBlockchainTopSeq_ = @@ -997,6 +1001,7 @@ Config Config::readFromFile(const std::string& fileName) { } result.maxUncorrectedBlock_ = params.count(PARAM_NAME_MAX_UNCORRECTED_BLOCK) ? params.get(PARAM_NAME_MAX_UNCORRECTED_BLOCK) : DEFAULT_MAX_UNCORRECTED_BLOCK; + readRollbackGuardSettings(params, result.maxAutoRollbackDepth_, result.allowDeepRollback_); result.generateFork_ = params.count(PARAM_NAME_GENERATE_FORK) ? params.get(PARAM_NAME_GENERATE_FORK) : false; if (params.count(PARAM_NAME_SYNC_ON)) { @@ -1201,6 +1206,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) { } } +void Config::readRollbackGuardSettings(const boost::property_tree::ptree& params, uint64_t& maxDepth, bool& allowDeep) { + maxDepth = params.count(PARAM_NAME_MAX_AUTO_ROLLBACK) ? params.get(PARAM_NAME_MAX_AUTO_ROLLBACK) : DEFAULT_MAX_AUTO_ROLLBACK; + allowDeep = params.count(PARAM_NAME_ALLOW_DEEP_ROLLBACK) ? params.get(PARAM_NAME_ALLOW_DEEP_ROLLBACK) : false; +} + void Config::readBackfillSettings(const boost::property_tree::ptree& data, StorageData& storage) { if (data.count("backfill")) { const auto text = data.get("backfill"); diff --git a/client/config/config.hpp b/client/config/config.hpp index 55c762f50..ed7a69afd 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -31,6 +31,8 @@ const std::string DEFAULT_PATH_TO_PRIVATE_KEY = "NodePrivate.txt"; const uint32_t DEFAULT_MIN_NEIGHBOURS = Neighbourhood::kMinNeighbours; const uint32_t DEFAULT_MAX_NEIGHBOURS = Neighbourhood::kMaxNeighbours; const uint64_t DEFAULT_MAX_UNCORRECTED_BLOCK = 0; +// keep in step with Consensus::MaxAutoRollbackDepth (solver/include/solver/consensus.hpp) +const uint64_t DEFAULT_MAX_AUTO_ROLLBACK = 100; const uint32_t DEFAULT_CONNECTION_BANDWIDTH = 1 << 19; const uint32_t DEFAULT_OBSERVER_WAIT_TIME = 5 * 60 * 1000; // ms const uint32_t DEFAULT_ROUND_ELAPSE_TIME = 1000 * 60; // ms @@ -327,6 +329,23 @@ class Config { return maxUncorrectedBlock_; } + // longest-chain guard: the most own confirmed blocks the node will roll back automatically when it + // adopts a competing chain during sync, before it stops and waits for the operator. + uint64_t maxAutoRollbackDepth() const { + return maxAutoRollbackDepth_; + } + + // longest-chain guard override: accept a deliberate deep switch without the depth bound (for one + // completed rollback episode; then it is off until the node restarts). + bool allowDeepRollback() const { + return allowDeepRollback_; + } + + // --resolve-fork: truncate the chain to the fork point recorded in fork-held.json, then stop + bool resolveFork() const { + return resolveFork_; + } + bool traverseNAT() const { return traverseNAT_; } @@ -395,6 +414,9 @@ class Config { static SnapshotData parseSnapshotData(const boost::property_tree::ptree& config); // the backfill keys of a [storage] section (backfill, backfill_batch, backfill_rate) static void readBackfillSettings(const boost::property_tree::ptree& storage, StorageData& data); + // the [params] keys of the bound on automatic chain reorganisations: max_auto_rollback (default + // DEFAULT_MAX_AUTO_ROLLBACK, 0 = no depth bound) and allow_deep_rollback (default false) + static void readRollbackGuardSettings(const boost::property_tree::ptree& params, uint64_t& maxDepth, bool& allowDeep); private: void setLoggerSettings(const boost::property_tree::ptree& config); @@ -430,6 +452,9 @@ class Config { bool restrictNeighbours_ = false; uint64_t connectionBandwidth_ = DEFAULT_CONNECTION_BANDWIDTH; uint64_t maxUncorrectedBlock_ = DEFAULT_MAX_UNCORRECTED_BLOCK; + uint64_t maxAutoRollbackDepth_ = DEFAULT_MAX_AUTO_ROLLBACK; + bool allowDeepRollback_ = false; + bool resolveFork_ = false; bool symmetric_ = false; EndpointData hostAddressEp_; diff --git a/client/include/client/cmdlineargs.hpp b/client/include/client/cmdlineargs.hpp index bfadfda9d..349a62b88 100644 --- a/client/include/client/cmdlineargs.hpp +++ b/client/include/client/cmdlineargs.hpp @@ -7,6 +7,7 @@ struct cmdline { static constexpr const char* argSeed = "seed"; static constexpr const char* argDumpKeys = "dumpkeys"; static constexpr const char* argSetBCTop = "set-bc-top"; + static constexpr const char* argResolveFork = "resolve-fork"; static constexpr const char* argBalChange = "bal-change"; #ifdef _WIN32 static constexpr const char* argInstall = "install"; diff --git a/client/src/main.cpp b/client/src/main.cpp index 6989d85f5..19acf500b 100644 --- a/client/src/main.cpp +++ b/client/src/main.cpp @@ -56,6 +56,7 @@ int main(int argc, char* argv[]) { ("recreate-index", "recreate index.db") (cmdline::argSeed, "enter with seed instead of keys") (cmdline::argSetBCTop, po::value(), "all blocks in blockchain with higher sequence will be removed") + (cmdline::argResolveFork, "a node that reports FORK HELD: remove the blocks above the fork point recorded in fork-held.json (only if the block there is still the recorded one), then exit; at the next start the node follows the other chain. Check with other nodes first (the node must be stopped)") (cmdline::argBalChange, po::value(), "prints all changes in account balance in log file") ("disable-auto-shutdown", "node will be prohibited to shutdown in case of fatal errors") ("version", "show node version") diff --git a/client/src/peer.cpp b/client/src/peer.cpp index 33ae2e656..10e0c2eae 100644 --- a/client/src/peer.cpp +++ b/client/src/peer.cpp @@ -144,7 +144,7 @@ bool Peer::onInit(const char*) { } bool Peer::onRun(const char*) { - if (vm_.count(cmdline::argSetBCTop) == 0) { + if (vm_.count(cmdline::argSetBCTop) == 0 && vm_.count(cmdline::argResolveFork) == 0) { cslog() << "Running Node"; node_->run(); } diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 109033fa7..dfbe0153e 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -81,6 +81,8 @@ class Storage final { // a quick start asked to read from a block after the last one stored: its checkpoint is // ahead of the blocks on disk StartAfterLastBlock = 7, + // --resolve-fork: the block at the requested top is not the one recorded (set_truncate_check) + TruncateMismatch = 8, UnknownError = 255, }; @@ -152,6 +154,11 @@ class Storage final { * In case of fail the error info can be received with methods: \ref last_error, * \ref last_error_message, \ref db_last_error() and \ref db_last_error_message() */ + // The next open() with a new blockchain top (--set-bc-top) cuts the chain only if the stored block + // at that top has this hash; otherwise it removes nothing and fails with TruncateMismatch. An + // empty hash (the default) checks nothing. Used by --resolve-fork. + void set_truncate_check(const PoolHash& expectedTopHash); + // dbBackend: "berkeleydb" (default), "rocksdb", or "lmdb" in /lmdb, whose removed blocks // LMDB reuses the space of (pruned storage) bool open(const ::std::string& path_to_base = ::std::string{}, diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index bd049743d..b8d1765bd 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -218,15 +218,29 @@ class Storage::priv { > PoolCache; PoolCache pools_cache; static const size_t cacheSize = 50; + // set_truncate_check(): the hash the block at the new blockchain top must have + PoolHash truncate_expected_hash; + // the cache keeps one pool per sequence and per hash: a pool saved at a sequence whose previous + // pool was removed (a chain reorganisation) replaces the cached one, instead of being refused by + // the unique index and leaving the removed pool to be returned by pool_load(sequence) void pools_cache_insert(const cs::Sequence& seq, const PoolHash &hash, const Pool &pool) { - if (pools_cache.size() == cacheSize) { + pools_cache_erase(seq, hash); + if (pools_cache.size() >= cacheSize) { auto random = pools_cache.begin(); pools_cache.erase(random); } pools_cache.insert({seq, hash, pool}); } + // forget a pool removed from the top of the chain (by sequence and by hash) + void pools_cache_erase(const cs::Sequence& seq, const PoolHash &hash) { + pools_cache.get().erase(seq); + if (!hash.is_empty()) { + pools_cache.get().erase(hash); + } + } + friend class ::csdb::Storage; private signals: @@ -465,6 +479,10 @@ Storage::WeakPtr Storage::weak_ptr() const noexcept { return d; } +void Storage::set_truncate_check(const PoolHash& expectedTopHash) { + d->truncate_expected_hash = expectedTopHash; +} + Storage::Error Storage::last_error() const { return last_error_map(d.get()).last_error_; } @@ -537,6 +555,22 @@ bool Storage::open(const OpenOptions& opt, OpenCallback callback) { } if (opt.newBlockchainTop != cs::kWrongSequence) { + if (!d->truncate_expected_hash.is_empty()) { + const PoolHash expected = d->truncate_expected_hash; + d->truncate_expected_hash = PoolHash{}; + cs::Bytes topBinary; + PoolHash stored; + if (d->db->get(static_cast(opt.newBlockchainTop), &topBinary)) { + stored = csdb::Pool::hash_from_binary(std::move(topBinary)); + } + if (stored != expected) { + d->set_last_error(TruncateMismatch, "block %llu is %s, not the recorded %s: nothing removed", + static_cast(opt.newBlockchainTop), + stored.is_empty() ? "not stored" : stored.to_string().c_str(), + expected.to_string().c_str()); + return false; + } + } auto seqToRemove = static_cast(opt.newBlockchainTop + 1); auto seqLast = seqToRemove; { @@ -1068,6 +1102,7 @@ Pool Storage::pool_remove_last() { bool found = write_queue_pop(res); if (found) { + d->pools_cache_erase(res.sequence(), res.hash()); std::unique_lock lock(d->data_lock); --d->count_pool; d->last_hash = res.previous_hash(); @@ -1099,6 +1134,7 @@ Pool Storage::pool_remove_last() { // error nearly impossible /*bool ok =*/ d->db->remove(last_hash().to_binary()); + d->pools_cache_erase(res.sequence(), res.hash()); --d->count_pool; d->last_hash = res.previous_hash(); @@ -1134,6 +1170,7 @@ bool Storage::pool_remove_last_repair(cs::Sequence test_sequence, const csdb::Po if (!write_queue_pop_last(test_hash, queued)) { return false; } + d->pools_cache_erase(test_sequence, test_hash); if (queued) { return true; } @@ -1198,6 +1235,7 @@ bool Storage::pool_remove_(cs::Sequence testSequence) { if (!write_queue_pop_last(testHash, queued)) { return false; } + d->pools_cache_erase(testSequence, testHash); if (queued) { return true; } diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index a2819cc27..d6d81a1cb 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -80,6 +80,7 @@ add_library(csnode include/csnode/snapshot_bootstrap.hpp include/csnode/snapshot_fetcher.hpp include/csnode/blockverifier.hpp + include/csnode/forkguardstate.hpp include/csnode/backwardsync.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -134,6 +135,7 @@ add_library(csnode src/snapshot_bootstrap.cpp src/snapshot_fetcher.cpp src/blockverifier.cpp + src/forkguardstate.cpp src/backwardsync.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index b872e38e5..f32da4a8a 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -3,7 +3,9 @@ #include #include +#include #include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include #include +#include #include @@ -242,6 +245,119 @@ class BlockChain { static void setBootstrap(csdb::Pool& block, bool is_bootstrap); static bool isBootstrap(const csdb::Pool& block); + // Longest-chain guard. It is called from inside removeLastBlock() and compromiseLastBlock(), so + // every automatic rollback of the node's own last block passes through it, whatever the caller + // (the sync fork path, the uncertain-replacement path, the antifork cache and the bad-hash reply). + // It returns true while the resulting head would stay at or above rollbackHighWater_ - + // max_auto_rollback, where rollbackHighWater_ is the head captured at the start of the current + // rollback episode (the first automatic removal from any path) and kept in reorg-episode.json next + // to the database, so a restart does not give a fresh full depth. Once the limit would be crossed + // it records the fork (fork-held.json), logs a prominent alert and returns false: the chain stays + // intact, the node reports FORK HELD and the operator decides (--resolve-fork, --set-bc-top). The + // state clears when the episode ends (the head climbs above the reference) or the operator + // truncates. An operator accepts a deliberate deep switch with [params] allow_deep_rollback or the + // CS_ALLOW_DEEP_ROLLBACK environment variable, for one episode. Local only: it changes how far + // this node unwinds, never block content. + bool guardAutoRollback(); + + // True if the competing block carries round confirmations signed by the confidants the node + // already trusts - the confidants of its own current last block - exactly as BlockVerifier checks + // them. A non-bootstrap competing block that fails this check never triggers a rollback: it is not + // vouched for by any authority this node recognises (a self-signed block vouches only for itself). + // A bootstrap block carries no such confirmations and always fails here (BlockVerifier accepts it + // without them); competingBlockMayTriggerUnwind() sends it to isBootstrapAuthenticatedLocally(). + // Strict for the unwind decision: a confirmation mask of more than one entry, of the same size as our + // last block's confidants, with as many signatures as it marks, and a majority of them confirming. + bool hasTrustedConfirmations(const csdb::Pool& competing) const; + + // True if a bootstrap-flagged competing block is authenticated locally: its own signatures verify + // against its confidants AND those confidants are exactly the confidants of a round table this node + // itself accepted (the current round or the two before it). A bootstrap flag alone is a bit anyone + // can set, so an unauthenticated bootstrap block never triggers an automatic unwind. + bool isBootstrapAuthenticatedLocally(const csdb::Pool& competing) const; + + // The single gate for an automatic unwind in response to a competing block: the operator override, + // or a non-bootstrap block with trusted confirmations, or a locally authenticated bootstrap block. + // A block that passes is still subject to the depth bound inside removeLastBlock(). + bool competingBlockMayTriggerUnwind(const csdb::Pool& competing) const; + + // The operator has approved a deliberate deep switch: [params] allow_deep_rollback=true or the + // CS_ALLOW_DEEP_ROLLBACK environment variable, until one episode has completed with it (then it is + // off for the rest of the run). It lifts both the authority gate and the depth bound. (--resolve-fork + // approves nothing as wide: see approvedBranch_.) + bool deepRollbackApproved() const; + // CS_ALLOW_DEEP_ROLLBACK is set to a non-empty value + static bool deepRollbackEnvSet(); + + // The local files of the guard live in `dir` (the folder next to the database). Loads the + // persisted episode reference and the fork record; an episode the chain has already passed is + // dropped. An empty dir keeps everything in memory only. Called by init(). + void setGuardStateDir(const std::string& dir); + + // FORK HELD: the node refused to unwind for a competing chain, the episode has not ended and the + // operator has not dismissed it. + bool forkHeld() const; + // the record while FORK HELD, else nothing + std::optional forkHeldRecord() const; + // the record, held or keeping a dismissal + std::optional forkRecord() const; + + // The operator keeps this node's chain: FORK HELD ends, the record keeps the competing branch as + // dismissed (still refused, never reported again; another branch is). False if no fork is held. + bool dismissFork(); + // the dismiss-fork file next to the database: dismissFork() and remove the file. Called every + // round, every few seconds from a timer and at start-up; true if a fork was dismissed. + bool checkDismissTrigger(); + + // A refusal for a competing block not vouched for by our confidants becomes FORK HELD only if the + // head has not moved since, at the second round start after it or a minute later (whichever is + // first); a head that moves drops it. Called at every round start (roundStarted) and from a timer. + // True if FORK HELD was raised now. + bool promotePendingHold(bool roundStarted); + bool forkHoldPending() const { + return pendingHold_.has_value(); + } + + // the sync reply being stored comes from this peer, which announced this height (for the fork + // record); cleared with clearSyncSource() + void setSyncSource(const cs::PublicKey& key, cs::Sequence announcedHeight); + void clearSyncSource(); + // resolves a peer key to "ip:port" for the fork record (set by the node; may return "") + void setPeerAddressResolver(std::function resolver); + + // --resolve-fork: init() truncates to `seq` only if the stored block there has `hash`, then clears + // the fork record. When the record's fork point was exact, it also records the competing block + // right after it (`competingHash` at seq + 1) as the one approved branch (resolve-approved.json). + void setResolveFork(cs::Sequence seq, const csdb::PoolHash& hash, const std::string& competingHash = std::string{}); + // the branch approved with --resolve-fork, while it is pending + std::optional approvedBranch() const { + return approvedBranch_; + } + // init() refused to truncate: the stored block at the requested sequence has another hash + bool truncationRefused() const { + return truncationRefused_; + } + +#ifdef TESTING + // test-only accessors for the longest-chain guard + void test_setLastSequence(cs::Sequence s) { lastSequence_ = s; } + cs::Sequence test_rollbackHighWater() const { return rollbackHighWater_; } + bool test_rollbackEpisode() const { return rollbackEpisode_; } + void test_resetRollbackEpisode() { + rollbackEpisode_ = false; rollbackHighWater_ = 0; approvedBranch_.reset(); deepOverrideSpent_ = false; pendingHold_.reset(); + std::lock_guard l(guardMutex_); forkRecord_.reset(); + } + // make `block` the node's last block (the deferred one), as getLastBlock() returns it + void test_setLastBlock(const csdb::Pool& block) { deferredBlock_ = block; lastSequence_ = block.sequence(); } + // let the episode end as if a block above the reference had been recorded + void test_onHeadAdvanced() { onHeadAdvanced(); checkApprovedBranch(); } + void test_setSelectionUnfinished(const csdb::PoolHash& lastPrev) { selectionFinished_ = false; lastPrevHash_ = lastPrev; antiForkMode_ = true; emittingRequest_ = 1; } + bool test_selectionFinished() const { return selectionFinished_; } + bool test_antiForkMode() const { return antiForkMode_; } + uint8_t test_emittingRequest() const { return emittingRequest_; } + csdb::PoolHash test_lastPrevHash() const { return lastPrevHash_; } +#endif + const cs::MultiWallets& multiWallets() const; /** @@ -595,6 +711,62 @@ public slots: mutable std::atomic uuid_ = 0; std::atomic lastSequence_; cs::Sequence blocksToBeRemoved_ = 0; + // longest-chain guard: a fixed reference for the current automatic-rollback episode. When the + // first automatic rollback of an episode starts, rollbackHighWater_ records the head we had then; + // the node then refuses to unwind automatically below rollbackHighWater_ - MaxAutoRollbackDepth. + // The reference is cleared only once the chain climbs back strictly above rollbackHighWater_, so + // alternating removals and appends cannot creep past the limit. Purely local bookkeeping; it + // never changes any block content. + bool rollbackEpisode_ = false; + cs::Sequence rollbackHighWater_ = 0; + cs::Sequence rollbackLowWater_ = 0; // lowest head reached in the current episode (for the log) + // longest-chain guard: where its files live (empty: memory only), the fork it refused (FORK + // HELD), the --resolve-fork / allow_deep_rollback approvals, the integrity repair in progress + std::string guardStateDir_; + mutable std::mutex guardMutex_; + std::optional forkRecord_; + std::chrono::steady_clock::time_point lastDismissedLog_{}; // dismissed branch offered again (hourly) + // --resolve-fork's approved branch (exact fork point only), pending until the node is past it + std::optional approvedBranch_; + std::string resolveCompetingHash_; + // the approved branch is settled: the node is past its block, or another block took that height + void checkApprovedBranch(); + void dropApprovedBranch(const char* why); + bool deepOverrideSpent_ = false; // allow_deep_rollback served one episode: off until restart + bool deepOverrideUsed_ = false; // the current episode ran with allow_deep_rollback + bool integrityRepair_ = false; // cacheLastBlocks() is removing blocks of incorrectBlocks_ + struct CompetingBlock { + cs::Sequence seq = 0; + csdb::PoolHash hash; + csdb::PoolHash previous; + bool vouched = false; // passed competingBlockMayTriggerUnwind() + }; + std::optional competing_; // the competing block being handled (for the record) + std::optional> syncSource_; + std::function peerAddressResolver_; + cs::Sequence resolveForkSeq_ = cs::kWrongSequence; + csdb::PoolHash resolveForkHash_; + bool truncationRefused_ = false; + // starts the episode at the current head if none is running (persisting its reference) + void beginRollbackEpisode(); + // the chain grew: ends the episode once the head is above its reference + void onHeadAdvanced(); + // records the refused fork (once per episode) and logs the alert; reason "depth" or "authority". + // False (and nothing raised) when the competing block belongs to a branch the operator dismissed. + // An "authority" refusal is only kept pending until promotePendingHold() sees the head stuck. + bool recordForkHeld(const char* reason); + cs::forkguard::ForkRecord buildForkRecord(const char* reason); + // FORK HELD now: episode, record, file, alert + void holdFork(cs::forkguard::ForkRecord record); + struct PendingHold { + cs::forkguard::ForkRecord record; + cs::Sequence head = 0; + std::chrono::steady_clock::time_point since; + unsigned rounds = 0; + }; + std::optional pendingHold_; + // removeLastBlock()/compromiseLastBlock() may go on: an integrity repair, or the guard allows it + bool mayUnwindLastBlock(); std::atomic_bool stop_ = false; // the quick-start checkpoint of the last init() is ahead of the blocks on disk bool checkpointAheadOfBlocks_ = false; diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp new file mode 100644 index 000000000..d3445260d --- /dev/null +++ b/csnode/include/csnode/forkguardstate.hpp @@ -0,0 +1,127 @@ +#pragma once + +#include +#include +#include +#include + +#include + +// Local files of the bound on automatic chain reorganisations, next to the block database (the +// parent folder of --db-path; the data folder in Docker). Purely local bookkeeping: nothing here is +// sent to other nodes or changes what a block contains. +// +// - reorg-episode.json: the fixed reference of the current automatic-rollback episode (the head the +// node had when the episode began). Kept across restarts, so a restart does not give a fresh full +// depth. Removed when the episode ends (the head is back above the reference) or the operator +// truncates the chain. +// - fork-held.json: written when the node refuses to unwind for a competing chain (beyond the depth +// bound, or a competing block not vouched for by the node's confidants). While its "held" is true +// the node reports FORK HELD. The operator either keeps this chain (the dismiss-fork file: the +// record then keeps the dismissed branch, still refused, and "held" turns false) or follows the +// other one (--resolve-fork / --set-bc-top, which remove the file). When the episode ends on its +// own, the file goes unless it holds a dismissal. +namespace cs::forkguard { + +inline constexpr const char* kEpisodeFile = "reorg-episode.json"; +inline constexpr const char* kForkFile = "fork-held.json"; +// created by the operator (any content) to dismiss the held fork and keep this node's chain; the +// running node picks it up within a round, acts, logs and deletes it +inline constexpr const char* kDismissTrigger = "dismiss-fork"; +// at most this many block hashes of dismissed branches are remembered +inline constexpr size_t kMaxBranchHashes = 256; + +// the folder of these files for a database path: its parent folder ("." for a relative "db") +std::string stateDirFor(const std::string& dbPath); + +struct Episode { + cs::Sequence highWater = 0; +}; + +// resolve-approved.json, written by --resolve-fork when the fork point was exact: the one block the +// operator chose, the competing block right after the fork point. If the node stores its own block at +// that height again before the recorded one arrives, the recorded block itself (that very hash, built +// on the fork point) makes the node give its own block up, never going to or below the fork point. +// Nothing else bypasses the usual checks. The approval ends once the node stores a block at that +// height and goes on, or a competing block that passes the usual checks is offered there. +inline constexpr const char* kApprovedFile = "resolve-approved.json"; +struct ApprovedBranch { + cs::Sequence forkPoint = 0; + cs::Sequence seq = 0; // forkPoint + 1 + std::string hash; // the recorded competing block +}; +bool saveApprovedBranch(const std::string& dir, const ApprovedBranch& branch); +std::optional loadApprovedBranch(const std::string& dir, bool* corrupt = nullptr); +void clearApprovedBranch(const std::string& dir); + +struct ForkRecord { + // FORK HELD: the node refuses a competing chain and the operator has not decided yet. False once + // the operator dismissed it (the record then keeps the dismissal) or the episode ended. + bool held = true; + std::string reason; // "depth" (the bound) or "authority" (not vouched for) + int64_t sinceUnix = 0; // first refusal of the episode, UTC seconds + cs::Sequence highWater = 0; // the episode's reference head + cs::Sequence ourHead = 0; // our head when the node refused + std::string ourHeadHash; + // the highest of our blocks the competing chain can share: exact when the competing block names + // it as its previous block, otherwise the competing chain diverges at or below it + cs::Sequence forkPoint = 0; + std::string forkPointHash; + bool forkPointExact = false; + cs::Sequence competingSeq = 0; // the refused block (0: not known, e.g. a bad-hash reply) + std::string competingHash; + std::string peerKey; // base58, empty when not known + std::string peerAddress; // ip:port, empty when not known + cs::Sequence peerHeight = 0; // the height the peer announced, 0 when not known + // the operator kept this node's chain (dismiss-fork): the first known block of the dismissed + // branch after the fork point, and the hashes of that branch seen since. A competing block that is + // one of them, or continues one of them, is still refused but raises FORK HELD no more. + int64_t dismissedUnix = 0; + cs::Sequence dismissedSeq = 0; + std::string dismissedHash; + std::vector branchHashes; + + bool hasDismissal() const { + return !branchHashes.empty(); + } + // the competing block (hash, previous hash) belongs to a dismissed branch + bool inDismissedBranch(const std::string& hash, const std::string& previous) const; + // remember one more block hash of a dismissed branch (the oldest go beyond kMaxBranchHashes, never + // dismissedHash) + void addBranchHash(const std::string& hash); +}; + +// An empty dir turns persistence off (the functions do nothing and load returns nothing). +bool saveEpisode(const std::string& dir, const Episode& episode); +// *corrupt (if given) is set when the file exists but cannot be read +std::optional loadEpisode(const std::string& dir, bool* corrupt = nullptr); +void clearEpisode(const std::string& dir); + +bool saveForkRecord(const std::string& dir, const ForkRecord& record); +std::optional loadForkRecord(const std::string& dir, bool* corrupt = nullptr); +void clearForkRecord(const std::string& dir); +// all three files: the database they describe was replaced (wiped, restored from a snapshot) +void clearAll(const std::string& dir); + +// --resolve-fork: the fork point to cut the chain at and the hash the stored block there must have, +// from the record in dir; nothing (and *error says why) when there is no record or it cannot be +// checked against the database +struct ResolvePlan { + cs::Sequence top = 0; + std::string hash; + ForkRecord record; +}; +std::optional planResolve(const std::string& dir, std::string* error); + +// "2026-11-19 10:04:05 UTC" +std::string utcTime(int64_t unixSeconds); + +// the operator's two choices for a held fork: keep this chain (the dismiss-fork file) or follow the +// other one (--resolve-fork, or --set-bc-top by hand), native and Docker +std::string remedy(const ForkRecord& record); + +// the periodic FORK HELD detail line: since when, where the chains diverge, the refused peer and the +// remedy +std::string summary(const ForkRecord& record); + +} // namespace cs::forkguard diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index ceb1fd570..6fe51844c 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -471,6 +471,15 @@ public slots: }; bool initCommon(); ChainInit initChain(); + // --resolve-fork: the fork point recorded in fork-held.json, checked and announced; none (logged) + // when there is nothing to resolve + std::optional resolveForkTop(); + // FORK HELD: the time of the last detail line (one a minute at most) + std::chrono::steady_clock::time_point lastForkHeldDetail_{}; + void logForkHeldDetail(const std::optional& forkHeld); + // the fork guard's timer (every 5 s, queued to the processor thread): dismiss-fork, pending holds + cs::Timer forkGuardTimer_; + void onForkGuardTick(); // snapshot.json of a restored folder against the digest of the loaded caches (design 6.8); // false with exit code 4 on a mismatch bool verifyAtStart(); diff --git a/csnode/include/csnode/poolsynchronizer.hpp b/csnode/include/csnode/poolsynchronizer.hpp index 7f5675c7b..12a28277f 100644 --- a/csnode/include/csnode/poolsynchronizer.hpp +++ b/csnode/include/csnode/poolsynchronizer.hpp @@ -30,6 +30,8 @@ class PoolSynchronizer { void stop(); cs::Sequence getMaxNeighbourSequence(); + // the last sequence this neighbour announced above ours, 0 if none + cs::Sequence announcedSequence(const cs::PublicKey& key) const; static const RoundNumber kRoundDifferentForSync = values::kDefaultMetaStorageMaxSize; void getSyncroMessage(const cs::PublicKey& sender, SyncroMessage msg); diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 149032d7b..deb00f63f 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include +#include #include #include #include @@ -240,12 +242,21 @@ bool BlockChain::init( // LMDB reuses the space of pruned blocks; BerkeleyDB's Recno never does ([storage] db_backend) cslog() << kLogPrefix << "block database: " << storageCfg.dbBackend << (storageCfg.dbBackend == "lmdb" ? " in " + path + "/lmdb" : std::string{}); + // --resolve-fork: the storage cuts the chain only if the block at the fork point is the recorded one + if (newBlockchainTop != cs::kWrongSequence && resolveForkSeq_ == newBlockchainTop) { + storage_.set_truncate_check(resolveForkHash_); + } + truncationRefused_ = false; if (!storage_.open(path, progress, newBlockchainTop, firstBlockToReadInDatabase, storageCfg.asyncWriteQueueSize, storageCfg.writeBatchSize, static_cast(storageCfg.rocksdbBlockCacheMb) << 20, static_cast(storageCfg.rocksdbMemtableMb) << 20, storageCfg.dbBackend)) { cserror() << kLogPrefix << "Couldn't open database at " << path; + truncationRefused_ = storage_.last_error() == csdb::Storage::TruncateMismatch; + if (truncationRefused_) { + cserror() << kLogPrefix << storage_.last_error_message(); + } checkpointAheadOfBlocks_ = successfulQuickStart && storage_.last_error() == csdb::Storage::StartAfterLastBlock; if (checkpointAheadOfBlocks_) { cserror() << kLogPrefix << "the QUICK START checkpoint continues after the last block on disk (" @@ -262,6 +273,18 @@ bool BlockChain::init( if (newBlockchainTop != cs::kWrongSequence) { if (trxIndex_) trxIndex_->trimToFloor(newBlockchainTop); + // the operator truncated: the rollback episode and FORK HELD end here + const std::string dir = cs::forkguard::stateDirFor(path); + cs::forkguard::clearForkRecord(dir); + cs::forkguard::clearEpisode(dir); + cs::forkguard::clearApprovedBranch(dir); + if (resolveForkSeq_ == newBlockchainTop && !resolveCompetingHash_.empty()) { + // --resolve-fork with an exact fork point: the recorded competing block right after it is + // the one approved branch. Normally the node simply syncs it forward from the fork point; + // the approval only lets that very branch replace a block stored above the fork point + // again before it arrives. Nothing else is relaxed. + cs::forkguard::saveApprovedBranch(dir, {newBlockchainTop, newBlockchainTop + 1, resolveCompetingHash_}); + } return true; } @@ -276,6 +299,8 @@ bool BlockChain::init( if (successfulQuickStart) { serializationManPtr_->clear(); } + // a new database: the rollback-bound files of a previous one (removed db/) do not apply + cs::forkguard::clearAll(cs::forkguard::stateDirFor(path)); writeGenesisBlock(); } else { @@ -323,6 +348,10 @@ bool BlockChain::init( serializationManPtr_->setCompletedFromGenesis(); } + // the longest-chain guard's files next to the database: a rollback episode (and FORK HELD) that + // was running when the node stopped goes on with the same reference + setGuardStateDir(cs::forkguard::stateDirFor(path)); + good_ = true; blocksToBeRemoved_ = totalLoaded - 1; // any amount to remave after start return true; @@ -829,6 +858,546 @@ csdb::Transaction BlockChain::loadTransaction(const csdb::TransactionID& transId return transaction; } +// Longest-chain guard: bound the depth of an automatic rollback while adopting a competing chain. +// A competing chain that asks us to unwind more than the configured bound of own blocks is either a +// deliberate network switch (a human must confirm it) or a chain that should not be followed: +// in both cases the safe local action is to stop unwinding and wait for the operator. The depth is +// measured against a fixed reference (the head when the episode started), not a counter that forward +// progress could reset, so alternating removals and appends cannot creep past it; the reference is +// kept in reorg-episode.json, so a restart does not reset it either. It is called from inside +// removeLastBlock() and compromiseLastBlock(), so every automatic removal is bounded whatever the +// caller, and rollbackHighWater_ is captured at the first removal from any path. +bool BlockChain::guardAutoRollback() { + beginRollbackEpisode(); + + // after --resolve-fork, never unwind to or below the fork point the operator cut the chain at + if (approvedBranch_.has_value() && lastSequence_.load() <= approvedBranch_->forkPoint) { + recordForkHeld("depth"); + return false; + } + + // operator override: accept a deliberate deep switch ([params] allow_deep_rollback or the + // CS_ALLOW_DEEP_ROLLBACK environment variable), for one completed episode + if (deepRollbackApproved()) { + deepOverrideUsed_ = true; // spent when this episode completes + return true; + } + const auto* config = cs::ConfigHolder::instance().config(); + const uint64_t maxDepth = (config != nullptr) ? config->maxAutoRollbackDepth() + : Consensus::MaxAutoRollbackDepth; + if (maxDepth == 0) { + return true; // max_auto_rollback=0: no depth bound (the authority gate still applies) + } + + // refuse the removal that would take the head below the floor (no wraparound when near genesis) + const cs::Sequence tip = lastSequence_.load(); + if (rollbackHighWater_ > maxDepth && tip <= rollbackHighWater_ - maxDepth) { + recordForkHeld("depth"); + return false; + } + return true; +} + +void BlockChain::beginRollbackEpisode() { + if (rollbackEpisode_) { + return; + } + const cs::Sequence tip = lastSequence_.load(); + rollbackEpisode_ = true; + rollbackHighWater_ = tip; // the head we had before this rollback episode began + rollbackLowWater_ = tip; + deepOverrideUsed_ = false; + cs::forkguard::saveEpisode(guardStateDir_, {rollbackHighWater_}); +} + +void BlockChain::onHeadAdvanced() { + if (pendingHold_.has_value() && lastSequence_.load() != pendingHold_->head) { + pendingHold_.reset(); // a refused block that did not stop the chain: no FORK HELD + } + // the rollback episode ends only once the chain climbs back strictly above the head it had when + // the episode began, so the depth cannot creep past the limit by alternating removals and appends + // below that head + if (!rollbackEpisode_ || lastSequence_.load() <= rollbackHighWater_) { + return; + } + rollbackEpisode_ = false; + // one line per finished episode: how deep the node actually unwound (measured in CI) + cslog() << kLogPrefix << "rollback episode ended: unwound " << (rollbackHighWater_ - rollbackLowWater_) + << " block(s) below #" << WithDelimiters(rollbackHighWater_); + if (deepOverrideUsed_ && !deepOverrideSpent_) { + deepOverrideSpent_ = true; + cserror() << kLogPrefix << "LONGEST-CHAIN GUARD: allow_deep_rollback (or CS_ALLOW_DEEP_ROLLBACK) has served " + << "one completed rollback episode and is now OFF until the node restarts. Remove it from " + << "config.ini (or the environment) now: left on, it would lift the bound again at the next start."; + } + deepOverrideUsed_ = false; + cs::forkguard::clearEpisode(guardStateDir_); + std::optional keep; + bool wasHeld = false; + { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value()) { + wasHeld = forkRecord_->held; + forkRecord_->held = false; + if (forkRecord_->hasDismissal()) { + keep = forkRecord_; // the dismissed branch stays refused and quiet + } + else { + forkRecord_.reset(); + } + } + } + if (keep.has_value()) { + cs::forkguard::saveForkRecord(guardStateDir_, *keep); + } + else { + cs::forkguard::clearForkRecord(guardStateDir_); + } + if (wasHeld) { + cswarning() << kLogPrefix << "FORK HELD cleared: the chain is back above #" << WithDelimiters(rollbackHighWater_); + } +} + +cs::forkguard::ForkRecord BlockChain::buildForkRecord(const char* reason) { + cs::forkguard::ForkRecord r; + { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value()) { + r = *forkRecord_; // keeps an earlier dismissal: that branch stays dismissed + } + } + r.held = true; + r.reason = reason; + r.sinceUnix = static_cast(std::time(nullptr)); + r.highWater = rollbackEpisode_ ? rollbackHighWater_ : lastSequence_.load(); + const cs::Sequence tip = lastSequence_.load(); + r.ourHead = tip; + r.ourHeadHash = getHashBySequence(tip).to_string(); + // the fork point: the highest of our blocks the competing chain can share + cs::Sequence candidate = tip; + bool candidateExcluded = true; // without a competing block, our head is the one disputed + r.forkPointExact = false; + r.competingSeq = 0; + r.competingHash.clear(); + if (competing_.has_value() && competing_->seq > 0) { + r.competingSeq = competing_->seq; + r.competingHash = competing_->hash.to_string(); + candidate = std::min(competing_->seq - 1, tip); + if (candidate + 1 == competing_->seq) { + // the competing block names its previous block: ours at that height is shared or not + r.forkPointExact = !competing_->previous.is_empty() && getHashBySequence(candidate) == competing_->previous; + candidateExcluded = !r.forkPointExact; + } + else { + candidateExcluded = false; // nothing known about our block at that height + } + } + r.forkPoint = (candidateExcluded && candidate > 0) ? candidate - 1 : candidate; + r.forkPointHash = getHashBySequence(r.forkPoint).to_string(); + r.peerKey.clear(); + r.peerAddress.clear(); + r.peerHeight = 0; + if (syncSource_.has_value()) { + const auto& key = syncSource_->first; + r.peerKey = EncodeBase58(key.data(), key.data() + key.size()); + r.peerHeight = syncSource_->second; + if (peerAddressResolver_) { + r.peerAddress = peerAddressResolver_(key); + } + } + return r; +} + +bool BlockChain::recordForkHeld(const char* reason) { + const std::string competingHash = competing_.has_value() ? competing_->hash.to_string() : std::string{}; + const std::string competingPrev = competing_.has_value() ? competing_->previous.to_string() : std::string{}; + { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value() && forkRecord_->hasDismissal() && competing_.has_value() && + forkRecord_->inDismissedBranch(competingHash, competingPrev)) { + // a branch the operator dismissed: still refused, never FORK HELD again. The branch is + // followed by its hashes only through blocks vouched for by our confidants, so blocks + // anyone can forge never grow (or evict from) that list. + bool grew = false; + if (competing_->vouched) { + const size_t before = forkRecord_->branchHashes.size(); + forkRecord_->addBranchHash(competingHash); + grew = forkRecord_->branchHashes.size() != before; + } + const auto now = std::chrono::steady_clock::now(); + if (lastDismissedLog_ == std::chrono::steady_clock::time_point{} || now - lastDismissedLog_ >= std::chrono::hours(1)) { + lastDismissedLog_ = now; + csinfo() << kLogPrefix << "competing branch at #" << WithDelimiters(forkRecord_->dismissedSeq) << " (" + << forkRecord_->dismissedHash << ") dismissed by the operator on " + << cs::forkguard::utcTime(forkRecord_->dismissedUnix) << "; still refused (block #" + << WithDelimiters(competing_->seq) << ")"; + } + if (grew) { + cs::forkguard::saveForkRecord(guardStateDir_, *forkRecord_); + } + return false; + } + if (forkRecord_.has_value() && forkRecord_->held) { + return true; // one record per episode: the first refusal + } + } + if (std::string(reason) == "authority") { + // any sync reply can carry a block not vouched for by our confidants (a peer on a minority + // branch): the block is refused at once, but FORK HELD (files, banner, healthcheck) is + // reported only if the head stays where it was (promotePendingHold), so that it does not flap + if (!pendingHold_.has_value()) { + pendingHold_ = PendingHold{buildForkRecord(reason), lastSequence_.load(), std::chrono::steady_clock::now(), 0}; + } + return true; + } + holdFork(buildForkRecord(reason)); + return true; +} + +void BlockChain::holdFork(cs::forkguard::ForkRecord r) { + pendingHold_.reset(); + beginRollbackEpisode(); // FORK HELD ends with the episode + r.highWater = rollbackHighWater_; + { + std::lock_guard l(guardMutex_); + forkRecord_ = r; + } + cs::forkguard::saveForkRecord(guardStateDir_, r); + + const auto* config = cs::ConfigHolder::instance().config(); + const uint64_t maxDepth = (config != nullptr) ? config->maxAutoRollbackDepth() : Consensus::MaxAutoRollbackDepth; + if (r.reason == "depth") { + cserror() << kLogPrefix << "LONGEST-CHAIN GUARD: a competing chain requires rolling back more than " + << maxDepth << " of our own confirmed blocks (reference head #" << WithDelimiters(rollbackHighWater_) + << ", now #" << WithDelimiters(r.ourHead) << "). Keeping our chain: FORK HELD, operator review required."; + } + else if (r.reason == "authority") { + cserror() << kLogPrefix << "a competing block #" << WithDelimiters(r.competingSeq) + << " is not vouched for by this node's confidants and the head has not moved since. Keeping our chain: " + << "FORK HELD, operator review required."; + } + else { + cserror() << kLogPrefix << "the fork record " << cs::forkguard::kForkFile << " could not be read: keeping our " + << "chain, FORK HELD, operator review required."; + } + cserror() << kLogPrefix << cs::forkguard::summary(r); +} + +bool BlockChain::promotePendingHold(bool roundStarted) { + if (!pendingHold_.has_value()) { + return false; + } + if (lastSequence_.load() != pendingHold_->head) { + pendingHold_.reset(); // the head moved: the refused block was no fork this node is stuck on + return false; + } + if (roundStarted) { + ++pendingHold_->rounds; + } + const bool stuck = pendingHold_->rounds >= 2 || + std::chrono::steady_clock::now() - pendingHold_->since >= std::chrono::minutes(1); + if (!stuck) { + return false; + } + holdFork(pendingHold_->record); + return true; +} + +bool BlockChain::dismissFork() { + cs::forkguard::ForkRecord r; + { + std::lock_guard l(guardMutex_); + if (!forkRecord_.has_value() || !forkRecord_->held) { + return false; + } + forkRecord_->held = false; + forkRecord_->dismissedUnix = static_cast(std::time(nullptr)); + // the first block of that branch after the fork point, as far as this node has seen it + forkRecord_->dismissedSeq = forkRecord_->competingSeq; + forkRecord_->dismissedHash = forkRecord_->competingHash; + forkRecord_->addBranchHash(forkRecord_->competingHash); + if (!forkRecord_->hasDismissal()) { + // no competing block known (a bad-hash reply): nothing to recognise it by, just clear it + forkRecord_.reset(); + r.held = false; + } + else { + r = *forkRecord_; + } + } + if (r.hasDismissal()) { + cs::forkguard::saveForkRecord(guardStateDir_, r); + cswarning() << kLogPrefix << "FORK HELD dismissed by the operator: keeping this chain; the competing branch at #" + << WithDelimiters(r.dismissedSeq) << " (" << r.dismissedHash << ") stays refused"; + } + else { + cs::forkguard::clearForkRecord(guardStateDir_); + cswarning() << kLogPrefix << "FORK HELD dismissed by the operator: keeping this chain"; + } + return true; +} + +bool BlockChain::checkDismissTrigger() { + if (guardStateDir_.empty()) { + return false; + } + namespace fs = std::filesystem; + std::error_code ec; + const fs::path trigger = fs::path(guardStateDir_) / cs::forkguard::kDismissTrigger; + if (!fs::exists(trigger, ec)) { + return false; + } + const bool dismissed = dismissFork(); + if (!dismissed) { + cslog() << kLogPrefix << "the " << cs::forkguard::kDismissTrigger << " file is there but no fork is held: nothing to dismiss"; + } + fs::remove(trigger, ec); + if (ec) { + cserror() << kLogPrefix << "cannot remove " << trigger.string() << ": " << ec.message(); + } + return dismissed; +} + +bool BlockChain::mayUnwindLastBlock() { + // the start-up integrity repair removes blocks of this node's own database that failed the hash + // check (cacheLastBlocks, incorrectBlocks_): a local repair, not a network reorg, so not bounded + return integrityRepair_ || guardAutoRollback(); +} + +bool BlockChain::forkHeld() const { + std::lock_guard l(guardMutex_); + return forkRecord_.has_value() && forkRecord_->held; +} + +std::optional BlockChain::forkHeldRecord() const { + std::lock_guard l(guardMutex_); + if (forkRecord_.has_value() && forkRecord_->held) { + return forkRecord_; + } + return std::nullopt; +} + +std::optional BlockChain::forkRecord() const { + std::lock_guard l(guardMutex_); + return forkRecord_; +} + +void BlockChain::setSyncSource(const cs::PublicKey& key, cs::Sequence announcedHeight) { + syncSource_ = std::make_pair(key, announcedHeight); +} + +void BlockChain::clearSyncSource() { + syncSource_.reset(); +} + +void BlockChain::setPeerAddressResolver(std::function resolver) { + peerAddressResolver_ = std::move(resolver); +} + +void BlockChain::setResolveFork(cs::Sequence seq, const csdb::PoolHash& hash, const std::string& competingHash) { + resolveForkSeq_ = seq; + resolveForkHash_ = hash; + resolveCompetingHash_ = competingHash; +} + +void BlockChain::setGuardStateDir(const std::string& dir) { + guardStateDir_ = dir; + const cs::Sequence tip = lastSequence_.load(); + bool corrupt = false; + if (const auto e = cs::forkguard::loadEpisode(dir, &corrupt)) { + if (tip <= e->highWater) { + rollbackEpisode_ = true; + rollbackHighWater_ = e->highWater; + rollbackLowWater_ = tip; + cswarning() << kLogPrefix << "rollback episode resumed: reference head #" << WithDelimiters(e->highWater) + << ", head #" << WithDelimiters(tip); + } + else { + cs::forkguard::clearEpisode(dir); + } + } + else if (corrupt) { + // fail safe: an unreadable reference must not lift the bound, so a new episode starts here + cserror() << kLogPrefix << cs::forkguard::kEpisodeFile << " in " << dir << " cannot be read: a rollback " + << "episode starts at the current head #" << WithDelimiters(tip) << " instead"; + rollbackEpisode_ = false; + beginRollbackEpisode(); + } + corrupt = false; + if (auto r = cs::forkguard::loadForkRecord(dir, &corrupt)) { + if (!rollbackEpisode_) { + r->held = false; // the chain is past that episode + } + if (r->held || r->hasDismissal()) { + { + std::lock_guard l(guardMutex_); + forkRecord_ = *r; + } + if (r->held) { + cserror() << kLogPrefix << cs::forkguard::summary(*r); + } + else { + cs::forkguard::saveForkRecord(dir, *r); + } + } + else { + cs::forkguard::clearForkRecord(dir); + } + } + else if (corrupt) { + // the record is lost: report FORK HELD with what is known and keep refusing through the episode + cserror() << kLogPrefix << cs::forkguard::kForkFile << " in " << dir << " cannot be read"; + { + std::lock_guard l(guardMutex_); + forkRecord_.reset(); + } + cs::forkguard::ForkRecord r; + r.reason = "unreadable"; + r.sinceUnix = static_cast(std::time(nullptr)); + r.ourHead = tip; + r.ourHeadHash = getHashBySequence(tip).to_string(); + r.forkPoint = tip; // unknown; no hash, so --resolve-fork refuses and the operator cuts by hand + holdFork(r); + } + corrupt = false; + approvedBranch_ = cs::forkguard::loadApprovedBranch(dir, &corrupt); + if (corrupt) { + cserror() << kLogPrefix << cs::forkguard::kApprovedFile << " in " << dir << " cannot be read: no branch is " + << "approved; removing it"; + cs::forkguard::clearApprovedBranch(dir); + } + if (approvedBranch_.has_value()) { + cslog() << kLogPrefix << "branch approved with --resolve-fork: block #" << WithDelimiters(approvedBranch_->seq) + << " (" << approvedBranch_->hash << ") after the fork point #" << WithDelimiters(approvedBranch_->forkPoint); + checkApprovedBranch(); + } + // a dismiss-fork file left while the node was stopped + checkDismissTrigger(); +} + +void BlockChain::dropApprovedBranch(const char* why) { + if (!approvedBranch_.has_value()) { + return; + } + cslog() << kLogPrefix << "branch approved with --resolve-fork (block #" << WithDelimiters(approvedBranch_->seq) + << "): " << why << "; the approval ends"; + approvedBranch_.reset(); + cs::forkguard::clearApprovedBranch(guardStateDir_); +} + +void BlockChain::checkApprovedBranch() { + if (!approvedBranch_.has_value()) { + return; + } + const cs::Sequence tip = lastSequence_.load(); + if (tip < approvedBranch_->seq) { + return; + } + if (getHashBySequence(approvedBranch_->seq).to_string() == approvedBranch_->hash) { + dropApprovedBranch("the node follows it"); + } + else if (tip > approvedBranch_->seq) { + dropApprovedBranch("the node went on with another block at that height"); + } +} + +bool BlockChain::deepRollbackEnvSet() { + const char* v = std::getenv("CS_ALLOW_DEEP_ROLLBACK"); + return v != nullptr && *v != '\0'; +} + +bool BlockChain::deepRollbackApproved() const { + if (deepOverrideSpent_) { + return false; + } + const auto* config = cs::ConfigHolder::instance().config(); + return deepRollbackEnvSet() || (config != nullptr && config->allowDeepRollback()); +} + +// A competing block legitimately asks us to roll back only if it is vouched for by authority this node +// already recognises. We verify its round confirmations against the confidants of our own current last +// block, exactly as BlockVerifier does for a block arriving from the network. Checking the block against +// the confidants it declares itself (as a self-signature check would) proves nothing: any block +// names its own keys and signs for itself. A legitimate micro-fork passes, because the honest competing +// block at the same height was produced by the same deterministically elected trusted group whose keys +// confirm our last block; a block without confirmations from that group fails and never triggers an +// unwind. Bootstrap blocks carry no such confirmations and fail here; they go through +// isBootstrapAuthenticatedLocally() instead. +bool BlockChain::hasTrustedConfirmations(const csdb::Pool& competing) const { + // Strict pre-checks for the unwind decision only; BlockVerifier::verifyConfirmations keeps the + // exceptions it has for normal block validation (sequence < 2, one-entry masks), which must not be + // enough to make this node unwind its own chain. + const auto fail = [&competing](const char* why) { + csdebug() << kLogPrefix << "competing block #" << competing.sequence() + << " not confirmed by our last block's confidants: " << why; + return false; + }; + if (!competing.is_valid() || competing.sequence() < 2) { + return fail("no confirmations to check"); + } + // BlockVerifier::verifyConfirmations accepts a bootstrap block without checking any confirmation + if (BlockChain::isBootstrap(competing)) { + return fail("bootstrap block carries no confirmations"); + } + const auto trustedConfidants = getLastBlock().confidants(); + if (competing.numberConfirmations() == 0 || competing.roundConfirmations().empty()) { + return fail("no round confirmations"); + } + const auto confMask = cs::Utils::bitsToMask(competing.numberConfirmations(), competing.roundConfirmationMask()); + if (confMask.size() <= 1 || confMask.size() != trustedConfidants.size()) { + return fail("confirmation mask does not match our last block's confidants"); + } + const size_t confirming = NodeUtils::realTrustedValue(confMask); + if (competing.roundConfirmations().size() != confirming) { + return fail("confirmation count does not match its mask"); + } + if (confirming * 2 <= trustedConfidants.size()) { + return fail("fewer than a majority of our confidants confirm it"); + } + std::string reason; + if (!cs::BlockVerifier::verifyConfirmations(competing, trustedConfidants, reason)) { + return fail(reason.c_str()); + } + return true; +} + +bool BlockChain::isBootstrapAuthenticatedLocally(const csdb::Pool& competing) const { + std::string reason; + if (!cs::BlockVerifier::verifySignatures(competing, reason)) { + csdebug() << kLogPrefix << "bootstrap block #" << competing.sequence() << " signatures: " << reason; + return false; + } + const auto& confidants = competing.confidants(); + if (confidants.empty()) { + return false; + } + // a round table this node accepted itself (its bootstrap table was checked when it was taken) + const auto& conveyer = cs::Conveyer::instance(); + if (conveyer.currentRoundTable().confidants == confidants) { + return true; + } + const cs::RoundNumber current = conveyer.currentRoundNumber(); + for (cs::RoundNumber back = 0; back < 3 && back <= current; ++back) { + const cs::RoundTable* table = conveyer.roundTable(current - back); + if (table != nullptr && table->confidants == confidants) { + return true; + } + } + csdebug() << kLogPrefix << "bootstrap block #" << competing.sequence() + << " names confidants of no round table this node accepted"; + return false; +} + +bool BlockChain::competingBlockMayTriggerUnwind(const csdb::Pool& competing) const { + if (deepRollbackApproved()) { + return true; // operator path: a deliberate switch to a chain confirmed by other confidants + } + if (BlockChain::isBootstrap(competing)) { + return isBootstrapAuthenticatedLocally(competing); + } + return hasTrustedConfirmations(competing); +} + // - remove the last block from the top of blockchain // - remove pair (hash, sequence) from cache (blockHashes_) // - decrement the last sequence by 1 @@ -838,6 +1407,15 @@ void BlockChain::removeLastBlock() { csmeta(csdebug) << kLogPrefix << "There are no blocks, allowed to be removed"; return; } + // longest-chain guard: every caller of removeLastBlock is an automatic reorg/sync removal or the + // start-up integrity repair (the operator truncates with --set-bc-top / --resolve-fork, at start-up, + // in the storage, not here), so bound the depth here, once, for all of the automatic ones. When the + // limit is reached the block is kept and the caller sees its own last block unchanged; the node + // reports FORK HELD until the episode ends or the operator truncates. + if (!mayUnwindLastBlock()) { + cswarning() << kLogPrefix << "longest-chain guard kept the last block, automatic rollback stopped"; + return; + } //--blocksToBeRemoved_; cs::Sequence remove_seq = lastSequence_; csdb::PoolHash remove_hash = blockHashes_->find(remove_seq); @@ -909,6 +1487,9 @@ void BlockChain::removeLastBlock() { blockHashes_->remove(remove_hash); } --lastSequence_; + if (rollbackEpisode_ && lastSequence_.load() < rollbackLowWater_) { + rollbackLowWater_ = lastSequence_.load(); + } csmeta(csdebug) << kLogPrefix << "done"; } @@ -928,6 +1509,14 @@ bool BlockChain::compromiseLastBlock(const csdb::PoolHash& desired_hash) { return false; } + // longest-chain guard: marking the last block uncertain is the first step of an automatic unwind, + // so it is bounded here too. Beyond the depth limit the block is kept and the caller falls through + // to its keep-own-chain branch. + if (!mayUnwindLastBlock()) { + cswarning() << kLogPrefix << "longest-chain guard kept the last block, not compromising it"; + return false; + } + const auto seq = last_block.sequence(); const auto current = last_block.hash(); const auto desired = desired_hash; @@ -1924,6 +2513,11 @@ std::optional BlockChain::recordBlock(csdb::Pool& pool, bool isTrust lastSequence_ = deferredBlock_.sequence(); } + // forward progress: ends the rollback episode (and FORK HELD) once the head is above its reference, + // and settles the branch approved with --resolve-fork + onHeadAdvanced(); + checkApprovedBranch(); + //csdetails() << kLogPrefix << "Pool #" << deferredBlock_.sequence() << ": " << cs::Utils::byteStreamToHex(deferredBlock_.to_binary().data(), deferredBlock_.to_binary().size()); emit storeBlockEvent(pool); if constexpr (false && (pool.transactions_count() > 0 || pool.sequence() % 10 == 0)) {//log code @@ -2265,6 +2859,20 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { if (poolSequence == lastSequence) { csdebug() << kLogPrefix << "poolSequence == lastSequence"; + // after --resolve-fork: the node stored its own block right after the fork point again, and the + // recorded competing block itself (the very hash the operator approved, built on the same + // fork point block as the own one) is offered: give up the own block, down to the fork point and never below, so the + // recorded block is appended next. No other block bypasses the usual checks. + if (approvedBranch_.has_value() && poolSequence == approvedBranch_->seq && + pool.hash().to_string() == approvedBranch_->hash && getLastHash().to_string() != approvedBranch_->hash && + !pool.previous_hash().is_empty() && pool.previous_hash() == getLastBlock().previous_hash()) { + cslog() << kLogPrefix << "the block #" << WithDelimiters(poolSequence) << " approved with --resolve-fork is offered: " + << "giving up the own block at that height"; + competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash()}; + removeLastBlock(); + competing_.reset(); + return false; + } if (isLastBlockUncertain() && pool.sequence() == uncertainSequence_) { cslog() << kLogPrefix << "review replacement for uncertain block " << WithDelimiters(poolSequence); if (pool.hash() == desiredHash_) { @@ -2281,7 +2889,9 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { } else { csdebug() << kLogPrefix << "the desired last block has the different content, drop it and remove own last block"; + competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash()}; removeLastBlock(); + competing_.reset(); return false; } } @@ -2308,13 +2918,42 @@ bool BlockChain::storeBlock(csdb::Pool& pool, cs::PoolStoreType type) { cserror() << "BLOCKCHAIN> new pool\'s prev. hash is empty, don\'t write it, do not any harm to our blockchain"; return false; } - if (compromiseLastBlock(pool.previous_hash())) { + // the competing block being handled, for the fork record if the node refuses to unwind + competing_ = CompetingBlock{poolSequence, pool.hash(), pool.previous_hash(), competingBlockMayTriggerUnwind(pool)}; + struct ClearCompeting { + std::optional& c; + ~ClearCompeting() { c.reset(); } + } clearCompeting{competing_}; + // after --resolve-fork, only a competing block that passes the usual checks ends the approval + // of the recorded branch; an unauthenticated offer changes nothing + if (approvedBranch_.has_value() && (poolSequence == approvedBranch_->seq || poolSequence == approvedBranch_->seq + 1) && + pool.hash().to_string() != approvedBranch_->hash && competingBlockMayTriggerUnwind(pool)) { + dropApprovedBranch("another block vouched for by this node's confidants was offered at that height"); + } + if (!competingBlockMayTriggerUnwind(pool)) { + // a competing block not vouched for by authority this node recognises (no strict + // confirmations by our last block's confidants, or a bootstrap flag that is not + // authenticated locally): never unwind our own chain for it, not even by marking our + // last block uncertain (which would let a later replacement remove it). The node + // reports FORK HELD until its head moves on or the operator decides. + if (recordForkHeld("authority")) { + cslog() << kLogPrefix << "competing block #" << poolSequence + << " is not authenticated by this node's confidants, keep own last block"; + } + else { + csdebug() << kLogPrefix << "competing block #" << poolSequence + << " of a branch dismissed by the operator, keep own last block"; + } + } + else if (compromiseLastBlock(pool.previous_hash())) { csdebug() << kLogPrefix << "compromise own last block and cancel store operation"; } else { //if (lastSequence + 5ULL < cs::Conveyer::instance().currentRoundNumber()) { // arrangeBlocksInCache(); //} + // authenticated competing block: unwind, but only within the depth bound that + // removeLastBlock enforces; beyond it the block is kept and the operator decides. csdebug() << kLogPrefix << "remove own last block and cancel store operation"; removeLastBlock(); } @@ -2797,6 +3436,27 @@ void BlockChain::cacheLastBlocks() { cs::Sequence lastSeq; while (!incorrectBlocks_.empty() || !selectionFinished_) { csinfo() << kLogPrefix << "Starting blocks transferring cycle"; + // removals while incorrectBlocks_ is not empty repair this node's own database (blocks that + // failed the hash check at start-up): a local repair, not a network reorg, so the guard does + // not bound them. The flag lives for this iteration only. + integrityRepair_ = !incorrectBlocks_.empty(); + struct EndRepair { + bool& f; + ~EndRepair() { f = false; } + } endRepair{integrityRepair_}; + // longest-chain guard, for the rest of the selection: stop unwinding own blocks once a deeper + // automatic rollback than allowed would be needed; leave the chain intact, abandon this + // antifork episode cleanly and wait for the operator. (guardAutoRollback measures depth + // against the fixed reference head via lastSequence_, so this counts real own-block removals.) + if (!integrityRepair_ && !guardAutoRollback()) { + cswarning() << kLogPrefix << "longest-chain guard stopped the antifork rollback; keeping own chain"; + incorrectBlocks_.clear(); + selectionFinished_ = true; + lastPrevHash_ = csdb::PoolHash(); + emittingRequest_ = 0; + antiForkMode_ = false; + return; + } auto lastBlock = getLastBlock(); lastSeq = lastBlock.sequence(); csinfo() << kLogPrefix << "now dealing with " << lastSeq; diff --git a/csnode/src/forkguardstate.cpp b/csnode/src/forkguardstate.cpp new file mode 100644 index 000000000..96398fa1f --- /dev/null +++ b/csnode/src/forkguardstate.cpp @@ -0,0 +1,414 @@ +#include + +#include +#include +#include +#include +#include + +#include +#ifdef _WIN32 +#include +#else +#include +#include +#endif + +#include +#include + +namespace cs::forkguard { + +namespace fs = std::filesystem; +using boost::property_tree::ptree; + +namespace { + +// flushes a written file (and, on POSIX, the folder entry of a rename) to the disk +bool syncFile(std::FILE* f) { + if (std::fflush(f) != 0) { + return false; + } +#ifdef _WIN32 + return _commit(_fileno(f)) == 0; +#else + return ::fsync(::fileno(f)) == 0; +#endif +} + +void syncDir(const fs::path& dir) { +#ifndef _WIN32 + const int fd = ::open(dir.c_str(), O_RDONLY); + if (fd >= 0) { + ::fsync(fd); + ::close(fd); + } +#else + (void)dir; +#endif +} + +// a small file written whole: to a temporary name first, flushed to the disk, then renamed over the +// old one, so a crash leaves either the old file or the new one +bool writeTree(const std::string& dir, const char* name, const ptree& tree) { + if (dir.empty()) { + return false; + } + std::error_code ec; + const fs::path target = fs::path(dir) / name; + const fs::path tmp = fs::path(dir) / (std::string(name) + ".tmp"); + std::string text; + try { + std::ostringstream out; + boost::property_tree::write_json(out, tree); + text = out.str(); + } + catch (const std::exception&) { + return false; + } + std::FILE* f = std::fopen(tmp.string().c_str(), "wb"); + if (f == nullptr) { + return false; + } + const bool written = std::fwrite(text.data(), 1, text.size(), f) == text.size() && syncFile(f); + const bool closed = std::fclose(f) == 0; + if (!written || !closed) { + fs::remove(tmp, ec); + return false; + } + fs::rename(tmp, target, ec); + if (ec) { + fs::remove(tmp, ec); + return false; + } + syncDir(fs::path(dir)); + return true; +} + +// the file's tree; nothing when it does not exist. A file that exists but cannot be read sets +// *corrupt (an empty file, bad JSON, or larger than any record) +std::optional readTree(const std::string& dir, const char* name, bool* corrupt) { + if (corrupt) { + *corrupt = false; + } + if (dir.empty()) { + return std::nullopt; + } + const fs::path path = fs::path(dir) / name; + std::error_code ec; + if (!fs::exists(path, ec)) { + return std::nullopt; + } + if (!fs::is_regular_file(path, ec) || fs::file_size(path, ec) > 64 * 1024 || fs::file_size(path, ec) == 0) { + if (corrupt) { + *corrupt = true; + } + return std::nullopt; + } + try { + std::ifstream in(path); + ptree tree; + boost::property_tree::read_json(in, tree); + return tree; + } + catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } + return std::nullopt; + } +} + +void removeFile(const std::string& dir, const char* name) { + if (dir.empty()) { + return; + } + std::error_code ec; + fs::remove(fs::path(dir) / name, ec); +} + +} // namespace + +std::string stateDirFor(const std::string& dbPath) { + fs::path p(dbPath); + // "db/" and "db" name the same folder + while (!p.empty() && !p.has_filename() && p.has_parent_path() && p != p.parent_path()) { + p = p.parent_path(); + } + const fs::path parent = p.parent_path(); + return parent.empty() ? std::string(".") : parent.string(); +} + +bool saveEpisode(const std::string& dir, const Episode& episode) { + ptree tree; + tree.put("high_water", episode.highWater); + return writeTree(dir, kEpisodeFile, tree); +} + +std::optional loadEpisode(const std::string& dir, bool* corrupt) { + const auto tree = readTree(dir, kEpisodeFile, corrupt); + if (!tree) { + return std::nullopt; + } + try { + Episode e; + e.highWater = tree->get("high_water"); + return e; + } + catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } + return std::nullopt; + } +} + +void clearEpisode(const std::string& dir) { + removeFile(dir, kEpisodeFile); +} + +bool saveApprovedBranch(const std::string& dir, const ApprovedBranch& b) { + ptree tree; + tree.put("fork_point", b.forkPoint); + tree.put("block", b.seq); + tree.put("hash", b.hash); + return writeTree(dir, kApprovedFile, tree); +} + +std::optional loadApprovedBranch(const std::string& dir, bool* corrupt) { + const auto tree = readTree(dir, kApprovedFile, corrupt); + if (!tree) { + return std::nullopt; + } + try { + ApprovedBranch b; + b.forkPoint = tree->get("fork_point"); + b.seq = tree->get("block"); + b.hash = tree->get("hash"); + if (b.hash.empty() || b.seq != b.forkPoint + 1) { + if (corrupt) { + *corrupt = true; + } + return std::nullopt; + } + return b; + } + catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } + return std::nullopt; + } +} + +void clearApprovedBranch(const std::string& dir) { + removeFile(dir, kApprovedFile); +} + +bool ForkRecord::inDismissedBranch(const std::string& hash, const std::string& previous) const { + for (const auto& h : branchHashes) { + if ((!hash.empty() && h == hash) || (!previous.empty() && h == previous)) { + return true; + } + } + return false; +} + +void ForkRecord::addBranchHash(const std::string& hash) { + if (hash.empty()) { + return; + } + for (const auto& h : branchHashes) { + if (h == hash) { + return; + } + } + branchHashes.push_back(hash); + // the oldest go first, but never the dismissed block itself: forged blocks cannot push it out + while (branchHashes.size() > kMaxBranchHashes) { + auto victim = branchHashes.begin(); + if (*victim == dismissedHash && branchHashes.size() > 1) { + ++victim; + } + branchHashes.erase(victim); + } +} + +bool saveForkRecord(const std::string& dir, const ForkRecord& r) { + ptree tree; + tree.put("held", r.held); + tree.put("reason", r.reason); + tree.put("since", utcTime(r.sinceUnix)); + tree.put("since_unix", r.sinceUnix); + tree.put("high_water", r.highWater); + tree.put("our_head", r.ourHead); + tree.put("our_head_hash", r.ourHeadHash); + tree.put("fork_point", r.forkPoint); + tree.put("fork_point_hash", r.forkPointHash); + tree.put("fork_point_exact", r.forkPointExact); + tree.put("competing_block", r.competingSeq); + tree.put("competing_hash", r.competingHash); + tree.put("peer_key", r.peerKey); + tree.put("peer_address", r.peerAddress); + tree.put("peer_height", r.peerHeight); + if (r.hasDismissal()) { + tree.put("dismissed", utcTime(r.dismissedUnix)); + tree.put("dismissed_unix", r.dismissedUnix); + tree.put("dismissed_block", r.dismissedSeq); + tree.put("dismissed_hash", r.dismissedHash); + ptree hashes; + for (const auto& h : r.branchHashes) { + ptree item; + item.put("", h); + hashes.push_back(std::make_pair("", item)); + } + tree.add_child("dismissed_branch_hashes", hashes); + } + return writeTree(dir, kForkFile, tree); +} + +std::optional loadForkRecord(const std::string& dir, bool* corrupt) { + const auto tree = readTree(dir, kForkFile, corrupt); + if (!tree) { + return std::nullopt; + } + try { + ForkRecord r; + r.held = tree->get("held", true); + r.reason = tree->get("reason", ""); + r.sinceUnix = tree->get("since_unix", 0); + r.highWater = tree->get("high_water"); + r.ourHead = tree->get("our_head"); + r.ourHeadHash = tree->get("our_head_hash", ""); + r.forkPoint = tree->get("fork_point"); + r.forkPointHash = tree->get("fork_point_hash", ""); + r.forkPointExact = tree->get("fork_point_exact", false); + r.competingSeq = tree->get("competing_block", 0); + r.competingHash = tree->get("competing_hash", ""); + r.peerKey = tree->get("peer_key", ""); + r.peerAddress = tree->get("peer_address", ""); + r.peerHeight = tree->get("peer_height", 0); + r.dismissedUnix = tree->get("dismissed_unix", 0); + r.dismissedSeq = tree->get("dismissed_block", 0); + r.dismissedHash = tree->get("dismissed_hash", ""); + if (const auto hashes = tree->get_child_optional("dismissed_branch_hashes")) { + for (const auto& item : *hashes) { + r.addBranchHash(item.second.get_value()); + } + } + return r; + } + catch (const std::exception&) { + if (corrupt) { + *corrupt = true; + } + return std::nullopt; + } +} + +void clearForkRecord(const std::string& dir) { + removeFile(dir, kForkFile); +} + +void clearAll(const std::string& dir) { + removeFile(dir, kEpisodeFile); + removeFile(dir, kForkFile); + removeFile(dir, kApprovedFile); +} + +std::optional planResolve(const std::string& dir, std::string* error) { + bool corrupt = false; + const auto record = loadForkRecord(dir, &corrupt); + if (corrupt) { + if (error) { + *error = std::string("the fork record (") + kForkFile + ") in " + dir + + " cannot be read: nothing removed. Use --set-bc-top after checking with other nodes"; + } + return std::nullopt; + } + if (!record.has_value()) { + if (error) { + *error = std::string("no fork record (") + kForkFile + ") in " + dir + + ": this node holds no refused fork, nothing to do. To cut the chain at a known block, use --set-bc-top "; + } + return std::nullopt; + } + if (!record->held) { + if (error) { + *error = record->hasDismissal() + ? "the fork was dismissed on " + utcTime(record->dismissedUnix) + + ": nothing held, nothing removed. To cut the chain at a known block, use --set-bc-top " + : std::string("nothing held, nothing removed. To cut the chain at a known block, use --set-bc-top "); + } + return std::nullopt; + } + if (record->forkPointHash.empty()) { + if (error) { + *error = "the fork record has no hash for block #" + std::to_string(record->forkPoint) + + " and cannot be checked against the database: nothing removed. Use --set-bc-top "; + } + return std::nullopt; + } + return ResolvePlan{record->forkPoint, record->forkPointHash, *record}; +} + +std::string utcTime(int64_t unixSeconds) { + const std::time_t t = static_cast(unixSeconds); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &t); +#else + gmtime_r(&t, &tm); +#endif + char buf[32] = {}; + std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S UTC", &tm); + return buf; +} + +std::string remedy(const ForkRecord& r) { + std::ostringstream os; + os << "first check the head and the block at #" << r.forkPoint << " with other nodes or the monitor. If this node's " + << "chain is the right one: create the file " << kDismissTrigger << " next to the db folder (native: touch " + << kDismissTrigger << " in the node folder; Docker: docker compose exec node touch /data/" << kDismissTrigger + << "); the running node picks it up within a few seconds, keeps refusing that branch and stops reporting it. If the other chain is the right one: stop " + << "the node and run once node --config-file config.ini --resolve-fork (Docker: docker compose stop && docker " + << "compose run --rm node resolve-fork && docker compose up -d), then start it as usual; by hand: --set-bc-top " + << r.forkPoint << " (Docker: docker compose run --rm node set-bc-top " << r.forkPoint << ")."; + return os.str(); +} + +std::string summary(const ForkRecord& r) { + std::ostringstream os; + os << "FORK HELD since " << utcTime(r.sinceUnix) << ": "; + if (r.reason == "authority") { + os << "a competing block is not vouched for by this node's confidants"; + } + else if (r.reason == "unreadable") { + os << "the fork record " << kForkFile << " could not be read; the fork point is not known (check the chain " + << "with other nodes)"; + } + else { + os << "a competing chain asks to unwind beyond the automatic bound (reference head #" << r.highWater << ")"; + } + os << ". Fork point: block #" << r.forkPoint << (r.forkPointExact ? "" : " or lower"); + if (!r.forkPointHash.empty()) { + os << " (" << r.forkPointHash << ")"; + } + os << "; our head #" << r.ourHead; + if (r.competingSeq != 0) { + os << ", refused block #" << r.competingSeq; + } + if (!r.peerKey.empty() || !r.peerAddress.empty()) { + os << " from " << (r.peerKey.empty() ? std::string("?") : r.peerKey); + if (!r.peerAddress.empty()) { + os << " (" << r.peerAddress << ")"; + } + } + if (r.peerHeight != 0) { + os << ", peer height #" << r.peerHeight; + } + os << ". The node keeps its chain. What to do: " << remedy(r); + return os.str(); +} + +} // namespace cs::forkguard diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index f903688e5..6a11ba977 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -123,6 +123,20 @@ Node::Node(cs::config::Observer& observer) cs::Connector::connect(&blockChain_.lastBlockReplacedEvent, this, static_cast(&Node::logStateDigest)); cs::Connector::connect(&blockChain_.uncertainBlock, this, &Node::sendBlockRequestToConfidants); cs::Connector::connect(&blockChain_.orderNecessaryBlock, this, &Node::sendNecessaryBlockRequest); + // the fork record names the address of the refused peer when the routing table knows it + blockChain_.setPeerAddressResolver([this](const cs::PublicKey& key) -> std::string { + if (!transport_) { + return {}; + } + std::vector peers; + transport_->getKnownPeers(peers); + for (const auto& p : peers) { + if (p.key == key && !p.ip.empty()) { + return p.ip + ":" + std::to_string(p.port); + } + } + return {}; + }); cs::Connector::connect(&stat_.accountInitiationRequest, this, &Node::accountInitiationRequest); cs::Connector::connect(&blockChain_.successfullQuickStartEvent, this, &Node::onSuccessQS); initPoolSynchronizer(); @@ -142,6 +156,7 @@ Node::Node(cs::config::Observer& observer) cs::Connector::connect(&poolSynchronizer_->sendRequest, this, &Node::sendBlockRequest); cs::Connector::connect(&snapshotTickTimer_.timeOut, this, &Node::onSnapshotTick); cs::Connector::connect(&backfillTimer_.timeOut, this, &Node::onBackfillTick); + cs::Connector::connect(&forkGuardTimer_.timeOut, this, &Node::onForkGuardTick); globalPublicKey_.fill(0); globalPublicKey_.at(31) = 7U; @@ -236,14 +251,85 @@ bool Node::initCommon() { return true; } +void Node::logForkHeldDetail(const std::optional& forkHeld) { + if (forkHeld.has_value()) { + const auto now = std::chrono::steady_clock::now(); + if (lastForkHeldDetail_ == std::chrono::steady_clock::time_point{} || now - lastForkHeldDetail_ >= std::chrono::minutes(1)) { + lastForkHeldDetail_ = now; + cswarning() << " " << cs::forkguard::summary(*forkHeld); + } + } + else { + lastForkHeldDetail_ = {}; + } +} + +// every few seconds on the processor thread, rounds or not (an isolated node has none): the +// dismiss-fork file, a pending refusal the head did not move past, the FORK HELD detail line +void Node::onForkGuardTick() { + if (!good_ || bootstrapping_) { + return; + } + blockChain_.checkDismissTrigger(); + blockChain_.promotePendingHold(false); + logForkHeldDetail(blockChain_.forkHeldRecord()); +} + +std::optional Node::resolveForkTop() { + const auto* config = cs::ConfigHolder::instance().config(); + if (config->newBlockchainTop()) { + cserror() << "--resolve-fork and --set-bc-top together: use one of them"; + return std::nullopt; + } + std::string error; + const auto plan = cs::forkguard::planResolve(cs::forkguard::stateDirFor(config->getPathToDB()), &error); + if (!plan.has_value()) { + cserror() << "--resolve-fork: " << error; + return std::nullopt; + } + cslog() << cs::forkguard::summary(plan->record); + cslog() << "--resolve-fork: removing every block above #" << WithDelimiters(plan->top) << " (" << plan->hash + << "), our head was #" << WithDelimiters(plan->record.ourHead) << ", if the block at #" + << WithDelimiters(plan->top) << " is still the recorded one"; + if (plan->record.forkPointExact && plan->record.competingSeq == plan->top + 1) { + blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash), plan->record.competingHash); + } + else { + cswarning() << "--resolve-fork: the fork point is an upper bound (the other chain diverges at #" + << WithDelimiters(plan->top) << " or lower). If it diverges lower, the node reports FORK HELD " + << "again with a lower fork point: then find the last common block with the monitor or other " + << "nodes and cut there with --set-bc-top ."; + blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash)); + } + return plan->top; +} + Node::ChainInit Node::initChain() { - if (cs::ConfigHolder::instance().config()->newBlockchainTop()) { + const auto* initConfig = cs::ConfigHolder::instance().config(); + if (initConfig->newBlockchainTop() || initConfig->resolveFork()) { + cs::Sequence newTop = initConfig->newBlockchainTopSeq(); + if (initConfig->resolveFork()) { + const auto resolved = resolveForkTop(); + if (!resolved.has_value()) { + exitCode_ = 3; + return ChainInit::Failed; + } + newTop = *resolved; + } if (!blockChain_.init( - cs::ConfigHolder::instance().config()->getPathToDB(), + initConfig->getPathToDB(), &cachesSerializationManager_, initialConfidants_, - cs::ConfigHolder::instance().config()->newBlockchainTopSeq()) + newTop) ) { + if (blockChain_.truncationRefused()) { + // nothing was removed: the database is as it was, the quick-start data stays + cserror() << "--resolve-fork: the block at the recorded fork point is not the recorded one (the " + << "chain changed since the fork was recorded): nothing removed. Check the node's head " + << "with other nodes or the monitor; if it must still be cut, use --set-bc-top ."; + exitCode_ = 3; + return ChainInit::Failed; + } if (isStopRequested()) { cslog() << "Init aborted by user; preserving QUICK START state."; cs::CheckpointHead headInfo; @@ -284,6 +370,10 @@ Node::ChainInit Node::initChain() { solver_->smart_contracts().rehydrateContractDbCache(blockChain_); solver_->smart_contracts().validateRestoredStatesAgainstChain(blockChain_); } + if (initConfig->resolveFork()) { + cslog() << "--resolve-fork: the chain now ends at block #" << WithDelimiters(newTop) + << " and the fork record is cleared. Start the node as usual: it syncs from there."; + } return ChainInit::StopAfterInit; } if (!blockChain_.init( @@ -410,6 +500,8 @@ bool Node::initRunning() { initBootstrapRP(initialConfidants_); EventReport::sendRunningStatus(*this, Running::Status::Run); startBackfill(); + // the fork guard: dismiss-fork and pending holds also without rounds (an isolated node) + forkGuardTimer_.start(5000, cs::Timer::Type::Standard, cs::RunPolicy::CallQueuePolicy); return true; } @@ -530,7 +622,7 @@ bool Node::startBootstrap() { const std::string dbPath = config.getPathToDB(); const std::string backend = config.getStorageSettings().dbBackend; bool removedTmp = false; - if (const auto skip = sb::notNeeded(cfg, ".", dbPath, backend, config.newBlockchainTop(), &removedTmp)) { + if (const auto skip = sb::notNeeded(cfg, ".", dbPath, backend, config.newBlockchainTop() || config.resolveFork(), &removedTmp)) { if (skip->why == sb::Skip::Backend) { cswarning() << "SNAPSHOT: " << skip->text << ": syncing from block 0"; } @@ -867,6 +959,7 @@ void Node::stop() { { std::lock_guard lock(backfillTimerMutex_); backfillTimer_.stop(); + forkGuardTimer_.stop(); if (backwardSync_) { backwardSync_->stop(); } @@ -2440,6 +2533,12 @@ void Node::getBlockRequest(const uint8_t* data, const size_t size, const cs::Pub void Node::getBlockReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { csinfo() << __func__ << " from " << cs::Utils::byteStreamToHex(sender); + // the blocks of this reply come from `sender`: named in the fork record should the node refuse one + blockChain_.setSyncSource(sender, poolSynchronizer_->announcedSequence(sender)); + struct ClearSource { + BlockChain& chain; + ~ClearSource() { chain.clearSyncSource(); } + } clearSource{blockChain_}; bool isSyncOn = poolSynchronizer_->isSyncroStarted(); bool isBlockchainUncertain = blockChain_.isLastBlockUncertain(); @@ -4716,6 +4815,16 @@ void Node::onRoundStart(const cs::RoundTable& roundTable, bool updateRound) { line1 << "TRUSTED [" << cs::numeric_cast(myConfidantIndex_) << "]"; status_ = cs::NodeStatus::Trusted; } + // the operator keeps this chain: a dismiss-fork file next to the database, picked up every round + // (and by the fork guard timer); a refusal the head did not move past becomes FORK HELD + blockChain_.checkDismissTrigger(); + blockChain_.promotePendingHold(true); + // the node refused to unwind for a competing chain and holds its own: on every round line, so it + // cannot be missed (the detail, with the two remedies, once a minute below) + const auto forkHeld = blockChain_.forkHeldRecord(); + if (forkHeld.has_value()) { + line1 << " | FORK HELD"; + } line1 << ' '; @@ -4730,6 +4839,7 @@ void Node::onRoundStart(const cs::RoundTable& roundTable, bool updateRound) { csdebug() << " Node key " << KeyToBase58(nodeIdKey_); cslog() << " Last written sequence = " << WithDelimiters(blockChain_.getLastSeq()) << ", neighbour nodes = " << transport_->getNeighboursCount() << (backwardSync_ && backwardSync_->stats().started ? ", oldest stored = " + std::to_string(blockChain_.getFirstStoredSequence()) : std::string{}); + logForkHeldDetail(forkHeld); std::ostringstream line2; diff --git a/csnode/src/poolsynchronizer.cpp b/csnode/src/poolsynchronizer.cpp index 87dbc7298..cb0a4495f 100644 --- a/csnode/src/poolsynchronizer.cpp +++ b/csnode/src/poolsynchronizer.cpp @@ -276,6 +276,11 @@ void PoolSynchronizer::onStoreBlockTimeElapsed() { // //} +cs::Sequence PoolSynchronizer::announcedSequence(const cs::PublicKey& key) const { + const auto it = neighbours_.find(key); + return it == neighbours_.end() ? 0 : it->second; +} + cs::Sequence PoolSynchronizer::getMaxNeighbourSequence() { cs::Sequence maxSequence = 0; for (auto& it : neighbours_) { diff --git a/csnode/src/snapshot_importer.cpp b/csnode/src/snapshot_importer.cpp index 83849699f..a33ba76eb 100644 --- a/csnode/src/snapshot_importer.cpp +++ b/csnode/src/snapshot_importer.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -334,6 +335,8 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife } syncPath(nodeDir / "snapshot.json", false); syncPath(nodeDir, true); + // the bound on automatic reorganisations kept its files for the database this one replaces + cs::forkguard::clearAll(cs::forkguard::stateDirFor(dbPath.string())); result.ok = true; return result; } diff --git a/docs/CHANGES.md b/docs/CHANGES.md index cc214dc2f..a8369230a 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -45,6 +45,7 @@ lines under the new release. | PR | Change | |---|---| | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | +| #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). A block not vouched for is refused at once but reported only if the head then stays put (no flapping); unreadable guard files fail safe; the files go when the database is replaced. `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | | #186 | ci: the comments of `ci/testnet/contracts.py` say why contract-to-contract calls fail: executor 1518 fails every `invokeExternalContract`; the node does not refuse `@UsingContract` methods (comments only) | diff --git a/docs/differences.md b/docs/differences.md index 4f74d09c0..b8e22d884 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -266,6 +266,87 @@ consensus rules. ## Consensus fixes +- **Bounded automatic rollback.** When a node is behind and syncs a competing chain that forks below + its current head, it rolls back its own blocks down to the fork point before taking the other + chain. The official node puts no limit on that depth. The community node stops the automatic + rollback once it would unwind more than `max_auto_rollback` of its own confirmed blocks (default + 100) below the head it had when the rollback started: it leaves its chain intact, logs a prominent + alert and waits for the operator. The bound covers every automatic removal path — the sync fork + path, the uncertain-block replacement, the antifork cache and the bad-hash reply — because the + depth check lives inside the two functions they all go through. The depth is measured against a + fixed reference, so forward progress cannot reset it and a node cannot be made to creep past the + limit by alternating removals and appends; the reference is kept in `reorg-episode.json` next to + the database, so a restart does not reset it either. A reorg this deep is a rare event — a + deliberate switch to another network, or a rejected one — and the choice is left to a human + instead of happening silently. The node keeps running and does not crash-loop. The start-up + integrity repair (blocks of the node's own database that fail the hash check, removed when the + operator chooses to resolve them) is a local repair, not a reorg, and is not bounded. + + Before any of this, the competing block itself must be vouched for by authority the node already + recognises, or the node does not unwind at all, not even by one block. A competing block that is + **not a bootstrap block** needs round confirmations from a majority of the confidants of the node's + own last block, checked as the block verifier checks a block from the network, but more strictly + (no one-entry confirmation masks). An honest micro-fork passes, because the honest competing block + at the same height was produced by the same elected trusted group. A **bootstrap block** carries no + such confirmations (the verifier accepts it without them, so this check refuses it), and its + bootstrap flag alone proves nothing, so it passes only when it is authenticated locally: its own + signatures verify and its confidants are exactly those of a round table this node itself accepted + in the last few rounds. Otherwise the node keeps its chain and the operator decides. Everything + that passes is still subject to the depth bound. + + **FORK HELD.** Whenever the node refuses to unwind (beyond the bound, or a competing block not + vouched for), it records the fork in `fork-held.json` next to the database — the fork point (the + highest of its blocks the other chain can share, with its hash, exact when the competing block + names it as its previous block), its head, the refused block, the peer and its announced height, + the time and the episode's reference — and reports **FORK HELD** until the episode ends or the + operator decides: `| FORK HELD` on every round line (`R- NORMAL | FORK HELD`), a detail line + with the two remedies once a minute, and an unhealthy Docker healthcheck. The state clears by + itself when the head climbs back above the reference. A competing block not vouched for is + refused at once, but reported as FORK HELD only when the head stays at the same block after it + (two round starts, or a minute), so a peer on a minority branch cannot make it flap. An unreadable + guard file is reported and fails safe (a new rollback episode at the head; FORK HELD kept), the + files are written with a flush to the disk, and they are removed when the database is replaced (a + new empty database, a snapshot restore). The operator, after checking with other + nodes or the monitor, either keeps this chain — a `dismiss-fork` file next to the database, which + the running node picks up within a few seconds: the record keeps that branch (its first known block and + the hashes seen since) as dismissed, the node still refuses it but no longer reports it, and a + different branch is reported again — or follows the other one: `--resolve-fork` (no argument) + cuts the chain at the recorded fork point, only if the block there is still the recorded one; the + node then syncs forward from there. It relaxes nothing else: when the fork point is exact, only + the recorded competing block right after it — that very hash — may replace a block the node + stored again right after the fork point; nothing is ever unwound to or below the fork point, every + other block meets the usual checks, and an unauthenticated offer changes nothing. The approval + ends once the node goes on past that height, or a competing block that passes the usual checks + is offered there. When the fork point is + only an upper bound and the other chain diverges lower, the node reports FORK HELD again with a + lower fork point; `--set-bc-top ` then cuts by hand at the last common block found with + the monitor or other nodes. Both are one-shot + and exit; neither advice is ever "restart", since a restart alone changes nothing. See "If your + node reports FORK HELD" in `docs/quick-start.md`. + + This is a local safety bound only: it changes how far this node unwinds on its own, never what a + block contains or what the node accepts in normal operation, so it is invisible to other nodes and a + mixed official/community network behaves exactly as before. Two settings tune it, both in `[params]`: + `max_auto_rollback=` sets the bound (default 100; 0 removes the depth bound, the authority check + stays), and `allow_deep_rollback=true` accepts a deliberate switch: it lifts both the depth bound + and the check above, since the other chain's blocks are confirmed by confidants this node does not + hold. The `CS_ALLOW_DEEP_ROLLBACK` environment variable does the same. It is the last resort, after + `--resolve-fork` and `--set-bc-top`: once one rollback episode has completed with it, the node + turns it off for the rest of the run and says so loudly; remove it from the configuration then. + + **After a partition, and a lagging node.** When both sides of a split keep producing blocks, each + side's blocks are confirmed by its own confidants. On reconnection a node on the shorter side does + not follow the longer chain on its own, whatever the depth, because that chain's blocks are not + confirmed by any confidants it holds: it keeps its chain, reports FORK HELD and the operator + decides. The official node unwinds to the longer chain without asking. A single node or a minority + that cannot reach consensus while cut off builds no branch and simply syncs forward when it + reconnects. The same holds for a node that lags behind with a divergent last block it never + replaced (for example, it missed a bootstrap round): when the network's next block is not + confirmed by the confidants of that block, the node waits for the operator instead of unwinding + it. This is the deliberate choice of this change; `--resolve-fork` is the remedy. The opt-in + community bootstrap fallback (PR #188, not merged yet) expects the longer chain to win after a + partition; with this change that switch is an operator decision too. + - **Two blocks of one round.** When a trusted node does not get one stage-2 in time, it fetches it with a request that takes exactly as long as the others wait for its stage-3. Depending on a few milliseconds, some trusted nodes finish the round with every signature while the others retry diff --git a/docs/quick-start.md b/docs/quick-start.md index 8d2b4a6dc..b77e5931d 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -421,7 +421,57 @@ On Windows, type the password in a console window (`node.exe` reads it from the a pipe). The plain key and the `.encrypted-...` copy keep the folder's permissions (Windows has no mode 600): keep the `csnode` folder readable by your user only. -## 11. Troubleshooting +## 11. If your node reports FORK HELD + +``` +=========== R-12'345'678.0 NORMAL | FORK HELD =========== + Last written sequence = 12'345'577, neighbour nodes = 14 + FORK HELD since 2026-11-19 10:04:05 UTC: a competing block is not vouched for by this node's confidants. Fork point: block #12345576 or lower; our head #12345577, refused block #12345578 from 7Q4e... (203.0.113.7:9510), peer height #12349000. The node keeps its chain. What to do: ... +``` + +**What it means.** Another chain asked this node to roll back its own confirmed blocks, and the node +refused: either the rollback would go deeper than `max_auto_rollback` blocks (100 by default), or +the other chain's blocks are not confirmed by the confidants this node trusts. The node keeps its +chain and waits for you. Nothing is lost and nothing was changed. The state shows on every round +line (`| FORK HELD`), as a detail line once a minute, in `fork-held.json` next to the `db` folder +(the `csnode` folder; `~/.credits` in Docker), and as an unhealthy container in `docker ps`. It +clears by itself if the node's chain grows past the point where the rollback started. + +A restart changes nothing: the node remembers the fork and the rollback limit across restarts, in +`reorg-episode.json`, `fork-held.json` and `resolve-approved.json` next to the `db` folder. They +describe that database: if you delete `db/` by hand to start again, delete these three files too +(a snapshot restore and Docker's `CREDITS_RESET=1` remove them for you). + +A block of another chain that the node refuses while its own chain goes on is not reported: FORK +HELD appears only when the node stays at the same block after the refusal (two rounds, or a minute). + +**1. Check before acting.** Compare the node's head (`Last written sequence`) and the block at the +fork point with other nodes: the [monitor](https://monitor.credits.com), the +[technical Telegram](https://t.me/creditstechnical), or a node you trust +(`credits-wallet status` on it). The question is which chain the network is on. + +**2. Then choose one.** + +| | Your chain is the right one | The other chain is the right one | +|---|---|---| +| What it does | The node keeps its chain. That branch stays refused but is no longer reported; a different branch would be reported again. The node keeps running. | The node removes its blocks above the recorded fork point (only if the block there is still the one it recorded) and exits; at the next start it syncs the other chain from there. | +| Native (node running) | in the `csnode` folder: `touch dismiss-fork` (Windows: `type nul > dismiss-fork`); the node picks it up within a few seconds and deletes it | stop the node (`Ctrl+C` once, or `sudo systemctl stop credits-node`), then `./node --config-file config.ini --resolve-fork` (Windows: `node.exe ...`), then start it as usual | +| Docker | `docker compose exec node touch /data/dismiss-fork` | `docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d` | + +`--resolve-fork` refuses, and removes nothing, when there is no fork record or when the block at +the recorded fork point is no longer the recorded one. To cut the chain at a block of your choice +instead, use `--set-bc-top ` (the last block to keep) with the node stopped; Docker: +`docker compose run --rm node set-bc-top `. If the detail line says +`Fork point: block #N or lower`, the node knows only an upper bound: should the other chain fork +lower, the node reports FORK HELD again with a lower fork point. Then find the last common block +with the monitor or other nodes and cut there with `--set-bc-top`. + +**Last resort.** `allow_deep_rollback=true` in `[params]` of `config.ini` (or the +`CS_ALLOW_DEEP_ROLLBACK` environment variable) lets the node follow any competing chain without the +limit or the confidants check. It turns itself off after one completed rollback and logs that you +must remove it; remove it then. + +## 12. Troubleshooting | Problem | What to do | |---|---| @@ -431,6 +481,7 @@ mode 600): keep the `csnode` folder readable by your user only. | `the block database is X, config.ini asks for Y` (exit code 3) | `config.ini` names another database than the one in `db/`. Set `db_backend` back to X, or convert on purpose with `./node --convert-db` (Docker: `CREDITS_CONVERT_DB=yes`). See [storage.md](storage.md#conversion). | | Snapshot download stopped | Run it again: `./node --download-snapshot` (node stopped) resumes, or run the set-up again. | | The node stops with exit code 4, `snapshot.mismatch` exists | The restored state did not match the snapshot. Start over with an empty `db/`, `caches/` and `qs/` (Docker: `CREDITS_RESET=1` once). | +| `FORK HELD` on the round lines; the container is unhealthy | See [If your node reports FORK HELD](#11-if-your-node-reports-fork-held). | | Disk full | A full node needs 250 to 300 GB with RocksDB. Free space or move to a bigger disk, or turn the backfill off (`backfill=off` in `[storage]`, `CREDITS_BACKFILL=off` in Docker before the first start). | | Linux: `GLIBC_2.38 not found` | Use the `linux-x64-glibc2.31` package. | | macOS: "cannot be opened" | `xattr -dr com.apple.quarantine csnode` | diff --git a/packaging/docker/README.md b/packaging/docker/README.md index b59686cd3..0bab1a5cb 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -84,6 +84,37 @@ password asks again. The password can also be piped: 1 GB of free memory for a few seconds: `CREDITS_MEMORY` must be at least about `1.2g` (default `4g`). A plain key is as exposed as the machine it is on. +## FORK HELD + +When the node refuses a competing chain (a rollback deeper than `max_auto_rollback`, 100 blocks by +default, or blocks not confirmed by the confidants it trusts), it keeps its chain, writes +`fork-held.json` to the data directory, shows `| FORK HELD` on every round line with a detail line +once a minute, and the healthcheck reports the container unhealthy with the two choices. Check with +other nodes or the monitor first, then: + +```sh +# this node's chain is the right one: it keeps refusing that branch and stops reporting it +docker compose exec node touch /data/dismiss-fork # stopped container: docker compose run --rm node dismiss-fork +# the other chain is the right one: cut at the recorded fork point, then follow the other chain +docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d +# or cut by hand at a block of your choice (the last one to keep) +docker compose stop && docker compose run --rm node set-bc-top && docker compose up -d +``` + +`resolve-fork` removes nothing when there is no record, the fork was dismissed, or the block at the +recorded fork point is no longer the recorded one. With a key encrypted by the node, `resolve-fork` +and `set-bc-top` start the node, which asks the key's password in the terminal (`docker compose run` +is interactive). See "If your node reports FORK HELD" in `docs/quick-start.md`. + +The files `reorg-episode.json`, `fork-held.json` and `resolve-approved.json` in `~/.credits` describe +the database in `db/`: `CREDITS_RESET=1` and a snapshot restore remove them; if you delete `db/` by +hand, delete them too. + +While a fork is held the container is reported unhealthy. Plain Docker Compose with `restart: +unless-stopped` (this file) does not restart it for that, which is what you want: a restart changes +nothing. Docker Swarm, Kubernetes liveness probes or an autoheal container would restart it in a +loop: there, do not act on this healthcheck, or watch `fork-held.json` instead. + ## Changing the storage | Change | Result | diff --git a/packaging/docker/entrypoint.sh b/packaging/docker/entrypoint.sh index 48ccb8f88..008979a0b 100755 --- a/packaging/docker/entrypoint.sh +++ b/packaging/docker/entrypoint.sh @@ -58,7 +58,33 @@ case "${1:-}" in # configuration or database [ -f keys/NodePrivate.txt ] || { log "no $DATA/keys/NodePrivate.txt: copy NodePrivate.txt and NodePublic.txt into the keys folder (~/.credits/keys on the host), owned by uid 1000, then run decrypt-key again"; exit 2; } exec "$OPT/node" --decryptkey --db-path db --public-key-file keys/NodePublic.txt --private-key-file keys/NodePrivate.txt ;; - *) log "unknown command '$1': the only command is decrypt-key (docker compose run --rm node decrypt-key)"; exit 2 ;; + resolve-fork|set-bc-top) + # a node that reports FORK HELD, the other chain being the right one (check with other nodes or + # the monitor first): resolve-fork cuts the chain at the fork point the node recorded in + # fork-held.json (only if the block there is still the recorded one), set-bc-top at a + # block given by hand; the node then exits and syncs from there at the next start. Stop the + # container first: docker compose stop && docker compose run --rm node resolve-fork + [ -f config.ini ] && [ -d db ] || { log "no config.ini or db in $DATA: nothing to cut"; exit 2; } + if [ "$1" = resolve-fork ]; then + node_args=(--resolve-fork) + else + [[ "${2:-}" =~ ^[0-9]+$ ]] || { log "set-bc-top needs the sequence of the last block to keep: docker compose run --rm node set-bc-top "; exit 2; } + node_args=(--set-bc-top "$2") + fi + mkdir -p log + rc=0 + "$OPT/node" --db-path db --config-file config.ini --public-key-file keys/NodePublic.txt \ + --private-key-file keys/NodePrivate.txt "${node_args[@]}" || rc=$? + [ "$rc" = 0 ] && log "done: start the node again with docker compose up -d" || log "$1 failed (exit $rc): nothing to do or nothing removed, see the lines above" + exit "$rc" ;; + dismiss-fork) + # a node that reports FORK HELD, this node's chain being the right one: the node keeps refusing + # that branch and stops reporting it. A running node picks the file up within a few seconds + # (docker compose exec node touch /data/dismiss-fork does the same); a stopped one at its start + touch dismiss-fork + log "dismiss-fork left in $DATA: the node dismisses the held fork within a few seconds, or at its next start" + exit 0 ;; + *) log "unknown command '$1': the commands are decrypt-key, resolve-fork, set-bc-top and dismiss-fork (docker compose run --rm node )"; exit 2 ;; esac # a data directory with a database but no config.ini (removed by hand): without CREDITS_DB_BACKEND, # config.ini is written for the backend already there, not for the default @@ -74,7 +100,8 @@ fi rm -rf .restore-tmp .digest-before if [ -e .restore-incomplete ]; then log "an interrupted snapshot restore of an earlier release left part of a snapshot: removing it" - rm -rf db caches qs roundstat.dat smartcontracts.dat snapshot.json snapshot.verified + rm -rf db caches qs roundstat.dat smartcontracts.dat snapshot.json snapshot.verified \ + reorg-episode.json fork-held.json resolve-approved.json rm -f .restore-incomplete fi @@ -149,7 +176,7 @@ if [ "$RESET" = 1 ] && { has_data || [ -e caches ] || [ -e qs ]; }; then old=old-$(date -u +%Y%m%d-%H%M%S) mkdir "$old" for p in db caches qs roundstat.dat smartcontracts.dat snapshot.json snapshot.verified snapshot.mismatch snapshot.sources.json \ - snapshot-tmp snapshot-download snapshot .storage; do + snapshot-tmp snapshot-download snapshot .storage reorg-episode.json fork-held.json resolve-approved.json; do [ ! -e "$p" ] || mv "$p" "$old/" done log "CREDITS_RESET=1: the previous state is in $DATA/$old; remove CREDITS_RESET from .env" diff --git a/packaging/docker/healthcheck.sh b/packaging/docker/healthcheck.sh index 9745b71d5..1b3554d91 100755 --- a/packaging/docker/healthcheck.sh +++ b/packaging/docker/healthcheck.sh @@ -1,5 +1,16 @@ #!/bin/sh # Healthy while the last written block advances: fails when it has not changed for 10 minutes. +# Unhealthy while the node reports FORK HELD (fork-held.json with "held": "true"): it refused a +# competing chain and waits for the operator. +fork=/data/fork-held.json +if [ -f "$fork" ] && grep -q '"held": "true"' "$fork"; then + at=$(sed -nE 's/.*"fork_point": "([0-9]+)".*/\1/p' "$fork") + since=$(sed -nE 's/.*"since": "([^"]+)".*/\1/p' "$fork") + echo "FORK HELD since $since: the node refused a competing chain (fork point #$at) and keeps its own." + echo "Check with other nodes or the monitor first. Our chain is right: docker compose exec node touch /data/dismiss-fork." + echo "The other chain is right: docker compose stop && docker compose run --rm node resolve-fork && docker compose up -d" + exit 1 +fi # the two newest log files, the older first: with the newest first, a rotated log made the last line # come from the old file, a block of hours ago (unhealthy while the node was at the head) seq=$(cat $(ls -tr /data/log/log_*.txt 2>/dev/null | tail -2) 2>/dev/null \ diff --git a/packaging/docker/snapshot-restore.sh b/packaging/docker/snapshot-restore.sh index c98825f12..1c6d50787 100755 --- a/packaging/docker/snapshot-restore.sh +++ b/packaging/docker/snapshot-restore.sh @@ -55,6 +55,8 @@ for p in db caches qs roundstat.dat smartcontracts.dat; do [ ! -e "$tmp/x/$p" ] || { rm -rf "${data:?}/$p"; mv "$tmp/x/$p" "$data/$p"; } done mv "$tmp/latest.json" "$data/snapshot.json" +# the node's files about the database this one replaces (rollback bound, FORK HELD) +rm -f "$data/reorg-episode.json" "$data/fork-held.json" "$data/resolve-approved.json" rm -f "$data/.restore-incomplete" rm -rf "$tmp" echo "snapshot restored: block $seq from $file" diff --git a/packaging/docker/test/test-entrypoint.sh b/packaging/docker/test/test-entrypoint.sh index 51a2bcd97..94cf7975c 100755 --- a/packaging/docker/test/test-entrypoint.sh +++ b/packaging/docker/test/test-entrypoint.sh @@ -3,7 +3,8 @@ # wallet on the PATH with its data on the volume, the token cache on request (CREDITS_TOKENS_CACHE), # the snapshot download on request (--download-snapshot) and the [snapshot] settings, for pruned and # full nodes, the refusal after a snapshot mismatch, storage changes, and a key encrypted by the node -# (the start stops, the decrypt-key command). +# (the start stops, the decrypt-key command), the fork commands (resolve-fork, set-bc-top, +# dismiss-fork) and the FORK HELD healthcheck. # Usage: test-entrypoint.sh set -euo pipefail here=$(cd "$(dirname "$0")/.." && pwd) @@ -297,4 +298,29 @@ set +e; out=$(command_run "$d" frobnicate); rc=$?; set -e [ "$rc" = 2 ] && grep -q "unknown command 'frobnicate'" <<<"$out" || { echo "$out"; fail "unknown command: exit $rc"; } pass "an encrypted key stops the start; decrypt-key makes it plain" +# 18. FORK HELD: resolve-fork and set-bc-top run the node once with the cut and exit, dismiss-fork +# leaves the trigger file for the node; the healthcheck is unhealthy while fork-held.json holds a fork +d=$(newdata 18) +run "$d" -e CREDITS_SNAPSHOT=off >/dev/null +mkdir -p "$d/db" +fork_run() { local data=$1; shift; docker run --rm --label credits-entrypoint-test --user "$(id -u)" -e STUB_SECONDS=0 -v "$data:/data" "$image" "$@" 2>&1; } +rm -f "$d/stub-args" +out=$(fork_run "$d" resolve-fork) || { echo "$out"; fail "resolve-fork"; } +grep -q -- '--resolve-fork' "$d/stub-args" && grep -q -- '--config-file config.ini' "$d/stub-args" || fail "resolve-fork arguments: $(cat "$d/stub-args")" +rm -f "$d/stub-args" +out=$(fork_run "$d" set-bc-top 1234) || { echo "$out"; fail "set-bc-top"; } +grep -q -- '--set-bc-top 1234' "$d/stub-args" || fail "set-bc-top arguments: $(cat "$d/stub-args")" +set +e; out=$(fork_run "$d" set-bc-top); rc=$?; set -e +[ "$rc" = 2 ] && grep -q 'set-bc-top needs the sequence' <<<"$out" || { echo "$out"; fail "set-bc-top without a sequence: exit $rc"; } +out=$(fork_run "$d" dismiss-fork) || { echo "$out"; fail "dismiss-fork"; } +[ -e "$d/dismiss-fork" ] || fail "dismiss-fork left no trigger file" +health() { docker run --rm --label credits-entrypoint-test --user "$(id -u)" --entrypoint /opt/credits/healthcheck.sh -v "$1:/data" "$image" 2>&1; } +printf '{\n "held": "true",\n "since": "2026-11-19 10:00:00 UTC",\n "fork_point": "4899"\n}\n' > "$d/fork-held.json" +set +e; out=$(health "$d"); rc=$?; set -e +[ "$rc" = 1 ] && grep -q 'FORK HELD since 2026-11-19 10:00:00 UTC' <<<"$out" && grep -q 'fork point #4899' <<<"$out" \ + && grep -q 'resolve-fork' <<<"$out" && grep -q 'dismiss-fork' <<<"$out" || { echo "$out"; fail "healthcheck with a fork held: exit $rc"; } +printf '{\n "held": "false",\n "fork_point": "4899"\n}\n' > "$d/fork-held.json" +out=$(health "$d") || { echo "$out"; fail "healthcheck with a dismissed fork"; } +pass "resolve-fork, set-bc-top and dismiss-fork; the healthcheck reports FORK HELD" + echo "PASS: entrypoint" diff --git a/packaging/mainnet/config.ini b/packaging/mainnet/config.ini index 1df11d87c..ece621622 100644 --- a/packaging/mainnet/config.ini +++ b/packaging/mainnet/config.ini @@ -12,6 +12,17 @@ compatible_version=true ; states that do not match the chain are fetched from the confidants once the node is live. ; false: no re-execution while syncing, every state is fetched (no executor needed to sync) sync_contract_execution=true +; longest-chain guard: the most of the node's own confirmed blocks it will roll back automatically +; when it adopts a competing chain during sync. Beyond this depth it keeps its chain, logs an alert, +; reports FORK HELD and waits for the operator (a reorg this deep is a deliberate network switch or a +; rejected one); see "If your node reports FORK HELD" in docs/quick-start.md. A competing block must +; also be confirmed by confidants this node holds. Default 100; 0 removes the depth bound only. Local +; only, block content is unchanged. To follow the other chain prefer node --resolve-fork (or +; --set-bc-top ); allow_deep_rollback=true (or the CS_ALLOW_DEEP_ROLLBACK environment +; variable) is the last resort: it lifts the bound for one completed episode, then the node turns it +; off until the next start; remove it again afterwards. +; max_auto_rollback=100 +; allow_deep_rollback=false [host_input] port=9510 diff --git a/solver/include/solver/consensus.hpp b/solver/include/solver/consensus.hpp index 86d8d455b..9f1043514 100644 --- a/solver/include/solver/consensus.hpp +++ b/solver/include/solver/consensus.hpp @@ -52,6 +52,15 @@ class Consensus { /** @brief The minimum trusted nodes to start consensus */ const static unsigned int MinTrustedNodes = 4; + /** @brief Default maximum number of own blocks the node will automatically roll back when it + * adopts a competing chain during sync; overridden by [params] max_auto_rollback in config.ini. + * Beyond this depth the automatic rollback is halted and the node waits for the operator: a reorg + * this deep is either a deliberate network switch (to be confirmed by a human) or a competing + * longer chain (to be rejected). Purely local: it changes only how far this node unwinds on its + * own, never the block contents other nodes see. Set high enough not to interfere with ordinary + * micro-forks; raise it (or set allow_deep_rollback) to accept a deeper deliberate switch. */ + constexpr static uint64_t MaxAutoRollbackDepth = 100; + /** @brief The maximum trusted nodes to take part in consensus */ const static unsigned int MaxTrustedNodes = 25; diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index e9f15a4c9..6b04cf97f 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -12,6 +12,21 @@ #include #include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include #include #include @@ -317,3 +332,856 @@ TEST(BlockChain, ConsensusSettingsDigestCoversEverySetting) { EXPECT_NE(changed.consensusSettingsDigest(), digest) << "change " << i; } } + +namespace { +// a clean default config so the guard reads the default bound and no deep-rollback override, +// independent of any config a prior test in the binary may have set. +void installDefaultConfig() { + cs::ConfigHolder::instance().setConfig(Config{}); +} + +void setEnv(const char* name, const char* value) { +#ifdef _WIN32 + _putenv_s(name, value); // an empty value removes the variable on Windows +#else + ::setenv(name, value, 1); +#endif +} + +void unsetEnv(const char* name) { +#ifdef _WIN32 + _putenv_s(name, ""); +#else + ::unsetenv(name); +#endif +} + +// an empty folder of its own under the system temporary folder, removed with the object +struct TempDir { + std::filesystem::path path; + explicit TempDir(const std::string& name) { + path = std::filesystem::temp_directory_path() / ("csnode-guard-" + name + "-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + std::filesystem::remove_all(path); + std::filesystem::create_directories(path); + } + ~TempDir() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + std::string str() const { return path.string(); } +}; +} // namespace + +static_assert(DEFAULT_MAX_AUTO_ROLLBACK == Consensus::MaxAutoRollbackDepth, + "the config default and the consensus constant of the rollback bound must stay in step"); +static_assert(Consensus::MaxAutoRollbackDepth == 100, "owner decision: default max_auto_rollback is 100"); + +// The longest-chain guard bounds an automatic rollback at Consensus::MaxAutoRollbackDepth below the +// head captured when the rollback episode began, and that reference is fixed for the episode. +TEST(BlockChain, LongestChainGuardBoundsRollbackDepth) { + installDefaultConfig(); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + + const cs::Sequence head = 5000; + const uint64_t depth = Consensus::MaxAutoRollbackDepth; + chain.test_setLastSequence(head); + + // the first call captures the high-water reference at the current head + ASSERT_TRUE(chain.guardAutoRollback()); + EXPECT_TRUE(chain.test_rollbackEpisode()); + EXPECT_EQ(chain.test_rollbackHighWater(), head); + + // unwinding step by step is allowed right up to the floor (head - depth), and the reference does + // not move as the head drops, so forward progress cannot reset it + chain.test_setLastSequence(head - depth + 1); + EXPECT_TRUE(chain.guardAutoRollback()); + EXPECT_EQ(chain.test_rollbackHighWater(), head); + EXPECT_FALSE(chain.forkHeld()); + + // the removal that would take the head to or below the floor is refused: FORK HELD + chain.test_setLastSequence(head - depth); + EXPECT_FALSE(chain.guardAutoRollback()); + EXPECT_TRUE(chain.forkHeld()); + + // refusing sets no permanent state: a fresh episode from a new head works again + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(head - depth); + EXPECT_TRUE(chain.guardAutoRollback()); + EXPECT_EQ(chain.test_rollbackHighWater(), head - depth); + EXPECT_FALSE(chain.forkHeld()); +} + +// Near genesis the bound never underflows: a head below the depth can always be unwound to genesis. +TEST(BlockChain, LongestChainGuardNoUnderflowNearGenesis) { + installDefaultConfig(); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + + chain.test_setLastSequence(10); + for (cs::Sequence s = 10; s > 0; --s) { + chain.test_setLastSequence(s); + EXPECT_TRUE(chain.guardAutoRollback()) << "seq " << s; + } +} + +// [params] max_auto_rollback and allow_deep_rollback, and the CS_ALLOW_DEEP_ROLLBACK variable +TEST(BlockChain, RollbackGuardConfigParsing) { + const auto parse = [](const std::string& params, uint64_t& depth, bool& allow) { + std::istringstream in("[params]\n" + params); + boost::property_tree::ptree tree; + boost::property_tree::read_ini(in, tree); + Config::readRollbackGuardSettings(tree.get_child("params"), depth, allow); + }; + uint64_t depth = 7; + bool allow = true; + parse("node_type=client\n", depth, allow); + EXPECT_EQ(depth, 100u); + EXPECT_FALSE(allow); + parse("max_auto_rollback=250\nallow_deep_rollback=true\n", depth, allow); + EXPECT_EQ(depth, 250u); + EXPECT_TRUE(allow); + parse("max_auto_rollback=0\nallow_deep_rollback=false\n", depth, allow); + EXPECT_EQ(depth, 0u); + EXPECT_FALSE(allow); + EXPECT_EQ(Config{}.maxAutoRollbackDepth(), 100u); + EXPECT_FALSE(Config{}.allowDeepRollback()); + + unsetEnv("CS_ALLOW_DEEP_ROLLBACK"); + EXPECT_FALSE(BlockChain::deepRollbackEnvSet()); + setEnv("CS_ALLOW_DEEP_ROLLBACK", ""); + EXPECT_FALSE(BlockChain::deepRollbackEnvSet()); // empty: not set + setEnv("CS_ALLOW_DEEP_ROLLBACK", "1"); + EXPECT_TRUE(BlockChain::deepRollbackEnvSet()); + unsetEnv("CS_ALLOW_DEEP_ROLLBACK"); +} + +// allow_deep_rollback (here the environment variable) lifts the bound for one completed episode, +// then it is off for the rest of the run +TEST(BlockChain, DeepRollbackOverrideServesOneEpisode) { + installDefaultConfig(); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + setEnv("CS_ALLOW_DEEP_ROLLBACK", "1"); + + chain.test_setLastSequence(5000); + EXPECT_TRUE(chain.deepRollbackApproved()); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4000); // far below the bound: allowed by the override + EXPECT_TRUE(chain.guardAutoRollback()); + EXPECT_FALSE(chain.forkHeld()); + + // the episode completes: the override is spent + chain.test_setLastSequence(5001); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.test_rollbackEpisode()); + EXPECT_FALSE(chain.deepRollbackApproved()); + chain.test_setLastSequence(5001); + ASSERT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(5001 - Consensus::MaxAutoRollbackDepth); + EXPECT_FALSE(chain.guardAutoRollback()); + unsetEnv("CS_ALLOW_DEEP_ROLLBACK"); + chain.test_resetRollbackEpisode(); +} + +// The episode reference is kept next to the database: a restart neither gives a fresh full depth nor +// forgets FORK HELD +TEST(BlockChain, RollbackReferenceSurvivesRestart) { + installDefaultConfig(); + TempDir dir("restart"); + const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + BlockChain chain(genesis, start); + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(5000); + chain.setGuardStateDir(dir.str()); + ASSERT_TRUE(chain.guardAutoRollback()); + ASSERT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + chain.test_setLastSequence(4950); + ASSERT_TRUE(chain.guardAutoRollback()); + + // "restart" at head 4950: the reference is still 5000, not 4950 + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(4950); + chain.setGuardStateDir(dir.str()); + EXPECT_TRUE(chain.test_rollbackEpisode()); + EXPECT_EQ(chain.test_rollbackHighWater(), 5000u); + chain.test_setLastSequence(4901); + EXPECT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); + EXPECT_FALSE(chain.guardAutoRollback()); + ASSERT_TRUE(chain.forkHeld()); + ASSERT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + + // restart again: still FORK HELD, same record + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(4900); + chain.setGuardStateDir(dir.str()); + EXPECT_TRUE(chain.forkHeld()); + ASSERT_TRUE(chain.forkHeldRecord().has_value()); + EXPECT_EQ(chain.forkHeldRecord()->highWater, 5000u); + EXPECT_EQ(chain.forkHeldRecord()->ourHead, 4900u); + + // the chain climbs above the reference: the episode and FORK HELD end, both files go + chain.test_setLastSequence(5001); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.forkHeld()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + + // a restart above an old reference drops it + ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {4000})); + chain.test_resetRollbackEpisode(); + chain.setGuardStateDir(dir.str()); + EXPECT_FALSE(chain.test_rollbackEpisode()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + chain.setGuardStateDir(""); +} + +// the fork record and the episode in their files, and the --resolve-fork plan +TEST(BlockChain, ForkGuardFilesRoundTrip) { + TempDir dir("files"); + EXPECT_EQ(cs::forkguard::stateDirFor("db"), "."); + EXPECT_EQ(cs::forkguard::stateDirFor("db/"), "."); + EXPECT_EQ(cs::forkguard::stateDirFor("/data/db"), "/data"); + + std::string error; + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("no fork record"), std::string::npos); + + cs::forkguard::ForkRecord r; + r.reason = "depth"; + r.sinceUnix = 1795000000; + r.highWater = 5000; + r.ourHead = 4900; + r.forkPoint = 4899; + r.forkPointExact = false; + r.competingSeq = 4901; + r.competingHash = "AB"; + r.peerKey = "key"; + r.peerAddress = "1.2.3.4:9510"; + r.peerHeight = 6000; + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), r)); + // no hash for the fork point: cannot be checked, refused + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("cannot be checked"), std::string::npos); + + r.forkPointHash = "CD"; + r.dismissedUnix = 1795000100; + r.dismissedSeq = 4901; + r.dismissedHash = "AB"; + r.addBranchHash("AB"); + r.addBranchHash("EF"); + r.addBranchHash("AB"); // once only + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), r)); + { + // the Docker healthcheck reads these lines with grep and sed + std::ifstream in(dir.path / cs::forkguard::kForkFile); + const std::string text((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + EXPECT_NE(text.find("\"held\": \"true\""), std::string::npos) << text; + EXPECT_NE(text.find("\"fork_point\": \"4899\""), std::string::npos) << text; + EXPECT_NE(text.find("\"since\": \"2026-"), std::string::npos) << text; + } + const auto back = cs::forkguard::loadForkRecord(dir.str()); + ASSERT_TRUE(back.has_value()); + EXPECT_TRUE(back->held); + EXPECT_EQ(back->reason, "depth"); + EXPECT_EQ(back->highWater, 5000u); + EXPECT_EQ(back->forkPoint, 4899u); + EXPECT_EQ(back->forkPointHash, "CD"); + EXPECT_EQ(back->competingSeq, 4901u); + EXPECT_EQ(back->peerAddress, "1.2.3.4:9510"); + EXPECT_EQ(back->peerHeight, 6000u); + EXPECT_EQ(back->dismissedSeq, 4901u); + EXPECT_EQ(back->branchHashes, (std::vector{"AB", "EF"})); + EXPECT_TRUE(back->inDismissedBranch("XX", "EF")); + EXPECT_FALSE(back->inDismissedBranch("XX", "YY")); + const auto plan = cs::forkguard::planResolve(dir.str(), &error); + ASSERT_TRUE(plan.has_value()); + EXPECT_EQ(plan->top, 4899u); + EXPECT_EQ(plan->hash, "CD"); + + // the detail line names both choices + const auto line = cs::forkguard::summary(r); + EXPECT_NE(line.find("FORK HELD since 2026-"), std::string::npos); + EXPECT_NE(line.find("--resolve-fork"), std::string::npos); + EXPECT_NE(line.find("--set-bc-top 4899"), std::string::npos); + EXPECT_NE(line.find(cs::forkguard::kDismissTrigger), std::string::npos); + EXPECT_NE(line.find("peer height #6000"), std::string::npos); + + cs::forkguard::clearForkRecord(dir.str()); + EXPECT_FALSE(cs::forkguard::loadForkRecord(dir.str()).has_value()); +} + +namespace { +struct GuardKeys { + std::vector pub; + std::vector priv; +}; + +GuardKeys guardKeys(size_t n) { + EXPECT_TRUE(cscrypto::cryptoInit()); + GuardKeys k; + for (size_t i = 0; i < n; ++i) { + cs::PublicKey p; + k.priv.push_back(cscrypto::generateKeyPair(p)); + k.pub.push_back(p); + } + return k; +} + +// a block of `seq` whose confidants are `keys`, all of them signing; when `prev` is given it carries +// round confirmations by the confidants of `prev` selected in `confirmMask` (bit i = confidant i) +csdb::Pool guardBlock(cs::Sequence seq, const GuardKeys& keys, const GuardKeys* prev = nullptr, + uint64_t confirmMask = 0, bool bootstrap = false, bool corruptSignature = false, + const csdb::PoolHash& previousHash = csdb::PoolHash{}, const std::string& payload = "payload") { + csdb::Pool p(previousHash, seq); + p.set_confidants(keys.pub); + p.add_number_trusted(static_cast(keys.pub.size())); + p.add_real_trusted((1ull << keys.pub.size()) - 1); + p.add_user_field(200, payload); // an id no block field uses + BlockChain::setBootstrap(p, bootstrap); + if (prev) { + cs::Bytes trustedToHash; + cs::ODataStream stream(trustedToHash); + stream << seq; + stream << keys.pub; + const cs::Hash h = cscrypto::calculateHash(trustedToHash.data(), trustedToHash.size()); + std::vector confirmations; + for (size_t i = 0; i < prev->priv.size(); ++i) { + if (confirmMask >> i & 1) { + confirmations.push_back(cscrypto::generateSignature(prev->priv[i], h.data(), h.size())); + } + } + p.add_number_confirmations(static_cast(prev->pub.size())); + p.add_confirmation_mask(confirmMask); + p.add_round_confirmations(confirmations); + } + auto probe = p.clone(); + EXPECT_TRUE(probe.compose()); + cs::Hash hash{}; + const auto hb = probe.hash().to_binary(); + std::copy(hb.begin(), hb.end(), hash.begin()); + std::vector sigs; + for (const auto& k : keys.priv) { + sigs.push_back(cscrypto::generateSignature(k, hash.data(), hash.size())); + } + if (corruptSignature) { + sigs.front()[0] ^= 0xff; + } + p.set_signatures(sigs); + EXPECT_TRUE(p.compose()); + return csdb::Pool::from_binary(p.to_binary()); +} + +BlockChain& guardChain() { + static const csdb::Address genesis = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"); + static const csdb::Address start = csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002"); + static BlockChain chain(genesis, start); + return chain; +} + +// an authority refusal becomes FORK HELD when the head stays put over two round starts +void stayStuck(BlockChain& chain) { + EXPECT_FALSE(chain.promotePendingHold(true)); + EXPECT_TRUE(chain.promotePendingHold(true)); +} + +// the guard chain in a clean state: no episode, no record, no files +BlockChain& freshGuardChain(const std::string& dir = std::string{}) { + installDefaultConfig(); + auto& chain = guardChain(); + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); + cs::forkguard::clearForkRecord(dir); + cs::forkguard::clearEpisode(dir); + cs::forkguard::clearApprovedBranch(dir); + chain.setGuardStateDir(dir); + return chain; +} + +csdb::PoolHash someHash(char c) { + csdb::Pool p(csdb::PoolHash{}, 1); + p.add_user_field(1, std::string(10, c)); + EXPECT_TRUE(p.compose()); + return p.hash(); +} +} // namespace + +// A non-bootstrap competing block may make the node unwind only with confirmations by a majority of +// the confidants of the node's own last block; stricter than block validation (no one-entry masks). +TEST(BlockChain, CompetingBlockNeedsStrictTrustedConfirmations) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + chain.test_setLastBlock(guardBlock(20, ours)); + + EXPECT_TRUE(chain.hasTrustedConfirmations(guardBlock(21, next, &ours, 0b111))); + EXPECT_TRUE(chain.hasTrustedConfirmations(guardBlock(21, next, &ours, 0b011))); // 2 of 3 + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next, &ours, 0b001))); // a minority + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next))); // none + const auto strangers = guardKeys(3); + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next, &strangers, 0b111))); // keys we do not trust + const auto four = guardKeys(4); + EXPECT_FALSE(chain.hasTrustedConfirmations(guardBlock(21, next, &four, 0b1111))); // mask of another size + EXPECT_TRUE(chain.competingBlockMayTriggerUnwind(guardBlock(21, next, &ours, 0b111))); + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(guardBlock(21, next, &strangers, 0b111))); +} + +// BlockVerifier accepts a bootstrap block without looking at confirmations: hasTrustedConfirmations +// must not, even when the block carries valid confirmations by our confidants +TEST(BlockChain, BootstrapBlockHasNoTrustedConfirmations) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + chain.test_setLastBlock(guardBlock(40, ours)); + ASSERT_TRUE(chain.hasTrustedConfirmations(guardBlock(41, next, &ours, 0b111))); + const auto flagged = guardBlock(41, next, &ours, 0b111, true); + ASSERT_TRUE(BlockChain::isBootstrap(flagged)); + EXPECT_FALSE(chain.hasTrustedConfirmations(flagged)); +} + +// The bootstrap flag is a bit anyone can set: a bootstrap-flagged competing block passes the gate +// only when it is authenticated locally (valid own signatures AND the confidants of a round table this +// node accepted), never on the flag alone, and never more easily than a normal block. +TEST(BlockChain, BootstrapFlagAloneNeverTriggersUnwind) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + const auto others = guardKeys(3); + chain.test_setLastBlock(guardBlock(30, ours)); + + auto& conveyer = cs::Conveyer::instance(); + cs::RoundTable unrelated; + unrelated.round = 1000; + unrelated.confidants = guardKeys(3).pub; + conveyer.setRound(unrelated.round); + conveyer.setTable(unrelated); + + const auto flagged = guardBlock(31, others, nullptr, 0, true); + ASSERT_TRUE(BlockChain::isBootstrap(flagged)); + EXPECT_FALSE(chain.isBootstrapAuthenticatedLocally(flagged)); + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(flagged)); + // the same block without the flag fails too: the flag never opens a gate a normal block cannot pass + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(guardBlock(31, others))); + + // the node took a round table naming exactly these confidants: the bootstrap block is authenticated + cs::RoundTable taken; + taken.round = 1001; + taken.confidants = others.pub; + conveyer.setRound(taken.round); + conveyer.setTable(taken); + EXPECT_TRUE(chain.isBootstrapAuthenticatedLocally(flagged)); + EXPECT_TRUE(chain.competingBlockMayTriggerUnwind(flagged)); + // ... but not with a bad signature + EXPECT_FALSE(chain.competingBlockMayTriggerUnwind(guardBlock(31, others, nullptr, 0, true, true))); + + // leave the conveyer on a table that matches nothing here + unrelated.round = 1002; + conveyer.setRound(unrelated.round); + conveyer.setTable(unrelated); +} + +// storeBlock: a competing block the node refuses (not vouched for, or beyond the depth bound) leaves +// the head and the uncertain flag untouched, and raises FORK HELD with a fork record +TEST(BlockChain, RefusedCompetingBlockLeavesTheChainUntouched) { + TempDir dir("store"); + auto& chain = freshGuardChain(dir.str()); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + const auto strangers = guardKeys(3); + + // not vouched for by our confidants + const auto own = guardBlock(20, ours); + chain.test_setLastBlock(own); + auto foreign = guardBlock(21, next, &strangers, 0b111, false, false, someHash('x')); + EXPECT_FALSE(chain.storeBlock(foreign, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 20u); + EXPECT_EQ(chain.getLastHash(), own.hash()); + EXPECT_FALSE(chain.isLastBlockUncertain()); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); + ASSERT_TRUE(chain.forkHeld()); + auto rec = chain.forkHeldRecord(); + EXPECT_EQ(rec->reason, "authority"); + EXPECT_EQ(rec->competingSeq, 21u); + EXPECT_EQ(rec->competingHash, foreign.hash().to_string()); + EXPECT_EQ(rec->ourHead, 20u); + EXPECT_EQ(rec->forkPoint, 19u); // our #20 is not the competing block's previous one + EXPECT_FALSE(rec->forkPointExact); + EXPECT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + + // vouched for, but beyond the depth bound + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); + chain.test_setLastSequence(200); + ASSERT_TRUE(chain.guardAutoRollback()); // reference head #200 + const auto deep = guardBlock(100, ours); + chain.test_setLastBlock(deep); + chain.setBlocksToBeRemoved(1); + auto vouched = guardBlock(101, next, &ours, 0b111, false, false, someHash('y')); + ASSERT_TRUE(chain.competingBlockMayTriggerUnwind(vouched)); + EXPECT_FALSE(chain.storeBlock(vouched, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 100u); + EXPECT_EQ(chain.getLastHash(), deep.hash()); + EXPECT_FALSE(chain.isLastBlockUncertain()); + ASSERT_TRUE(chain.forkHeld()); + EXPECT_EQ(chain.forkHeldRecord()->reason, "depth"); + EXPECT_EQ(chain.forkHeldRecord()->highWater, 200u); + chain.test_resetRollbackEpisode(); +} + +// The operator keeps this chain (the dismiss-fork file): FORK HELD ends, the branch stays refused and +// is not reported again, even after a restart; another branch is reported again +TEST(BlockChain, DismissedForkStaysRefusedAndQuiet) { + TempDir dir("dismiss"); + auto& chain = freshGuardChain(dir.str()); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + const auto strangers = guardKeys(3); + const auto own = guardBlock(50, ours); + chain.test_setLastBlock(own); + + auto branch = guardBlock(51, next, &strangers, 0b111, false, false, someHash('a')); + EXPECT_FALSE(chain.storeBlock(branch, cs::PoolStoreType::Synced)); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); + ASSERT_TRUE(chain.forkHeld()); + + // no trigger: nothing happens + EXPECT_FALSE(chain.checkDismissTrigger()); + EXPECT_TRUE(chain.forkHeld()); + // the trigger file: dismissed and consumed + { std::ofstream(dir.path / cs::forkguard::kDismissTrigger) << "\n"; } + EXPECT_TRUE(chain.checkDismissTrigger()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kDismissTrigger)); + EXPECT_FALSE(chain.forkHeld()); + const auto rec = cs::forkguard::loadForkRecord(dir.str()); + ASSERT_TRUE(rec.has_value()); + EXPECT_FALSE(rec->held); + EXPECT_EQ(rec->dismissedSeq, 51u); + EXPECT_EQ(rec->dismissedHash, branch.hash().to_string()); + + // the same branch again: still refused, the chain untouched, no FORK HELD + EXPECT_FALSE(chain.storeBlock(branch, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 50u); + EXPECT_EQ(chain.getLastHash(), own.hash()); + EXPECT_FALSE(chain.forkHeld()); + // a trigger with nothing held is consumed and does nothing + { std::ofstream(dir.path / cs::forkguard::kDismissTrigger) << "\n"; } + EXPECT_FALSE(chain.checkDismissTrigger()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kDismissTrigger)); + + // restart: the dismissal is in the record; the branch, continued by its next block, stays quiet + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); + chain.test_setLastBlock(guardBlock(51, ours, nullptr, 0, false, false, own.hash())); + chain.setGuardStateDir(dir.str()); + EXPECT_FALSE(chain.forkHeld()); + ASSERT_TRUE(chain.forkRecord().has_value()); + EXPECT_TRUE(chain.forkRecord()->hasDismissal()); + auto continued = guardBlock(52, next, &strangers, 0b111, false, false, branch.hash()); + EXPECT_FALSE(chain.storeBlock(continued, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 51u); + EXPECT_FALSE(chain.forkHeld()); + + // a different branch is reported again, and the earlier dismissal is kept + auto other = guardBlock(52, next, &strangers, 0b111, false, false, someHash('b'), "other"); + EXPECT_FALSE(chain.storeBlock(other, cs::PoolStoreType::Synced)); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); + ASSERT_TRUE(chain.forkHeld()); + EXPECT_EQ(chain.forkHeldRecord()->competingHash, other.hash().to_string()); + EXPECT_TRUE(chain.forkHeldRecord()->inDismissedBranch(branch.hash().to_string(), "")); + EXPECT_FALSE(chain.forkHeldRecord()->inDismissedBranch(continued.hash().to_string(), "")); // not vouched for: not added + chain.setGuardStateDir(""); + chain.test_resetRollbackEpisode(); +} + +// The antifork selection stopped by the guard leaves a clean state: no pending request, selection +// finished, antifork mode off, the chain as it was, FORK HELD +TEST(BlockChain, AntiforkSelectionAbandonedByTheGuard) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + chain.test_setLastSequence(300); + ASSERT_TRUE(chain.guardAutoRollback()); // reference head #300 + const auto low = guardBlock(200, ours); + chain.test_setLastBlock(low); + chain.test_setSelectionUnfinished(someHash('s')); + ASSERT_TRUE(chain.getIncorrectBlockNumbers()->empty()); + + chain.cacheLastBlocks(); + EXPECT_TRUE(chain.test_selectionFinished()); + EXPECT_FALSE(chain.test_antiForkMode()); + EXPECT_EQ(chain.test_emittingRequest(), 0); + EXPECT_TRUE(chain.test_lastPrevHash().is_empty()); + EXPECT_TRUE(chain.getIncorrectBlockNumbers()->empty()); + EXPECT_EQ(chain.getLastSeq(), 200u); + EXPECT_EQ(chain.getLastHash(), low.hash()); + EXPECT_TRUE(chain.forkHeld()); + chain.test_resetRollbackEpisode(); +} + +// The start-up integrity repair (blocks of incorrectBlocks_) is a local repair, not a network reorg: +// the guard does not bound it and raises nothing, even deep below an episode's reference +TEST(BlockChain, IntegrityRepairIsNotBoundedByTheGuard) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + chain.test_setLastSequence(300); + ASSERT_TRUE(chain.guardAutoRollback()); // reference head #300: #200 would be refused + chain.test_setLastBlock(guardBlock(200, ours)); + chain.addIncorrectBlockNumber(200); + chain.setBlocksToBeRemoved(1); + + chain.cacheLastBlocks(); + EXPECT_EQ(chain.getLastSeq(), 199u); // the incorrect block was removed + EXPECT_TRUE(chain.getIncorrectBlockNumbers()->empty()); + EXPECT_FALSE(chain.forkHeld()); + chain.test_resetRollbackEpisode(); +} + +// --resolve-fork approves only the recorded block: it alone (that very hash, built on the fork point) +// may replace a block the node stored again right after the fork point; a forged block pointing at it, +// or any other unauthenticated offer, is refused and does not cancel the approval; nothing is ever +// unwound to or below the fork point; the approval clears once the node is past that height +TEST(BlockChain, ResolveForkApprovesOnlyTheRecordedBlock) { + TempDir dir("approve"); + const auto ours = guardKeys(3); + const auto theirs = guardKeys(3); + const auto strangers = guardKeys(3); + const auto forkBlock = guardBlock(60, ours); // #60, the fork point + const auto recorded = guardBlock(61, theirs, &strangers, 0b111, false, false, forkBlock.hash()); // their #61 + const cs::forkguard::ApprovedBranch approval{60, 61, recorded.hash().to_string()}; + const auto ownAgain = guardBlock(61, ours, &ours, 0b111, false, false, forkBlock.hash()); // our #61, stored again + + // unauthenticated offers at #61 and #62: refused as usual, and the approval stays + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(forkBlock); + chain.setGuardStateDir(dir.str()); + ASSERT_TRUE(chain.approvedBranch().has_value()); + EXPECT_FALSE(chain.deepRollbackApproved()); // nothing as wide as allow_deep_rollback + auto other = guardBlock(61, strangers, &theirs, 0b111, false, false, someHash('o')); + EXPECT_FALSE(chain.storeBlock(other, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 60u); + EXPECT_EQ(chain.getLastHash(), forkBlock.hash()); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); + EXPECT_TRUE(chain.forkHeld()); + EXPECT_TRUE(chain.approvedBranch().has_value()); + EXPECT_TRUE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + } + // a forged #62 pointing at the recorded hash, while the node holds its own #61 again: refused, the + // own block kept, the approval kept + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(ownAgain); + chain.setGuardStateDir(dir.str()); + auto forged = guardBlock(62, strangers, &theirs, 0b111, false, false, recorded.hash()); + EXPECT_FALSE(chain.storeBlock(forged, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 61u); + EXPECT_EQ(chain.getLastHash(), ownAgain.hash()); + EXPECT_FALSE(chain.isLastBlockUncertain()); + EXPECT_FALSE(chain.forkHeld()); // refused at once, reported once the head stays + EXPECT_TRUE(chain.forkHoldPending()); + stayStuck(chain); + EXPECT_TRUE(chain.forkHeld()); + EXPECT_TRUE(chain.approvedBranch().has_value()); + } + // nothing is unwound to or below the fork point while the approval is pending + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(forkBlock); + chain.setGuardStateDir(dir.str()); + chain.setBlocksToBeRemoved(1); + EXPECT_FALSE(chain.guardAutoRollback()); + chain.removeLastBlock(); + EXPECT_EQ(chain.getLastSeq(), 60u); + EXPECT_EQ(chain.getLastHash(), forkBlock.hash()); + EXPECT_TRUE(chain.approvedBranch().has_value()); + } + // the recorded block: it replaces the own #61, down to the fork point and no further, then the + // node appends it and the approval clears + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(ownAgain); + chain.setGuardStateDir(dir.str()); + chain.setBlocksToBeRemoved(1); + auto offered = recorded; + EXPECT_FALSE(chain.storeBlock(offered, cs::PoolStoreType::Synced)); + EXPECT_EQ(chain.getLastSeq(), 60u); // own #61 removed, the fork point kept + EXPECT_FALSE(chain.forkHeld()); + EXPECT_TRUE(chain.approvedBranch().has_value()); + EXPECT_FALSE(chain.guardAutoRollback()); // and never below it + + chain.test_setLastBlock(recorded); // appended as the next block + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.approvedBranch().has_value()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + } + // a competing block vouched for by this node's confidants at that height ends the approval + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(ownAgain); + chain.setGuardStateDir(dir.str()); + auto vouched = guardBlock(62, theirs, &ours, 0b111, false, false, someHash('v')); + ASSERT_TRUE(chain.competingBlockMayTriggerUnwind(vouched)); + chain.storeBlock(vouched, cs::PoolStoreType::Synced); + EXPECT_FALSE(chain.approvedBranch().has_value()); + } + // the node goes past that height with another block: the approval clears too + { + auto& chain = freshGuardChain(dir.str()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), approval)); + chain.test_setLastBlock(forkBlock); + chain.setGuardStateDir(dir.str()); + chain.test_setLastBlock(guardBlock(62, ours)); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.approvedBranch().has_value()); + } + freshGuardChain(); +} + +// --resolve-fork acts only on a held fork: a dismissed record is refused, with the remedy +TEST(BlockChain, ResolvePlanRefusesADismissedRecord) { + TempDir dir("plan-dismissed"); + cs::forkguard::ForkRecord r; + r.held = false; + r.reason = "authority"; + r.forkPoint = 99; + r.forkPointHash = "AB"; + r.dismissedUnix = 1795000000; + r.dismissedSeq = 100; + r.dismissedHash = "CD"; + r.addBranchHash("CD"); + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), r)); + std::string error; + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("dismissed on 2026-"), std::string::npos) << error; + EXPECT_NE(error.find("--set-bc-top "), std::string::npos) << error; +} + +// an unreadable (or empty) file of each kind is reported, never read as "nothing there": the episode +// fails safe (a new one at the head), the fork record stays FORK HELD, an approval is dropped +TEST(BlockChain, CorruptGuardFilesFailSafe) { + for (const std::string content : {std::string("{ not json"), std::string()}) { + SCOPED_TRACE(content.empty() ? "empty" : "garbage"); + TempDir dir("corrupt"); + const auto write = [&](const char* name) { std::ofstream(dir.path / name, std::ios::binary) << content; }; + bool corrupt = false; + + write(cs::forkguard::kEpisodeFile); + EXPECT_FALSE(cs::forkguard::loadEpisode(dir.str(), &corrupt).has_value()); + EXPECT_TRUE(corrupt); + write(cs::forkguard::kForkFile); + EXPECT_FALSE(cs::forkguard::loadForkRecord(dir.str(), &corrupt).has_value()); + EXPECT_TRUE(corrupt); + std::string error; + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); + EXPECT_NE(error.find("cannot be read"), std::string::npos) << error; + write(cs::forkguard::kApprovedFile); + EXPECT_FALSE(cs::forkguard::loadApprovedBranch(dir.str(), &corrupt).has_value()); + EXPECT_TRUE(corrupt); + // a missing file is not corrupt + EXPECT_FALSE(cs::forkguard::loadEpisode((dir.path / "none").string(), &corrupt).has_value()); + EXPECT_FALSE(corrupt); + + auto& chain = freshGuardChain(); + chain.test_setLastSequence(700); + write(cs::forkguard::kEpisodeFile); + write(cs::forkguard::kForkFile); + write(cs::forkguard::kApprovedFile); + chain.setGuardStateDir(dir.str()); + EXPECT_TRUE(chain.test_rollbackEpisode()); // a new episode at the head + EXPECT_EQ(chain.test_rollbackHighWater(), 700u); + const auto episode = cs::forkguard::loadEpisode(dir.str(), &corrupt); + ASSERT_TRUE(episode.has_value()); + EXPECT_EQ(episode->highWater, 700u); + ASSERT_TRUE(chain.forkHeld()); // the lost record: still FORK HELD + EXPECT_EQ(chain.forkHeldRecord()->reason, "unreadable"); + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error).has_value()); // no hash to check + EXPECT_FALSE(chain.approvedBranch().has_value()); // no approval + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kApprovedFile)); + freshGuardChain(); + } +} + +// a restart below the floor of the persisted reference refuses at once; and the files go together +// when the database they describe is replaced +TEST(BlockChain, RestartBelowTheFloorAndCleanup) { + TempDir dir("floor"); + auto& chain = freshGuardChain(); + ASSERT_TRUE(cs::forkguard::saveEpisode(dir.str(), {5000})); + chain.test_setLastSequence(4800); // already below 5000 - 100 + chain.setGuardStateDir(dir.str()); + EXPECT_EQ(chain.test_rollbackHighWater(), 5000u); + EXPECT_FALSE(chain.guardAutoRollback()); + EXPECT_TRUE(chain.forkHeld()); + ASSERT_TRUE(cs::forkguard::saveApprovedBranch(dir.str(), {10, 11, "AB"})); + for (const char* f : {cs::forkguard::kEpisodeFile, cs::forkguard::kForkFile, cs::forkguard::kApprovedFile}) { + EXPECT_TRUE(std::filesystem::exists(dir.path / f)) << f; + } + cs::forkguard::clearAll(dir.str()); + for (const char* f : {cs::forkguard::kEpisodeFile, cs::forkguard::kForkFile, cs::forkguard::kApprovedFile}) { + EXPECT_FALSE(std::filesystem::exists(dir.path / f)) << f; + } + freshGuardChain(); +} + +// a block not vouched for, refused while the chain goes on: no FORK HELD, no file, no flap +TEST(BlockChain, AuthorityRefusalDoesNotFlap) { + TempDir dir("flap"); + auto& chain = freshGuardChain(dir.str()); + const auto ours = guardKeys(3); + const auto next = guardKeys(3); + const auto strangers = guardKeys(3); + const auto own = guardBlock(70, ours); + chain.test_setLastBlock(own); + auto stray = guardBlock(71, next, &strangers, 0b111, false, false, someHash('f')); + EXPECT_FALSE(chain.storeBlock(stray, cs::PoolStoreType::Synced)); // refused at once + EXPECT_TRUE(chain.forkHoldPending()); + EXPECT_FALSE(chain.forkHeld()); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kForkFile)); + EXPECT_FALSE(std::filesystem::exists(dir.path / cs::forkguard::kEpisodeFile)); + EXPECT_FALSE(chain.promotePendingHold(true)); // one round start: not yet + + // the next block of our chain: the refusal is forgotten + chain.test_setLastBlock(guardBlock(71, ours, &ours, 0b111, false, false, own.hash())); + chain.test_onHeadAdvanced(); + EXPECT_FALSE(chain.forkHoldPending()); + EXPECT_FALSE(chain.promotePendingHold(true)); + EXPECT_FALSE(chain.promotePendingHold(true)); + EXPECT_FALSE(chain.forkHeld()); + for (const char* f : {cs::forkguard::kEpisodeFile, cs::forkguard::kForkFile, cs::forkguard::kApprovedFile}) { + EXPECT_FALSE(std::filesystem::exists(dir.path / f)) << f; + } + freshGuardChain(); +} + +// forged blocks cannot push the dismissed block out of the remembered hashes +TEST(BlockChain, DismissedHashIsNeverEvicted) { + cs::forkguard::ForkRecord r; + r.dismissedHash = "D"; + r.addBranchHash("D"); + for (size_t i = 0; i < cs::forkguard::kMaxBranchHashes * 2; ++i) { + r.addBranchHash("F" + std::to_string(i)); + } + EXPECT_EQ(r.branchHashes.size(), cs::forkguard::kMaxBranchHashes); + EXPECT_TRUE(r.inDismissedBranch("D", "")); + EXPECT_TRUE(r.inDismissedBranch("x", "D")); + EXPECT_FALSE(r.inDismissedBranch("F0", "")); // the oldest others did go +} diff --git a/tests/csnode/storage_writer_tests.cpp b/tests/csnode/storage_writer_tests.cpp index 8c54e5809..b74b52acf 100644 --- a/tests/csnode/storage_writer_tests.cpp +++ b/tests/csnode/storage_writer_tests.cpp @@ -218,3 +218,204 @@ TEST(StorageWriter, RemovingTheLastQueuedBlockEndsTheFailure) { EXPECT_TRUE(db->has(pools[0].hash())); storage.close(); } + +// A chain reorganisation below the deferred block: the node removes stored blocks and saves the +// competing ones at the same sequences. pool_load(sequence) must then return the new blocks, not the +// removed ones still held by the storage's read cache (the stale block made the next consistency +// check fail for ever, and the node stopped two blocks above the fork point). +namespace { + +void checkReplacedBlocksAreLoaded(bool asyncWrites) { + auto db = std::make_shared(); + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.writeBatchSize = 100; + opt.asyncWrites = asyncWrites; + ASSERT_TRUE(storage.open(opt)); + + const auto own = chain(5); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + for (cs::Sequence s = 0; s < 5; ++s) ASSERT_EQ(storage.pool_load(s).hash(), own[s].hash()); + + // unwind to #2, then the competing #3 and #4 + ASSERT_EQ(storage.pool_remove_last().hash(), own[4].hash()); + ASSERT_EQ(storage.pool_remove_last().hash(), own[3].hash()); + std::vector other; + csdb::PoolHash prev = own[2].hash(); + for (cs::Sequence s = 3; s < 5; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(100, static_cast('z' - s))); + ASSERT_TRUE(p.compose()); + ASSERT_NE(p.hash(), own[s].hash()); + prev = p.hash(); + other.push_back(p); + ASSERT_TRUE(storage.pool_save(p)); + } + + EXPECT_EQ(storage.pool_load(3).hash(), other[0].hash()); + EXPECT_EQ(storage.pool_load(4).hash(), other[1].hash()); + EXPECT_EQ(storage.pool_load(4).previous_hash(), storage.pool_load(3).hash()); + EXPECT_FALSE(storage.pool_load(own[3].hash()).is_valid()); + EXPECT_FALSE(storage.pool_load(own[4].hash()).is_valid()); + storage.close(); +} + +} // namespace + +TEST(StorageWriter, BlocksSavedAfterAnUnwindReplaceTheCachedOnes) { + checkReplacedBlocksAreLoaded(false); +} + +TEST(StorageWriter, BlocksSavedAfterAnUnwindReplaceTheCachedOnesAsync) { + checkReplacedBlocksAreLoaded(true); +} + +// The recovery path of BlockChain::removeLastBlock(): when the removed block cannot be read back, the +// node removes it with pool_remove_last_repair(). That path must forget the block in the read cache +// too, and a node restarted afterwards (the storage rescan fills the cache again) must read the blocks +// saved after the removal, not the removed ones. +namespace { + +csdb::Pool competing(const csdb::PoolHash& prev, cs::Sequence seq) { + csdb::Pool p(prev, seq); + p.add_user_field(1, std::string(100, static_cast('z' - seq))); + EXPECT_TRUE(p.compose()); + return p; +} + +void checkRepairedBlockIsForgotten(bool asyncWrites) { + auto db = std::make_shared(); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.writeBatchSize = 100; + opt.asyncWrites = asyncWrites; + + const auto own = chain(4); + csdb::Pool other; + { + csdb::Storage storage; + ASSERT_TRUE(storage.open(opt)); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + ASSERT_TRUE(storage.flush()); + ASSERT_EQ(storage.pool_load(3).hash(), own[3].hash()); // #3 now in the read cache + + ASSERT_TRUE(storage.pool_remove_last_repair(3, own[3].hash())); + EXPECT_EQ(storage.size(), 3u); + EXPECT_EQ(storage.last_hash(), own[2].hash()); + EXPECT_FALSE(storage.pool_load(3).is_valid()) << "the repaired block is still read from the cache"; + EXPECT_FALSE(storage.pool_load(own[3].hash()).is_valid()); + + other = competing(own[2].hash(), 3); + ASSERT_NE(other.hash(), own[3].hash()); + ASSERT_TRUE(storage.pool_save(other)); + EXPECT_EQ(storage.pool_load(3).hash(), other.hash()); + ASSERT_TRUE(storage.flush()); + storage.close(); + } + + // restart: the rescan reads every stored block and fills the cache from the database + csdb::Storage storage; + ASSERT_TRUE(storage.open(opt)); + EXPECT_EQ(storage.size(), 4u); + EXPECT_EQ(storage.last_hash(), other.hash()); + EXPECT_EQ(storage.pool_load(3).hash(), other.hash()); + EXPECT_EQ(storage.pool_load(other.hash()).sequence(), 3u); + EXPECT_FALSE(storage.pool_load(own[3].hash()).is_valid()); + storage.close(); +} + +} // namespace + +TEST(StorageWriter, RepairedLastBlockIsForgottenAndRestartReadsItsReplacement) { + checkRepairedBlockIsForgotten(false); +} + +TEST(StorageWriter, RepairedLastBlockIsForgottenAndRestartReadsItsReplacementAsync) { + checkRepairedBlockIsForgotten(true); +} + +// --resolve-fork cuts the chain through the --set-bc-top path, but only when the stored block at the +// fork point is the recorded one: then every block above it goes and the fork point stays the last +// block; otherwise nothing is removed and the open fails with TruncateMismatch. +TEST(StorageWriter, ResolveForkTruncatesExactlyAtTheRecordedForkPoint) { + auto db = std::make_shared(); + const auto own = chain(10); + { + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + ASSERT_TRUE(storage.open(opt)); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + ASSERT_TRUE(storage.flush()); + storage.close(); + } + ASSERT_EQ(db->stored(), 10u); + + // a record that no longer matches (another hash at #6): refused, nothing removed + { + csdb::Storage storage; + storage.set_truncate_check(own[5].hash()); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = 6; + EXPECT_FALSE(storage.open(opt)); + EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); + EXPECT_EQ(db->stored(), 10u); + } + // a fork point that is not stored at all: refused too + { + csdb::Storage storage; + storage.set_truncate_check(own[6].hash()); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = 12; + EXPECT_FALSE(storage.open(opt)); + EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); + EXPECT_EQ(db->stored(), 10u); + } + // a fork point below the oldest stored block (pruned): refused too + { + auto pruned = std::make_shared(); + { + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = pruned; + ASSERT_TRUE(storage.open(opt)); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + ASSERT_TRUE(storage.flush()); + storage.close(); + } + ASSERT_TRUE(pruned->remove(own[1].hash().to_binary())); + csdb::Storage storage; + storage.set_truncate_check(own[1].hash()); + csdb::Storage::OpenOptions opt; + opt.db = pruned; + opt.newBlockchainTop = 1; + EXPECT_FALSE(storage.open(opt)); + EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); + EXPECT_EQ(pruned->stored(), 9u); + } + // the recorded block: everything above #6 goes, #6 stays + { + csdb::Storage storage; + storage.set_truncate_check(own[6].hash()); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = 6; + ASSERT_TRUE(storage.open(opt)); + storage.close(); + } + EXPECT_EQ(db->stored(), 7u); + for (cs::Sequence s = 0; s <= 6; ++s) EXPECT_TRUE(db->has(own[s].hash())) << s; + for (cs::Sequence s = 7; s < 10; ++s) EXPECT_FALSE(db->has(own[s].hash())) << s; + + // and the storage opens normally afterwards, ending at the fork point + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + ASSERT_TRUE(storage.open(opt)); + EXPECT_EQ(storage.size(), 7u); + EXPECT_EQ(storage.last_hash(), own[6].hash()); + storage.close(); +}