diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 554ae8700..1fde3c11b 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -180,8 +180,9 @@ # network restarted, as after a power cut; the nodes must agree again # CHECKPOINT_AHEAD_AT (default 0 = off): at that sequence node CHECKPOINT_AHEAD_NODE (default 3, # a checkpoint a minute, 3 kept) is killed, started once offline with --set-bc-top a few blocks -# before its second newest checkpoint, and killed again: its two newest checkpoints are then -# ahead of the blocks on disk. It is restarted on every exit, at most 5 times: it must set +# before its second newest checkpoint, and killed again. The cut must have removed its two +# newest checkpoints; they are put back, as from a backup, and are then ahead of the blocks +# on disk. It is restarted on every exit, at most 5 times: it must set # those checkpoints aside, quick-start from the older one and end with the same state digests # APPLY_FAIL_AT (default 0 = off, a CREDITS_TESTNET build): at that sequence node # APPLY_FAIL_NODE (default 3) is told to fail applying a block 20 blocks later to its caches @@ -1540,6 +1541,8 @@ PY section == "[host_input]" && /^port=/ { print "port=" port; next } { print }' "$ca_dir/config.ini" > "$ca_dir/config-offline.ini" ca_before=$(grep -ac 'Changing consensus settings to' "$ca_dir/node.log" || true) + rm -rf "$ca_dir/qs-keep"; mkdir -p "$ca_dir/qs-keep" + for c in $ca_cps; do cp -a "$ca_dir/qs/$c" "$ca_dir/qs-keep/$c"; done (cd "$ca_dir" && exec "$BIN" --db-path db --config-file config-offline.ini --set-bc-top "$ca_top" > stdout.log 2>&1) & ca_pid=$! ca_t0=$(date +%s) @@ -1547,7 +1550,17 @@ PY || ! kill -0 "$ca_pid" 2>/dev/null || [ $(( $(date +%s) - ca_t0 )) -ge 120 ]; do sleep 1; done kill -KILL "$ca_pid" 2>/dev/null || true; wait "$ca_pid" 2>/dev/null || true pkill -KILL -f "cs.testnet.node=n$ca " 2>/dev/null || true - echo "checkpoint-ahead test: n$ca cut to block $ca_top, its checkpoints $(tr '\n' ' ' <<<"$ca_cps")are ahead of it" + # the cut removes the checkpoints above the new top itself; put them back (as a restored + # backup of qs/ would), so that the start must set them aside + for c in $ca_cps; do + if [ -d "$ca_dir/qs/$c" ]; then + echo "FAIL: n$ca: --set-bc-top $ca_top left the checkpoint qs/$c above the new top" + exit 1 + fi + cp -a "$ca_dir/qs-keep/$c" "$ca_dir/qs/$c" + done + rm -rf "$ca_dir/qs-keep" + echo "checkpoint-ahead test: n$ca cut to block $ca_top, which removed its checkpoints $(tr '\n' ' ' <<<"$ca_cps")above it; put back, they are ahead of it" ca_lines=$(wc -l < "$ca_dir/node.log") ca_restarts=0 start_node "$ca" diff --git a/client/config/config.cpp b/client/config/config.cpp index 78a804e5a..041820150 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -136,6 +136,7 @@ const std::string ARG_NAME_ENCRYPT_KEY_FILE = "encryptkey"; const std::string ARG_NAME_RECREATE_INDEX = "recreate-index"; const std::string ARG_NAME_NEW_BC_TOP = "set-bc-top"; const std::string ARG_NAME_RESOLVE_FORK = "resolve-fork"; +const std::string ARG_NAME_TO_LOCATED = "to-located"; const std::string ARG_NAME_BAL_CHANGE = "bal-change"; const std::string ARG_NAME_DISABLE_AUTO_SHUTDOWN = "disable-auto-shutdown"; @@ -281,6 +282,7 @@ Config Config::read(po::variables_map& vm, bool atStart) { if (atStart) { result.resolveFork_ = vm.count(ARG_NAME_RESOLVE_FORK) > 0; + result.resolveToLocated_ = vm.count(ARG_NAME_TO_LOCATED) > 0; if (vm.count(ARG_NAME_NEW_BC_TOP)) { result.newBlockchainTop_ = true; result.newBlockchainTopSeq_ = diff --git a/client/config/config.hpp b/client/config/config.hpp index ed7a69afd..b72d4544d 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -345,6 +345,10 @@ class Config { bool resolveFork() const { return resolveFork_; } + // --resolve-fork --to-located: cut at the located but unverified last common block instead + bool resolveToLocated() const { + return resolveToLocated_; + } bool traverseNAT() const { return traverseNAT_; @@ -455,6 +459,7 @@ class Config { uint64_t maxAutoRollbackDepth_ = DEFAULT_MAX_AUTO_ROLLBACK; bool allowDeepRollback_ = false; bool resolveFork_ = false; + bool resolveToLocated_ = false; bool symmetric_ = false; EndpointData hostAddressEp_; diff --git a/client/include/client/cmdlineargs.hpp b/client/include/client/cmdlineargs.hpp index 349a62b88..c5c5601d5 100644 --- a/client/include/client/cmdlineargs.hpp +++ b/client/include/client/cmdlineargs.hpp @@ -8,6 +8,7 @@ struct cmdline { static constexpr const char* argDumpKeys = "dumpkeys"; static constexpr const char* argSetBCTop = "set-bc-top"; static constexpr const char* argResolveFork = "resolve-fork"; + static constexpr const char* argToLocated = "to-located"; static constexpr const char* argBalChange = "bal-change"; #ifdef _WIN32 static constexpr const char* argInstall = "install"; diff --git a/client/src/main.cpp b/client/src/main.cpp index 19acf500b..d64c581f5 100644 --- a/client/src/main.cpp +++ b/client/src/main.cpp @@ -57,6 +57,7 @@ int main(int argc, char* argv[]) { (cmdline::argSeed, "enter with seed instead of keys") (cmdline::argSetBCTop, po::value(), "all blocks in blockchain with higher sequence will be removed") (cmdline::argResolveFork, "a node that reports FORK HELD: remove the blocks above the fork point recorded in fork-held.json (only if the block there is still the recorded one), then exit; at the next start the node follows the other chain. Check with other nodes first (the node must be stopped)") + (cmdline::argToLocated, "with --resolve-fork: cut at the last common block the node located with a peer (fork-held.json, \"located_point\") even though it could not verify it, instead of the recorded fork point") (cmdline::argBalChange, po::value(), "prints all changes in account balance in log file") ("disable-auto-shutdown", "node will be prohibited to shutdown in case of fatal errors") ("version", "show node version") diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index dfbe0153e..c30dfdc6b 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -83,6 +83,10 @@ class Storage final { StartAfterLastBlock = 7, // --resolve-fork: the block at the requested top is not the one recorded (set_truncate_check) TruncateMismatch = 8, + // --set-bc-top / --resolve-fork on a storage that does not start at block 0 (pruned, or a + // backfill in progress) with no quick-start checkpoint left at or below the new top: after the + // cut the node could neither quick-start nor slow-start (set_truncate_needs_checkpoint) + TruncateNoCheckpoint = 9, UnknownError = 255, }; @@ -158,6 +162,12 @@ class Storage final { // at that top has this hash; otherwise it removes nothing and fails with TruncateMismatch. An // empty hash (the default) checks nothing. Used by --resolve-fork. void set_truncate_check(const PoolHash& expectedTopHash); + // The next open() with a new blockchain top refuses the cut (TruncateNoCheckpoint, nothing removed) + // when no quick-start checkpoint remains at or below that top (checkpointAtOrBelowTop false) and the + // storage does not start at block 0. true (the default) checks nothing. + void set_truncate_needs_checkpoint(bool checkpointAtOrBelowTop); + // after an open() that cut the chain: the last block stored before the cut (0: nothing was cut) + cs::Sequence truncated_last() const; // dbBackend: "berkeleydb" (default), "rocksdb", or "lmdb" in /lmdb, whose removed blocks // LMDB reuses the space of (pruned storage) diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index b8d1765bd..26ad61683 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -220,6 +220,10 @@ class Storage::priv { static const size_t cacheSize = 50; // set_truncate_check(): the hash the block at the new blockchain top must have PoolHash truncate_expected_hash; + // set_truncate_needs_checkpoint(): false refuses a cut of a storage that does not start at block 0 + bool truncate_checkpoint_below = true; + // the last block stored before the last cut + cs::Sequence truncated_last = 0; // the cache keeps one pool per sequence and per hash: a pool saved at a sequence whose previous // pool was removed (a chain reorganisation) replaces the cached one, instead of being refused by @@ -483,6 +487,14 @@ void Storage::set_truncate_check(const PoolHash& expectedTopHash) { d->truncate_expected_hash = expectedTopHash; } +void Storage::set_truncate_needs_checkpoint(bool checkpointAtOrBelowTop) { + d->truncate_checkpoint_below = checkpointAtOrBelowTop; +} + +cs::Sequence Storage::truncated_last() const { + return d->truncated_last; +} + Storage::Error Storage::last_error() const { return last_error_map(d.get()).last_error_; } @@ -573,6 +585,7 @@ bool Storage::open(const OpenOptions& opt, OpenCallback callback) { } auto seqToRemove = static_cast(opt.newBlockchainTop + 1); auto seqLast = seqToRemove; + uint32_t seqFirst = 0; { Database::IteratorPtr it = d->db->new_iterator(); it->seek_to_last(); @@ -582,7 +595,21 @@ bool Storage::open(const OpenOptions& opt, OpenCallback callback) { seqLast = key; } } + it->seek_to_first(); + if (it->is_valid()) { + seqFirst = it->key(); + } + } + const bool checkpointBelow = d->truncate_checkpoint_below; + d->truncate_checkpoint_below = true; + if (!checkpointBelow && seqFirst > 0 && seqLast >= seqToRemove) { + d->set_last_error(TruncateNoCheckpoint, + "the storage starts at block %u (pruned, or older blocks not fetched yet) and no quick-start " + "checkpoint is left at or below block %llu: after the cut the node could not start; nothing removed", + seqFirst, static_cast(opt.newBlockchainTop)); + return false; } + d->truncated_last = seqLast >= seqToRemove ? seqLast : 0; auto blocksCount = (seqLast >= seqToRemove) ? (seqLast - seqToRemove + 1) : 0; if(seqToRemove <= seqLast) { diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index d6d81a1cb..0072edb35 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -81,6 +81,7 @@ add_library(csnode include/csnode/snapshot_fetcher.hpp include/csnode/blockverifier.hpp include/csnode/forkguardstate.hpp + include/csnode/forklocator.hpp include/csnode/backwardsync.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -136,6 +137,7 @@ add_library(csnode src/snapshot_fetcher.cpp src/blockverifier.cpp src/forkguardstate.cpp + src/forklocator.cpp src/backwardsync.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index f32da4a8a..e4c823d7e 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -269,6 +269,13 @@ class BlockChain { // Strict for the unwind decision: a confirmation mask of more than one entry, of the same size as our // last block's confidants, with as many signatures as it marks, and a majority of them confirming. bool hasTrustedConfirmations(const csdb::Pool& competing) const; + // the same checks against the given confidants + bool confirmedByConfidants(const csdb::Pool& competing, const std::vector& trustedConfidants) const; + // The fork point search (ForkLocator): `block` (a peer's, not stored) is built on our stored block + // at its sequence - 1, its signatures verify against its own confidants, and its round + // confirmations, by a majority, against the confidants of that block of ours, as hasTrustedConfirmations + // checks them. A bootstrap block never passes (it carries no confirmations). Read-only. + bool verifiesAfterOurBlock(const csdb::Pool& block) const; // True if a bootstrap-flagged competing block is authenticated locally: its own signatures verify // against its confidants AND those confidants are exactly the confidants of a round table this node @@ -333,10 +340,25 @@ class BlockChain { std::optional approvedBranch() const { return approvedBranch_; } + // The search for the last common block (cs::forkguard::ForkLocator, driven by the node while FORK + // HELD) has an answer for the held record `expected` (the record it searched for; nothing is + // changed if the record changed since). `located` is the last common block the peer's answers + // give (`hash` is our stored hash there, checked again here): the fork point only when `verified` + // (the peer's next block verified against our confidants there), otherwise a hint; or, with + // `deeper`, the lowest block searched, where the peer's block already differs (a hint). Only the + // record (fork-held.json) changes: nothing is unwound, nothing approved. + // True if the record was updated. + bool applyLocatedForkPoint(const cs::forkguard::ForkRecord& expected, cs::Sequence located, const std::string& hash, + bool deeper, const std::string& peerKey, bool verified = false); // init() refused to truncate: the stored block at the requested sequence has another hash bool truncationRefused() const { return truncationRefused_; } + // init() refused to truncate: the storage does not start at block 0 and no quick-start checkpoint + // is left at or below the requested top, so the node could not start after the cut + bool truncationNoCheckpoint() const { + return truncationNoCheckpoint_; + } #ifdef TESTING // test-only accessors for the longest-chain guard @@ -747,6 +769,7 @@ public slots: cs::Sequence resolveForkSeq_ = cs::kWrongSequence; csdb::PoolHash resolveForkHash_; bool truncationRefused_ = false; + bool truncationNoCheckpoint_ = false; // starts the episode at the current head if none is running (persisting its reference) void beginRollbackEpisode(); // the chain grew: ends the episode once the head is above its reference @@ -808,6 +831,9 @@ public slots: } bool tryQuickStart(cs::CachesSerializationManager*, std::set& initialConfidants); + // after a cut at `top` (--set-bc-top, --resolve-fork): the block hashes and the quick-start + // checkpoints above it go, so that the next start finds nothing beyond the stored blocks + void dropStateAboveTop(cs::Sequence top, cs::CachesSerializationManager* serializationManPtr); bool bindSerializationManToCaches(cs::CachesSerializationManager*, std::set& initialConfidants); cs::CachesSerializationManager* serializationManPtr_ = nullptr; diff --git a/csnode/include/csnode/blockhashes.hpp b/csnode/include/csnode/blockhashes.hpp index b251d224c..447afc796 100644 --- a/csnode/include/csnode/blockhashes.hpp +++ b/csnode/include/csnode/blockhashes.hpp @@ -34,6 +34,10 @@ class BlockHashes { bool remove(cs::Sequence); bool remove(const csdb::PoolHash& hash); + // after a cut at `top`: removes the entries of every block above it, through `last` (the last + // block stored before the cut; gaps included) and on up to the first missing sequence after it. + // Returns the number of blocks removed + size_t removeAbove(cs::Sequence top, cs::Sequence last); // The first failure of either database is reported once and leaves the file blockhashes.rebuild // in the caches folder: the next start rebuilds both databases from the stored blocks. diff --git a/csnode/include/csnode/caches_serialization_manager.hpp b/csnode/include/csnode/caches_serialization_manager.hpp index 31132eb8c..a32386cdf 100644 --- a/csnode/include/csnode/caches_serialization_manager.hpp +++ b/csnode/include/csnode/caches_serialization_manager.hpp @@ -41,6 +41,16 @@ std::optional readCheckpointHead(const std::filesystem::path& di bool isCheckpointOnChain(const CheckpointHead& head, const std::function& chainHashAt); +// After the chain was cut at `top` (--set-bc-top, --resolve-fork): sets aside, as ".bad" in +// the quick-start folder `root`, every checkpoint that continues above `top` (its head, or its name +// for a periodic checkpoint without head.bin, is above it; a qs/0 without head.bin cannot be placed +// and goes too), with its .prev backup. The next start then quick-starts from the highest checkpoint +// at or below the new top, or slow-starts. Returns the number of checkpoints set aside. +size_t dropCheckpointsAbove(const std::filesystem::path& root, cs::Sequence top); +// a checkpoint (not a .prev backup) that does not continue above `top` is in `root`: a node whose +// storage does not start at block 0 can start after a cut at `top` only with one +bool hasCheckpointAtOrBelow(const std::filesystem::path& root, cs::Sequence top); + class CachesSerializationManager { public: CachesSerializationManager(); @@ -81,6 +91,10 @@ class CachesSerializationManager { void setCompletedFromGenesis(); bool isLoadedFromCompletedSnapshot() const; void pruneCheckpoints(size_t keep); + // dropCheckpointsAbove() on this node's quick-start folder + size_t dropAbove(cs::Sequence top); + // hasCheckpointAtOrBelow() on this node's quick-start folder + bool hasCheckpointAtOrBelow(cs::Sequence top) const; private: struct Impl; diff --git a/csnode/include/csnode/forkguardstate.hpp b/csnode/include/csnode/forkguardstate.hpp index d3445260d..0fe83425c 100644 --- a/csnode/include/csnode/forkguardstate.hpp +++ b/csnode/include/csnode/forkguardstate.hpp @@ -68,6 +68,26 @@ struct ForkRecord { cs::Sequence forkPoint = 0; std::string forkPointHash; bool forkPointExact = false; + // the search for the last common block (ForkLocator) while FORK HELD: located is true once it has + // an answer, found from the upper bound forkPoint had before (upperBound) with the peer locatedWith. + // - verified (locatedVerified): the peer's block right after the located point is built on our + // block there, and its signatures and round confirmations verify against the confidants of our + // block there, as a synced block's would. Only then is the point the fork point: forkPoint and + // forkPointHash are set to it and forkPointExact turns true (the competing block is still the + // one first refused, so no approval follows from it). + // - not verified (a bootstrap block, or confirmations that do not verify): a hint only. forkPoint + // stays the upper bound; locatedPoint/locatedHash keep the hint, and --resolve-fork cuts there + // only with --to-located. + // - deeper: the peer's block at deeperThan, the lowest one searched, already differs from ours; a + // hint as well (a peer can claim it), forkPoint stays the upper bound + bool located = false; + bool locatedVerified = false; + cs::Sequence locatedPoint = 0; + std::string locatedHash; + bool deeper = false; + cs::Sequence upperBound = 0; + cs::Sequence deeperThan = 0; + std::string locatedWith; cs::Sequence competingSeq = 0; // the refused block (0: not known, e.g. a bad-hash reply) std::string competingHash; std::string peerKey; // base58, empty when not known @@ -81,6 +101,16 @@ struct ForkRecord { std::string dismissedHash; std::vector branchHashes; + // "exact", "located (unverified)", "deeper" (below deeperThan, unverified) or "upper bound" + std::string forkPointStatus() const { + if (forkPointExact) { + return "exact"; + } + if (located && !deeper && !locatedHash.empty()) { + return "located (unverified)"; + } + return deeper ? "deeper" : "upper bound"; + } bool hasDismissal() const { return !branchHashes.empty(); } @@ -106,12 +136,19 @@ void clearAll(const std::string& dir); // --resolve-fork: the fork point to cut the chain at and the hash the stored block there must have, // from the record in dir; nothing (and *error says why) when there is no record or it cannot be // checked against the database +// toLocated (--resolve-fork --to-located): the located but unverified last common block instead of +// the recorded fork point struct ResolvePlan { cs::Sequence top = 0; std::string hash; ForkRecord record; + bool toLocated = false; }; -std::optional planResolve(const std::string& dir, std::string* error); +std::optional planResolve(const std::string& dir, std::string* error, bool toLocated = false); + +// what --resolve-fork shows before it cuts: our head, the upper bound, the located point and whether +// it is verified, where the cut goes and how many blocks it removes +std::string resolveOverview(const ResolvePlan& plan); // "2026-11-19 10:04:05 UTC" std::string utcTime(int64_t unixSeconds); diff --git a/csnode/include/csnode/forklocator.hpp b/csnode/include/csnode/forklocator.hpp new file mode 100644 index 000000000..0c7f7dd88 --- /dev/null +++ b/csnode/include/csnode/forklocator.hpp @@ -0,0 +1,180 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace cs::forkguard { + +// While FORK HELD with only an upper bound of the fork point ("block #N or lower"), the node asks the +// peer of the refused block for some of its blocks at heights at or below that bound, with the usual +// BlockRequest, and compares their hashes with its own: the last block both chains share is the +// highest height where they match. A k-ary search (a few heights per request) from the bound down to +// a floor (`Settings::maxDepth` blocks below it, never below the first stored block). +// +// Read-only and local: it stores nothing, unwinds nothing and changes nothing the node accepts. The +// peer's blocks serve only to locate the point: a block's hash is computed from its content, a +// "match" is a block identical to ours, and the answer is checked to hold together (matches strictly +// below mismatches, consecutive blocks chained, the first differing block of the peer built on our +// block at the fork point). A peer that lies can at worst make the search fail or move the point +// within [floor, bound]. So the answer is taken as the fork point only when the peer's block right +// after it verifies as a synced block would after our block there (PeerBlock::verified: signatures, +// and round confirmations by the confidants of our block at the point); otherwise it stays a hint +// that --resolve-fork follows only on the operator's explicit request (--to-located). The cut still +// checks that our stored block at the point is the recorded one. +// +// One request at a time, at most one per Settings::minInterval (the peer refuses a second request +// within 5 s of its reply as a duplicate). Single-threaded: the node calls it from its processor +// thread. The network and our chain are reached only through Hooks. +class ForkLocator { +public: + using Clock = std::chrono::steady_clock; + + struct Settings { + cs::Sequence maxDepth = 2000; // blocks searched below the upper bound + size_t probesPerRequest = 8; // heights asked for in one request, at least 2 + Clock::duration minInterval = std::chrono::seconds(6); + Clock::duration replyTimeout = std::chrono::seconds(20); + size_t maxAttempts = 8; // requests in a row without a usable reply before giving up + // requests in a row refused as duplicates before giving up. The peer refuses a request as a + // duplicate while it is answering another request of this node (its forward sync asks the same + // peer again and again while FORK HELD), so each such refusal costs a minInterval: a busy peer can + // stretch the search to a few minutes + size_t maxDuplicates = 50; + // times the floor may be raised because the peer does not store the lowest height asked for + // (a pruned peer) before giving up + size_t maxFloorRaises = 16; + }; + + struct Hooks { + // our stored block hash at a height (hex), empty when not stored + std::function ourHash; + // sends a BlockRequest for these heights to the peer; false when it cannot (not a neighbour) + std::function&)> sendRequest; + }; + + // a block of the peer, as far as the search needs it (hashes in hex). verified: the node checked + // it as a synced block after OUR block at seq - 1: built on it (previous is our hash there), its + // signatures verify against its confidants and its round confirmations against the confidants of + // our block at seq - 1 (never so for a bootstrap block, which carries no confirmations) + struct PeerBlock { + cs::Sequence seq = 0; + std::string hash; + std::string previous; + bool verified = false; + }; + + enum class Status { + Idle, // never started, or stopped + Searching, + Exact, // forkPoint is the last common block, forkPointHash our hash there; verified() says + // whether the peer's next block was verified against our confidants there (only + // then may the node take it as the fork point; otherwise it is a hint) + Deeper, // the chains already differ at forkPoint (the floor): the fork is below it + Failed // why() says why; the node may start again later + }; + + ForkLocator(Hooks hooks, Settings settings); + + // starts a search: the chains are known to differ above `upper` (the block at upper + 1 is not + // shared); the search goes down to `floor` at most (raised to upper - maxDepth, and to block 1: + // the genesis block is shared by definition and never asked for) + void start(const cs::PublicKey& peer, cs::Sequence upper, cs::Sequence floor, Clock::time_point now); + void stop(); + + // sends the next request when one is due, or gives up after Settings::maxAttempts + void tick(Clock::time_point now); + + // a request of this search to `sender` waits for its reply, and the reply holds no more blocks than + // were asked for, each of a different height asked for: the reply is for the search + bool expects(const cs::PublicKey& sender, const std::vector& replySeqs) const; + // the heights the request waiting for its reply asked for (0: none waits) + size_t inFlightCount() const { + return inFlight_.size(); + } + // the reply to the request in flight (blocks not asked for are ignored) + void onReply(const cs::PublicKey& sender, const std::vector& blocks, Clock::time_point now); + // the peer said it will not answer the request in flight: + // - NoSuchBlocks: it does not store the lowest height asked for (a pruned peer). While no match is + // known the floor goes up past it (to the next height asked for), at most maxFloorRaises times; + // otherwise a failed attempt. + // - Duplicate: it answered another request of this node a moment ago (e.g. of the forward sync): + // tried again, up to maxDuplicates times in a row. + // - Other (an incorrect request): a failed attempt. + enum class Refusal { NoSuchBlocks, Duplicate, Other }; + void onRefused(const cs::PublicKey& sender, Clock::time_point now, Refusal why = Refusal::Other); + + Status status() const { + return status_; + } + bool searching() const { + return status_ == Status::Searching; + } + const cs::PublicKey& peer() const { + return peer_; + } + cs::Sequence upper() const { + return upper_; + } + cs::Sequence floor() const { + return floor_; + } + cs::Sequence forkPoint() const { + return forkPoint_; + } + const std::string& forkPointHash() const { + return forkPointHash_; + } + bool verified() const { + return verified_; + } + const std::string& why() const { + return why_; + } + size_t requestsSent() const { + return requests_; + } + + // the heights the next request asks for (exposed for tests) + std::vector nextProbes() const; + +private: + bool matches(cs::Sequence seq) const; + void fail(const std::string& why); + void evaluate(); + + Hooks hooks_; + Settings settings_; + Status status_ = Status::Idle; + cs::PublicKey peer_{}; + cs::Sequence upper_ = 0; + cs::Sequence floor_ = 0; + // highest height known to match (none yet: floor_ not confirmed) and lowest known to differ + std::optional lo_; + cs::Sequence hi_ = 0; + std::map known_; // the peer's blocks received + std::vector inFlight_; // the heights of the request waiting for its reply + Clock::time_point sentAt_{}; + Clock::time_point lastSent_{}; + bool everSent_ = false; + size_t attempts_ = 0; + size_t duplicates_ = 0; + size_t floorRaises_ = 0; + size_t requests_ = 0; + cs::Sequence forkPoint_ = 0; + std::string forkPointHash_; + bool verified_ = false; + std::string why_; +}; + +// the depth the search covers below the upper bound for a max_auto_rollback setting: 20 times it, +// between 1000 and 10000 blocks; 2000 when max_auto_rollback is 0 (no depth bound) +cs::Sequence locateDepth(uint64_t maxAutoRollback); + +} // namespace cs::forkguard diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index 6fe51844c..0173aabe6 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -480,6 +481,14 @@ public slots: // the fork guard's timer (every 5 s, queued to the processor thread): dismiss-fork, pending holds cs::Timer forkGuardTimer_; void onForkGuardTick(); + // FORK HELD with only an upper bound of the fork point: the search for the last common block with + // the peer of the refused block (read-only; processor thread), for the record forkLocatorFor_; + // after a failed search, a new one starts at forkLocatorRetryAt_ + std::unique_ptr forkLocator_; + std::optional forkLocatorFor_; + std::chrono::steady_clock::time_point forkLocatorRetryAt_{}; + void tickForkLocator(); + void startForkLocator(const cs::forkguard::ForkRecord& record); // snapshot.json of a restored folder against the digest of the loaded caches (design 6.8); // false with exit code 4 on a mismatch bool verifyAtStart(); diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index deb00f63f..7d7c42987 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -247,6 +247,11 @@ bool BlockChain::init( storage_.set_truncate_check(resolveForkHash_); } truncationRefused_ = false; + truncationNoCheckpoint_ = false; + if (newBlockchainTop != cs::kWrongSequence && serializationManPtr != nullptr) { + // a storage that does not start at block 0 can start after the cut only from a checkpoint + storage_.set_truncate_needs_checkpoint(serializationManPtr->hasCheckpointAtOrBelow(newBlockchainTop)); + } if (!storage_.open(path, progress, newBlockchainTop, firstBlockToReadInDatabase, storageCfg.asyncWriteQueueSize, storageCfg.writeBatchSize, static_cast(storageCfg.rocksdbBlockCacheMb) << 20, @@ -254,7 +259,8 @@ bool BlockChain::init( storageCfg.dbBackend)) { cserror() << kLogPrefix << "Couldn't open database at " << path; truncationRefused_ = storage_.last_error() == csdb::Storage::TruncateMismatch; - if (truncationRefused_) { + truncationNoCheckpoint_ = storage_.last_error() == csdb::Storage::TruncateNoCheckpoint; + if (truncationRefused_ || truncationNoCheckpoint_) { cserror() << kLogPrefix << storage_.last_error_message(); } checkpointAheadOfBlocks_ = successfulQuickStart && storage_.last_error() == csdb::Storage::StartAfterLastBlock; @@ -273,6 +279,7 @@ bool BlockChain::init( if (newBlockchainTop != cs::kWrongSequence) { if (trxIndex_) trxIndex_->trimToFloor(newBlockchainTop); + dropStateAboveTop(newBlockchainTop, serializationManPtr); // the operator truncated: the rollback episode and FORK HELD end here const std::string dir = cs::forkguard::stateDirFor(path); cs::forkguard::clearForkRecord(dir); @@ -357,6 +364,26 @@ bool BlockChain::init( return true; } +// The chain was cut at `top` at start-up (--set-bc-top, --resolve-fork): the storage no longer has +// the blocks above it, so neither may what describes them. The block hashes databases forget them +// (else a checkpoint above the top would still look "on this chain"), and every quick-start +// checkpoint above the top goes (the shutdown checkpoint qs/0 was saved at the old head: the next +// start would load it and then stop, its blocks being gone). The next start quick-starts from the +// highest checkpoint at or below the top, or slow-starts. +void BlockChain::dropStateAboveTop(cs::Sequence top, cs::CachesSerializationManager* serializationManPtr) { + size_t hashes = 0; + if (blockHashes_) { + hashes = blockHashes_->removeAbove(top, storage_.truncated_last()); + blockHashes_->flush(); + } + size_t checkpoints = 0; + if (serializationManPtr != nullptr) { + checkpoints = serializationManPtr->dropAbove(top); + } + cslog() << kLogPrefix << "chain cut at #" << WithDelimiters(top) << ": " << hashes << " block hash(es) and " + << checkpoints << " quick-start checkpoint(s) above it removed"; +} + bool BlockChain::isGood() const { return good_; } @@ -1103,6 +1130,64 @@ bool BlockChain::promotePendingHold(bool roundStarted) { return true; } +bool BlockChain::applyLocatedForkPoint(const cs::forkguard::ForkRecord& expected, cs::Sequence located, + const std::string& hash, bool deeper, const std::string& peerKey, bool verified) { + // our block at the located point must still be the one the search compared (read before the lock: + // the hash lookups take the storage lock) + if (!deeper && (hash.empty() || located > lastSequence_.load() || getHashBySequence(located).to_string() != hash)) { + cswarning() << kLogPrefix << "fork point search: block #" << WithDelimiters(located) + << " changed since the search compared it, the record is left as it was"; + return false; + } + cs::forkguard::ForkRecord r; + { + std::lock_guard l(guardMutex_); + if (!forkRecord_.has_value() || !forkRecord_->held || forkRecord_->located || + forkRecord_->sinceUnix != expected.sinceUnix || forkRecord_->forkPoint != expected.forkPoint || + forkRecord_->competingHash != expected.competingHash) { + return false; // another record (or none) by now: the answer is for a fork no longer held + } + forkRecord_->located = true; + forkRecord_->upperBound = forkRecord_->forkPoint; + forkRecord_->locatedWith = peerKey; + if (deeper) { + forkRecord_->deeper = true; + forkRecord_->deeperThan = located; + } + else { + forkRecord_->locatedPoint = located; + forkRecord_->locatedHash = hash; + forkRecord_->locatedVerified = verified; + if (verified) { + // the other branch provably continues from our block there: that is the fork point + forkRecord_->forkPoint = located; + forkRecord_->forkPointHash = hash; + forkRecord_->forkPointExact = true; + } + } + r = *forkRecord_; + } + cs::forkguard::saveForkRecord(guardStateDir_, r); + if (deeper) { + cserror() << kLogPrefix << "fork point search: a peer's block already differs from ours at #" << WithDelimiters(located) + << ", " << (r.upperBound - located) << " blocks below the upper bound #" << WithDelimiters(r.upperBound) + << " (not verified): the fork may be deeper; --resolve-fork still cuts at the upper bound"; + } + else if (verified) { + cswarning() << kLogPrefix << "fork point search: the last common block is #" << WithDelimiters(located) << " (" + << hash << "), verified, located from the upper bound #" << WithDelimiters(r.upperBound) + << "; --resolve-fork now cuts there"; + } + else { + cswarning() << kLogPrefix << "fork point search: a peer's blocks place the last common block at #" + << WithDelimiters(located) << ", but its next block could not be verified against our confidants: " + << "a hint only. --resolve-fork cuts at the upper bound #" << WithDelimiters(r.upperBound) + << "; --resolve-fork --to-located at #" << WithDelimiters(located); + } + cserror() << kLogPrefix << cs::forkguard::summary(r); + return true; +} + bool BlockChain::dismissFork() { cs::forkguard::ForkRecord r; { @@ -1324,12 +1409,33 @@ bool BlockChain::deepRollbackApproved() const { // unwind. Bootstrap blocks carry no such confirmations and fail here; they go through // isBootstrapAuthenticatedLocally() instead. bool BlockChain::hasTrustedConfirmations(const csdb::Pool& competing) const { + return confirmedByConfidants(competing, getLastBlock().confidants()); +} + +bool BlockChain::verifiesAfterOurBlock(const csdb::Pool& block) const { + if (!block.is_valid() || block.sequence() < 2) { + return false; + } + const csdb::Pool parent = loadBlock(block.sequence() - 1); + if (!parent.is_valid() || block.previous_hash() != parent.hash()) { + return false; + } + std::string reason; + if (!cs::BlockVerifier::verifySignatures(block, reason)) { + csdebug() << kLogPrefix << "block #" << block.sequence() << " signatures: " << reason; + return false; + } + // a bootstrap block is refused here: it carries no confirmations to check + return confirmedByConfidants(block, parent.confidants()); +} + +bool BlockChain::confirmedByConfidants(const csdb::Pool& competing, const std::vector& trustedConfidants) const { // Strict pre-checks for the unwind decision only; BlockVerifier::verifyConfirmations keeps the // exceptions it has for normal block validation (sequence < 2, one-entry masks), which must not be // enough to make this node unwind its own chain. const auto fail = [&competing](const char* why) { csdebug() << kLogPrefix << "competing block #" << competing.sequence() - << " not confirmed by our last block's confidants: " << why; + << " not confirmed by our confidants: " << why; return false; }; if (!competing.is_valid() || competing.sequence() < 2) { @@ -1339,13 +1445,12 @@ bool BlockChain::hasTrustedConfirmations(const csdb::Pool& competing) const { if (BlockChain::isBootstrap(competing)) { return fail("bootstrap block carries no confirmations"); } - const auto trustedConfidants = getLastBlock().confidants(); if (competing.numberConfirmations() == 0 || competing.roundConfirmations().empty()) { return fail("no round confirmations"); } const auto confMask = cs::Utils::bitsToMask(competing.numberConfirmations(), competing.roundConfirmationMask()); if (confMask.size() <= 1 || confMask.size() != trustedConfidants.size()) { - return fail("confirmation mask does not match our last block's confidants"); + return fail("confirmation mask does not match our confidants"); } const size_t confirming = NodeUtils::realTrustedValue(confMask); if (competing.roundConfirmations().size() != confirming) { diff --git a/csnode/src/blockhashes.cpp b/csnode/src/blockhashes.cpp index c0fee9529..3a2c7794a 100644 --- a/csnode/src/blockhashes.cpp +++ b/csnode/src/blockhashes.cpp @@ -90,6 +90,18 @@ bool BlockHashes::remove(const csdb::PoolHash& hash) { return true; } +size_t BlockHashes::removeAbove(cs::Sequence top, cs::Sequence last) { + size_t removed = 0; + cs::Sequence seq = top + 1; + for (; seq <= last; ++seq) { + removed += remove(seq) ? 1 : 0; + } + for (; remove(seq); ++seq) { + ++removed; + } + return removed; +} + bool BlockHashes::rebuildRequested() const { std::error_code ec; return std::filesystem::exists(path_ + rebuildMarker, ec); diff --git a/csnode/src/caches_serialization_manager.cpp b/csnode/src/caches_serialization_manager.cpp index 68c1e85f0..79cce0506 100644 --- a/csnode/src/caches_serialization_manager.cpp +++ b/csnode/src/caches_serialization_manager.cpp @@ -652,6 +652,109 @@ bool isCheckpointOnChain(const CheckpointHead& head, const std::function" and its crash-recovery backup ".prev" (restored +// as "" at the next start when "" is missing); ".tmp" and ".bad" are never loaded +struct CheckpointDir { + std::filesystem::path path; + cs::Sequence version = 0; + bool prev = false; + std::optional head; + // the block it continues after: its head, or its name for a periodic one without head.bin; none + // for a qs/0 without head.bin (cannot be placed) + std::optional top() const { + if (head.has_value()) { + return std::max(head->sequence, version); + } + if (version == 0) { + return std::nullopt; + } + return version; + } +}; + +std::vector checkpointDirs(const std::filesystem::path& root) { + namespace fs = std::filesystem; + std::vector dirs; + std::error_code ec; + if (!fs::is_directory(root, ec)) { + return dirs; + } + for (const auto& entry : fs::directory_iterator(root, ec)) { + if (!entry.is_directory(ec)) { + continue; + } + std::string name = entry.path().filename().string(); + const std::string prevExt = ".prev"; + bool prev = false; + if (name.size() > prevExt.size() && name.compare(name.size() - prevExt.size(), prevExt.size(), prevExt) == 0) { + name.resize(name.size() - prevExt.size()); + prev = true; + } + if (name.empty() || name.size() > 19 || !std::all_of(name.begin(), name.end(), [](char c) { return c >= '0' && c <= '9'; })) { + continue; + } + CheckpointDir d; + d.path = entry.path(); + d.version = static_cast(std::stoull(name)); + d.prev = prev; + d.head = readCheckpointHead(entry.path()); + dirs.push_back(std::move(d)); + } + return dirs; +} +} // namespace + +size_t dropCheckpointsAbove(const std::filesystem::path& root, cs::Sequence top) { + namespace fs = std::filesystem; + size_t dropped = 0; + for (const auto& d : checkpointDirs(root)) { + const auto at = d.top(); + if (at.has_value() && *at <= top) { + continue; + } + // set aside as ".bad" (kept for inspection, never loaded), replacing an older one + std::error_code ec; + const fs::path bad = d.path.string() + ".bad"; + fs::remove_all(bad, ec); + fs::rename(d.path, bad, ec); + if (ec) { + cswarning() << "CachesSerializationManager: cannot set " << d.path << " aside (above the new top " << top + << "): " << ec.message() << "; removing it"; + fs::remove_all(d.path, ec); + if (ec) { + cserror() << "CachesSerializationManager: cannot remove " << d.path << ": " << ec.message(); + continue; + } + } + ++dropped; + cslog() << "CachesSerializationManager: " << d.path.string() << " continues above the new top " << top + << ", set aside as " << bad.string(); + } + if (dropped > 0) { + fsyncDir(root); + } + return dropped; +} + +bool hasCheckpointAtOrBelow(const std::filesystem::path& root, cs::Sequence top) { + for (const auto& d : checkpointDirs(root)) { + const auto at = d.top(); + if (!d.prev && at.has_value() && *at <= top) { + return true; + } + } + return false; +} + +size_t CachesSerializationManager::dropAbove(cs::Sequence top) { + return dropCheckpointsAbove(pImpl_->kQuickStartRoot, top); +} + +bool CachesSerializationManager::hasCheckpointAtOrBelow(cs::Sequence top) const { + return cs::hasCheckpointAtOrBelow(pImpl_->kQuickStartRoot, top); +} + bool CachesSerializationManager::load() { return load(ChainVerifier{}); } diff --git a/csnode/src/forkguardstate.cpp b/csnode/src/forkguardstate.cpp index 96398fa1f..4f90707fa 100644 --- a/csnode/src/forkguardstate.cpp +++ b/csnode/src/forkguardstate.cpp @@ -246,6 +246,20 @@ bool saveForkRecord(const std::string& dir, const ForkRecord& r) { tree.put("fork_point", r.forkPoint); tree.put("fork_point_hash", r.forkPointHash); tree.put("fork_point_exact", r.forkPointExact); + tree.put("fork_point_status", r.forkPointStatus()); + if (r.located) { + tree.put("fork_point_located", true); + tree.put("fork_point_upper_bound", r.upperBound); + tree.put("fork_point_located_with", r.locatedWith); + tree.put("located_verified", r.locatedVerified); + if (!r.locatedHash.empty()) { + tree.put("located_point", r.locatedPoint); + tree.put("located_hash", r.locatedHash); + } + if (r.deeper) { + tree.put("fork_point_below", r.deeperThan); + } + } tree.put("competing_block", r.competingSeq); tree.put("competing_hash", r.competingHash); tree.put("peer_key", r.peerKey); @@ -283,6 +297,16 @@ std::optional loadForkRecord(const std::string& dir, bool* corrupt) r.forkPoint = tree->get("fork_point"); r.forkPointHash = tree->get("fork_point_hash", ""); r.forkPointExact = tree->get("fork_point_exact", false); + r.located = tree->get("fork_point_located", false); + r.upperBound = tree->get("fork_point_upper_bound", 0); + r.locatedWith = tree->get("fork_point_located_with", ""); + r.locatedVerified = r.located && tree->get("located_verified", false); + r.locatedPoint = tree->get("located_point", 0); + r.locatedHash = tree->get("located_hash", ""); + if (const auto below = tree->get_optional("fork_point_below")) { + r.deeper = r.located && !r.locatedVerified; + r.deeperThan = *below; + } r.competingSeq = tree->get("competing_block", 0); r.competingHash = tree->get("competing_hash", ""); r.peerKey = tree->get("peer_key", ""); @@ -316,7 +340,7 @@ void clearAll(const std::string& dir) { removeFile(dir, kApprovedFile); } -std::optional planResolve(const std::string& dir, std::string* error) { +std::optional planResolve(const std::string& dir, std::string* error, bool toLocated) { bool corrupt = false; const auto record = loadForkRecord(dir, &corrupt); if (corrupt) { @@ -342,6 +366,17 @@ std::optional planResolve(const std::string& dir, std::string* erro } return std::nullopt; } + if (toLocated) { + if (!record->located || record->deeper || record->locatedHash.empty()) { + if (error) { + *error = "--to-located: the node has located no last common block (fork point status: " + + record->forkPointStatus() + "): nothing removed. Run --resolve-fork without it to cut at " + "the recorded fork point, or --set-bc-top "; + } + return std::nullopt; + } + return ResolvePlan{record->locatedPoint, record->locatedHash, *record, true}; + } if (record->forkPointHash.empty()) { if (error) { *error = "the fork record has no hash for block #" + std::to_string(record->forkPoint) + @@ -349,7 +384,29 @@ std::optional planResolve(const std::string& dir, std::string* erro } return std::nullopt; } - return ResolvePlan{record->forkPoint, record->forkPointHash, *record}; + return ResolvePlan{record->forkPoint, record->forkPointHash, *record, false}; +} + +std::string resolveOverview(const ResolvePlan& plan) { + const auto& r = plan.record; + std::ostringstream os; + os << "our head #" << r.ourHead << "; fork point #" << r.forkPoint << " (" << r.forkPointStatus() << ")"; + if (r.located) { + os << ", upper bound #" << r.upperBound; + if (r.deeper) { + os << ", a peer's block differs already at #" << r.deeperThan << " (not verified)"; + } + else if (!r.locatedHash.empty()) { + os << ", located #" << r.locatedPoint << (r.locatedVerified ? " (verified)" : " (NOT verified)"); + } + } + else { + os << ", not located"; + } + const cs::Sequence removed = r.ourHead > plan.top ? r.ourHead - plan.top : 0; + os << ". Cutting at #" << plan.top << (plan.toLocated ? " (--to-located)" : "") << ": " << removed + << " block(s) removed"; + return os.str(); } std::string utcTime(int64_t unixSeconds) { @@ -394,6 +451,18 @@ std::string summary(const ForkRecord& r) { if (!r.forkPointHash.empty()) { os << " (" << r.forkPointHash << ")"; } + if (r.located && r.locatedVerified) { + os << ", the last common block, verified (located from #" << r.upperBound << ")"; + } + else if (r.located && r.deeper) { + os << "; a peer's block differs already at #" << r.deeperThan << " (not verified): the fork may be deeper, " + << "find the last common block with other nodes"; + } + else if (r.located && !r.locatedHash.empty()) { + os << "; a peer's blocks place the last common block at #" << r.locatedPoint + << ", NOT verified: --resolve-fork cuts at #" << r.forkPoint << ", --resolve-fork --to-located at #" + << r.locatedPoint; + } os << "; our head #" << r.ourHead; if (r.competingSeq != 0) { os << ", refused block #" << r.competingSeq; diff --git a/csnode/src/forklocator.cpp b/csnode/src/forklocator.cpp new file mode 100644 index 000000000..c9245ed1f --- /dev/null +++ b/csnode/src/forklocator.cpp @@ -0,0 +1,271 @@ +#include + +#include +#include +#include + +namespace cs::forkguard { + +ForkLocator::ForkLocator(Hooks hooks, Settings settings) +: hooks_(std::move(hooks)) +, settings_(settings) { + settings_.probesPerRequest = std::max(settings_.probesPerRequest, 2); + settings_.maxAttempts = std::max(settings_.maxAttempts, 1); +} + +void ForkLocator::start(const cs::PublicKey& peer, cs::Sequence upper, cs::Sequence floor, Clock::time_point now) { + (void)now; + status_ = Status::Searching; + peer_ = peer; + upper_ = upper; + const cs::Sequence byDepth = upper > settings_.maxDepth ? upper - settings_.maxDepth : 0; + // never block 0: both chains start from the same genesis block, and a peer may serve block 0 in a + // form that hashes differently (seen with the official node), which would read as a fork below it + floor_ = std::min(std::max({floor, byDepth, cs::Sequence{1}}), std::max(upper, cs::Sequence{1})); + lo_.reset(); + hi_ = upper + 1; + known_.clear(); + inFlight_.clear(); + sentAt_ = {}; + lastSent_ = {}; + everSent_ = false; + attempts_ = 0; + duplicates_ = 0; + floorRaises_ = 0; + requests_ = 0; + forkPoint_ = 0; + forkPointHash_.clear(); + verified_ = false; + why_.clear(); +} + +void ForkLocator::stop() { + status_ = Status::Idle; + inFlight_.clear(); +} + +std::vector ForkLocator::nextProbes() const { + std::vector probes; + if (status_ != Status::Searching) { + return probes; + } + const bool hiKnown = known_.count(hi_) != 0; + // the heights still open: (lo, hi), or [floor, hi) while no match is known (the floor itself must + // be seen to match, else the fork is deeper) + const cs::Sequence a = lo_.has_value() ? *lo_ + 1 : floor_; + if (hi_ == 0 || a > hi_ - 1) { + if (!hiKnown) { + probes.push_back(hi_); // the first differing block, to check it is built on ours at lo + } + return probes; + } + const cs::Sequence b = hi_ - 1; + const cs::Sequence open = b - a + 1; + const size_t k = settings_.probesPerRequest; + if (open + (hiKnown ? 0 : 1) <= k) { + for (cs::Sequence s = a; s <= b; ++s) { + probes.push_back(s); + } + if (!hiKnown) { + probes.push_back(hi_); + } + return probes; + } + // k heights spread over the open range, the lowest first (the floor, while no match is known) + for (size_t i = 0; i < k; ++i) { + const cs::Sequence s = a + static_cast((static_cast(open) * i) / k); + if (probes.empty() || probes.back() != s) { + probes.push_back(s); + } + } + return probes; +} + +void ForkLocator::tick(Clock::time_point now) { + if (status_ != Status::Searching) { + return; + } + if (!inFlight_.empty()) { + if (now - sentAt_ < settings_.replyTimeout) { + return; + } + inFlight_.clear(); + if (++attempts_ >= settings_.maxAttempts) { + fail("no reply from the peer"); + return; + } + } + if (everSent_ && now - lastSent_ < settings_.minInterval) { + return; + } + const auto probes = nextProbes(); + if (probes.empty()) { + evaluate(); + return; + } + everSent_ = true; + lastSent_ = now; + if (!hooks_.sendRequest || !hooks_.sendRequest(peer_, probes)) { + if (++attempts_ >= settings_.maxAttempts) { + fail("the peer is not connected"); + } + return; + } + inFlight_ = probes; + sentAt_ = now; + ++requests_; +} + +bool ForkLocator::expects(const cs::PublicKey& sender, const std::vector& replySeqs) const { + if (status_ != Status::Searching || inFlight_.empty() || sender != peer_ || replySeqs.empty() || + replySeqs.size() > inFlight_.size()) { + return false; + } + std::vector seen; + seen.reserve(replySeqs.size()); + for (const auto s : replySeqs) { + if (std::find(inFlight_.begin(), inFlight_.end(), s) == inFlight_.end() || + std::find(seen.begin(), seen.end(), s) != seen.end()) { + return false; + } + seen.push_back(s); + } + return true; +} + +void ForkLocator::onReply(const cs::PublicKey& sender, const std::vector& blocks, Clock::time_point now) { + (void)now; + if (status_ != Status::Searching || inFlight_.empty() || sender != peer_) { + return; + } + size_t usable = 0; + for (const auto& b : blocks) { + if (b.hash.empty() || std::find(inFlight_.begin(), inFlight_.end(), b.seq) == inFlight_.end()) { + continue; + } + known_[b.seq] = b; + ++usable; + } + inFlight_.clear(); + duplicates_ = 0; + if (usable == 0) { + if (++attempts_ >= settings_.maxAttempts) { + fail("no usable reply from the peer"); + } + return; + } + attempts_ = 0; + evaluate(); +} + +void ForkLocator::onRefused(const cs::PublicKey& sender, Clock::time_point now, Refusal why) { + (void)now; + if (status_ != Status::Searching || inFlight_.empty() || sender != peer_) { + return; + } + const std::vector asked = std::move(inFlight_); + inFlight_.clear(); + if (why == Refusal::Duplicate) { + if (++duplicates_ >= settings_.maxDuplicates) { + fail("the peer refuses the requests as duplicates"); + } + return; + } + if (why == Refusal::NoSuchBlocks && !lo_.has_value() && !asked.empty() && asked.front() >= floor_) { + // the peer does not store the lowest height asked for: search above it (the heights between it + // and the next one asked for were never asked, so skip to that one) + if (++floorRaises_ > settings_.maxFloorRaises) { + fail("the peer does not store the blocks to search"); + return; + } + floor_ = asked.size() > 1 ? asked[1] : asked.front() + 1; + if (floor_ >= hi_) { + fail("the peer does not store the blocks to search"); + } + return; + } + if (++attempts_ >= settings_.maxAttempts) { + fail("the peer refuses the requests"); + } +} + +bool ForkLocator::matches(cs::Sequence seq) const { + const auto it = known_.find(seq); + if (it == known_.end() || !hooks_.ourHash) { + return false; + } + const std::string ours = hooks_.ourHash(seq); + return !ours.empty() && ours == it->second.hash; +} + +void ForkLocator::fail(const std::string& why) { + status_ = Status::Failed; + why_ = why; + inFlight_.clear(); +} + +void ForkLocator::evaluate() { + if (status_ != Status::Searching) { + return; + } + // the peer's blocks hold together: two consecutive heights are chained + for (auto it = known_.begin(); it != known_.end(); ++it) { + const auto next = std::next(it); + if (next != known_.end() && next->first == it->first + 1 && next->second.previous != it->second.hash) { + fail("the peer's blocks #" + std::to_string(it->first) + " and #" + std::to_string(next->first) + " are not chained"); + return; + } + } + // one chain shares a prefix with ours: every match lies below every difference + std::optional lo = lo_; + cs::Sequence hi = hi_; + for (const auto& [seq, block] : known_) { + if (matches(seq)) { + if (!lo.has_value() || seq > *lo) { + lo = seq; + } + } + else if (seq < hi) { + hi = seq; + } + } + if (lo.has_value() && *lo >= hi) { + fail("the peer's block #" + std::to_string(*lo) + " is ours, but the chains differ at #" + std::to_string(hi) + + " below it"); + return; + } + lo_ = lo; + hi_ = hi; + if (!lo_.has_value() && hi_ <= floor_) { + status_ = Status::Deeper; + forkPoint_ = floor_; + return; + } + if (lo_.has_value() && hi_ == *lo_ + 1) { + const auto first = known_.find(hi_); + if (first == known_.end()) { + return; // the next request asks for it + } + const std::string ours = hooks_.ourHash ? hooks_.ourHash(*lo_) : std::string{}; + if (ours.empty() || first->second.previous != ours) { + fail("the peer's block #" + std::to_string(hi_) + " is not built on our block #" + std::to_string(*lo_)); + return; + } + status_ = Status::Exact; + forkPoint_ = *lo_; + forkPointHash_ = ours; + verified_ = first->second.verified; + } +} + +cs::Sequence locateDepth(uint64_t maxAutoRollback) { + if (maxAutoRollback == 0) { + return 2000; + } + if (maxAutoRollback >= 500) { + return 10000; + } + return static_cast(std::max(maxAutoRollback * 20, 1000)); +} + +} // namespace cs::forkguard diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 6a11ba977..d0d79e5d1 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -272,9 +272,102 @@ void Node::onForkGuardTick() { } blockChain_.checkDismissTrigger(); blockChain_.promotePendingHold(false); + tickForkLocator(); logForkHeldDetail(blockChain_.forkHeldRecord()); } +void Node::startForkLocator(const cs::forkguard::ForkRecord& record) { + cs::PublicKey peer{}; + cs::Bytes key; + if (!DecodeBase58(record.peerKey, key) || key.size() != peer.size()) { + return; + } + std::copy(key.begin(), key.end(), peer.begin()); + if (!forkLocator_) { + cs::forkguard::ForkLocator::Hooks hooks; + hooks.ourHash = [this](cs::Sequence seq) { + if (seq > blockChain_.getLastSeq()) { + return std::string{}; + } + const auto hash = blockChain_.getHashBySequence(seq); + return hash.is_empty() ? std::string{} : hash.to_string(); + }; + hooks.sendRequest = [this](const cs::PublicKey& target, const std::vector& sequences) { + if (!transport_->hasNeighbour(target)) { + return false; + } + sendBlockRequestPacket(target, sequences); + return true; + }; + cs::forkguard::ForkLocator::Settings settings; + const auto* config = cs::ConfigHolder::instance().config(); + settings.maxDepth = cs::forkguard::locateDepth(config != nullptr ? config->maxAutoRollbackDepth() + : Consensus::MaxAutoRollbackDepth); + forkLocator_ = std::make_unique(std::move(hooks), settings); + } + forkLocatorFor_ = record; + forkLocatorRetryAt_ = {}; + forkLocator_->start(peer, record.forkPoint, blockChain_.getFirstStoredSequence(), std::chrono::steady_clock::now()); + cslog() << "FORK HELD: looking for the last common block with " << record.peerKey << ", from #" + << WithDelimiters(record.forkPoint) << " down to #" << WithDelimiters(forkLocator_->floor()) + << " at most (read-only)"; +} + +// FORK HELD with only an upper bound of the fork point: find the last common block (read-only), and +// put it in the record, so that --resolve-fork cuts there in one run +void Node::tickForkLocator() { + const auto held = blockChain_.forkHeldRecord(); + const bool wanted = held.has_value() && !held->located && !held->forkPointExact && !held->peerKey.empty() && + held->reason != "unreadable"; + if (!wanted) { + if (forkLocator_) { + forkLocator_->stop(); + } + forkLocatorFor_.reset(); + return; + } + const bool same = forkLocatorFor_.has_value() && forkLocatorFor_->sinceUnix == held->sinceUnix && + forkLocatorFor_->forkPoint == held->forkPoint && forkLocatorFor_->competingHash == held->competingHash; + if (!same) { + startForkLocator(*held); + if (!forkLocator_ || !forkLocatorFor_.has_value()) { + return; + } + } + const auto now = std::chrono::steady_clock::now(); + using Status = cs::forkguard::ForkLocator::Status; + switch (forkLocator_->status()) { + case Status::Searching: + forkLocator_->tick(now); + break; + case Status::Exact: + case Status::Deeper: { + const bool deeper = forkLocator_->status() == Status::Deeper; + const cs::Sequence point = forkLocator_->forkPoint(); + const std::string hash = forkLocator_->forkPointHash(); + const bool verified = forkLocator_->verified(); + cslog() << "FORK HELD: fork point search done after " << forkLocator_->requestsSent() << " request(s)" + << (deeper ? "" : (verified ? ", next block verified" : ", next block NOT verified")); + forkLocator_->stop(); + blockChain_.applyLocatedForkPoint(*forkLocatorFor_, point, hash, deeper, forkLocatorFor_->peerKey, verified); + break; + } + case Status::Failed: + if (forkLocatorRetryAt_ == std::chrono::steady_clock::time_point{}) { + cswarning() << "FORK HELD: the search for the last common block stopped: " << forkLocator_->why() + << "; the fork point stays an upper bound, a new search starts in 2 minutes"; + forkLocatorRetryAt_ = now + std::chrono::minutes(2); + } + else if (now >= forkLocatorRetryAt_) { + startForkLocator(*held); + } + break; + case Status::Idle: + startForkLocator(*held); + break; + } +} + std::optional Node::resolveForkTop() { const auto* config = cs::ConfigHolder::instance().config(); if (config->newBlockchainTop()) { @@ -282,18 +375,41 @@ std::optional Node::resolveForkTop() { return std::nullopt; } std::string error; - const auto plan = cs::forkguard::planResolve(cs::forkguard::stateDirFor(config->getPathToDB()), &error); + if (config->resolveToLocated() && !config->resolveFork()) { + cserror() << "--to-located goes with --resolve-fork"; + return std::nullopt; + } + const auto plan = cs::forkguard::planResolve(cs::forkguard::stateDirFor(config->getPathToDB()), &error, + config->resolveToLocated()); if (!plan.has_value()) { cserror() << "--resolve-fork: " << error; return std::nullopt; } cslog() << cs::forkguard::summary(plan->record); + // before anything is removed: the bound, the located point and its status, the cut + cswarning() << "--resolve-fork: " << cs::forkguard::resolveOverview(*plan); cslog() << "--resolve-fork: removing every block above #" << WithDelimiters(plan->top) << " (" << plan->hash << "), our head was #" << WithDelimiters(plan->record.ourHead) << ", if the block at #" << WithDelimiters(plan->top) << " is still the recorded one"; - if (plan->record.forkPointExact && plan->record.competingSeq == plan->top + 1) { + if (plan->toLocated) { + // the operator's explicit choice of the located point (verified or not); no branch is approved + cswarning() << "--resolve-fork --to-located: cutting at #" << WithDelimiters(plan->top) << ", the last common " + << "block located with " << plan->record.locatedWith << " from the upper bound #" + << WithDelimiters(plan->record.upperBound) + << (plan->record.locatedVerified ? " (verified)" : " (NOT verified: the operator's choice)"); + blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash)); + } + else if (plan->record.forkPointExact && !plan->record.located && plan->record.competingSeq == plan->top + 1) { blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash), plan->record.competingHash); } + else if (plan->record.forkPointExact) { + // located and verified by the node (the last common block): the cut goes there; the competing + // block first refused is higher up, so no branch is approved + cslog() << "--resolve-fork: #" << WithDelimiters(plan->top) << " is the last common block, verified, located " + << "from the upper bound #" << WithDelimiters(plan->record.upperBound) << "; the node syncs the other " + << "chain from there at the next start"; + blockChain_.setResolveFork(plan->top, csdb::PoolHash::from_string(plan->hash)); + } else { cswarning() << "--resolve-fork: the fork point is an upper bound (the other chain diverges at #" << WithDelimiters(plan->top) << " or lower). If it diverges lower, the node reports FORK HELD " @@ -322,6 +438,17 @@ Node::ChainInit Node::initChain() { initialConfidants_, newTop) ) { + if (blockChain_.truncationNoCheckpoint()) { + // nothing was removed: a pruned node (or one still fetching its older blocks) cannot + // slow-start, so it needs a quick-start checkpoint at or below the new top + cserror() << (initConfig->resolveFork() ? "--resolve-fork" : "--set-bc-top") << ": this node's database " + << "does not start at block 0 (pruned, or its older blocks are not fetched yet) and no " + << "quick-start checkpoint (qs/) is left at or below block #" << WithDelimiters(newTop) + << ": after the cut it could not start. Nothing removed. Cut at a block at or above the oldest " + << "checkpoint, or start again from a snapshot."; + exitCode_ = 3; + return ChainInit::Failed; + } if (blockChain_.truncationRefused()) { // nothing was removed: the database is as it was, the quick-start data stays cserror() << "--resolve-fork: the block at the recorded fork point is not the recorded one (the " @@ -2355,6 +2482,14 @@ void Node::getSyncroMessage(const uint8_t* data, const size_t size, const cs::Pu if (backwardSync_) { backwardSync_->onSyncroMessage(sender, static_cast(msg)); } + if (forkLocator_ && (msg == cs::SyncroMessage::NoSuchBlocks || msg == cs::SyncroMessage::DuplicatedRequest || + msg == cs::SyncroMessage::IncorrectRequest)) { + using Refusal = cs::forkguard::ForkLocator::Refusal; + forkLocator_->onRefused(sender, std::chrono::steady_clock::now(), + msg == cs::SyncroMessage::DuplicatedRequest ? Refusal::Duplicate + : msg == cs::SyncroMessage::NoSuchBlocks ? Refusal::NoSuchBlocks + : Refusal::Other); + } poolSynchronizer_->getSyncroMessage(sender, static_cast(msg)); } @@ -2543,8 +2678,9 @@ void Node::getBlockReply(const uint8_t* data, const size_t size, const cs::Publi bool isSyncOn = poolSynchronizer_->isSyncroStarted(); bool isBlockchainUncertain = blockChain_.isLastBlockUncertain(); const bool isBackfill = backwardSync_ && backwardSync_->hasRequestTo(sender); + const bool isForkSearch = forkLocator_ && forkLocator_->searching() && forkLocator_->peer() == sender; - if (!isSyncOn && !isBlockchainUncertain && !isBackfill/* && poolSynchronizer_->getTargetSequence() == 0ULL*/) { + if (!isSyncOn && !isBlockchainUncertain && !isBackfill && !isForkSearch/* && poolSynchronizer_->getTargetSequence() == 0ULL*/) { csdebug() << "NODE> Get block reply> Pool sync has already finished"; return; } @@ -2563,6 +2699,41 @@ void Node::getBlockReply(const uint8_t* data, const size_t size, const cs::Publi return; } + // the reply to the fork point search (FORK HELD): read for the hashes only, never stored + // (expects(): no more blocks than asked for, each of a different height asked for, so every block + // is checked once at most) + if (isForkSearch && poolsBlock.size() <= forkLocator_->inFlightCount()) { + std::vector seqs; + seqs.reserve(poolsBlock.size()); + for (const auto& b : poolsBlock) { + seqs.push_back(b.sequence()); + } + if (forkLocator_->expects(sender, seqs)) { + std::vector blocks; + blocks.reserve(poolsBlock.size()); + for (const auto& b : poolsBlock) { + // a block built on ours but not ours is also checked as a synced block after it + // (signatures, and confirmations by the confidants of our block): only such a block + // makes the located point the fork point + const cs::Sequence seq = b.sequence(); + const csdb::PoolHash hash = b.hash(); + const bool child = seq > 0 && seq <= blockChain_.getLastSeq() + 1 && + b.previous_hash() == blockChain_.getHashBySequence(seq - 1) && + (seq > blockChain_.getLastSeq() || hash != blockChain_.getHashBySequence(seq)); + blocks.push_back({seq, hash.to_string(), b.previous_hash().to_string(), + child && blockChain_.verifiesAfterOurBlock(b)}); + } + forkLocator_->onReply(sender, blocks, std::chrono::steady_clock::now()); + tickForkLocator(); + return; + } + } + // a reply from the searched peer that is not for the search, while nothing else waits for one + if (isForkSearch && !isSyncOn && !isBlockchainUncertain && !isBackfill) { + csdebug() << "NODE> Get block reply> not the reply the fork point search waits for, dropped"; + return; + } + // The "necessary block" request is disabled (sendNecessaryBlockRequest sends nothing), so no reply // to it can be legitimate: neededSequence_ stays 0, and a one-block reply for block 0 from any peer // used to reach getNecessaryBlockRequest, erase end() of an empty vector and cache the peer's block. diff --git a/docs/CHANGES.md b/docs/CHANGES.md index a8369230a..9d6fddd32 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -44,6 +44,7 @@ lines under the new release. | PR | Change | |---|---| +| #192 | csnode: FORK HELD follow-ups to #189. A cut at start-up (`--resolve-fork`, `--set-bc-top`) also removes the block hashes above the new top and sets aside (`qs/.bad`) the quick-start checkpoints above it; on a pruned node, or one still fetching its older blocks, a cut with no checkpoint left at or below the new top is refused, since such a node cannot slow-start; so the first start after it no longer stops (exit 255) on the shutdown checkpoint saved at the old head. While FORK HELD with only an upper bound of the fork point, the node finds the last common block itself: it asks the peer of the refused block for a few of its blocks below the bound (usual block requests, one every 6 s at most, down to 20 × `max_auto_rollback` blocks), compares hashes; when the peer's next block verifies against the confidants of the node's own block there (signatures and confirmations, as for a synced block), the point is recorded as exact in `fork-held.json` and `--resolve-fork` cuts there in one run. An unverified point (a bootstrap block, a wrong answer) or a fork deeper than the search is only a hint: `--resolve-fork` cuts at the upper bound, and at the hint only with `--resolve-fork --to-located` (Docker `resolve-fork to-located`); it prints the bound, the located point, its status and the blocks it removes before cutting. Read-only: nothing is stored, unwound or approved. Local only | | #190 | ci: the snapshot workflow's monitor check sends its own User-Agent; the monitor answers 403 to urllib's default one, which failed the snapshot runs | | #189 | csnode: bound automatic chain reorganisations. The node stops unwinding its own confirmed blocks once it would roll back more than `[params] max_auto_rollback` (default 100) below the head it had when the rollback began, on every automatic removal path; the reference is kept in `reorg-episode.json`, so a restart does not reset it. A competing block triggers a rollback only if it is vouched for locally: confirmations by a majority of the node's own last confidants, or, for a bootstrap block, its own signatures plus confidants of a round table the node accepted. When it refuses, the node keeps its chain and reports **FORK HELD** until the operator decides: a marker on every round line, a detail line once a minute, `fork-held.json` next to the database and an unhealthy Docker healthcheck. The operator either keeps this chain (a `dismiss-fork` file, picked up by the running node: that branch stays refused and is not reported again) or follows the other one (`node --resolve-fork` cuts the chain at the recorded fork point if the block there is still the recorded one; `--set-bc-top ` by hand; Docker commands `resolve-fork`, `set-bc-top`, `dismiss-fork`). A block not vouched for is refused at once but reported only if the head then stays put (no flapping); unreadable guard files fail safe; the files go when the database is replaced. `allow_deep_rollback` / `CS_ALLOW_DEEP_ROLLBACK` stays as the last resort and turns itself off after one completed episode. The start-up integrity repair is not bounded. Local only; block content is unchanged (see `docs/differences.md`, "Consensus fixes", and "If your node reports FORK HELD" in `docs/quick-start.md`). Also csdb: the storage read cache forgets a block removed from the top of the chain, so a block saved at the same sequence afterwards is the one read back (before, a node that unwound stored blocks could keep reading the removed block and stop a few blocks above the fork point) | | #187 | docs: `AGENTS.md` and `CLAUDE.md` for coding agents: build, tests, CI, conventions and the mainnet constraints (documentation only) | diff --git a/docs/differences.md b/docs/differences.md index b8e22d884..941084236 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -318,9 +318,22 @@ consensus rules. other block meets the usual checks, and an unauthenticated offer changes nothing. The approval ends once the node goes on past that height, or a competing block that passes the usual checks is offered there. When the fork point is - only an upper bound and the other chain diverges lower, the node reports FORK HELD again with a - lower fork point; `--set-bc-top ` then cuts by hand at the last common block found with - the monitor or other nodes. Both are one-shot + only an upper bound, the node looks for the last common block itself while FORK HELD: it asks the + peer of the refused block for some of its blocks at or below the bound (the usual block request, a + few heights per request, one request every few seconds at most) and compares their hashes with its + own. The search is read-only (nothing stored, unwound or approved) and bounded to 20 times + `max_auto_rollback` blocks (1,000 to 10,000; 2,000 when it is 0). The point it finds becomes the + fork point (`"fork_point_status": "exact"` in `fork-held.json`, and `--resolve-fork` cuts there + in one run) only if the peer's next block verifies as a synced block would after the node's own + block there: signatures, and round confirmations by a majority of that block's confidants. + Otherwise (a bootstrap block, or a peer that answers wrongly) the point is only a hint + (`"located (unverified)"`, or `"deeper"` beyond the bound). `--resolve-fork` then still cuts at the + upper bound, and cuts at the hint only with `--to-located`; it prints the bound, the located point, + its status and the number of blocks it removes before cutting. `--set-bc-top ` still cuts by + hand at a block of the operator's choice. After either cut the quick-start checkpoints above the + new top are set aside (`qs/.bad`), so the first start after it does not stop on one. A node + whose storage does not start at block 0 (pruned, or a backfill in progress) cannot slow-start, so + on such a node a cut with no checkpoint left at or below the new top is refused and removes nothing. Both are one-shot and exit; neither advice is ever "restart", since a restart alone changes nothing. See "If your node reports FORK HELD" in `docs/quick-start.md`. diff --git a/docs/quick-start.md b/docs/quick-start.md index b77e5931d..b858bd8e9 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -461,10 +461,46 @@ fork point with other nodes: the [monitor](https://monitor.credits.com), the `--resolve-fork` refuses, and removes nothing, when there is no fork record or when the block at the recorded fork point is no longer the recorded one. To cut the chain at a block of your choice instead, use `--set-bc-top ` (the last block to keep) with the node stopped; Docker: -`docker compose run --rm node set-bc-top `. If the detail line says -`Fork point: block #N or lower`, the node knows only an upper bound: should the other chain fork -lower, the node reports FORK HELD again with a lower fork point. Then find the last common block -with the monitor or other nodes and cut there with `--set-bc-top`. +`docker compose run --rm node set-bc-top `. + +**The fork point.** `--resolve-fork` now finds the last common block itself; the manual +`--set-bc-top` remains. When the node first refuses, it may know only an upper bound of the fork +point (`Fork point: block #N or lower`), for example after a long partition. While FORK HELD it +then asks the peer of the refused block for a few of its blocks below that bound, compares their +hashes with its own and narrows down to the last block both chains share. This only reads: nothing +is stored, removed or approved. The search covers 20 times `max_auto_rollback` blocks (2,000 by +default, between 1,000 and 10,000). A peer could answer wrongly, so the node takes the point it +finds as the fork point only when it can verify the peer's next block as it would verify a synced +block: built on this node's block at that point, signed by its confidants and confirmed by a +majority of the confidants of this node's block there. After a minute or two, one of these: + +- **Verified.** The detail line says `Fork point: block #M (...), the last common block, verified` + and `fork-held.json` has `"fork_point_status": "exact"`. `--resolve-fork` cuts there in one run. + A verified point proves that a branch of the other chain forks there, but the branch the node + refused might fork higher up. Cutting there may then remove more good blocks than needed. The + cost is only a resync of those blocks. +- **Not verified** (for example, the peer's next block is a bootstrap block, which carries no + confirmations to check). The line says `a peer's blocks place the last common block at #M, NOT + verified` (`"fork_point_status": "located (unverified)"`). `--resolve-fork` still cuts at the + upper bound N. Once you have checked block M with other nodes or the monitor, cut there with + `--resolve-fork --to-located` (Docker: `docker compose run --rm node resolve-fork to-located`), or + with `--set-bc-top M`. +- **Deeper.** The peer's block already differs at the lowest block searched (`"fork_point_status": + "deeper"`). `--resolve-fork` cuts at the upper bound. Find the last common block with the monitor or + other nodes and cut there with `--set-bc-top`. + +Before it removes anything, `--resolve-fork` prints the node's head, the upper bound, the located +block and whether it is verified, where it cuts and how many blocks that removes. Run it after the +detail line shows the search result. Run before, it cuts at the upper bound; if the other chain +forks lower, the node reports FORK HELD again with a lower fork point. + +After a cut (`--resolve-fork` or `--set-bc-top`) the node also sets aside the quick-start +checkpoints above the new last block, as `qs/.bad`. The next start then uses an older checkpoint +or reads the chain from the start, and does not stop on a checkpoint that no longer matches the +blocks on disk. A pruned node, or one still fetching its older blocks, cannot read the chain from +the start. On such a node the cut is refused, and nothing is removed, when no checkpoint is left at +or below the new last block. Cut at or above the oldest checkpoint in `qs/` instead, or start again +from a snapshot. **Last resort.** `allow_deep_rollback=true` in `[params]` of `config.ini` (or the `CS_ALLOW_DEEP_ROLLBACK` environment variable) lets the node follow any competing chain without the diff --git a/packaging/docker/README.md b/packaging/docker/README.md index 0bab1a5cb..077139e30 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -102,7 +102,12 @@ docker compose stop && docker compose run --rm node set-bc-top && doc ``` `resolve-fork` removes nothing when there is no record, the fork was dismissed, or the block at the -recorded fork point is no longer the recorded one. With a key encrypted by the node, `resolve-fork` +recorded fork point is no longer the recorded one. When the node knows only an upper bound of the +fork point, it finds the last common block itself within a minute or two. If it can verify the +point (the detail line says `the last common block, verified`), `resolve-fork` cuts there. If it +cannot (`NOT verified`), `resolve-fork` cuts at the upper bound. After checking the located block +with other nodes, `docker compose run --rm node resolve-fork to-located` cuts at it instead, without +asking. With a key encrypted by the node, `resolve-fork` and `set-bc-top` start the node, which asks the key's password in the terminal (`docker compose run` is interactive). See "If your node reports FORK HELD" in `docs/quick-start.md`. diff --git a/packaging/docker/entrypoint.sh b/packaging/docker/entrypoint.sh index 008979a0b..743d7fa16 100755 --- a/packaging/docker/entrypoint.sh +++ b/packaging/docker/entrypoint.sh @@ -67,6 +67,12 @@ case "${1:-}" in [ -f config.ini ] && [ -d db ] || { log "no config.ini or db in $DATA: nothing to cut"; exit 2; } if [ "$1" = resolve-fork ]; then node_args=(--resolve-fork) + # resolve-fork to-located: the located (unverified) last common block, see fork-held.json + case "${2:-}" in + "") ;; + to-located|--to-located) node_args+=(--to-located) ;; + *) log "resolve-fork takes no argument but to-located: docker compose run --rm node resolve-fork [to-located]"; exit 2 ;; + esac else [[ "${2:-}" =~ ^[0-9]+$ ]] || { log "set-bc-top needs the sequence of the last block to keep: docker compose run --rm node set-bc-top "; exit 2; } node_args=(--set-bc-top "$2") diff --git a/packaging/docker/test/test-entrypoint.sh b/packaging/docker/test/test-entrypoint.sh index 94cf7975c..8997afaed 100755 --- a/packaging/docker/test/test-entrypoint.sh +++ b/packaging/docker/test/test-entrypoint.sh @@ -308,6 +308,11 @@ rm -f "$d/stub-args" out=$(fork_run "$d" resolve-fork) || { echo "$out"; fail "resolve-fork"; } grep -q -- '--resolve-fork' "$d/stub-args" && grep -q -- '--config-file config.ini' "$d/stub-args" || fail "resolve-fork arguments: $(cat "$d/stub-args")" rm -f "$d/stub-args" +out=$(fork_run "$d" resolve-fork to-located) || { echo "$out"; fail "resolve-fork to-located"; } +grep -q -- '--resolve-fork --to-located' "$d/stub-args" || fail "resolve-fork to-located arguments: $(cat "$d/stub-args")" +set +e; out=$(fork_run "$d" resolve-fork 123); rc=$?; set -e +[ "$rc" = 2 ] && grep -q 'resolve-fork takes no argument but to-located' <<<"$out" || { echo "$out"; fail "resolve-fork with a stray argument: exit $rc"; } +rm -f "$d/stub-args" out=$(fork_run "$d" set-bc-top 1234) || { echo "$out"; fail "set-bc-top"; } grep -q -- '--set-bc-top 1234' "$d/stub-args" || fail "set-bc-top arguments: $(cat "$d/stub-args")" set +e; out=$(fork_run "$d" set-bc-top); rc=$?; set -e @@ -321,6 +326,6 @@ set +e; out=$(health "$d"); rc=$?; set -e && grep -q 'resolve-fork' <<<"$out" && grep -q 'dismiss-fork' <<<"$out" || { echo "$out"; fail "healthcheck with a fork held: exit $rc"; } printf '{\n "held": "false",\n "fork_point": "4899"\n}\n' > "$d/fork-held.json" out=$(health "$d") || { echo "$out"; fail "healthcheck with a dismissed fork"; } -pass "resolve-fork, set-bc-top and dismiss-fork; the healthcheck reports FORK HELD" +pass "resolve-fork (also to-located), set-bc-top and dismiss-fork; the healthcheck reports FORK HELD" echo "PASS: entrypoint" diff --git a/tests/csnode/blockchain_tests.cpp b/tests/csnode/blockchain_tests.cpp index 6b04cf97f..1034e85f2 100644 --- a/tests/csnode/blockchain_tests.cpp +++ b/tests/csnode/blockchain_tests.cpp @@ -948,6 +948,40 @@ TEST(BlockChain, IntegrityRepairIsNotBoundedByTheGuard) { chain.test_resetRollbackEpisode(); } +// The fork point search takes a located point as the fork point only if the peer's block right after +// it verifies as a synced block would after our block there: built on it, signed by its confidants, +// confirmed by a majority of our block's confidants. A bootstrap block (no confirmations), a block +// confirmed by others, or one built on another block stays unverified +TEST(BlockChain, LocatedForkPointNeedsAVerifiedNextBlock) { + auto& chain = freshGuardChain(); + const auto ours = guardKeys(3); + const auto theirs = guardKeys(3); + const auto strangers = guardKeys(3); + const auto forkBlock = guardBlock(80, ours); + chain.test_setLastBlock(forkBlock); // our block at the located point + + const auto verified = guardBlock(81, theirs, &ours, 0b111, false, false, forkBlock.hash()); + EXPECT_TRUE(chain.verifiesAfterOurBlock(verified)); + const auto majority = guardBlock(81, theirs, &ours, 0b011, false, false, forkBlock.hash()); + EXPECT_TRUE(chain.verifiesAfterOurBlock(majority)); + + const auto bootstrap = guardBlock(81, theirs, nullptr, 0, true, false, forkBlock.hash()); + EXPECT_FALSE(chain.verifiesAfterOurBlock(bootstrap)); + const auto bootstrapConfirmed = guardBlock(81, theirs, &ours, 0b111, true, false, forkBlock.hash()); + EXPECT_FALSE(chain.verifiesAfterOurBlock(bootstrapConfirmed)); + const auto byStrangers = guardBlock(81, theirs, &strangers, 0b111, false, false, forkBlock.hash()); + EXPECT_FALSE(chain.verifiesAfterOurBlock(byStrangers)); + const auto minority = guardBlock(81, theirs, &ours, 0b001, false, false, forkBlock.hash()); + EXPECT_FALSE(chain.verifiesAfterOurBlock(minority)); + const auto badSignature = guardBlock(81, theirs, &ours, 0b111, false, true, forkBlock.hash()); + EXPECT_FALSE(chain.verifiesAfterOurBlock(badSignature)); + const auto elsewhere = guardBlock(81, theirs, &ours, 0b111, false, false, someHash('e')); + EXPECT_FALSE(chain.verifiesAfterOurBlock(elsewhere)); + const auto noParent = guardBlock(90, theirs, &ours, 0b111, false, false, forkBlock.hash()); + EXPECT_FALSE(chain.verifiesAfterOurBlock(noParent)); // we store no block at #89 + chain.test_resetRollbackEpisode(); +} + // --resolve-fork approves only the recorded block: it alone (that very hash, built on the fork point) // may replace a block the node stored again right after the fork point; a forged block pointing at it, // or any other unauthenticated offer, is refused and does not cancel the approval; nothing is ever diff --git a/tests/csnode/blockhashes_tests.cpp b/tests/csnode/blockhashes_tests.cpp index 6254e7aff..dedee1a45 100644 --- a/tests/csnode/blockhashes_tests.cpp +++ b/tests/csnode/blockhashes_tests.cpp @@ -182,3 +182,32 @@ TEST(BlockHashes, RepairBottomMixed) { hashes.flush(); hashes.close(); } + +// After a cut at the start (--set-bc-top, --resolve-fork) the entries of every block above the new top +// go, through the last block stored before the cut even across a gap, and on past it up to the first +// missing sequence (entries written ahead of the storage); those at or below the top stay. +TEST(BlockHashes, RemoveAboveTheTopAcrossGaps) { + auto dir = freshDir("cut"); + cs::BlockHashes hashes(dir.string()); + for (cs::Sequence s = 0; s < 30; ++s) { + hashes.update(block(s)); + } + ASSERT_TRUE(hashes.remove(cs::Sequence{15})); // a gap above the top + EXPECT_EQ(hashes.removeAbove(10, 25), 18u); // 11..14, 16..29 + for (cs::Sequence s = 0; s <= 10; ++s) { + EXPECT_EQ(hashes.find(s), block(s).hash()) << s; + } + for (cs::Sequence s = 11; s < 30; ++s) { + EXPECT_TRUE(hashes.find(s).is_empty()) << s; + EXPECT_EQ(hashes.find(block(s).hash()), cs::kWrongSequence) << s; + } + EXPECT_EQ(hashes.size(), 11u); + EXPECT_TRUE(hashes.consistentWith(0, 10, [](cs::Sequence s) { return block(s).hash(); })); + // the last block before the cut not known (0): up to the first missing sequence + for (cs::Sequence s = 11; s < 20; ++s) { + hashes.update(block(s)); + } + EXPECT_EQ(hashes.removeAbove(10, 0), 9u); + EXPECT_EQ(hashes.size(), 11u); + hashes.close(); +} diff --git a/tests/csnode/forklocator_tests.cpp b/tests/csnode/forklocator_tests.cpp new file mode 100644 index 000000000..d13a1f531 --- /dev/null +++ b/tests/csnode/forklocator_tests.cpp @@ -0,0 +1,749 @@ +#define TESTING + +#include "gtest/gtest.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// The search for the last common block while FORK HELD (cs::forkguard::ForkLocator), the record it +// fills in, and the checkpoints a chain cut leaves behind. + +namespace { + +using Locator = cs::forkguard::ForkLocator; +using Status = Locator::Status; +using namespace std::chrono_literals; + +// two chains sharing blocks 0..fork: ours ("O") and the peer's ("P") above it +struct Chains { + cs::Sequence fork = 0; + cs::Sequence ourHead = 0; + cs::Sequence peerHead = 0; + + static std::string common(cs::Sequence s) { + return "C" + std::to_string(s); + } + std::string ours(cs::Sequence s) const { + if (s > ourHead) { + return {}; + } + return s <= fork ? common(s) : "O" + std::to_string(s); + } + std::string theirs(cs::Sequence s) const { + return s <= fork ? common(s) : "P" + std::to_string(s); + } + // verifiable: the node can verify the peer's first block after the fork point against its own + // confidants (false: e.g. a bootstrap block) + bool verifiable = true; + Locator::PeerBlock peerBlock(cs::Sequence s) const { + return {s, theirs(s), s == 0 ? std::string{} : theirs(s - 1), verifiable && s > 0 && s <= fork + 1}; + } +}; + +cs::PublicKey key(uint8_t b) { + cs::PublicKey k{}; + k.fill(b); + return k; +} + +// a locator against `chains`, the peer answering with `answer` (by default its own chain); runs until +// the search ends (or 200 ticks), 7 s apart +struct Harness { + Chains chains; + Locator::Settings settings; + std::vector> requests; + bool connected = true; + std::function(const std::vector&)> answer; + std::optional locator; + Locator::Clock::time_point now = Locator::Clock::time_point{} + 1h; + + Locator& make() { + Locator::Hooks hooks; + hooks.ourHash = [this](cs::Sequence s) { return chains.ours(s); }; + hooks.sendRequest = [this](const cs::PublicKey&, const std::vector& seqs) { + if (!connected) { + return false; + } + requests.push_back(seqs); + return true; + }; + locator.emplace(std::move(hooks), settings); + return *locator; + } + + std::vector honest(const std::vector& seqs) const { + std::vector out; + for (auto s : seqs) { + if (s <= chains.peerHead) { + out.push_back(chains.peerBlock(s)); + } + } + return out; + } + + Status run(const cs::PublicKey& peer = key(7)) { + size_t answered = 0; + for (int i = 0; i < 200 && locator->searching(); ++i) { + locator->tick(now); + if (requests.size() > answered) { + answered = requests.size(); + const auto& seqs = requests.back(); + auto blocks = answer ? answer(seqs) : honest(seqs); + std::vector got; + for (const auto& b : blocks) { + got.push_back(b.seq); + } + if (!answer && !got.empty()) { + EXPECT_TRUE(locator->expects(peer, got)); + } + locator->onReply(peer, blocks, now); + } + now += 7s; + } + return locator->status(); + } +}; + +} // namespace + +// a fork 397 blocks under the upper bound: found exactly, in a few requests of a few blocks each +TEST(ForkLocator, FindsTheExactForkPoint) { + Harness h; + h.chains = {533, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Exact); + EXPECT_EQ(loc.forkPoint(), 533u); + EXPECT_EQ(loc.forkPointHash(), Chains::common(533)); + EXPECT_TRUE(loc.verified()); + EXPECT_LE(h.requests.size(), 8u); + for (const auto& r : h.requests) { + EXPECT_LE(r.size(), h.settings.probesPerRequest); + for (auto s : r) { + EXPECT_LE(s, 931u); // never above the bound + 1 + } + } +} + +// the bound itself is the fork point (the chains differ right above it), or the fork is at the floor +TEST(ForkLocator, ForkAtTheBoundOrAtTheFloor) { + { + Harness h; + h.chains = {930, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Exact); + EXPECT_EQ(loc.forkPoint(), 930u); + } + { + Harness h; + h.chains = {500, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 500, h.now); // the first stored block is 500 + EXPECT_EQ(h.run(), Status::Exact); + EXPECT_EQ(loc.forkPoint(), 500u); + } +} + +// a fork below the bound of the search: "deeper", at the lowest block searched +TEST(ForkLocator, DeepForkBeyondTheBound) { + Harness h; + h.settings.maxDepth = 2000; + h.chains = {100, 5001, 6000}; + auto& loc = h.make(); + loc.start(key(7), 5000, 0, h.now); + EXPECT_EQ(loc.floor(), 3000u); + EXPECT_EQ(h.run(), Status::Deeper); + EXPECT_EQ(loc.forkPoint(), 3000u); + EXPECT_EQ(h.requests.size(), 1u); // the floor is in the first request + + // a pruned node: never below its first stored block + Harness p; + p.chains = {100, 5001, 6000}; + auto& pruned = p.make(); + pruned.start(key(7), 5000, 4500, p.now); + EXPECT_EQ(p.run(), Status::Deeper); + EXPECT_EQ(pruned.forkPoint(), 4500u); + + // the genesis block is never asked for, even when a peer would answer it with another hash + Harness g; + g.chains = {533, 931, 1500}; + g.answer = [&](const std::vector& seqs) { + auto out = g.honest(seqs); + for (auto& b : out) { + EXPECT_NE(b.seq, 0u); + if (b.seq == 0) { + b.hash = "other genesis"; + } + } + return out; + }; + auto& genesis = g.make(); + genesis.start(key(7), 930, 0, g.now); + EXPECT_EQ(genesis.floor(), 1u); + EXPECT_EQ(g.run(), Status::Exact); + EXPECT_EQ(genesis.forkPoint(), 533u); + + EXPECT_EQ(cs::forkguard::locateDepth(100), 2000u); + EXPECT_EQ(cs::forkguard::locateDepth(10), 1000u); + EXPECT_EQ(cs::forkguard::locateDepth(0), 2000u); + EXPECT_EQ(cs::forkguard::locateDepth(1000000), 10000u); +} + +// a peer that lies cannot make the search say anything that does not hold together +TEST(ForkLocator, LyingPeer) { + // our own blocks at every height, also above the bound: the chains cannot differ there and match + // above it + { + Harness h; + h.chains = {533, 931, 1500}; + h.answer = [&](const std::vector& seqs) { + std::vector out; + for (auto s : seqs) { + out.push_back({s, h.chains.ours(s), s ? h.chains.ours(s - 1) : std::string{}}); + } + return out; + }; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Failed); + EXPECT_NE(loc.why().find("chains differ"), std::string::npos) << loc.why(); + } + // matches above a difference + { + Harness h; + h.chains = {533, 931, 1500}; + h.answer = [&](const std::vector& seqs) { + std::vector out; + for (auto s : seqs) { + // the lowest height differs, the others are ours + const bool lie = s != seqs.front(); + out.push_back({s, lie ? h.chains.ours(s) : "X" + std::to_string(s), "Y"}); + } + return out; + }; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Failed); + } + // the right answer, but the first block above the fork point is not built on ours + { + Harness h; + h.chains = {533, 931, 1500}; + h.answer = [&](const std::vector& seqs) { + auto out = h.honest(seqs); + for (auto& b : out) { + if (b.seq == 534) { + b.previous = "forged"; + } + } + return out; + }; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Failed); + // "not built on our block #533", or "#533 and #534 are not chained" when both came together + EXPECT_NE(loc.why().find("#534"), std::string::npos) << loc.why(); + } + // consecutive blocks that are not chained + { + Harness h; + h.chains = {533, 931, 1500}; + h.answer = [&](const std::vector& seqs) { + auto out = h.honest(seqs); + for (auto& b : out) { + b.previous = "broken"; + } + return out; + }; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Failed); + } + // blocks nobody asked for are not taken; a peer that only sends those makes the search give up + { + Harness h; + h.chains = {533, 931, 1500}; + h.answer = [&](const std::vector&) { + return std::vector{h.chains.peerBlock(1200)}; + }; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_FALSE(loc.expects(key(7), {1200})); + h.locator->tick(h.now); + EXPECT_FALSE(loc.expects(key(7), {1200})); + EXPECT_FALSE(loc.expects(key(8), {h.requests.back().front()})); // another peer + EXPECT_TRUE(loc.expects(key(7), {h.requests.back().front()})); + EXPECT_EQ(h.run(), Status::Failed); + EXPECT_NE(loc.why().find("no usable reply"), std::string::npos) << loc.why(); + } +} + +// no peer to ask, or a peer that never answers: the search gives up, nothing is decided +TEST(ForkLocator, NoPeer) { + { + Harness h; + h.chains = {533, 931, 1500}; + h.connected = false; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Failed); + EXPECT_NE(loc.why().find("not connected"), std::string::npos) << loc.why(); + EXPECT_EQ(loc.requestsSent(), 0u); + } + { + Harness h; + h.chains = {533, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + for (int i = 0; i < 200 && loc.searching(); ++i) { + loc.tick(h.now); + h.now += 7s; + } + EXPECT_EQ(loc.status(), Status::Failed); + EXPECT_NE(loc.why().find("no reply"), std::string::npos) << loc.why(); + EXPECT_EQ(loc.requestsSent(), h.settings.maxAttempts); + } + { + // refusals (duplicate request, no such blocks) count as failed attempts too + Harness h; + h.chains = {533, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + for (int i = 0; i < 200 && loc.searching(); ++i) { + loc.tick(h.now); + loc.onRefused(key(8), h.now); // another peer: ignored + loc.onRefused(key(7), h.now); + h.now += 7s; + } + EXPECT_EQ(loc.status(), Status::Failed); + EXPECT_NE(loc.why().find("refuses"), std::string::npos) << loc.why(); + } + { + // a duplicate-request refusal is tried again, more often, then given up as well + Harness h; + h.chains = {533, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + for (size_t i = 0; i < h.settings.maxAttempts + 1; ++i) { + loc.tick(h.now); + loc.onRefused(key(7), h.now, Locator::Refusal::Duplicate); + h.now += 7s; + } + EXPECT_TRUE(loc.searching()); + for (int i = 0; i < 200 && loc.searching(); ++i) { + loc.tick(h.now); + loc.onRefused(key(7), h.now, Locator::Refusal::Duplicate); + h.now += 7s; + } + EXPECT_EQ(loc.status(), Status::Failed); + EXPECT_NE(loc.why().find("duplicates"), std::string::npos) << loc.why(); + } +} + +// a reply with more blocks than asked for, or one height twice, is not for the search (each block is +// checked once at most) +TEST(ForkLocator, DuplicatedBlocksReplyIsDropped) { + Harness h; + h.chains = {533, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + loc.tick(h.now); + ASSERT_EQ(h.requests.size(), 1u); + const auto asked = h.requests.back(); + EXPECT_EQ(loc.inFlightCount(), asked.size()); + EXPECT_TRUE(loc.expects(key(7), asked)); + EXPECT_FALSE(loc.expects(key(7), {asked[0], asked[0]})); + auto tooMany = asked; + tooMany.push_back(asked.back()); + EXPECT_FALSE(loc.expects(key(7), tooMany)); + EXPECT_TRUE(loc.expects(key(7), {asked[1], asked[0]})); // any order +} + +// a pruned peer does not store the lowest height asked for: the floor goes up past it instead of +// counting a failed attempt, a limited number of times +TEST(ForkLocator, PrunedPeerRaisesTheFloor) { + Harness h; + h.chains = {533, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + loc.tick(h.now); + const auto first = h.requests.back(); + loc.onRefused(key(7), h.now, Locator::Refusal::NoSuchBlocks); + EXPECT_TRUE(loc.searching()); + EXPECT_EQ(loc.floor(), first[1]); + // then the peer answers: the search ends exactly, above the raised floor + h.requests.clear(); // the refused request is not answered + EXPECT_EQ(h.run(), Status::Exact); + EXPECT_EQ(loc.forkPoint(), 533u); + + // a peer that never has the blocks: given up after maxFloorRaises + Harness p; + p.chains = {533, 931, 1500}; + p.settings.maxFloorRaises = 3; + auto& none = p.make(); + none.start(key(7), 930, 0, p.now); + for (int i = 0; i < 50 && none.searching(); ++i) { + none.tick(p.now); + none.onRefused(key(7), p.now, Locator::Refusal::NoSuchBlocks); + p.now += 7s; + } + EXPECT_EQ(none.status(), Status::Failed); + EXPECT_NE(none.why().find("does not store"), std::string::npos) << none.why(); +} + +// one request at a time, at most one per minInterval +TEST(ForkLocator, RateLimited) { + Harness h; + h.chains = {533, 931, 1500}; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + loc.tick(h.now); + ASSERT_EQ(h.requests.size(), 1u); + loc.tick(h.now + 1s); + EXPECT_EQ(h.requests.size(), 1u); // waits for the reply + loc.onReply(key(7), h.honest(h.requests.back()), h.now + 1s); + loc.tick(h.now + 2s); + EXPECT_EQ(h.requests.size(), 1u); // replied, but too early for the next one + loc.tick(h.now + 6s); + EXPECT_EQ(h.requests.size(), 2u); + loc.stop(); + loc.tick(h.now + 60s); + EXPECT_EQ(h.requests.size(), 2u); + EXPECT_EQ(loc.status(), Status::Idle); +} + +namespace { +struct TempFolder { + std::filesystem::path path; + explicit TempFolder(const std::string& name) { + path = std::filesystem::temp_directory_path() / + ("csnode-locator-" + name + "-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + std::filesystem::remove_all(path); + std::filesystem::create_directories(path); + } + ~TempFolder() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + std::string str() const { + return path.string(); + } +}; + +cs::forkguard::ForkRecord heldRecord() { + cs::forkguard::ForkRecord r; + r.reason = "authority"; + r.sinceUnix = 1795000000; + r.highWater = 931; + r.ourHead = 931; + r.forkPoint = 930; + r.forkPointHash = "C930"; + r.competingSeq = 932; + r.competingHash = "P932"; + r.peerKey = "key"; + return r; +} +} // namespace + +// a lying peer that reports a deeper fork point: it can send blocks of its own below the real fork, +// chained and consistent, but it cannot make the block after its claimed point verify against our +// confidants there: the point is found but not verified (a hint only), and a bootstrap-flagged block +// after the point stays unverified too +TEST(ForkLocator, LyingPeerReportingADeeperPointIsNotVerified) { + Harness h; + h.chains = {533, 931, 1500}; + Chains fake = h.chains; + fake.fork = 300; // the liar's chain parts from ours at #300 + h.answer = [&](const std::vector& seqs) { + std::vector out; + for (auto s : seqs) { + auto b = fake.peerBlock(s); + b.verified = false; // its forged blocks never verify against our confidants + out.push_back(b); + } + return out; + }; + auto& loc = h.make(); + loc.start(key(7), 930, 0, h.now); + EXPECT_EQ(h.run(), Status::Exact); + EXPECT_EQ(loc.forkPoint(), 300u); + EXPECT_FALSE(loc.verified()); + + // an honest peer whose block after the fork point is a bootstrap block: located, not verified + Harness b; + b.chains = {533, 931, 1500}; + b.chains.verifiable = false; + auto& boot = b.make(); + boot.start(key(7), 930, 0, b.now); + EXPECT_EQ(b.run(), Status::Exact); + EXPECT_EQ(boot.forkPoint(), 533u); + EXPECT_FALSE(boot.verified()); +} + +// the record of a located fork point and what --resolve-fork does with it: a verified point is the +// fork point and the cut goes there; an unverified one is a hint, the cut stays at the upper bound +// unless the operator asks for it (--to-located); "deeper" is a hint as well +TEST(ForkLocator, LocatedRecordAndResolvePlan) { + TempFolder dir("record"); + std::string error; + + auto r = heldRecord(); + r.located = true; + r.locatedVerified = true; + r.upperBound = 930; + r.locatedWith = "key"; + r.locatedPoint = 533; + r.locatedHash = "C533"; + r.forkPoint = 533; + r.forkPointHash = "C533"; + r.forkPointExact = true; + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), r)); + { + std::ifstream in(dir.path / cs::forkguard::kForkFile); + const std::string text((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + EXPECT_NE(text.find("\"fork_point_status\": \"exact\""), std::string::npos) << text; + EXPECT_NE(text.find("\"fork_point_upper_bound\": \"930\""), std::string::npos) << text; + EXPECT_NE(text.find("\"located_verified\": \"true\""), std::string::npos) << text; + } + auto back = cs::forkguard::loadForkRecord(dir.str()); + ASSERT_TRUE(back.has_value()); + EXPECT_TRUE(back->located); + EXPECT_TRUE(back->locatedVerified); + EXPECT_TRUE(back->forkPointExact); + EXPECT_FALSE(back->deeper); + EXPECT_EQ(back->upperBound, 930u); + EXPECT_EQ(back->locatedPoint, 533u); + EXPECT_EQ(back->locatedWith, "key"); + auto plan = cs::forkguard::planResolve(dir.str(), &error); + ASSERT_TRUE(plan.has_value()) << error; + EXPECT_EQ(plan->top, 533u); // a verified point: cut exactly there + EXPECT_EQ(plan->hash, "C533"); + auto overview = cs::forkguard::resolveOverview(*plan); + EXPECT_NE(overview.find("our head #931"), std::string::npos) << overview; + EXPECT_NE(overview.find("upper bound #930"), std::string::npos) << overview; + EXPECT_NE(overview.find("located #533 (verified)"), std::string::npos) << overview; + EXPECT_NE(overview.find("Cutting at #533: 398 block(s) removed"), std::string::npos) << overview; + auto line = cs::forkguard::summary(*back); + EXPECT_NE(line.find("Fork point: block #533 (C533), the last common block, verified (located from #930)"), std::string::npos) << line; + EXPECT_NE(line.find("--set-bc-top 533"), std::string::npos) << line; + + // not verified: the fork point stays the upper bound; the located point is a hint + auto hint = heldRecord(); + hint.located = true; + hint.upperBound = 930; + hint.locatedWith = "key"; + hint.locatedPoint = 300; + hint.locatedHash = "C300"; + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), hint)); + back = cs::forkguard::loadForkRecord(dir.str()); + ASSERT_TRUE(back.has_value()); + EXPECT_FALSE(back->locatedVerified); + EXPECT_FALSE(back->forkPointExact); + EXPECT_EQ(back->forkPointStatus(), "located (unverified)"); + plan = cs::forkguard::planResolve(dir.str(), &error); + ASSERT_TRUE(plan.has_value()) << error; + EXPECT_EQ(plan->top, 930u); // no deeper cut without the operator's confirmation + EXPECT_EQ(plan->hash, "C930"); + EXPECT_FALSE(plan->toLocated); + overview = cs::forkguard::resolveOverview(*plan); + EXPECT_NE(overview.find("located #300 (NOT verified)"), std::string::npos) << overview; + EXPECT_NE(overview.find("Cutting at #930: 1 block(s) removed"), std::string::npos) << overview; + plan = cs::forkguard::planResolve(dir.str(), &error, true); // --to-located + ASSERT_TRUE(plan.has_value()) << error; + EXPECT_EQ(plan->top, 300u); + EXPECT_EQ(plan->hash, "C300"); + EXPECT_TRUE(plan->toLocated); + overview = cs::forkguard::resolveOverview(*plan); + EXPECT_NE(overview.find("Cutting at #300 (--to-located): 631 block(s) removed"), std::string::npos) << overview; + line = cs::forkguard::summary(*back); + EXPECT_NE(line.find("Fork point: block #930 or lower (C930)"), std::string::npos) << line; + EXPECT_NE(line.find("NOT verified"), std::string::npos) << line; + EXPECT_NE(line.find("--to-located at #300"), std::string::npos) << line; + + // deeper: a hint as well; --resolve-fork cuts at the upper bound, --to-located has no point + auto deep = heldRecord(); + deep.located = true; + deep.deeper = true; + deep.upperBound = 930; + deep.deeperThan = 100; + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), deep)); + back = cs::forkguard::loadForkRecord(dir.str()); + ASSERT_TRUE(back.has_value()); + EXPECT_TRUE(back->deeper); + EXPECT_EQ(back->deeperThan, 100u); + EXPECT_EQ(back->forkPointStatus(), "deeper"); + plan = cs::forkguard::planResolve(dir.str(), &error); + ASSERT_TRUE(plan.has_value()) << error; + EXPECT_EQ(plan->top, 930u); + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error, true).has_value()); + EXPECT_NE(error.find("--to-located"), std::string::npos) << error; + line = cs::forkguard::summary(*back); + EXPECT_NE(line.find("differs already at #100 (not verified)"), std::string::npos) << line; + + // --to-located without any located point + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.str(), heldRecord())); + EXPECT_FALSE(cs::forkguard::planResolve(dir.str(), &error, true).has_value()); +} + +namespace { +// a BlockChain holding a fork after a depth refusal at 4900 (reference 5000), fork point an upper +// bound, with its last block (the only one the test chain can name) at 4800 +struct HeldChain { + TempFolder dir{"chain"}; + BlockChain chain{csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000001"), + csdb::Address::from_string("0000000000000000000000000000000000000000000000000000000000000002")}; + cs::forkguard::ForkRecord held; + std::string hash; + HeldChain() { + cs::ConfigHolder::instance().setConfig(Config{}); + chain.test_resetRollbackEpisode(); + chain.setGuardStateDir(dir.str()); + chain.test_setLastSequence(5000); + EXPECT_TRUE(chain.guardAutoRollback()); + chain.test_setLastSequence(4900); + EXPECT_FALSE(chain.guardAutoRollback()); + held = *chain.forkHeldRecord(); + csdb::Pool block(csdb::PoolHash{}, 4800); + block.add_user_field(200, std::string("located")); + EXPECT_TRUE(block.compose()); + chain.test_setLastBlock(block); + hash = block.hash().to_string(); + } + ~HeldChain() { + chain.test_resetRollbackEpisode(); + cs::forkguard::clearAll(dir.str()); + } +}; +} // namespace + +// the node puts the answer in the held record only if it is still the record searched for and our +// block at the point is still the one compared; a verified answer becomes the fork point, an +// unverified one stays a hint; nothing else changes +TEST(ForkLocator, BlockChainTakesTheAnswer) { + { + HeldChain h; + auto& chain = h.chain; + ASSERT_FALSE(h.held.forkPointExact); + // another record than the one searched for: ignored + auto other = h.held; + other.sinceUnix += 1; + EXPECT_FALSE(chain.applyLocatedForkPoint(other, 4800, h.hash, false, "peer", true)); + // our block there is not the one compared: ignored + EXPECT_FALSE(chain.applyLocatedForkPoint(h.held, 4800, "00", false, "peer", true)); + EXPECT_FALSE(chain.forkHeldRecord()->located); + + EXPECT_TRUE(chain.applyLocatedForkPoint(h.held, 4800, h.hash, false, "peer", true)); + const auto located = chain.forkHeldRecord(); + ASSERT_TRUE(located.has_value()); + EXPECT_TRUE(located->held); + EXPECT_TRUE(located->located); + EXPECT_TRUE(located->locatedVerified); + EXPECT_TRUE(located->forkPointExact); + EXPECT_EQ(located->forkPoint, 4800u); + EXPECT_EQ(located->forkPointHash, h.hash); + EXPECT_EQ(located->upperBound, h.held.forkPoint); + EXPECT_EQ(located->competingSeq, h.held.competingSeq); // the refused block stays the one refused + EXPECT_EQ(chain.getLastSeq(), 4800u); // nothing unwound + EXPECT_FALSE(chain.approvedBranch().has_value()); // nothing approved + const auto plan = cs::forkguard::planResolve(h.dir.str(), nullptr); + ASSERT_TRUE(plan.has_value()); + EXPECT_EQ(plan->top, 4800u); // the verified point cuts exactly + // once located, a second answer changes nothing + EXPECT_FALSE(chain.applyLocatedForkPoint(h.held, 4800, h.hash, true, "peer")); + } + { + // not verified (a lying peer's deeper point, a bootstrap block after it): a hint only + HeldChain h; + auto& chain = h.chain; + EXPECT_TRUE(chain.applyLocatedForkPoint(h.held, 4800, h.hash, false, "peer", false)); + const auto hint = chain.forkHeldRecord(); + ASSERT_TRUE(hint.has_value()); + EXPECT_TRUE(hint->located); + EXPECT_FALSE(hint->locatedVerified); + EXPECT_FALSE(hint->forkPointExact); + EXPECT_EQ(hint->forkPoint, h.held.forkPoint); // still the upper bound + EXPECT_EQ(hint->locatedPoint, 4800u); + std::string error; + // no deeper cut without confirmation: the recorded upper bound, or nothing (this test chain + // has no hash for it) + auto plan = cs::forkguard::planResolve(h.dir.str(), &error); + if (plan.has_value()) { + EXPECT_EQ(plan->top, h.held.forkPoint); + } + plan = cs::forkguard::planResolve(h.dir.str(), &error, true); + ASSERT_TRUE(plan.has_value()) << error; + EXPECT_EQ(plan->top, 4800u); + } +} + +namespace { +void writeHead(const std::filesystem::path& dir, cs::Sequence seq) { + std::filesystem::create_directories(dir); + std::ofstream f(dir / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + f.write(reinterpret_cast(&magic), sizeof(magic)); + const uint64_t s = seq; + f.write(reinterpret_cast(&s), sizeof(s)); + const cs::Bytes hash(32, 0xab); + const uint16_t hh = static_cast(hash.size()); + f.write(reinterpret_cast(&hh), sizeof(hh)); + f.write(reinterpret_cast(hash.data()), hh); + const uint16_t ph = 0; + f.write(reinterpret_cast(&ph), sizeof(ph)); +} +} // namespace + +// after a cut at 950: every checkpoint above it is set aside as .bad (the shutdown checkpoint +// qs/0 at the old head first), with its .prev backup; those at or below it stay +TEST(ForkLocator, ChainCutDropsCheckpointsAboveTheTop) { + TempFolder qs("qs"); + writeHead(qs.path / "0", 980); + writeHead(qs.path / "900", 900); + writeHead(qs.path / "960", 960); + writeHead(qs.path / "960.prev", 960); + writeHead(qs.path / "0.bad", 980); + std::filesystem::create_directories(qs.path / "500"); // legacy, no head.bin: named by its block + std::filesystem::create_directories(qs.path / "2000"); + std::ofstream(qs.path / "notes.txt") << "not a checkpoint"; + + EXPECT_TRUE(cs::hasCheckpointAtOrBelow(qs.path, 950)); + EXPECT_FALSE(cs::hasCheckpointAtOrBelow(qs.path, 400)); // 500 and 900 continue above it + EXPECT_EQ(cs::dropCheckpointsAbove(qs.path, 950), 4u); + for (const char* gone : {"0", "960", "960.prev", "2000"}) { + EXPECT_FALSE(std::filesystem::exists(qs.path / gone)) << gone; + } + // set aside, not deleted (an older 0.bad is replaced) + for (const char* aside : {"0.bad", "960.bad", "960.prev.bad", "2000.bad"}) { + EXPECT_TRUE(std::filesystem::exists(qs.path / aside)) << aside; + } + EXPECT_EQ(cs::readCheckpointHead(qs.path / "0.bad")->sequence, 980u); + for (const char* kept : {"900", "500", "notes.txt"}) { + EXPECT_TRUE(std::filesystem::exists(qs.path / kept)) << kept; + } + // set-aside checkpoints never count: a cut at 400 would leave a pruned node no checkpoint + EXPECT_FALSE(cs::hasCheckpointAtOrBelow(qs.path, 400)); + // a cut above every checkpoint removes nothing; a missing folder is no error + EXPECT_EQ(cs::dropCheckpointsAbove(qs.path, 5000), 0u); + EXPECT_EQ(cs::dropCheckpointsAbove(qs.path / "none", 10), 0u); + // a shutdown checkpoint at the new top itself stays + writeHead(qs.path / "0", 950); + EXPECT_EQ(cs::dropCheckpointsAbove(qs.path, 950), 0u); + EXPECT_TRUE(std::filesystem::exists(qs.path / "0")); +} diff --git a/tests/csnode/storage_writer_tests.cpp b/tests/csnode/storage_writer_tests.cpp index b74b52acf..5f31177b9 100644 --- a/tests/csnode/storage_writer_tests.cpp +++ b/tests/csnode/storage_writer_tests.cpp @@ -15,6 +15,9 @@ #include "memorydb.hpp" +#include +#include + // The asynchronous block writer of csdb::Storage (used with RocksDB), on an in-memory database whose // put_batch can be held or made to fail. namespace { @@ -419,3 +422,119 @@ TEST(StorageWriter, ResolveForkTruncatesExactlyAtTheRecordedForkPoint) { EXPECT_EQ(storage.last_hash(), own[6].hash()); storage.close(); } + +// A storage that does not start at block 0 (pruned, or a backfill in progress) cannot slow-start: a +// cut with no quick-start checkpoint left at or below the new top is refused and removes nothing +// (TruncateNoCheckpoint); with one it goes ahead; a storage from block 0 is cut either way. +TEST(StorageWriter, PrunedCutBelowEveryCheckpointIsRefused) { + const auto own = chain(10); + const auto make = [&](bool pruned) { + auto db = std::make_shared(); + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + EXPECT_TRUE(storage.open(opt)); + for (const auto& p : own) EXPECT_TRUE(storage.pool_save(p)); + EXPECT_TRUE(storage.flush()); + storage.close(); + if (pruned) { + for (cs::Sequence s = 0; s < 3; ++s) EXPECT_TRUE(db->remove(own[s].hash().to_binary())); + } + return db; + }; + const auto cut = [&](const std::shared_ptr& db, bool checkpointBelow) { + csdb::Storage storage; + storage.set_truncate_needs_checkpoint(checkpointBelow); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = 5; + const bool ok = storage.open(opt); + const auto error = storage.last_error(); + const auto last = storage.truncated_last(); + storage.close(); + return std::make_tuple(ok, error, last); + }; + { + auto db = make(true); + const auto [ok, error, last] = cut(db, false); + EXPECT_FALSE(ok); + EXPECT_EQ(error, csdb::Storage::TruncateNoCheckpoint); + EXPECT_EQ(db->stored(), 7u); // nothing removed + (void)last; + } + { + auto db = make(true); + const auto [ok, error, last] = cut(db, true); + EXPECT_TRUE(ok); + EXPECT_EQ(last, 9u); + EXPECT_EQ(db->stored(), 3u); // #3..#5 + (void)error; + } + { + auto db = make(false); + const auto [ok, error, last] = cut(db, false); + EXPECT_TRUE(ok); + EXPECT_EQ(last, 9u); + EXPECT_EQ(db->stored(), 6u); + (void)error; + } +} + +// --resolve-fork --to-located: the plan is the located (unverified) point and its hash; when the block +// stored there is no longer that one, the cut is refused (TruncateMismatch) and nothing is removed +TEST(StorageWriter, ToLocatedCutChecksTheLocatedBlock) { + const auto own = chain(10); + auto db = std::make_shared(); + { + csdb::Storage storage; + csdb::Storage::OpenOptions opt; + opt.db = db; + ASSERT_TRUE(storage.open(opt)); + for (const auto& p : own) ASSERT_TRUE(storage.pool_save(p)); + ASSERT_TRUE(storage.flush()); + storage.close(); + } + const auto dir = std::filesystem::temp_directory_path() / + ("csnode-tolocated-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + std::filesystem::create_directories(dir); + cs::forkguard::ForkRecord r; + r.reason = "authority"; + r.ourHead = 9; + r.forkPoint = 8; + r.forkPointHash = own[8].hash().to_string(); + r.located = true; + r.upperBound = 8; + r.locatedPoint = 4; + r.locatedHash = own[3].hash().to_string(); // not the block stored at #4 (it changed since) + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.string(), r)); + std::string error; + const auto plan = cs::forkguard::planResolve(dir.string(), &error, true); + ASSERT_TRUE(plan.has_value()) << error; + EXPECT_EQ(plan->top, 4u); + { + csdb::Storage storage; + storage.set_truncate_check(csdb::PoolHash::from_string(plan->hash)); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = plan->top; + EXPECT_FALSE(storage.open(opt)); + EXPECT_EQ(storage.last_error(), csdb::Storage::TruncateMismatch); + EXPECT_EQ(db->stored(), 10u); + } + // the block that is stored there: cut + r.locatedHash = own[4].hash().to_string(); + ASSERT_TRUE(cs::forkguard::saveForkRecord(dir.string(), r)); + const auto good = cs::forkguard::planResolve(dir.string(), &error, true); + ASSERT_TRUE(good.has_value()) << error; + { + csdb::Storage storage; + storage.set_truncate_check(csdb::PoolHash::from_string(good->hash)); + csdb::Storage::OpenOptions opt; + opt.db = db; + opt.newBlockchainTop = good->top; + EXPECT_TRUE(storage.open(opt)); + storage.close(); + } + EXPECT_EQ(db->stored(), 5u); + std::filesystem::remove_all(dir); +}