Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions ci/testnet/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,9 @@
# network restarted, as after a power cut; the nodes must agree again
# CHECKPOINT_AHEAD_AT (default 0 = off): at that sequence node CHECKPOINT_AHEAD_NODE (default 3,
# a checkpoint a minute, 3 kept) is killed, started once offline with --set-bc-top a few blocks
# before its second newest checkpoint, and killed again: its two newest checkpoints are then
# ahead of the blocks on disk. It is restarted on every exit, at most 5 times: it must set
# before its second newest checkpoint, and killed again. The cut must have removed its two
# newest checkpoints; they are put back, as from a backup, and are then ahead of the blocks
# on disk. It is restarted on every exit, at most 5 times: it must set
# those checkpoints aside, quick-start from the older one and end with the same state digests
# APPLY_FAIL_AT (default 0 = off, a CREDITS_TESTNET build): at that sequence node
# APPLY_FAIL_NODE (default 3) is told to fail applying a block 20 blocks later to its caches
Expand Down Expand Up @@ -1540,14 +1541,26 @@ PY
section == "[host_input]" && /^port=/ { print "port=" port; next }
{ print }' "$ca_dir/config.ini" > "$ca_dir/config-offline.ini"
ca_before=$(grep -ac 'Changing consensus settings to' "$ca_dir/node.log" || true)
rm -rf "$ca_dir/qs-keep"; mkdir -p "$ca_dir/qs-keep"
for c in $ca_cps; do cp -a "$ca_dir/qs/$c" "$ca_dir/qs-keep/$c"; done
(cd "$ca_dir" && exec "$BIN" --db-path db --config-file config-offline.ini --set-bc-top "$ca_top" </dev/null >> stdout.log 2>&1) &
ca_pid=$!
ca_t0=$(date +%s)
until [ "$(grep -ac 'Changing consensus settings to' "$ca_dir/node.log" || true)" -gt "${ca_before:-0}" ] \
|| ! kill -0 "$ca_pid" 2>/dev/null || [ $(( $(date +%s) - ca_t0 )) -ge 120 ]; do sleep 1; done
kill -KILL "$ca_pid" 2>/dev/null || true; wait "$ca_pid" 2>/dev/null || true
pkill -KILL -f "cs.testnet.node=n$ca " 2>/dev/null || true
echo "checkpoint-ahead test: n$ca cut to block $ca_top, its checkpoints $(tr '\n' ' ' <<<"$ca_cps")are ahead of it"
# the cut removes the checkpoints above the new top itself; put them back (as a restored
# backup of qs/ would), so that the start must set them aside
for c in $ca_cps; do
if [ -d "$ca_dir/qs/$c" ]; then
echo "FAIL: n$ca: --set-bc-top $ca_top left the checkpoint qs/$c above the new top"
exit 1
fi
cp -a "$ca_dir/qs-keep/$c" "$ca_dir/qs/$c"
done
rm -rf "$ca_dir/qs-keep"
echo "checkpoint-ahead test: n$ca cut to block $ca_top, which removed its checkpoints $(tr '\n' ' ' <<<"$ca_cps")above it; put back, they are ahead of it"
ca_lines=$(wc -l < "$ca_dir/node.log")
ca_restarts=0
start_node "$ca"
Expand Down
2 changes: 2 additions & 0 deletions client/config/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ const std::string ARG_NAME_ENCRYPT_KEY_FILE = "encryptkey";
const std::string ARG_NAME_RECREATE_INDEX = "recreate-index";
const std::string ARG_NAME_NEW_BC_TOP = "set-bc-top";
const std::string ARG_NAME_RESOLVE_FORK = "resolve-fork";
const std::string ARG_NAME_TO_LOCATED = "to-located";
const std::string ARG_NAME_BAL_CHANGE = "bal-change";
const std::string ARG_NAME_DISABLE_AUTO_SHUTDOWN = "disable-auto-shutdown";

Expand Down Expand Up @@ -281,6 +282,7 @@ Config Config::read(po::variables_map& vm, bool atStart) {

if (atStart) {
result.resolveFork_ = vm.count(ARG_NAME_RESOLVE_FORK) > 0;
result.resolveToLocated_ = vm.count(ARG_NAME_TO_LOCATED) > 0;
if (vm.count(ARG_NAME_NEW_BC_TOP)) {
result.newBlockchainTop_ = true;
result.newBlockchainTopSeq_ =
Expand Down
5 changes: 5 additions & 0 deletions client/config/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,10 @@ class Config {
bool resolveFork() const {
return resolveFork_;
}
// --resolve-fork --to-located: cut at the located but unverified last common block instead
bool resolveToLocated() const {
return resolveToLocated_;
}

bool traverseNAT() const {
return traverseNAT_;
Expand Down Expand Up @@ -455,6 +459,7 @@ class Config {
uint64_t maxAutoRollbackDepth_ = DEFAULT_MAX_AUTO_ROLLBACK;
bool allowDeepRollback_ = false;
bool resolveFork_ = false;
bool resolveToLocated_ = false;
bool symmetric_ = false;

EndpointData hostAddressEp_;
Expand Down
1 change: 1 addition & 0 deletions client/include/client/cmdlineargs.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ struct cmdline {
static constexpr const char* argDumpKeys = "dumpkeys";
static constexpr const char* argSetBCTop = "set-bc-top";
static constexpr const char* argResolveFork = "resolve-fork";
static constexpr const char* argToLocated = "to-located";
static constexpr const char* argBalChange = "bal-change";
#ifdef _WIN32
static constexpr const char* argInstall = "install";
Expand Down
1 change: 1 addition & 0 deletions client/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ int main(int argc, char* argv[]) {
(cmdline::argSeed, "enter with seed instead of keys")
(cmdline::argSetBCTop, po::value<uint64_t>(), "all blocks in blockchain with higher sequence will be removed")
(cmdline::argResolveFork, "a node that reports FORK HELD: remove the blocks above the fork point recorded in fork-held.json (only if the block there is still the recorded one), then exit; at the next start the node follows the other chain. Check with other nodes first (the node must be stopped)")
(cmdline::argToLocated, "with --resolve-fork: cut at the last common block the node located with a peer (fork-held.json, \"located_point\") even though it could not verify it, instead of the recorded fork point")
(cmdline::argBalChange, po::value<std::string>(), "prints all changes in account balance in log file")
("disable-auto-shutdown", "node will be prohibited to shutdown in case of fatal errors")
("version", "show node version")
Expand Down
10 changes: 10 additions & 0 deletions csdb/include/csdb/storage.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ class Storage final {
StartAfterLastBlock = 7,
// --resolve-fork: the block at the requested top is not the one recorded (set_truncate_check)
TruncateMismatch = 8,
// --set-bc-top / --resolve-fork on a storage that does not start at block 0 (pruned, or a
// backfill in progress) with no quick-start checkpoint left at or below the new top: after the
// cut the node could neither quick-start nor slow-start (set_truncate_needs_checkpoint)
TruncateNoCheckpoint = 9,
UnknownError = 255,
};

Expand Down Expand Up @@ -158,6 +162,12 @@ class Storage final {
// at that top has this hash; otherwise it removes nothing and fails with TruncateMismatch. An
// empty hash (the default) checks nothing. Used by --resolve-fork.
void set_truncate_check(const PoolHash& expectedTopHash);
// The next open() with a new blockchain top refuses the cut (TruncateNoCheckpoint, nothing removed)
// when no quick-start checkpoint remains at or below that top (checkpointAtOrBelowTop false) and the
// storage does not start at block 0. true (the default) checks nothing.
void set_truncate_needs_checkpoint(bool checkpointAtOrBelowTop);
// after an open() that cut the chain: the last block stored before the cut (0: nothing was cut)
cs::Sequence truncated_last() const;

// dbBackend: "berkeleydb" (default), "rocksdb", or "lmdb" in <path>/lmdb, whose removed blocks
// LMDB reuses the space of (pruned storage)
Expand Down
27 changes: 27 additions & 0 deletions csdb/src/storage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,10 @@ class Storage::priv {
static const size_t cacheSize = 50;
// set_truncate_check(): the hash the block at the new blockchain top must have
PoolHash truncate_expected_hash;
// set_truncate_needs_checkpoint(): false refuses a cut of a storage that does not start at block 0
bool truncate_checkpoint_below = true;
// the last block stored before the last cut
cs::Sequence truncated_last = 0;

// the cache keeps one pool per sequence and per hash: a pool saved at a sequence whose previous
// pool was removed (a chain reorganisation) replaces the cached one, instead of being refused by
Expand Down Expand Up @@ -483,6 +487,14 @@ void Storage::set_truncate_check(const PoolHash& expectedTopHash) {
d->truncate_expected_hash = expectedTopHash;
}

void Storage::set_truncate_needs_checkpoint(bool checkpointAtOrBelowTop) {
d->truncate_checkpoint_below = checkpointAtOrBelowTop;
}

cs::Sequence Storage::truncated_last() const {
return d->truncated_last;
}

Storage::Error Storage::last_error() const {
return last_error_map(d.get()).last_error_;
}
Expand Down Expand Up @@ -573,6 +585,7 @@ bool Storage::open(const OpenOptions& opt, OpenCallback callback) {
}
auto seqToRemove = static_cast<uint32_t>(opt.newBlockchainTop + 1);
auto seqLast = seqToRemove;
uint32_t seqFirst = 0;
{
Database::IteratorPtr it = d->db->new_iterator();
it->seek_to_last();
Expand All @@ -582,7 +595,21 @@ bool Storage::open(const OpenOptions& opt, OpenCallback callback) {
seqLast = key;
}
}
it->seek_to_first();
if (it->is_valid()) {
seqFirst = it->key();
}
}
const bool checkpointBelow = d->truncate_checkpoint_below;
d->truncate_checkpoint_below = true;
if (!checkpointBelow && seqFirst > 0 && seqLast >= seqToRemove) {
d->set_last_error(TruncateNoCheckpoint,
"the storage starts at block %u (pruned, or older blocks not fetched yet) and no quick-start "
"checkpoint is left at or below block %llu: after the cut the node could not start; nothing removed",
seqFirst, static_cast<unsigned long long>(opt.newBlockchainTop));
return false;
}
d->truncated_last = seqLast >= seqToRemove ? seqLast : 0;

auto blocksCount = (seqLast >= seqToRemove) ? (seqLast - seqToRemove + 1) : 0;
if(seqToRemove <= seqLast) {
Expand Down
2 changes: 2 additions & 0 deletions csnode/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ add_library(csnode
include/csnode/snapshot_fetcher.hpp
include/csnode/blockverifier.hpp
include/csnode/forkguardstate.hpp
include/csnode/forklocator.hpp
include/csnode/backwardsync.hpp
include/csnode/sendcachedata.hpp
include/csnode/eventreport.hpp
Expand Down Expand Up @@ -136,6 +137,7 @@ add_library(csnode
src/snapshot_fetcher.cpp
src/blockverifier.cpp
src/forkguardstate.cpp
src/forklocator.cpp
src/backwardsync.cpp
src/sendcachedata.cpp
src/eventreport.cpp
Expand Down
26 changes: 26 additions & 0 deletions csnode/include/csnode/blockchain.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,13 @@ class BlockChain {
// Strict for the unwind decision: a confirmation mask of more than one entry, of the same size as our
// last block's confidants, with as many signatures as it marks, and a majority of them confirming.
bool hasTrustedConfirmations(const csdb::Pool& competing) const;
// the same checks against the given confidants
bool confirmedByConfidants(const csdb::Pool& competing, const std::vector<cs::PublicKey>& trustedConfidants) const;
// The fork point search (ForkLocator): `block` (a peer's, not stored) is built on our stored block
// at its sequence - 1, its signatures verify against its own confidants, and its round
// confirmations, by a majority, against the confidants of that block of ours, as hasTrustedConfirmations
// checks them. A bootstrap block never passes (it carries no confirmations). Read-only.
bool verifiesAfterOurBlock(const csdb::Pool& block) const;

// True if a bootstrap-flagged competing block is authenticated locally: its own signatures verify
// against its confidants AND those confidants are exactly the confidants of a round table this node
Expand Down Expand Up @@ -333,10 +340,25 @@ class BlockChain {
std::optional<cs::forkguard::ApprovedBranch> approvedBranch() const {
return approvedBranch_;
}
// The search for the last common block (cs::forkguard::ForkLocator, driven by the node while FORK
// HELD) has an answer for the held record `expected` (the record it searched for; nothing is
// changed if the record changed since). `located` is the last common block the peer's answers
// give (`hash` is our stored hash there, checked again here): the fork point only when `verified`
// (the peer's next block verified against our confidants there), otherwise a hint; or, with
// `deeper`, the lowest block searched, where the peer's block already differs (a hint). Only the
// record (fork-held.json) changes: nothing is unwound, nothing approved.
// True if the record was updated.
bool applyLocatedForkPoint(const cs::forkguard::ForkRecord& expected, cs::Sequence located, const std::string& hash,
bool deeper, const std::string& peerKey, bool verified = false);
// init() refused to truncate: the stored block at the requested sequence has another hash
bool truncationRefused() const {
return truncationRefused_;
}
// init() refused to truncate: the storage does not start at block 0 and no quick-start checkpoint
// is left at or below the requested top, so the node could not start after the cut
bool truncationNoCheckpoint() const {
return truncationNoCheckpoint_;
}

#ifdef TESTING
// test-only accessors for the longest-chain guard
Expand Down Expand Up @@ -747,6 +769,7 @@ public slots:
cs::Sequence resolveForkSeq_ = cs::kWrongSequence;
csdb::PoolHash resolveForkHash_;
bool truncationRefused_ = false;
bool truncationNoCheckpoint_ = false;
// starts the episode at the current head if none is running (persisting its reference)
void beginRollbackEpisode();
// the chain grew: ends the episode once the head is above its reference
Expand Down Expand Up @@ -808,6 +831,9 @@ public slots:
}

bool tryQuickStart(cs::CachesSerializationManager*, std::set<cs::PublicKey>& initialConfidants);
// after a cut at `top` (--set-bc-top, --resolve-fork): the block hashes and the quick-start
// checkpoints above it go, so that the next start finds nothing beyond the stored blocks
void dropStateAboveTop(cs::Sequence top, cs::CachesSerializationManager* serializationManPtr);
bool bindSerializationManToCaches(cs::CachesSerializationManager*, std::set<cs::PublicKey>& initialConfidants);

cs::CachesSerializationManager* serializationManPtr_ = nullptr;
Expand Down
4 changes: 4 additions & 0 deletions csnode/include/csnode/blockhashes.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ class BlockHashes {

bool remove(cs::Sequence);
bool remove(const csdb::PoolHash& hash);
// after a cut at `top`: removes the entries of every block above it, through `last` (the last
// block stored before the cut; gaps included) and on up to the first missing sequence after it.
// Returns the number of blocks removed
size_t removeAbove(cs::Sequence top, cs::Sequence last);

// The first failure of either database is reported once and leaves the file blockhashes.rebuild
// in the caches folder: the next start rebuilds both databases from the stored blocks.
Expand Down
14 changes: 14 additions & 0 deletions csnode/include/csnode/caches_serialization_manager.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,16 @@ std::optional<CheckpointHead> readCheckpointHead(const std::filesystem::path& di

bool isCheckpointOnChain(const CheckpointHead& head, const std::function<cs::Bytes(cs::Sequence)>& chainHashAt);

// After the chain was cut at `top` (--set-bc-top, --resolve-fork): sets aside, as "<name>.bad" in
// the quick-start folder `root`, every checkpoint that continues above `top` (its head, or its name
// for a periodic checkpoint without head.bin, is above it; a qs/0 without head.bin cannot be placed
// and goes too), with its .prev backup. The next start then quick-starts from the highest checkpoint
// at or below the new top, or slow-starts. Returns the number of checkpoints set aside.
size_t dropCheckpointsAbove(const std::filesystem::path& root, cs::Sequence top);
// a checkpoint (not a .prev backup) that does not continue above `top` is in `root`: a node whose
// storage does not start at block 0 can start after a cut at `top` only with one
bool hasCheckpointAtOrBelow(const std::filesystem::path& root, cs::Sequence top);

class CachesSerializationManager {
public:
CachesSerializationManager();
Expand Down Expand Up @@ -81,6 +91,10 @@ class CachesSerializationManager {
void setCompletedFromGenesis();
bool isLoadedFromCompletedSnapshot() const;
void pruneCheckpoints(size_t keep);
// dropCheckpointsAbove() on this node's quick-start folder
size_t dropAbove(cs::Sequence top);
// hasCheckpointAtOrBelow() on this node's quick-start folder
bool hasCheckpointAtOrBelow(cs::Sequence top) const;

private:
struct Impl;
Expand Down
39 changes: 38 additions & 1 deletion csnode/include/csnode/forkguardstate.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,26 @@ struct ForkRecord {
cs::Sequence forkPoint = 0;
std::string forkPointHash;
bool forkPointExact = false;
// the search for the last common block (ForkLocator) while FORK HELD: located is true once it has
// an answer, found from the upper bound forkPoint had before (upperBound) with the peer locatedWith.
// - verified (locatedVerified): the peer's block right after the located point is built on our
// block there, and its signatures and round confirmations verify against the confidants of our
// block there, as a synced block's would. Only then is the point the fork point: forkPoint and
// forkPointHash are set to it and forkPointExact turns true (the competing block is still the
// one first refused, so no approval follows from it).
// - not verified (a bootstrap block, or confirmations that do not verify): a hint only. forkPoint
// stays the upper bound; locatedPoint/locatedHash keep the hint, and --resolve-fork cuts there
// only with --to-located.
// - deeper: the peer's block at deeperThan, the lowest one searched, already differs from ours; a
// hint as well (a peer can claim it), forkPoint stays the upper bound
bool located = false;
bool locatedVerified = false;
cs::Sequence locatedPoint = 0;
std::string locatedHash;
bool deeper = false;
cs::Sequence upperBound = 0;
cs::Sequence deeperThan = 0;
std::string locatedWith;
cs::Sequence competingSeq = 0; // the refused block (0: not known, e.g. a bad-hash reply)
std::string competingHash;
std::string peerKey; // base58, empty when not known
Expand All @@ -81,6 +101,16 @@ struct ForkRecord {
std::string dismissedHash;
std::vector<std::string> branchHashes;

// "exact", "located (unverified)", "deeper" (below deeperThan, unverified) or "upper bound"
std::string forkPointStatus() const {
if (forkPointExact) {
return "exact";
}
if (located && !deeper && !locatedHash.empty()) {
return "located (unverified)";
}
return deeper ? "deeper" : "upper bound";
}
bool hasDismissal() const {
return !branchHashes.empty();
}
Expand All @@ -106,12 +136,19 @@ void clearAll(const std::string& dir);
// --resolve-fork: the fork point to cut the chain at and the hash the stored block there must have,
// from the record in dir; nothing (and *error says why) when there is no record or it cannot be
// checked against the database
// toLocated (--resolve-fork --to-located): the located but unverified last common block instead of
// the recorded fork point
struct ResolvePlan {
cs::Sequence top = 0;
std::string hash;
ForkRecord record;
bool toLocated = false;
};
std::optional<ResolvePlan> planResolve(const std::string& dir, std::string* error);
std::optional<ResolvePlan> planResolve(const std::string& dir, std::string* error, bool toLocated = false);

// what --resolve-fork shows before it cuts: our head, the upper bound, the located point and whether
// it is verified, where the cut goes and how many blocks it removes
std::string resolveOverview(const ResolvePlan& plan);

// "2026-11-19 10:04:05 UTC"
std::string utcTime(int64_t unixSeconds);
Expand Down
Loading
Loading