From 88f40672ab64b1803178741b1c67697176e9fff9 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 10:15:03 +0100 Subject: [PATCH 1/6] snapshot: serve snapshots to p2p neighbours SnapshotProvider answers manifest and piece requests from the newest snapshot this node built: verified on load, files kept open, pieces read with pread, a global and a per-peer token bucket, two pieces per peer every 100 ms, at most serve_max_peers peers in 30 s, half the budget while trusted, manifests once per peer every 5 s. SnapshotService (from #98, without the HTTP server) builds the snapshot of each periodic checkpoint while serve is on and loads it into the provider. Snapshots use 512 KiB chunks and no contracts file. Co-Authored-By: Claude Opus 5.5 --- client/src/snapshot_export.cpp | 5 +- csdb/include/csdb/storage.hpp | 5 + csdb/src/storage.cpp | 8 + csnode/CMakeLists.txt | 4 + csnode/include/csnode/blockchain.hpp | 6 + csnode/include/csnode/node.hpp | 15 + csnode/include/csnode/snapshot_builder.hpp | 2 +- csnode/include/csnode/snapshot_provider.hpp | 117 +++++++ csnode/include/csnode/snapshot_service.hpp | 59 ++++ csnode/src/blockchain.cpp | 11 + csnode/src/node.cpp | 90 ++++- csnode/src/snapshot_builder.cpp | 29 +- csnode/src/snapshot_provider.cpp | 358 ++++++++++++++++++++ csnode/src/snapshot_service.cpp | 191 +++++++++++ tests/csnode/snapshot_provider_tests.cpp | 339 ++++++++++++++++++ 15 files changed, 1212 insertions(+), 27 deletions(-) create mode 100644 csnode/include/csnode/snapshot_provider.hpp create mode 100644 csnode/include/csnode/snapshot_service.hpp create mode 100644 csnode/src/snapshot_provider.cpp create mode 100644 csnode/src/snapshot_service.cpp create mode 100644 tests/csnode/snapshot_provider_tests.cpp diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index 7d6a2f6ab..a79e337a9 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -123,9 +123,8 @@ int run(const Config& config, const std::string& outDir) { in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) { return db->get(static_cast(seq), &out); }; - in.contracts = [&db](const std::function& visit) { - return db->forEachContractData(visit); - }; + // no contract records: the contracts database holds each contract's latest state, later than the + // checkpoint; the checkpoint carries the states at its block say() << "checkpoint " << S << ", blocks from " << std::max(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0) << ", into " << outDir << std::endl; diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 3eca33681..7d1332a2b 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -364,6 +364,11 @@ class Storage final { */ cs::Sequence first_sequence() const; + /** + * The stored bytes of block sequence, as the database holds them (no parsing, no pool cache). + */ + bool pool_bytes(cs::Sequence sequence, cs::Bytes& out) const; + /** * Returns the space of pruned pools to the file system. */ diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index c4b66a62b..a6f98f35e 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -1350,6 +1350,14 @@ bool Storage::pool_prune_batch(const std::vector& hashes) { return true; } +bool Storage::pool_bytes(cs::Sequence sequence, cs::Bytes& out) const { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + return d->db->get(static_cast(sequence), &out); +} + cs::Sequence Storage::first_sequence() const { if (!isOpen()) { d->set_last_error(NotOpen); diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index c20628610..0d23c1f8f 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -73,6 +73,8 @@ add_library(csnode include/csnode/snapshot_builder.hpp include/csnode/snapshot_importer.hpp include/csnode/snapshot_messages.hpp + include/csnode/snapshot_provider.hpp + include/csnode/snapshot_service.hpp include/csnode/blockverifier.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -118,6 +120,8 @@ add_library(csnode src/snapshot_builder.cpp src/snapshot_importer.cpp src/snapshot_messages.cpp + src/snapshot_provider.cpp + src/snapshot_service.cpp src/blockverifier.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 4495eb6d7..ca925b0d7 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -95,6 +95,10 @@ class BlockChain { // pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were // removed once a later periodic checkpoint made them unnecessary cs::Sequence getFirstStoredSequence() const; + // the stored bytes of a block, under the storage lock, for a snapshot: no parsing, no pool cache + bool blockBytes(cs::Sequence sequence, cs::Bytes& out) const; + // pruned storage: the blocks below this one are being or will be removed (0: nothing scheduled) + cs::Sequence pruneTarget() const; // return unique id of database if at least one unique block has written, otherwise (only genesis block) 0 uint64_t uuid() const; @@ -303,6 +307,8 @@ public signals: cs::StopReadingBlocksSignal stopReadingBlocksEvent; cs::SuccessQSSignal successfullQuickStartEvent; + // a periodic checkpoint of this block was saved (live), with its state digest + cs::Signal checkpointSavedEvent; public slots: diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index 072c772e2..b2dfffaf4 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -1,6 +1,7 @@ #ifndef NODE_HPP #define NODE_HPP +#include #include #include #include @@ -26,6 +27,12 @@ class Transport; +namespace cs { +class SnapshotProvider; +class SnapshotService; +struct SnapshotNodeState; +} + namespace cs { class SolverCore; } @@ -425,6 +432,9 @@ public slots: private: // snapshot messages dropped: malformed, or not expected by this node uint64_t snapshotMessagesDropped_ = 0; + // the provider's view of this node: last block, round, synced, trusted + void snapshotNodeState(cs::SnapshotNodeState& state) const; + void logSnapshotSummary(); bool init(); void initPoolSynchronizer(); @@ -497,6 +507,11 @@ public slots: std::unique_ptr api_; #endif + // snapshots served to p2p neighbours ([snapshot] serve) + std::unique_ptr snapshotProvider_; + std::unique_ptr snapshotService_; + std::chrono::steady_clock::time_point snapshotSummaryAt_{}; + uint32_t startPacketRequestPoint_ = 0; // ms timeout diff --git a/csnode/include/csnode/snapshot_builder.hpp b/csnode/include/csnode/snapshot_builder.hpp index 84b6d8831..86bfda068 100644 --- a/csnode/include/csnode/snapshot_builder.hpp +++ b/csnode/include/csnode/snapshot_builder.hpp @@ -31,7 +31,7 @@ struct SnapshotBuildInput { std::function blockBytes; std::function&)> contracts; std::string nodeVersion; - uint64_t chunkSize = 4ull << 20; + uint64_t chunkSize = 512ull << 10; // one p2p piece (design 4.3) const std::atomic* stop = nullptr; // checked between blocks: true aborts the build }; diff --git a/csnode/include/csnode/snapshot_provider.hpp b/csnode/include/csnode/snapshot_provider.hpp new file mode 100644 index 000000000..8fbf191fc --- /dev/null +++ b/csnode/include/csnode/snapshot_provider.hpp @@ -0,0 +1,117 @@ +#ifndef SNAPSHOT_PROVIDER_HPP +#define SNAPSHOT_PROVIDER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace cs { + +// what the node is doing when a snapshot request arrives +struct SnapshotNodeState { + cs::Sequence lastSeq = 0; + cs::RoundNumber round = 0; + bool synced = false; // design section 5.2 + bool trusted = false; // a confidant or the writer of the current round: half the budget +}; + +// Answers the snapshot requests of p2p neighbours from the newest snapshot this node built +// (docs/specs/2026-10-25-snapshot-p2p-design.md, section 5). Used on the node's processor thread; +// load() and unload() come from the snapshot service's thread. It keeps no state per requester +// beyond a table of the peers served in the last 30 s (at most 64), and every reply is bounded +// by a global token bucket and a per-peer one. +class SnapshotProvider { +public: + using Clock = std::function; + + struct Settings { + bool serve = false; + size_t rateKBps = 2000; // all peers together, KB/s; 0: unlimited + size_t maxPeers = 8; // peers served within 30 s + }; + + using NodeState = SnapshotNodeState; + + static constexpr size_t kMaxPeers = 64; + static constexpr auto kPeerWindow = std::chrono::seconds(30); + static constexpr auto kPieceWindow = std::chrono::milliseconds(100); + static constexpr size_t kPiecesPerWindow = 2; + static constexpr auto kManifestInterval = std::chrono::seconds(5); + static constexpr uint64_t kBurstBytes = 4ull << 20; + static constexpr uint32_t kMaxRetryMs = 5000; + + explicit SnapshotProvider(Clock clock = std::chrono::steady_clock::now); + ~SnapshotProvider(); + + // dir = snapshot/: its snapshot.json must parse, verify, name S, and its files must have the + // declared sizes; on failure the previous snapshot stays loaded + bool load(const std::filesystem::path& dir, std::string* error = nullptr); + void unload(); + std::optional loadedSeq() const; + + // nullopt: no answer (the peer asked again too soon, or the budget is spent; it asks again) + std::optional onManifestRequest(const cs::PublicKey& sender, const Settings& settings, + const NodeState& state); + snapshot_p2p::PieceReply onPieceRequest(const cs::PublicKey& sender, const snapshot_p2p::PieceRequest& request, + const Settings& settings, const NodeState& state); + + // a line about the traffic since the last call, empty when there was none + std::string takeSummary(); + // a peer that just received the last byte of a whole snapshot: once per peer and snapshot + std::function onPeerCompleted; + +private: + struct Loaded; + + struct Bucket { + double tokens = 0; + std::chrono::steady_clock::time_point at{}; + // refills at rate bytes/s up to burst; true and the cost taken when there is enough + bool take(double cost, double rate, double burst, std::chrono::steady_clock::time_point now, uint32_t& retryMs); + }; + + struct Peer { + std::chrono::steady_clock::time_point lastSeen{}; + std::chrono::steady_clock::time_point lastPiece{}; + std::chrono::steady_clock::time_point firstPiece{}; + std::chrono::steady_clock::time_point windowStart{}; + size_t piecesInWindow = 0; + std::chrono::steady_clock::time_point lastManifest{}; + bool manifestSent = false; + Bucket bucket; + bool bucketStarted = false; + cs::Sequence seq = 0; + uint64_t bytes = 0; // of seq + bool completed = false; + }; + + Peer& peer(const cs::PublicKey& key, std::chrono::steady_clock::time_point now); + size_t activePeers(std::chrono::steady_clock::time_point now, const cs::PublicKey& except) const; + + Clock clock_; + mutable std::mutex loadedLock_; + std::shared_ptr loaded_; + + // processor thread only + std::map peers_; + Bucket global_; + bool globalStarted_ = false; + uint64_t piecesServed_ = 0; + uint64_t bytesServed_ = 0; + uint64_t busyAnswers_ = 0; + uint64_t manifestsServed_ = 0; +}; + +} // namespace cs + +#endif // SNAPSHOT_PROVIDER_HPP diff --git a/csnode/include/csnode/snapshot_service.hpp b/csnode/include/csnode/snapshot_service.hpp new file mode 100644 index 000000000..2b3bae8ad --- /dev/null +++ b/csnode/include/csnode/snapshot_service.hpp @@ -0,0 +1,59 @@ +#ifndef SNAPSHOT_SERVICE_HPP +#define SNAPSHOT_SERVICE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +class BlockChain; + +namespace cs { + +// Snapshots served by this node to its p2p neighbours: every 10 s it reads [snapshot] from the +// configuration; while serve is on, it builds the snapshot of each new periodic checkpoint into +// snapshot/ at low priority (and of the latest one when serving starts) and hands the newest complete +// one to the provider, which answers the requests; serve off unloads it. Whether the node is synced +// is the provider's question, asked per request. +class SnapshotService { +public: + SnapshotService(BlockChain& blockchain, SnapshotProvider& provider, const cs::PrivateKey& key, std::string nodeVersion); + ~SnapshotService(); + + void start(); + void stop(); + // slot of BlockChain::checkpointSavedEvent: a new periodic checkpoint to build from + void onCheckpointSaved(cs::Sequence sequence); + +private: + void run(); + void buildLatest(); + // the newest snapshot/ with a snapshot.json, into the provider + void loadNewest(); + + BlockChain& blockchain_; + cs::PrivateKey key_; + std::string nodeVersion_; + std::filesystem::path root_ = "snapshot"; + SnapshotProvider& provider_; + bool serving_ = false; + std::thread thread_; + std::atomic stop_{false}; + std::atomic buildPending_{true}; + static constexpr int kMaxBuildAttempts = 5; + int failedBuilds_ = 0; + std::chrono::steady_clock::time_point retryAt_{}; + std::mutex m_; + std::condition_variable cv_; +}; + +} // namespace cs + +#endif // SNAPSHOT_SERVICE_HPP diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index d1daa1183..49304db0e 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1053,6 +1053,7 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { if (serializationManPtr_->save(pool.sequence(), head, digest)) { cslog() << cs::stateDigestLine(pool.sequence(), digest.digest, digest.wallets) << " checkpoint"; serializationManPtr_->pruneCheckpoints(sto.checkpointKeep); + emit checkpointSavedEvent(pool.sequence()); lastCheckpointWallClock_ = now; if (byCount) { schedulePruning(pool.sequence()); @@ -1064,6 +1065,16 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { } } +bool BlockChain::blockBytes(cs::Sequence sequence, cs::Bytes& out) const { + std::lock_guard lock(dbLock_); + return storage_.pool_bytes(sequence, out); +} + +cs::Sequence BlockChain::pruneTarget() const { + std::lock_guard lock(dbLock_); + return pruneUpTo_; +} + cs::CheckpointDigest BlockChain::checkpointDigest() const { cs::CheckpointDigest result; const auto& wallets = multiWallets(); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index de2ef8c84..15c52fa50 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -259,6 +262,28 @@ bool Node::init() { api_->run(); #endif // NODE_API + // snapshots served to p2p neighbours: [snapshot] serve, read again every 10 s (off by default) + { + std::string version; + std::ifstream versionFile("VERSION"); + if (!versionFile || !std::getline(versionFile, version) || version.empty() || version.size() > 64) { + version = "4.2." + std::to_string(Config::getNodeVersion()) + "." + std::to_string(Config::getMinorNodeVersion()); + } + for (auto& c : version) { + if (c < 0x20 || c >= 0x7f || c == '"' || c == '\\') { + c = '_'; + } + } + snapshotProvider_ = std::make_unique(); + snapshotProvider_->onPeerCompleted = [](const cs::PublicKey& key, cs::Sequence seq, uint64_t bytes, std::chrono::seconds took) { + cslog() << "SNAPSHOT: served block " << seq << " to " << EncodeBase58(key.data(), key.data() + key.size()) << ", " + << bytes / (1u << 20) << " MB in " << took.count() << " s"; + }; + snapshotService_ = std::make_unique(blockChain_, *snapshotProvider_, solver_->getPrivateKey(), version); + cs::Connector::connect(&blockChain_.checkpointSavedEvent, snapshotService_.get(), &cs::SnapshotService::onCheckpointSaved); + snapshotService_->start(); + } + if (!transport_->isGood()) { return false; } @@ -358,6 +383,11 @@ void Node::destroy() { api_->stop(); cswarning() << "[API STOPPED]"; + if (snapshotService_) { + snapshotService_->stop(); + cswarning() << "[SNAPSHOT SERVICE STOPPED]"; + } + solver_->finish(); cswarning() << "[SOLVER STOPPED]"; @@ -2162,17 +2192,53 @@ void logSnapshotDrop(uint64_t& dropped, const char* what, const cs::PublicKey& s } } // namespace +void Node::snapshotNodeState(cs::SnapshotNodeState& state) const { + state.lastSeq = blockChain_.getLastSeq(); + state.round = cs::Conveyer::instance().currentRoundNumber(); + // design 5.2: not syncing, not in a bootstrap round, at most 3 blocks behind a known round + state.synced = state.round != 0 && (!poolSynchronizer_ || !poolSynchronizer_->isSyncroStarted()) && !isBootstrapRound_ && + state.lastSeq + 3 >= state.round; + state.trusted = myLevel_ != Level::Normal; +} + +namespace { +cs::SnapshotProvider::Settings snapshotProviderSettings() { + const auto& cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + cs::SnapshotProvider::Settings settings; + settings.serve = cfg.serve; + settings.rateKBps = cfg.serveRateKbps; + settings.maxPeers = cfg.serveMaxPeers; + return settings; +} +} // namespace + +void Node::logSnapshotSummary() { + const auto now = std::chrono::steady_clock::now(); + if (now - snapshotSummaryAt_ < std::chrono::hours(1)) { + return; + } + snapshotSummaryAt_ = now; + const auto summary = snapshotProvider_->takeSummary(); + if (!summary.empty()) { + cslog() << "SNAPSHOT: in the last hour " << summary; + } +} + void Node::getSnapshotManifestRequest(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { cs::snapshot_p2p::ManifestRequest request; if (!cs::snapshot_p2p::decode(data, size, request)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed manifest request", sender); } - // this build does not serve snapshots yet - cs::snapshot_p2p::ManifestReply reply; - reply.status = cs::snapshot_p2p::ManifestStatus::NotServing; - reply.lastSeq = blockChain_.getLastSeq(); - reply.round = cs::Conveyer::instance().currentRoundNumber(); - sendSnapshotMessage(sender, MsgTypes::SnapshotManifestReply, cs::snapshot_p2p::encode(reply)); + if (!snapshotProvider_) { + return; + } + cs::SnapshotNodeState state; + snapshotNodeState(state); + const auto reply = snapshotProvider_->onManifestRequest(sender, snapshotProviderSettings(), state); + if (reply) { + sendSnapshotMessage(sender, MsgTypes::SnapshotManifestReply, cs::snapshot_p2p::encode(*reply)); + } + logSnapshotSummary(); } void Node::getSnapshotManifestReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { @@ -2189,12 +2255,14 @@ void Node::getSnapshotPieceRequest(const uint8_t* data, const size_t size, const if (!cs::snapshot_p2p::decode(data, size, request)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed piece request", sender); } - cs::snapshot_p2p::PieceReply reply; - reply.status = cs::snapshot_p2p::PieceStatus::NotServing; - reply.seq = request.seq; - reply.name = request.name; - reply.chunk = request.chunk; + if (!snapshotProvider_) { + return; + } + cs::SnapshotNodeState state; + snapshotNodeState(state); + const auto reply = snapshotProvider_->onPieceRequest(sender, request, snapshotProviderSettings(), state); sendSnapshotMessage(sender, MsgTypes::SnapshotPieceReply, cs::snapshot_p2p::encode(reply)); + logSnapshotSummary(); } void Node::getSnapshotPieceReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index f9fd80fbc..5b1938ea6 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -175,15 +175,6 @@ std::string utcNow() { return buf; } -bool readFile(const std::filesystem::path& path, std::vector& out) { - std::ifstream in(path, std::ios::binary); - if (!in) { - return false; - } - out.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); - return !in.bad(); -} - bool stopped(const SnapshotBuildInput& in) { return in.stop && in.stop->load(); } @@ -332,14 +323,28 @@ SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::fi if (!std::filesystem::exists(src, ec)) { continue; // tokens.dat and apihandler.dat exist only in builds with the API } - std::vector data; - if (!readFile(src, data)) { + // in pieces: walletscache.dat is about 180 MB on mainnet, and the builder runs inside the node + std::ifstream in(src, std::ios::binary); + if (!in) { return failWith("cannot read " + src.string()); } SnapshotFile f; f.name = "qs-" + name + ".lz4"; Lz4Writer w(tmp / f.name, m.chunkSize); - w.write(data); + std::vector piece(snapshot_lz4::kBlockSize); + while (in) { + if (stopped(input)) { + return failWith("stopped"); + } + in.read(reinterpret_cast(piece.data()), static_cast(piece.size())); + const auto n = in.gcount(); + if (n > 0) { + w.write(piece.data(), static_cast(n)); + } + } + if (in.bad()) { + return failWith("cannot read " + src.string()); + } if (!w.finish(f)) { return failWith("cannot write " + f.name); } diff --git a/csnode/src/snapshot_provider.cpp b/csnode/src/snapshot_provider.cpp new file mode 100644 index 000000000..9ae332f2e --- /dev/null +++ b/csnode/src/snapshot_provider.cpp @@ -0,0 +1,358 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; +using Clock = std::chrono::steady_clock; + +bool fail(std::string* error, const std::string& text) { + if (error) { + *error = text; + } + return false; +} + +bool powerOfTwo(uint64_t v) { + return v != 0 && (v & (v - 1)) == 0; +} + +uint32_t msUntil(Clock::time_point now, Clock::time_point when) { + const auto ms = std::chrono::duration_cast(when - now).count(); + return static_cast(std::clamp(ms, 1, SnapshotProvider::kMaxRetryMs)); +} +} // namespace + +struct SnapshotProvider::Loaded { + struct File { + int fd = -1; + uint64_t size = 0; + uint64_t chunks = 0; + }; + + SnapshotManifest manifest; + std::string json; + std::map files; + uint64_t totalBytes = 0; + + ~Loaded() { + for (auto& [name, f] : files) { + if (f.fd >= 0) { + ::close(f.fd); + } + } + } +}; + +bool SnapshotProvider::Bucket::take(double cost, double rate, double burst, Clock::time_point now, uint32_t& retryMs) { + const double elapsed = std::chrono::duration(now - at).count(); + tokens = std::min(burst, tokens + std::max(0.0, elapsed) * rate); + at = now; + if (tokens >= cost) { + tokens -= cost; + return true; + } + const double ms = std::ceil((cost - tokens) / rate * 1000.0); + retryMs = static_cast(std::clamp(ms, 1.0, static_cast(kMaxRetryMs))); + return false; +} + +SnapshotProvider::SnapshotProvider(Clock clock) +: clock_(std::move(clock)) { +} + +SnapshotProvider::~SnapshotProvider() = default; + +bool SnapshotProvider::load(const fs::path& dir, std::string* error) { + cs::Sequence seq = 0; + try { + size_t used = 0; + const auto name = dir.filename().string(); + seq = std::stoull(name, &used); + if (used != name.size()) { + return fail(error, "not a snapshot folder: " + dir.string()); + } + } + catch (const std::exception&) { + return fail(error, "not a snapshot folder: " + dir.string()); + } + + std::error_code ec; + const auto jsonPath = dir / "snapshot.json"; + const auto jsonSize = fs::file_size(jsonPath, ec); + if (ec || jsonSize > snapshot_p2p::kMaxManifestJson) { + return fail(error, "no snapshot.json of at most 256 KB in " + dir.string()); + } + std::ifstream in(jsonPath, std::ios::binary); + std::string json((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + if (!in.good() && !in.eof()) { + return fail(error, "cannot read " + jsonPath.string()); + } + + std::string why; + auto manifest = SnapshotManifest::fromJson(json, &why); + if (!manifest) { + return fail(error, jsonPath.string() + ": " + why); + } + if (manifest->seq != seq) { + return fail(error, jsonPath.string() + " is the snapshot of block " + std::to_string(manifest->seq)); + } + if (!manifest->verify()) { + return fail(error, jsonPath.string() + ": bad signature"); + } + if (manifest->chunkSize > snapshot_p2p::kMaxPieceBytes || !powerOfTwo(manifest->chunkSize)) { + return fail(error, jsonPath.string() + ": chunk size " + std::to_string(manifest->chunkSize) + " cannot be served"); + } + + auto loaded = std::make_shared(); + for (const auto& f : manifest->files) { + // the path comes from the manifest this node built and verified, never from a request + const auto path = dir / f.name; + Loaded::File entry; + entry.fd = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (entry.fd < 0) { + return fail(error, "cannot open " + path.string()); + } + loaded->files[f.name] = entry; // closed with loaded from here on + struct stat st {}; + if (::fstat(entry.fd, &st) != 0 || static_cast(st.st_size) != f.size) { + return fail(error, path.string() + " does not have the size of the manifest"); + } + loaded->files[f.name].size = f.size; + loaded->files[f.name].chunks = f.chunks.size(); + loaded->totalBytes += f.size; + } + loaded->manifest = std::move(*manifest); + loaded->json = std::move(json); + + std::lock_guard lock(loadedLock_); + loaded_ = std::move(loaded); + return true; +} + +void SnapshotProvider::unload() { + std::lock_guard lock(loadedLock_); + loaded_.reset(); +} + +std::optional SnapshotProvider::loadedSeq() const { + std::lock_guard lock(loadedLock_); + if (!loaded_) { + return std::nullopt; + } + return loaded_->manifest.seq; +} + +SnapshotProvider::Peer& SnapshotProvider::peer(const cs::PublicKey& key, Clock::time_point now) { + auto it = peers_.find(key); + if (it == peers_.end()) { + if (peers_.size() >= kMaxPeers) { + auto oldest = std::min_element(peers_.begin(), peers_.end(), + [](const auto& a, const auto& b) { return a.second.lastSeen < b.second.lastSeen; }); + peers_.erase(oldest); + } + it = peers_.emplace(key, Peer{}).first; + } + it->second.lastSeen = now; + return it->second; +} + +size_t SnapshotProvider::activePeers(Clock::time_point now, const cs::PublicKey& except) const { + size_t n = 0; + for (const auto& [key, p] : peers_) { + if (key != except && p.lastPiece != Clock::time_point{} && now - p.lastPiece < kPeerWindow) { + ++n; + } + } + return n; +} + +std::optional SnapshotProvider::onManifestRequest(const cs::PublicKey& sender, const Settings& settings, + const NodeState& state) { + using snapshot_p2p::ManifestStatus; + snapshot_p2p::ManifestReply reply; + reply.lastSeq = state.lastSeq; + reply.round = state.round; + reply.servingTrusted = state.trusted; + + std::shared_ptr loaded; + { + std::lock_guard lock(loadedLock_); + loaded = loaded_; + } + if (!settings.serve) { + reply.status = ManifestStatus::NotServing; + return reply; + } + if (!loaded) { + reply.status = ManifestStatus::Building; + return reply; + } + if (!state.synced) { + reply.status = ManifestStatus::NotSynced; + return reply; + } + + // the manifest is the large reply to a small request: once per peer every 5 s, and paid for + const auto now = clock_(); + auto& p = peer(sender, now); + if (p.manifestSent && now - p.lastManifest < kManifestInterval) { + return std::nullopt; + } + if (settings.rateKBps != 0) { + const double rate = settings.rateKBps * 1000.0 / (state.trusted ? 2 : 1); + const double burst = static_cast(kBurstBytes) / (state.trusted ? 2 : 1); + if (!globalStarted_) { + global_ = Bucket{burst, now}; + globalStarted_ = true; + } + uint32_t retry = 0; + if (!global_.take(static_cast(loaded->json.size()), rate, burst, now, retry)) { + ++busyAnswers_; + return std::nullopt; + } + } + p.manifestSent = true; + p.lastManifest = now; + ++manifestsServed_; + reply.status = ManifestStatus::Ready; + reply.manifestJson = loaded->json; + return reply; +} + +snapshot_p2p::PieceReply SnapshotProvider::onPieceRequest(const cs::PublicKey& sender, const snapshot_p2p::PieceRequest& request, + const Settings& settings, const NodeState& state) { + using snapshot_p2p::PieceStatus; + snapshot_p2p::PieceReply reply; + reply.seq = request.seq; + reply.name = request.name; + reply.chunk = request.chunk; + + std::shared_ptr loaded; + { + std::lock_guard lock(loadedLock_); + loaded = loaded_; + } + if (!settings.serve || !loaded) { + reply.status = PieceStatus::NotServing; + return reply; + } + const auto now = clock_(); + auto busy = [&](uint32_t retryMs) { + ++busyAnswers_; + reply.status = PieceStatus::Busy; + reply.retryMs = retryMs; + return reply; + }; + if (!state.synced) { + return busy(kMaxRetryMs); + } + const auto file = loaded->files.find(request.name); + if (request.seq != loaded->manifest.seq || file == loaded->files.end() || request.chunk >= file->second.chunks) { + reply.status = PieceStatus::Unknown; + return reply; + } + const uint64_t chunkSize = loaded->manifest.chunkSize; + const uint64_t offset = uint64_t(request.chunk) * chunkSize; + const auto length = static_cast(std::min(chunkSize, file->second.size - offset)); + + auto& p = peer(sender, now); + const bool active = p.lastPiece != Clock::time_point{} && now - p.lastPiece < kPeerWindow; + if (!active && activePeers(now, sender) >= std::max(settings.maxPeers, 1)) { + return busy(kMaxRetryMs); + } + if (now - p.windowStart >= kPieceWindow) { + p.windowStart = now; + p.piecesInWindow = 0; + } + if (p.piecesInWindow >= kPiecesPerWindow) { + return busy(msUntil(now, p.windowStart + kPieceWindow)); + } + if (settings.rateKBps != 0) { + const double rate = settings.rateKBps * 1000.0 / (state.trusted ? 2 : 1); + const double burst = static_cast(kBurstBytes) / (state.trusted ? 2 : 1); + if (!globalStarted_) { + global_ = Bucket{burst, now}; + globalStarted_ = true; + } + if (!p.bucketStarted) { + p.bucket = Bucket{burst / 2, now}; + p.bucketStarted = true; + } + // both buckets are checked before either is charged + Bucket peerTry = p.bucket; + Bucket globalTry = global_; + uint32_t retryPeer = 0; + uint32_t retryGlobal = 0; + const bool peerOk = peerTry.take(static_cast(length), rate / 2, burst / 2, now, retryPeer); + const bool globalOk = globalTry.take(static_cast(length), rate, burst, now, retryGlobal); + if (!peerOk || !globalOk) { + return busy(std::max(peerOk ? 0 : retryPeer, globalOk ? 0 : retryGlobal)); + } + p.bucket = peerTry; + global_ = globalTry; + } + + reply.data.resize(length); + size_t done = 0; + while (done < length) { + const auto n = ::pread(file->second.fd, reply.data.data() + done, length - done, static_cast(offset + done)); + if (n <= 0) { + reply.data.clear(); + reply.status = PieceStatus::NotServing; + return reply; + } + done += static_cast(n); + } + + ++p.piecesInWindow; + p.lastPiece = now; + if (p.seq != loaded->manifest.seq) { + p.seq = loaded->manifest.seq; + p.bytes = 0; + p.completed = false; + p.firstPiece = now; + } + p.bytes += length; + ++piecesServed_; + bytesServed_ += length; + if (!p.completed && p.bytes >= loaded->totalBytes) { + p.completed = true; + if (onPeerCompleted) { + onPeerCompleted(sender, p.seq, p.bytes, std::chrono::duration_cast(now - p.firstPiece)); + } + } + reply.status = PieceStatus::Ok; + return reply; +} + +std::string SnapshotProvider::takeSummary() { + if (piecesServed_ == 0 && manifestsServed_ == 0 && busyAnswers_ == 0) { + return {}; + } + const auto now = clock_(); + size_t recent = 0; + for (const auto& [key, p] : peers_) { + if (now - p.lastSeen < std::chrono::hours(1)) { + ++recent; + } + } + std::ostringstream out; + out << recent << " peers, " << manifestsServed_ << " manifests, " << piecesServed_ << " pieces, " << bytesServed_ / (1u << 20) + << " MB, " << busyAnswers_ << " busy answers"; + piecesServed_ = bytesServed_ = busyAnswers_ = manifestsServed_ = 0; + return out.str(); +} + +} // namespace cs diff --git a/csnode/src/snapshot_service.cpp b/csnode/src/snapshot_service.cpp new file mode 100644 index 000000000..098f3e33d --- /dev/null +++ b/csnode/src/snapshot_service.cpp @@ -0,0 +1,191 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; +constexpr auto kPeriod = std::chrono::seconds(10); + +bool isDigits(const std::string& s) { + return !s.empty() && s.size() <= 12 && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; }); +} + +// the highest periodic checkpoint that records its state (qs/0, the shutdown one, is not periodic) +cs::Sequence latestCheckpoint() { + cs::Sequence best = 0; + std::error_code ec; + for (const auto& entry : fs::directory_iterator("qs", ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::exists(entry.path() / CheckpointDigest::kFileName, ec)) { + best = std::max(best, std::stoull(name)); + } + } + return best; +} +} // namespace + +SnapshotService::SnapshotService(BlockChain& blockchain, SnapshotProvider& provider, const cs::PrivateKey& key, std::string nodeVersion) +: blockchain_(blockchain), key_(key), nodeVersion_(std::move(nodeVersion)), provider_(provider) { +} + +SnapshotService::~SnapshotService() { + stop(); +} + +void SnapshotService::start() { + stop_ = false; + thread_ = std::thread([this]() { run(); }); +} + +void SnapshotService::stop() { + if (!thread_.joinable()) { + return; + } + { + std::lock_guard lock(m_); + stop_ = true; + } + cv_.notify_all(); + thread_.join(); + provider_.unload(); +} + +void SnapshotService::onCheckpointSaved(cs::Sequence sequence) { + (void)sequence; + failedBuilds_ = 0; + buildPending_ = true; + cv_.notify_all(); +} + +void SnapshotService::run() { + // the builder works on this thread: below the node's other work + ::setpriority(PRIO_PROCESS, static_cast(::syscall(SYS_gettid)), 10); + while (!stop_) { + const bool serve = cs::ConfigHolder::instance().config()->getSnapshotSettings().serve; // the observer may swap the config + if (serve != serving_) { + serving_ = serve; + if (serve) { + loadNewest(); + buildPending_ = true; + } + else { + provider_.unload(); + cslog() << "SNAPSHOT: not serving ([snapshot] serve=off)"; + } + } + if (serve && retryAt_ != std::chrono::steady_clock::time_point{} && std::chrono::steady_clock::now() >= retryAt_) { + retryAt_ = {}; + buildPending_ = true; + } + if (serve && buildPending_.exchange(false)) { + buildLatest(); + } + std::unique_lock lock(m_); + cv_.wait_for(lock, kPeriod, [this]() { return stop_.load(); }); + } +} + +void SnapshotService::buildLatest() { + const cs::Sequence S = latestCheckpoint(); + std::error_code ec; + if (S == 0 || fs::exists(root_ / std::to_string(S) / "snapshot.json", ec)) { + return; // nothing new: no checkpoint with a digest yet, or already built + } + const auto cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + SnapshotBuildInput in; + in.seq = S; + in.checkpointDir = fs::path("qs") / std::to_string(S); + // pruning may be removing the lowest blocks right now (it follows the same checkpoint): start above it + in.firstStored = std::max(blockchain_.getFirstStoredSequence(), blockchain_.pruneTarget()); + in.snapshotBlocks = cfg.snapshotBlocks; + in.nodeVersion = nodeVersion_; + in.stop = &stop_; + in.blockBytes = [this](cs::Sequence seq, cs::Bytes& out) { + return blockchain_.blockBytes(seq, out) && !out.empty(); + }; + // no contract records: the contracts database holds the latest state of each contract, later than + // S, while the signed snapshot is the state at S; the checkpoint carries the states at S + // (smartcontracts.dat) and the node fetches any it lacks from the confidants + fs::create_directories(root_, ec); + // the node's own database shares the disk: a build that could fill it is not started + uint64_t estimate = 512ull << 20; + for (const auto& entry : fs::directory_iterator(in.checkpointDir, ec)) { + estimate += entry.is_regular_file(ec) ? entry.file_size(ec) / 4 : 0; + } + const auto space = fs::space(root_, ec); + if (!ec && space.available < 2 * estimate) { + cswarning() << "SNAPSHOT: not enough free disk for the snapshot of checkpoint " << S << " (" << space.available / (1u << 20) + << " MB free, about " << estimate / (1u << 20) << " MB needed twice)"; + return; + } + cslog() << "SNAPSHOT: building the snapshot of checkpoint " << S; + const auto started = std::chrono::steady_clock::now(); + const auto r = buildSnapshot(in, root_, key_); + if (!r.ok) { + // tried again a few times a minute apart (the block range may have been pruned under it) + const bool retry = !stop_ && ++failedBuilds_ < kMaxBuildAttempts; + cswarning() << "SNAPSHOT: no snapshot of checkpoint " << S << ": " << r.error << (retry ? ", trying again in a minute" : ""); + if (retry) { + retryAt_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + } + return; + } + failedBuilds_ = 0; + uint64_t bytes = 0; + for (const auto& f : r.manifest.files) { + bytes += f.size; + } + cslog() << "SNAPSHOT: built block " << S << ", " << r.manifest.files.size() << " files, " << bytes / (1u << 20) << " MB, state digest " + << r.manifest.stateDigest << " in " + << std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count() << " s"; + loadNewest(); +} + +void SnapshotService::loadNewest() { + std::error_code ec; + cs::Sequence best = 0; + for (const auto& entry : fs::directory_iterator(root_, ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::is_regular_file(entry.path() / "snapshot.json", ec)) { + best = std::max(best, std::stoull(name)); + } + } + if (best == 0) { + return; + } + if (provider_.loadedSeq() == best) { + return; + } + std::string error; + const auto dir = root_ / std::to_string(best); + if (!provider_.load(dir, &error)) { + cswarning() << "SNAPSHOT: " << dir.string() << " is not served: " << error; + return; + } + uint64_t bytes = 0; + size_t files = 0; + for (const auto& entry : fs::directory_iterator(dir, ec)) { + if (entry.is_regular_file(ec) && entry.path().filename() != "snapshot.json") { + bytes += entry.file_size(ec); + ++files; + } + } + cslog() << "SNAPSHOT: serving block " << best << " (" << files << " files, " << bytes / (1u << 20) << " MB)"; +} + +} // namespace cs diff --git a/tests/csnode/snapshot_provider_tests.cpp b/tests/csnode/snapshot_provider_tests.cpp new file mode 100644 index 000000000..1736fc35d --- /dev/null +++ b/tests/csnode/snapshot_provider_tests.cpp @@ -0,0 +1,339 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; +using namespace std::chrono_literals; +using cs::snapshot_p2p::ManifestStatus; +using cs::snapshot_p2p::PieceStatus; + +void putU16(std::ofstream& f, uint16_t v) { + f.write(reinterpret_cast(&v), sizeof(v)); +} + +// a snapshot of block 200 built by buildSnapshot, with 64 KiB chunks and a checkpoint file that +// does not compress, so that it spans several chunks +struct Fixture { + fs::path root; + cs::PrivateKey key; + cs::PublicKey pub; + fs::path dir; + std::chrono::steady_clock::time_point now = std::chrono::steady_clock::time_point{} + 1000h; + cs::SnapshotProvider provider{[this]() { return now; }}; + + explicit Fixture(const char* name) : root(fs::temp_directory_path() / name) { + fs::remove_all(root); + fs::create_directories(root); + EXPECT_TRUE(cscrypto::cryptoInit()); + key = cscrypto::generateKeyPair(pub); + + std::map blocks; + cs::Bytes head; + csdb::PoolHash prev; + for (cs::Sequence s = 0; s <= 200; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(100 + s % 50, static_cast('a' + s % 26))); + EXPECT_TRUE(p.compose()); + blocks[s] = p.to_binary(); + head = p.hash().to_binary(); + prev = p.hash(); + } + const auto qs = root / "qs" / "200"; + fs::create_directories(qs); + { + std::ofstream f(qs / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t s = 200; + f.write(reinterpret_cast(&magic), sizeof(magic)); + f.write(reinterpret_cast(&s), sizeof(s)); + putU16(f, static_cast(head.size())); + f.write(reinterpret_cast(head.data()), static_cast(head.size())); + putU16(f, 0); + } + { + std::mt19937 rng(7); + std::string noise(300000, '\0'); + for (auto& c : noise) { + c = static_cast(rng()); + } + std::ofstream(qs / "walletscache.dat", std::ios::binary) << noise; + } + std::ofstream(qs / "blockchain.dat", std::ios::binary) << "settings"; + cs::CheckpointDigest d; + d.digest.fill(7); + d.settings.fill(9); + d.wallets = 42; + EXPECT_TRUE(d.write(qs / cs::CheckpointDigest::kFileName)); + + cs::SnapshotBuildInput in; + in.seq = 200; + in.checkpointDir = qs; + in.snapshotBlocks = 150; + in.nodeVersion = "test"; + in.chunkSize = 64 << 10; + in.blockBytes = [&blocks](cs::Sequence s, cs::Bytes& out) { + auto it = blocks.find(s); + if (it == blocks.end()) return false; + out = it->second; + return true; + }; + const auto r = cs::buildSnapshot(in, root / "snapshot", key); + EXPECT_TRUE(r.ok) << r.error; + dir = r.dir; + } + ~Fixture() { + std::error_code ec; + fs::remove_all(root, ec); + } + + cs::SnapshotManifest manifest() const { + std::ifstream f(dir / "snapshot.json", std::ios::binary); + return *cs::SnapshotManifest::fromJson(std::string(std::istreambuf_iterator(f), std::istreambuf_iterator())); + } +}; + +cs::PublicKey peerKey(uint8_t n) { + cs::PublicKey k{}; + k.fill(n); + return k; +} + +cs::SnapshotProvider::Settings serving(size_t rateKBps = 0, size_t maxPeers = 8) { + cs::SnapshotProvider::Settings s; + s.serve = true; + s.rateKBps = rateKBps; + s.maxPeers = maxPeers; + return s; +} + +cs::SnapshotNodeState synced(bool trusted = false) { + cs::SnapshotNodeState s; + s.lastSeq = 1000; + s.round = 1001; + s.synced = true; + s.trusted = trusted; + return s; +} + +cs::snapshot_p2p::PieceRequest piece(const std::string& name, uint32_t chunk, cs::Sequence seq = 200) { + cs::snapshot_p2p::PieceRequest r; + r.seq = seq; + r.name = name; + r.chunk = chunk; + return r; +} +} // namespace + +TEST(SnapshotProvider, ManifestStatuses) { + Fixture fx("snapshot_provider_status"); + const auto a = peerKey(1); + + auto off = fx.provider.onManifestRequest(a, cs::SnapshotProvider::Settings{}, synced()); + ASSERT_TRUE(off); + EXPECT_EQ(off->status, ManifestStatus::NotServing); + EXPECT_EQ(off->lastSeq, 1000u); + + auto building = fx.provider.onManifestRequest(a, serving(), synced()); + ASSERT_TRUE(building); + EXPECT_EQ(building->status, ManifestStatus::Building); + + ASSERT_TRUE(fx.provider.load(fx.dir)); + EXPECT_EQ(fx.provider.loadedSeq(), 200u); + + cs::SnapshotNodeState behind = synced(); + behind.synced = false; + auto notSynced = fx.provider.onManifestRequest(a, serving(), behind); + ASSERT_TRUE(notSynced); + EXPECT_EQ(notSynced->status, ManifestStatus::NotSynced); + + auto ready = fx.provider.onManifestRequest(a, serving(), synced(true)); + ASSERT_TRUE(ready); + EXPECT_EQ(ready->status, ManifestStatus::Ready); + EXPECT_TRUE(ready->servingTrusted); + const auto parsed = cs::SnapshotManifest::fromJson(ready->manifestJson); + ASSERT_TRUE(parsed); + EXPECT_TRUE(parsed->verify()); + EXPECT_EQ(parsed->seq, 200u); + + // once per peer every 5 s + EXPECT_FALSE(fx.provider.onManifestRequest(a, serving(), synced())); + EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(2), serving(), synced())); + fx.now += 5s; + EXPECT_TRUE(fx.provider.onManifestRequest(a, serving(), synced())); + + fx.provider.unload(); + EXPECT_FALSE(fx.provider.loadedSeq()); +} + +TEST(SnapshotProvider, EveryChunkMatchesTheManifest) { + Fixture fx("snapshot_provider_chunks"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto m = fx.manifest(); + bool sawShortLast = false; + bool sawSeveralChunks = false; + uint8_t n = 1; + for (const auto& f : m.files) { + sawSeveralChunks = sawSeveralChunks || f.chunks.size() > 2; + for (uint32_t c = 0; c < f.chunks.size(); ++c) { + fx.now += 1s; // well inside every limit + const auto r = fx.provider.onPieceRequest(peerKey(n), piece(f.name, c), serving(), synced()); + ASSERT_EQ(r.status, PieceStatus::Ok) << f.name << " " << c; + EXPECT_EQ(cs::sha256Hex(r.data.data(), r.data.size()), f.chunks[c]) << f.name << " " << c; + if (c + 1 == f.chunks.size() && r.data.size() < m.chunkSize) { + sawShortLast = true; + } + EXPECT_EQ(r.name, f.name); + EXPECT_EQ(r.chunk, c); + } + } + EXPECT_TRUE(sawSeveralChunks); + EXPECT_TRUE(sawShortLast); +} + +TEST(SnapshotProvider, UnknownRequests) { + Fixture fx("snapshot_provider_unknown"); + const auto a = peerKey(1); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0), serving(), synced()).status, PieceStatus::NotServing); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto m = fx.manifest(); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0, 199), serving(), synced()).status, PieceStatus::Unknown); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("contracts.bin.lz4", 0), serving(), synced()).status, PieceStatus::Unknown); + const auto chunks = static_cast(m.file("blocks.bin.lz4")->chunks.size()); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", chunks), serving(), synced()).status, PieceStatus::Unknown); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0), cs::SnapshotProvider::Settings{}, synced()).status, + PieceStatus::NotServing); + cs::SnapshotNodeState behind = synced(); + behind.synced = false; + const auto r = fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0), serving(), behind); + EXPECT_EQ(r.status, PieceStatus::Busy); + EXPECT_EQ(r.retryMs, cs::SnapshotProvider::kMaxRetryMs); +} + +TEST(SnapshotProvider, TwoPiecesPerPeerEvery100ms) { + Fixture fx("snapshot_provider_window"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto a = peerKey(1); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 0), serving(), synced()).status, PieceStatus::Ok); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 1), serving(), synced()).status, PieceStatus::Ok); + fx.now += 40ms; + const auto third = fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 2), serving(), synced()); + EXPECT_EQ(third.status, PieceStatus::Busy); + EXPECT_GT(third.retryMs, 0u); + EXPECT_LE(third.retryMs, 60u); + // another peer is not held up + EXPECT_EQ(fx.provider.onPieceRequest(peerKey(2), piece("qs-walletscache.dat.lz4", 2), serving(), synced()).status, PieceStatus::Ok); + fx.now += 60ms; + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 2), serving(), synced()).status, PieceStatus::Ok); +} + +namespace { +// pieces of 64 KiB a peer gets, two every 100 ms, before the first busy answer +size_t piecesUntilBusy(Fixture& fx, const cs::PublicKey& peer, size_t rateKBps, bool trusted, uint32_t* retryMs = nullptr) { + for (size_t n = 0; n < 1000; ++n) { + if (n % 2 == 0) { + fx.now += 100ms; + } + const auto r = fx.provider.onPieceRequest(peer, piece("qs-walletscache.dat.lz4", 0), serving(rateKBps), synced(trusted)); + if (r.status == PieceStatus::Busy) { + if (retryMs) { + *retryMs = r.retryMs; + } + return n; + } + EXPECT_EQ(r.status, PieceStatus::Ok); + } + return 1000; +} +} // namespace + +TEST(SnapshotProvider, PerPeerAndGlobalBudget) { + Fixture fx("snapshot_provider_budget"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + // 100 KB/s: a peer may take half (50 KB/s) after a burst of 2 MiB, 32 pieces of 64 KiB + uint32_t retry = 0; + const auto first = piecesUntilBusy(fx, peerKey(1), 100, false, &retry); + EXPECT_GE(first, 32u); + EXPECT_LE(first, 36u); + EXPECT_GT(retry, 0u); + EXPECT_LE(retry, cs::SnapshotProvider::kMaxRetryMs); + // a second peer still gets its share of what the global burst has left (4 MiB - 2 MiB) + const auto second = piecesUntilBusy(fx, peerKey(2), 100, false); + EXPECT_GE(second, 28u); + // and then the global bucket is empty for everyone + EXPECT_LE(piecesUntilBusy(fx, peerKey(3), 100, false), 3u); +} + +TEST(SnapshotProvider, TrustedHalvesTheBudget) { + Fixture fx("snapshot_provider_trusted"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto n = piecesUntilBusy(fx, peerKey(1), 100, true); + EXPECT_GE(n, 16u); + EXPECT_LE(n, 18u); +} + +TEST(SnapshotProvider, MaxPeers) { + Fixture fx("snapshot_provider_maxpeers"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + EXPECT_EQ(fx.provider.onPieceRequest(peerKey(1), piece("blocks.bin.lz4", 0), serving(0, 1), synced()).status, PieceStatus::Ok); + const auto r = fx.provider.onPieceRequest(peerKey(2), piece("blocks.bin.lz4", 0), serving(0, 1), synced()); + EXPECT_EQ(r.status, PieceStatus::Busy); + EXPECT_EQ(r.retryMs, cs::SnapshotProvider::kMaxRetryMs); + fx.now += 31s; + EXPECT_EQ(fx.provider.onPieceRequest(peerKey(2), piece("blocks.bin.lz4", 0), serving(0, 1), synced()).status, PieceStatus::Ok); +} + +TEST(SnapshotProvider, PeerTableEvictsTheOldest) { + Fixture fx("snapshot_provider_table"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(0), serving(), synced())); + EXPECT_FALSE(fx.provider.onManifestRequest(peerKey(0), serving(), synced())); + for (uint8_t i = 1; i <= cs::SnapshotProvider::kMaxPeers; ++i) { + fx.now += 1ms; + EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(i), serving(), synced())); + } + // peer 0 was the oldest of 65: forgotten, so answered again within its 5 s + EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(0), serving(), synced())); +} + +TEST(SnapshotProvider, CompletedPeerAndSummary) { + Fixture fx("snapshot_provider_complete"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + int completed = 0; + fx.provider.onPeerCompleted = [&](const cs::PublicKey&, cs::Sequence seq, uint64_t, std::chrono::seconds) { + EXPECT_EQ(seq, 200u); + ++completed; + }; + for (const auto& f : fx.manifest().files) { + for (uint32_t c = 0; c < f.chunks.size(); ++c) { + fx.now += 1s; + ASSERT_EQ(fx.provider.onPieceRequest(peerKey(1), piece(f.name, c), serving(), synced()).status, PieceStatus::Ok); + } + } + EXPECT_EQ(completed, 1); + EXPECT_NE(fx.provider.takeSummary().find("1 peers"), std::string::npos); + EXPECT_TRUE(fx.provider.takeSummary().empty()); +} + +TEST(SnapshotProvider, ATamperedFileIsNotLoaded) { + Fixture fx("snapshot_provider_tampered"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + std::ofstream(fx.dir / "blocks.bin.lz4", std::ios::binary | std::ios::app) << "x"; + std::string error; + EXPECT_FALSE(fx.provider.load(fx.dir, &error)); + EXPECT_NE(error.find("size"), std::string::npos) << error; + EXPECT_EQ(fx.provider.loadedSeq(), 200u); // the one loaded before stays + EXPECT_FALSE(fx.provider.load(fx.root / "snapshot" / "not-a-number")); +} From 9645c06b308058a09a5f29e4c49653271a722d52 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 12:52:02 +0100 Subject: [PATCH 2/6] snapshot provider: steady clock alias that does not hide the injected clock Co-Authored-By: Claude Opus 5.5 --- csnode/src/snapshot_provider.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/csnode/src/snapshot_provider.cpp b/csnode/src/snapshot_provider.cpp index 9ae332f2e..31eb1c849 100644 --- a/csnode/src/snapshot_provider.cpp +++ b/csnode/src/snapshot_provider.cpp @@ -15,7 +15,7 @@ namespace cs { namespace { namespace fs = std::filesystem; -using Clock = std::chrono::steady_clock; +using SteadyClock = std::chrono::steady_clock; bool fail(std::string* error, const std::string& text) { if (error) { @@ -28,7 +28,7 @@ bool powerOfTwo(uint64_t v) { return v != 0 && (v & (v - 1)) == 0; } -uint32_t msUntil(Clock::time_point now, Clock::time_point when) { +uint32_t msUntil(SteadyClock::time_point now, SteadyClock::time_point when) { const auto ms = std::chrono::duration_cast(when - now).count(); return static_cast(std::clamp(ms, 1, SnapshotProvider::kMaxRetryMs)); } @@ -55,7 +55,7 @@ struct SnapshotProvider::Loaded { } }; -bool SnapshotProvider::Bucket::take(double cost, double rate, double burst, Clock::time_point now, uint32_t& retryMs) { +bool SnapshotProvider::Bucket::take(double cost, double rate, double burst, SteadyClock::time_point now, uint32_t& retryMs) { const double elapsed = std::chrono::duration(now - at).count(); tokens = std::min(burst, tokens + std::max(0.0, elapsed) * rate); at = now; @@ -154,7 +154,7 @@ std::optional SnapshotProvider::loadedSeq() const { return loaded_->manifest.seq; } -SnapshotProvider::Peer& SnapshotProvider::peer(const cs::PublicKey& key, Clock::time_point now) { +SnapshotProvider::Peer& SnapshotProvider::peer(const cs::PublicKey& key, SteadyClock::time_point now) { auto it = peers_.find(key); if (it == peers_.end()) { if (peers_.size() >= kMaxPeers) { @@ -168,10 +168,10 @@ SnapshotProvider::Peer& SnapshotProvider::peer(const cs::PublicKey& key, Clock:: return it->second; } -size_t SnapshotProvider::activePeers(Clock::time_point now, const cs::PublicKey& except) const { +size_t SnapshotProvider::activePeers(SteadyClock::time_point now, const cs::PublicKey& except) const { size_t n = 0; for (const auto& [key, p] : peers_) { - if (key != except && p.lastPiece != Clock::time_point{} && now - p.lastPiece < kPeerWindow) { + if (key != except && p.lastPiece != SteadyClock::time_point{} && now - p.lastPiece < kPeerWindow) { ++n; } } @@ -268,7 +268,7 @@ snapshot_p2p::PieceReply SnapshotProvider::onPieceRequest(const cs::PublicKey& s const auto length = static_cast(std::min(chunkSize, file->second.size - offset)); auto& p = peer(sender, now); - const bool active = p.lastPiece != Clock::time_point{} && now - p.lastPiece < kPeerWindow; + const bool active = p.lastPiece != SteadyClock::time_point{} && now - p.lastPiece < kPeerWindow; if (!active && activePeers(now, sender) >= std::max(settings.maxPeers, 1)) { return busy(kMaxRetryMs); } From 11192d15a12299dcc01a8dfec0831524c86b6874 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 13:14:38 +0100 Subject: [PATCH 3/6] run.sh: serving nodes take periodic checkpoints and must build and serve a snapshot; snapshot-p2p-serve scenario Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 6 +++++- ci/testnet/run.sh | 20 +++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 4fb0d3b59..18c5af90d 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -246,9 +246,13 @@ jobs: # slow nodes: 5% of one CPU for two trusted nodes - name: slow-nodes env: SLOW_NODES=3,6 SLOW_CPU=5 - # snapshot sharing announced in host_data: n3 and n5 serve, a stopped node lists them + # snapshot sharing: n3 (full) and n5 serve, build their snapshots and announce them in + # host_data; a stopped node lists them - name: snapshot-capability env: SNAPSHOT_SERVE_NODES=3,5 + # a pruned LMDB node and a full RocksDB one serve, the pruned one while it prunes + - name: snapshot-p2p-serve + env: NODE_STORAGE=berkeleydb,berkeleydb,berkeleydb,berkeleydb,berkeleydb,lmdb:pruned,rocksdb SNAPSHOT_SERVE_NODES=6,7 TARGET_SEQ=2600 # random datagrams to every node's p2p port - name: garbage-datagrams env: GARBAGE_AT=150 GARBAGE_RATE=300 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index e09468b90..1a2cf17e5 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -67,9 +67,10 @@ # snapshot with --import-snapshot into its own backend and joins; with SNAPSHOT_CORRUPT=1 the # blocks file is truncated first, so the restore must fail and that node syncs from block 0. # SNAPSHOT_FORMAT=1: packaging/docker/snapshot-make.sh and snapshot-restore.sh instead -# SNAPSHOT_SERVE_NODES (optional, comma separated): those nodes run with [snapshot] serve=on; -# at the end one other node of this build is stopped and its known_peers.json must list each -# of them with the snapshot capability, and no other node with it +# SNAPSHOT_SERVE_NODES (optional, comma separated): those nodes run with [snapshot] serve=on +# and a checkpoint every 1'000 blocks; each must build the snapshot of a checkpoint and serve +# it; at the end one other node of this build is stopped and its known_peers.json must list +# each of them with the snapshot capability, and no other node with it # STORAGE_SWITCH_AT (default 0 = off, needs PRUNE_NODE=6 and DB_BACKEND=lmdb): at that sequence # node 5 is restarted as a pruned node and node 6 as a full one # CONVERT_AT (default 0 = off, needs DB_BACKEND=berkeleydb): at that sequence n4 is converted to @@ -355,6 +356,10 @@ storage_section() { # that the kills hit the moments right after one lines="checkpoint_every=1000\ncheckpoint_every_minutes=1\ncheckpoint_keep=3\n" fi + if [[ ",$SNAPSHOT_SERVE_NODES," == *",$i,"* ]] && [[ "$lines" != *checkpoint_every=* ]]; then + # a serving node builds its snapshots from the periodic checkpoints + lines="${lines}checkpoint_every=1000\ncheckpoint_keep=2\n" + fi [ -n "$backend" ] && lines="${lines}db_backend=$backend\n" [ -n "$lines" ] && printf "\n[storage]\n$lines" return 0 @@ -1316,6 +1321,15 @@ if [ "$LOW_STAKE_NODES" -gt 0 ]; then echo "stake: none of the $LOW_STAKE_NODES nodes below the minimum stake was trusted after round $from" fi if [ -n "$SNAPSHOT_SERVE_NODES" ]; then + # every serving node built the snapshot of a checkpoint and serves it (the build follows the + # checkpoint at 1'000 on a thread of low priority: a minute at most) + for i in ${SNAPSHOT_SERVE_NODES//,/ }; do + for _ in $(seq 1 60); do grep -aq 'SNAPSHOT: serving block' "$WORK/n$i/node.log" && break; sleep 1; done + built=$(grep -a 'SNAPSHOT: built block' "$WORK/n$i/node.log" | tail -1 | sed -E 's/.*(built block [0-9]+[^,]*, [^,]*, [^,]*).*/\1/') + serving=$(grep -a 'SNAPSHOT: serving block' "$WORK/n$i/node.log" | tail -1 | sed -E 's/.*SNAPSHOT: //') + [ -n "$built" ] && [ -n "$serving" ] || { echo "FAIL: n$i did not build and serve a snapshot"; grep -a 'SNAPSHOT' "$WORK/n$i/node.log" | tail -5; exit 1; } + echo "snapshot: n$i $built; $serving" + done # known_peers.json is written at a clean stop: stop one node of this build that does not serve observer=0 for i in $(seq 1 "$NODES"); do From 9498aca1b8e65fff854c6b8e96e1af83f341e1e7 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 13:27:13 +0100 Subject: [PATCH 4/6] snapshot provider and service: review fixes The service resets its failure count and is woken under its lock, and a snapshot that does not load is removed and built again (at most five times). The provider evicts idle keys before a downloading peer, sends one manifest to anyone every 50 ms, counts each chunk once towards a completed download, and asks the kernel to keep the snapshot files cached. KB/s documented. Co-Authored-By: Claude Opus 5.5 --- client/config/config.hpp | 2 +- csnode/include/csnode/snapshot_provider.hpp | 6 ++++ csnode/include/csnode/snapshot_service.hpp | 2 +- csnode/src/node.cpp | 1 + csnode/src/snapshot_provider.cpp | 32 +++++++++++++++--- csnode/src/snapshot_service.cpp | 20 ++++++++--- docs/CHANGES.md | 2 ++ packaging/mainnet/config.ini | 3 +- tests/csnode/snapshot_provider_tests.cpp | 37 ++++++++++++++++----- 9 files changed, 84 insertions(+), 21 deletions(-) diff --git a/client/config/config.hpp b/client/config/config.hpp index a55896fe4..386869696 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -75,7 +75,7 @@ struct StorageData { // observer: every key but fetch* applies while the node runs. struct SnapshotData { bool serve = false; // build snapshots and answer snapshot requests - size_t serveRateKbps = 2000; // upload cap of all peers together; 0: unlimited + size_t serveRateKbps = 2000; // upload cap of all peers together, KB/s (2000: 2 MB/s); 0: unlimited size_t serveMaxPeers = 8; // peers served within 30 s; more get busy size_t snapshotBlocks = 101'000; // blocks before the checkpoint in a snapshot std::string fetch = "auto"; // auto: fetch at a start on an empty database; off: never diff --git a/csnode/include/csnode/snapshot_provider.hpp b/csnode/include/csnode/snapshot_provider.hpp index 8fbf191fc..03325249a 100644 --- a/csnode/include/csnode/snapshot_provider.hpp +++ b/csnode/include/csnode/snapshot_provider.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -47,6 +48,9 @@ class SnapshotProvider { static constexpr auto kPieceWindow = std::chrono::milliseconds(100); static constexpr size_t kPiecesPerWindow = 2; static constexpr auto kManifestInterval = std::chrono::seconds(5); + // between two manifest replies to anyone: peer keys cost nothing, so the per-key limit alone + // does not bound them + static constexpr auto kManifestGap = std::chrono::milliseconds(50); static constexpr uint64_t kBurstBytes = 4ull << 20; static constexpr uint32_t kMaxRetryMs = 5000; @@ -92,6 +96,7 @@ class SnapshotProvider { bool bucketStarted = false; cs::Sequence seq = 0; uint64_t bytes = 0; // of seq + std::vector chunksSeen; // of seq, over all files in manifest order: re-requests do not count bool completed = false; }; @@ -106,6 +111,7 @@ class SnapshotProvider { std::map peers_; Bucket global_; bool globalStarted_ = false; + std::chrono::steady_clock::time_point lastManifestAny_{}; uint64_t piecesServed_ = 0; uint64_t bytesServed_ = 0; uint64_t busyAnswers_ = 0; diff --git a/csnode/include/csnode/snapshot_service.hpp b/csnode/include/csnode/snapshot_service.hpp index 2b3bae8ad..71ac84629 100644 --- a/csnode/include/csnode/snapshot_service.hpp +++ b/csnode/include/csnode/snapshot_service.hpp @@ -48,7 +48,7 @@ class SnapshotService { std::atomic stop_{false}; std::atomic buildPending_{true}; static constexpr int kMaxBuildAttempts = 5; - int failedBuilds_ = 0; + std::atomic failedBuilds_{0}; // reset by onCheckpointSaved on the thread that stores blocks std::chrono::steady_clock::time_point retryAt_{}; std::mutex m_; std::condition_variable cv_; diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 15c52fa50..b1ee9c4e7 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -275,6 +275,7 @@ bool Node::init() { } } snapshotProvider_ = std::make_unique(); + snapshotSummaryAt_ = std::chrono::steady_clock::now(); snapshotProvider_->onPeerCompleted = [](const cs::PublicKey& key, cs::Sequence seq, uint64_t bytes, std::chrono::seconds took) { cslog() << "SNAPSHOT: served block " << seq << " to " << EncodeBase58(key.data(), key.data() + key.size()) << ", " << bytes / (1u << 20) << " MB in " << took.count() << " s"; diff --git a/csnode/src/snapshot_provider.cpp b/csnode/src/snapshot_provider.cpp index 31eb1c849..cce917578 100644 --- a/csnode/src/snapshot_provider.cpp +++ b/csnode/src/snapshot_provider.cpp @@ -39,12 +39,14 @@ struct SnapshotProvider::Loaded { int fd = -1; uint64_t size = 0; uint64_t chunks = 0; + uint64_t firstChunk = 0; // index of its first chunk among all files, in manifest order }; SnapshotManifest manifest; std::string json; std::map files; uint64_t totalBytes = 0; + uint64_t totalChunks = 0; ~Loaded() { for (auto& [name, f] : files) { @@ -129,8 +131,12 @@ bool SnapshotProvider::load(const fs::path& dir, std::string* error) { if (::fstat(entry.fd, &st) != 0 || static_cast(st.st_size) != f.size) { return fail(error, path.string() + " does not have the size of the manifest"); } + // pieces are read on the thread that handles consensus messages: keep the files in the page cache + ::posix_fadvise(entry.fd, 0, 0, POSIX_FADV_WILLNEED); loaded->files[f.name].size = f.size; loaded->files[f.name].chunks = f.chunks.size(); + loaded->files[f.name].firstChunk = loaded->totalChunks; + loaded->totalChunks += f.chunks.size(); loaded->totalBytes += f.size; } loaded->manifest = std::move(*manifest); @@ -158,8 +164,13 @@ SnapshotProvider::Peer& SnapshotProvider::peer(const cs::PublicKey& key, SteadyC auto it = peers_.find(key); if (it == peers_.end()) { if (peers_.size() >= kMaxPeers) { - auto oldest = std::min_element(peers_.begin(), peers_.end(), - [](const auto& a, const auto& b) { return a.second.lastSeen < b.second.lastSeen; }); + // a peer downloading now goes last: new keys (free to make up) must not push it out + auto downloading = [&](const Peer& p) { return p.lastPiece != SteadyClock::time_point{} && now - p.lastPiece < kPeerWindow; }; + auto oldest = std::min_element(peers_.begin(), peers_.end(), [&](const auto& a, const auto& b) { + const bool da = downloading(a.second); + const bool db = downloading(b.second); + return da != db ? !da : a.second.lastSeen < b.second.lastSeen; + }); peers_.erase(oldest); } it = peers_.emplace(key, Peer{}).first; @@ -204,8 +215,13 @@ std::optional SnapshotProvider::onManifestRequest(c return reply; } - // the manifest is the large reply to a small request: once per peer every 5 s, and paid for + // the manifest is the large reply to a small request: once per peer every 5 s, one to anyone + // every 50 ms, and paid for const auto now = clock_(); + if (lastManifestAny_ != SteadyClock::time_point{} && now - lastManifestAny_ < kManifestGap) { + ++busyAnswers_; + return std::nullopt; + } auto& p = peer(sender, now); if (p.manifestSent && now - p.lastManifest < kManifestInterval) { return std::nullopt; @@ -225,6 +241,7 @@ std::optional SnapshotProvider::onManifestRequest(c } p.manifestSent = true; p.lastManifest = now; + lastManifestAny_ = now; ++manifestsServed_; reply.status = ManifestStatus::Ready; reply.manifestJson = loaded->json; @@ -321,10 +338,15 @@ snapshot_p2p::PieceReply SnapshotProvider::onPieceRequest(const cs::PublicKey& s if (p.seq != loaded->manifest.seq) { p.seq = loaded->manifest.seq; p.bytes = 0; + p.chunksSeen.assign(static_cast(loaded->totalChunks), false); p.completed = false; p.firstPiece = now; } - p.bytes += length; + const size_t index = static_cast(file->second.firstChunk + request.chunk); + if (index < p.chunksSeen.size() && !p.chunksSeen[index]) { + p.chunksSeen[index] = true; + p.bytes += length; + } ++piecesServed_; bytesServed_ += length; if (!p.completed && p.bytes >= loaded->totalBytes) { @@ -350,7 +372,7 @@ std::string SnapshotProvider::takeSummary() { } std::ostringstream out; out << recent << " peers, " << manifestsServed_ << " manifests, " << piecesServed_ << " pieces, " << bytesServed_ / (1u << 20) - << " MB, " << busyAnswers_ << " busy answers"; + << " MiB, " << busyAnswers_ << " busy answers"; piecesServed_ = bytesServed_ = busyAnswers_ = manifestsServed_ = 0; return out.str(); } diff --git a/csnode/src/snapshot_service.cpp b/csnode/src/snapshot_service.cpp index 098f3e33d..bcbbbd94c 100644 --- a/csnode/src/snapshot_service.cpp +++ b/csnode/src/snapshot_service.cpp @@ -67,8 +67,12 @@ void SnapshotService::stop() { void SnapshotService::onCheckpointSaved(cs::Sequence sequence) { (void)sequence; - failedBuilds_ = 0; - buildPending_ = true; + { + // under the lock: the service thread cannot miss it between its check and its wait + std::lock_guard lock(m_); + failedBuilds_ = 0; + buildPending_ = true; + } cv_.notify_all(); } @@ -96,7 +100,7 @@ void SnapshotService::run() { buildLatest(); } std::unique_lock lock(m_); - cv_.wait_for(lock, kPeriod, [this]() { return stop_.load(); }); + cv_.wait_for(lock, kPeriod, [this]() { return stop_.load() || (serving_ && buildPending_.load()); }); } } @@ -174,7 +178,15 @@ void SnapshotService::loadNewest() { std::string error; const auto dir = root_ / std::to_string(best); if (!provider_.load(dir, &error)) { - cswarning() << "SNAPSHOT: " << dir.string() << " is not served: " << error; + // a snapshot of this node's own builder (it has snapshot.json): removed, so that the next + // round builds it again while its checkpoint is kept + const bool retry = ++failedBuilds_ < kMaxBuildAttempts; + cswarning() << "SNAPSHOT: " << dir.string() << " is not served: " << error << "; removed" + << (retry ? ", to be built again in a minute" : ""); + fs::remove_all(dir, ec); + if (retry) { + retryAt_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + } return; } uint64_t bytes = 0; diff --git a/docs/CHANGES.md b/docs/CHANGES.md index dd80f3bdc..bd38f7018 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -36,6 +36,8 @@ lines under the new release. |---|---| | #99 | config: the `[snapshot]` section takes the keys of p2p snapshot sharing (`serve`, `serve_rate_kbps`, `serve_max_peers`, `snapshot_blocks` from 1, `fetch`, `fetch_wait`, `fetch_fallback`, `min_sources` 2, `sources` as node keys, `sources_file`, `fetch_rate_kbps`); the HTTP service keys are gone and an unknown key is logged and ignored; nothing reads them yet | | #100 | net: four p2p messages for snapshot manifests and pieces (types 40-43) with bounded decoders; this build answers requests as not serving and sends none | +| #101 | net: `host_data` announces snapshot serving (bit 24, `[snapshot] serve=on` at start) and the community version (bits 32-47, `MINOR_NODE_VERSION`, now 13); `known_peers.json` at a clean stop lists the known peers with what they announce | +| #102 | snapshot: a node with `[snapshot] serve=on` builds the snapshot of each periodic checkpoint and serves it to its p2p neighbours, within `serve_rate_kbps` (KB/s), `serve_max_peers` and per-peer limits, at half budget while trusted; snapshots use 512 KiB chunks and no contracts file | ## community.12 — 2026-10-24 diff --git a/packaging/mainnet/config.ini b/packaging/mainnet/config.ini index 25c8f946f..fdd066ea5 100644 --- a/packaging/mainnet/config.ini +++ b/packaging/mainnet/config.ini @@ -67,9 +67,10 @@ reject_contract_consensus=true alarm_invalid_block=true big_bang=true -; Snapshots shared with the p2p neighbours (not active yet; the defaults are shown). +; Snapshots shared with the p2p neighbours (serve works; fetching at a first start comes later). Defaults shown. ;[snapshot] ;serve=off +; serve_rate_kbps and fetch_rate_kbps are in KB/s (2000 = 2 MB/s) ;serve_rate_kbps=2000 ;serve_max_peers=8 ;snapshot_blocks=101000 diff --git a/tests/csnode/snapshot_provider_tests.cpp b/tests/csnode/snapshot_provider_tests.cpp index 1736fc35d..2d4f126b1 100644 --- a/tests/csnode/snapshot_provider_tests.cpp +++ b/tests/csnode/snapshot_provider_tests.cpp @@ -168,9 +168,13 @@ TEST(SnapshotProvider, ManifestStatuses) { EXPECT_TRUE(parsed->verify()); EXPECT_EQ(parsed->seq, 200u); - // once per peer every 5 s + // once per peer every 5 s, one to anyone every 50 ms + fx.now += 50ms; EXPECT_FALSE(fx.provider.onManifestRequest(a, serving(), synced())); EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(2), serving(), synced())); + EXPECT_FALSE(fx.provider.onManifestRequest(peerKey(3), serving(), synced())); + fx.now += 50ms; + EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(3), serving(), synced())); fx.now += 5s; EXPECT_TRUE(fx.provider.onManifestRequest(a, serving(), synced())); @@ -295,17 +299,25 @@ TEST(SnapshotProvider, MaxPeers) { EXPECT_EQ(fx.provider.onPieceRequest(peerKey(2), piece("blocks.bin.lz4", 0), serving(0, 1), synced()).status, PieceStatus::Ok); } -TEST(SnapshotProvider, PeerTableEvictsTheOldest) { +TEST(SnapshotProvider, NewKeysDoNotEvictADownloader) { Fixture fx("snapshot_provider_table"); ASSERT_TRUE(fx.provider.load(fx.dir)); - EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(0), serving(), synced())); - EXPECT_FALSE(fx.provider.onManifestRequest(peerKey(0), serving(), synced())); - for (uint8_t i = 1; i <= cs::SnapshotProvider::kMaxPeers; ++i) { - fx.now += 1ms; + const auto downloader = peerKey(200); + const auto idle = peerKey(201); + EXPECT_TRUE(fx.provider.onManifestRequest(downloader, serving(), synced())); + EXPECT_EQ(fx.provider.onPieceRequest(downloader, piece("blocks.bin.lz4", 0), serving(), synced()).status, PieceStatus::Ok); + fx.now += 50ms; + EXPECT_TRUE(fx.provider.onManifestRequest(idle, serving(), synced())); + // 63 made-up keys fill the table to 65 entries + for (uint8_t i = 1; i < cs::SnapshotProvider::kMaxPeers; ++i) { + fx.now += 50ms; EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(i), serving(), synced())); } - // peer 0 was the oldest of 65: forgotten, so answered again within its 5 s - EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(0), serving(), synced())); + // the downloader is still known (its 5 s are not over), the idle key was the one pushed out + fx.now += 50ms; + EXPECT_FALSE(fx.provider.onManifestRequest(downloader, serving(), synced())); + fx.now += 50ms; + EXPECT_TRUE(fx.provider.onManifestRequest(idle, serving(), synced())); } TEST(SnapshotProvider, CompletedPeerAndSummary) { @@ -316,7 +328,14 @@ TEST(SnapshotProvider, CompletedPeerAndSummary) { EXPECT_EQ(seq, 200u); ++completed; }; - for (const auto& f : fx.manifest().files) { + const auto files = fx.manifest().files; + // the same chunk again and again does not make a whole snapshot + for (int i = 0; i < 100; ++i) { + fx.now += 1s; + ASSERT_EQ(fx.provider.onPieceRequest(peerKey(1), piece(files.front().name, 0), serving(), synced()).status, PieceStatus::Ok); + } + EXPECT_EQ(completed, 0); + for (const auto& f : files) { for (uint32_t c = 0; c < f.chunks.size(); ++c) { fx.now += 1s; ASSERT_EQ(fx.provider.onPieceRequest(peerKey(1), piece(f.name, c), serving(), synced()).status, PieceStatus::Ok); From f14681fabf1ea68943e015cd0bdbb7e2002543d7 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 13:29:09 +0100 Subject: [PATCH 5/6] snapshot provider: manifests to idle keys from a burst bucket, downloaders exempt Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_provider.hpp | 10 ++++--- csnode/src/snapshot_provider.cpp | 25 +++++++++++------ tests/csnode/snapshot_provider_tests.cpp | 31 +++++++++++++++++---- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/csnode/include/csnode/snapshot_provider.hpp b/csnode/include/csnode/snapshot_provider.hpp index 03325249a..5555f8122 100644 --- a/csnode/include/csnode/snapshot_provider.hpp +++ b/csnode/include/csnode/snapshot_provider.hpp @@ -48,9 +48,10 @@ class SnapshotProvider { static constexpr auto kPieceWindow = std::chrono::milliseconds(100); static constexpr size_t kPiecesPerWindow = 2; static constexpr auto kManifestInterval = std::chrono::seconds(5); - // between two manifest replies to anyone: peer keys cost nothing, so the per-key limit alone - // does not bound them - static constexpr auto kManifestGap = std::chrono::milliseconds(50); + // manifest replies to keys that are not downloading: peer keys cost nothing, so the per-key + // limit alone does not bound them; a peer downloading now is not held up by them + static constexpr double kManifestsPerSecond = 20; + static constexpr double kManifestBurst = 8; static constexpr uint64_t kBurstBytes = 4ull << 20; static constexpr uint32_t kMaxRetryMs = 5000; @@ -111,7 +112,8 @@ class SnapshotProvider { std::map peers_; Bucket global_; bool globalStarted_ = false; - std::chrono::steady_clock::time_point lastManifestAny_{}; + Bucket manifests_; + bool manifestsStarted_ = false; uint64_t piecesServed_ = 0; uint64_t bytesServed_ = 0; uint64_t busyAnswers_ = 0; diff --git a/csnode/src/snapshot_provider.cpp b/csnode/src/snapshot_provider.cpp index cce917578..af3caa879 100644 --- a/csnode/src/snapshot_provider.cpp +++ b/csnode/src/snapshot_provider.cpp @@ -215,17 +215,27 @@ std::optional SnapshotProvider::onManifestRequest(c return reply; } - // the manifest is the large reply to a small request: once per peer every 5 s, one to anyone - // every 50 ms, and paid for + // the manifest is the large reply to a small request: once per peer every 5 s, paid for, and + // to keys that are not downloading at most 20 a second (bursts of 8) const auto now = clock_(); - if (lastManifestAny_ != SteadyClock::time_point{} && now - lastManifestAny_ < kManifestGap) { - ++busyAnswers_; + const auto known = peers_.find(sender); + const bool downloading = known != peers_.end() && known->second.lastPiece != SteadyClock::time_point{} && + now - known->second.lastPiece < kPeerWindow; + if (known != peers_.end() && known->second.manifestSent && now - known->second.lastManifest < kManifestInterval) { return std::nullopt; } - auto& p = peer(sender, now); - if (p.manifestSent && now - p.lastManifest < kManifestInterval) { - return std::nullopt; + if (!downloading) { + if (!manifestsStarted_) { + manifests_ = Bucket{kManifestBurst, now}; + manifestsStarted_ = true; + } + uint32_t retry = 0; + if (!manifests_.take(1, kManifestsPerSecond, kManifestBurst, now, retry)) { + ++busyAnswers_; + return std::nullopt; + } } + auto& p = peer(sender, now); if (settings.rateKBps != 0) { const double rate = settings.rateKBps * 1000.0 / (state.trusted ? 2 : 1); const double burst = static_cast(kBurstBytes) / (state.trusted ? 2 : 1); @@ -241,7 +251,6 @@ std::optional SnapshotProvider::onManifestRequest(c } p.manifestSent = true; p.lastManifest = now; - lastManifestAny_ = now; ++manifestsServed_; reply.status = ManifestStatus::Ready; reply.manifestJson = loaded->json; diff --git a/tests/csnode/snapshot_provider_tests.cpp b/tests/csnode/snapshot_provider_tests.cpp index 2d4f126b1..28dba0801 100644 --- a/tests/csnode/snapshot_provider_tests.cpp +++ b/tests/csnode/snapshot_provider_tests.cpp @@ -168,13 +168,9 @@ TEST(SnapshotProvider, ManifestStatuses) { EXPECT_TRUE(parsed->verify()); EXPECT_EQ(parsed->seq, 200u); - // once per peer every 5 s, one to anyone every 50 ms - fx.now += 50ms; + // once per peer every 5 s EXPECT_FALSE(fx.provider.onManifestRequest(a, serving(), synced())); EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(2), serving(), synced())); - EXPECT_FALSE(fx.provider.onManifestRequest(peerKey(3), serving(), synced())); - fx.now += 50ms; - EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(3), serving(), synced())); fx.now += 5s; EXPECT_TRUE(fx.provider.onManifestRequest(a, serving(), synced())); @@ -299,6 +295,29 @@ TEST(SnapshotProvider, MaxPeers) { EXPECT_EQ(fx.provider.onPieceRequest(peerKey(2), piece("blocks.bin.lz4", 0), serving(0, 1), synced()).status, PieceStatus::Ok); } +TEST(SnapshotProvider, ManyKeysDoNotStarveADownloader) { + Fixture fx("snapshot_provider_manifests"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto downloader = peerKey(200); + EXPECT_TRUE(fx.provider.onManifestRequest(downloader, serving(), synced())); + EXPECT_EQ(fx.provider.onPieceRequest(downloader, piece("blocks.bin.lz4", 0), serving(), synced()).status, PieceStatus::Ok); + // made-up keys get a burst of 8 (the downloader took one), then 20 a second + size_t answered = 0; + for (uint8_t i = 1; i <= 40; ++i) { + answered += fx.provider.onManifestRequest(peerKey(i), serving(), synced()) ? 1 : 0; + } + EXPECT_EQ(answered, 7u); + fx.now += 1s; + answered = 0; + for (uint8_t i = 41; i <= 80; ++i) { + answered += fx.provider.onManifestRequest(peerKey(i), serving(), synced()) ? 1 : 0; + } + EXPECT_EQ(answered, 8u); + // the downloader is answered after its 5 s whatever the others did + fx.now += 4s; + EXPECT_TRUE(fx.provider.onManifestRequest(downloader, serving(), synced())); +} + TEST(SnapshotProvider, NewKeysDoNotEvictADownloader) { Fixture fx("snapshot_provider_table"); ASSERT_TRUE(fx.provider.load(fx.dir)); @@ -310,7 +329,7 @@ TEST(SnapshotProvider, NewKeysDoNotEvictADownloader) { EXPECT_TRUE(fx.provider.onManifestRequest(idle, serving(), synced())); // 63 made-up keys fill the table to 65 entries for (uint8_t i = 1; i < cs::SnapshotProvider::kMaxPeers; ++i) { - fx.now += 50ms; + fx.now += 50ms; // within 20 a second EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(i), serving(), synced())); } // the downloader is still known (its 5 s are not over), the idle key was the one pushed out From 0e1f8b0b518011a1ea17d2eef3ff9cb9df0cbe25 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 14:32:50 +0100 Subject: [PATCH 6/6] run.sh: serving checks without a silent exit; snapshot-p2p-serve serves from odd nodes (mixed runs) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 5 +++-- ci/testnet/run.sh | 12 +++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 18c5af90d..fb057c733 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -250,9 +250,10 @@ jobs: # host_data; a stopped node lists them - name: snapshot-capability env: SNAPSHOT_SERVE_NODES=3,5 - # a pruned LMDB node and a full RocksDB one serve, the pruned one while it prunes + # a pruned LMDB node and a full RocksDB one serve, the pruned one while it prunes (odd nodes: + # the even ones run node_alt_ref in a mixed run) - name: snapshot-p2p-serve - env: NODE_STORAGE=berkeleydb,berkeleydb,berkeleydb,berkeleydb,berkeleydb,lmdb:pruned,rocksdb SNAPSHOT_SERVE_NODES=6,7 TARGET_SEQ=2600 + env: NODE_STORAGE=berkeleydb,berkeleydb,berkeleydb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb SNAPSHOT_SERVE_NODES=5,7 TARGET_SEQ=2600 # random datagrams to every node's p2p port - name: garbage-datagrams env: GARBAGE_AT=150 GARBAGE_RATE=300 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 1a2cf17e5..d536ffa40 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -204,6 +204,12 @@ SNAPSHOT_AT=${SNAPSHOT_AT:-0} SNAPSHOT_FORMAT=${SNAPSHOT_FORMAT:-2} SNAPSHOT_CORRUPT=${SNAPSHOT_CORRUPT:-0} SNAPSHOT_SERVE_NODES=${SNAPSHOT_SERVE_NODES:-} +if [ -n "$BIN_ALT" ]; then + for i in ${SNAPSHOT_SERVE_NODES//,/ }; do + # the even nodes run BIN_ALT, which may not serve snapshots + [ $((i % 2)) -eq 1 ] || { echo "FAIL: SNAPSHOT_SERVE_NODES names n$i, an even node, which runs BIN_ALT"; exit 1; } + done +fi STORAGE_SWITCH_AT=${STORAGE_SWITCH_AT:-0} CONVERT_AT=${CONVERT_AT:-0} INDEX_LOSS_AT=${INDEX_LOSS_AT:-0} @@ -1325,9 +1331,9 @@ if [ -n "$SNAPSHOT_SERVE_NODES" ]; then # checkpoint at 1'000 on a thread of low priority: a minute at most) for i in ${SNAPSHOT_SERVE_NODES//,/ }; do for _ in $(seq 1 60); do grep -aq 'SNAPSHOT: serving block' "$WORK/n$i/node.log" && break; sleep 1; done - built=$(grep -a 'SNAPSHOT: built block' "$WORK/n$i/node.log" | tail -1 | sed -E 's/.*(built block [0-9]+[^,]*, [^,]*, [^,]*).*/\1/') - serving=$(grep -a 'SNAPSHOT: serving block' "$WORK/n$i/node.log" | tail -1 | sed -E 's/.*SNAPSHOT: //') - [ -n "$built" ] && [ -n "$serving" ] || { echo "FAIL: n$i did not build and serve a snapshot"; grep -a 'SNAPSHOT' "$WORK/n$i/node.log" | tail -5; exit 1; } + built=$({ grep -a 'SNAPSHOT: built block' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E 's/.*(built block [0-9]+[^,]*, [^,]*, [^,]*).*/\1/') + serving=$({ grep -a 'SNAPSHOT: serving block' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E 's/.*SNAPSHOT: //') + [ -n "$built" ] && [ -n "$serving" ] || { echo "FAIL: n$i did not build and serve a snapshot"; { grep -a 'SNAPSHOT' "$WORK/n$i/node.log" || true; } | tail -5; exit 1; } echo "snapshot: n$i $built; $serving" done # known_peers.json is written at a clean stop: stop one node of this build that does not serve