diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 4fb0d3b59..fb057c733 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -246,9 +246,14 @@ jobs: # slow nodes: 5% of one CPU for two trusted nodes - name: slow-nodes env: SLOW_NODES=3,6 SLOW_CPU=5 - # snapshot sharing announced in host_data: n3 and n5 serve, a stopped node lists them + # snapshot sharing: n3 (full) and n5 serve, build their snapshots and announce them in + # host_data; a stopped node lists them - name: snapshot-capability env: SNAPSHOT_SERVE_NODES=3,5 + # a pruned LMDB node and a full RocksDB one serve, the pruned one while it prunes (odd nodes: + # the even ones run node_alt_ref in a mixed run) + - name: snapshot-p2p-serve + env: NODE_STORAGE=berkeleydb,berkeleydb,berkeleydb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb SNAPSHOT_SERVE_NODES=5,7 TARGET_SEQ=2600 # random datagrams to every node's p2p port - name: garbage-datagrams env: GARBAGE_AT=150 GARBAGE_RATE=300 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index e09468b90..d536ffa40 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -67,9 +67,10 @@ # snapshot with --import-snapshot into its own backend and joins; with SNAPSHOT_CORRUPT=1 the # blocks file is truncated first, so the restore must fail and that node syncs from block 0. # SNAPSHOT_FORMAT=1: packaging/docker/snapshot-make.sh and snapshot-restore.sh instead -# SNAPSHOT_SERVE_NODES (optional, comma separated): those nodes run with [snapshot] serve=on; -# at the end one other node of this build is stopped and its known_peers.json must list each -# of them with the snapshot capability, and no other node with it +# SNAPSHOT_SERVE_NODES (optional, comma separated): those nodes run with [snapshot] serve=on +# and a checkpoint every 1'000 blocks; each must build the snapshot of a checkpoint and serve +# it; at the end one other node of this build is stopped and its known_peers.json must list +# each of them with the snapshot capability, and no other node with it # STORAGE_SWITCH_AT (default 0 = off, needs PRUNE_NODE=6 and DB_BACKEND=lmdb): at that sequence # node 5 is restarted as a pruned node and node 6 as a full one # CONVERT_AT (default 0 = off, needs DB_BACKEND=berkeleydb): at that sequence n4 is converted to @@ -203,6 +204,12 @@ SNAPSHOT_AT=${SNAPSHOT_AT:-0} SNAPSHOT_FORMAT=${SNAPSHOT_FORMAT:-2} SNAPSHOT_CORRUPT=${SNAPSHOT_CORRUPT:-0} SNAPSHOT_SERVE_NODES=${SNAPSHOT_SERVE_NODES:-} +if [ -n "$BIN_ALT" ]; then + for i in ${SNAPSHOT_SERVE_NODES//,/ }; do + # the even nodes run BIN_ALT, which may not serve snapshots + [ $((i % 2)) -eq 1 ] || { echo "FAIL: SNAPSHOT_SERVE_NODES names n$i, an even node, which runs BIN_ALT"; exit 1; } + done +fi STORAGE_SWITCH_AT=${STORAGE_SWITCH_AT:-0} CONVERT_AT=${CONVERT_AT:-0} INDEX_LOSS_AT=${INDEX_LOSS_AT:-0} @@ -355,6 +362,10 @@ storage_section() { # that the kills hit the moments right after one lines="checkpoint_every=1000\ncheckpoint_every_minutes=1\ncheckpoint_keep=3\n" fi + if [[ ",$SNAPSHOT_SERVE_NODES," == *",$i,"* ]] && [[ "$lines" != *checkpoint_every=* ]]; then + # a serving node builds its snapshots from the periodic checkpoints + lines="${lines}checkpoint_every=1000\ncheckpoint_keep=2\n" + fi [ -n "$backend" ] && lines="${lines}db_backend=$backend\n" [ -n "$lines" ] && printf "\n[storage]\n$lines" return 0 @@ -1316,6 +1327,15 @@ if [ "$LOW_STAKE_NODES" -gt 0 ]; then echo "stake: none of the $LOW_STAKE_NODES nodes below the minimum stake was trusted after round $from" fi if [ -n "$SNAPSHOT_SERVE_NODES" ]; then + # every serving node built the snapshot of a checkpoint and serves it (the build follows the + # checkpoint at 1'000 on a thread of low priority: a minute at most) + for i in ${SNAPSHOT_SERVE_NODES//,/ }; do + for _ in $(seq 1 60); do grep -aq 'SNAPSHOT: serving block' "$WORK/n$i/node.log" && break; sleep 1; done + built=$({ grep -a 'SNAPSHOT: built block' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E 's/.*(built block [0-9]+[^,]*, [^,]*, [^,]*).*/\1/') + serving=$({ grep -a 'SNAPSHOT: serving block' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E 's/.*SNAPSHOT: //') + [ -n "$built" ] && [ -n "$serving" ] || { echo "FAIL: n$i did not build and serve a snapshot"; { grep -a 'SNAPSHOT' "$WORK/n$i/node.log" || true; } | tail -5; exit 1; } + echo "snapshot: n$i $built; $serving" + done # known_peers.json is written at a clean stop: stop one node of this build that does not serve observer=0 for i in $(seq 1 "$NODES"); do diff --git a/client/config/config.hpp b/client/config/config.hpp index a55896fe4..386869696 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -75,7 +75,7 @@ struct StorageData { // observer: every key but fetch* applies while the node runs. struct SnapshotData { bool serve = false; // build snapshots and answer snapshot requests - size_t serveRateKbps = 2000; // upload cap of all peers together; 0: unlimited + size_t serveRateKbps = 2000; // upload cap of all peers together, KB/s (2000: 2 MB/s); 0: unlimited size_t serveMaxPeers = 8; // peers served within 30 s; more get busy size_t snapshotBlocks = 101'000; // blocks before the checkpoint in a snapshot std::string fetch = "auto"; // auto: fetch at a start on an empty database; off: never diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index 7d6a2f6ab..a79e337a9 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -123,9 +123,8 @@ int run(const Config& config, const std::string& outDir) { in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) { return db->get(static_cast(seq), &out); }; - in.contracts = [&db](const std::function& visit) { - return db->forEachContractData(visit); - }; + // no contract records: the contracts database holds each contract's latest state, later than the + // checkpoint; the checkpoint carries the states at its block say() << "checkpoint " << S << ", blocks from " << std::max(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0) << ", into " << outDir << std::endl; diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 3eca33681..7d1332a2b 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -364,6 +364,11 @@ class Storage final { */ cs::Sequence first_sequence() const; + /** + * The stored bytes of block sequence, as the database holds them (no parsing, no pool cache). + */ + bool pool_bytes(cs::Sequence sequence, cs::Bytes& out) const; + /** * Returns the space of pruned pools to the file system. */ diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index c4b66a62b..a6f98f35e 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -1350,6 +1350,14 @@ bool Storage::pool_prune_batch(const std::vector& hashes) { return true; } +bool Storage::pool_bytes(cs::Sequence sequence, cs::Bytes& out) const { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + return d->db->get(static_cast(sequence), &out); +} + cs::Sequence Storage::first_sequence() const { if (!isOpen()) { d->set_last_error(NotOpen); diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index c20628610..0d23c1f8f 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -73,6 +73,8 @@ add_library(csnode include/csnode/snapshot_builder.hpp include/csnode/snapshot_importer.hpp include/csnode/snapshot_messages.hpp + include/csnode/snapshot_provider.hpp + include/csnode/snapshot_service.hpp include/csnode/blockverifier.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -118,6 +120,8 @@ add_library(csnode src/snapshot_builder.cpp src/snapshot_importer.cpp src/snapshot_messages.cpp + src/snapshot_provider.cpp + src/snapshot_service.cpp src/blockverifier.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 4495eb6d7..ca925b0d7 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -95,6 +95,10 @@ class BlockChain { // pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were // removed once a later periodic checkpoint made them unnecessary cs::Sequence getFirstStoredSequence() const; + // the stored bytes of a block, under the storage lock, for a snapshot: no parsing, no pool cache + bool blockBytes(cs::Sequence sequence, cs::Bytes& out) const; + // pruned storage: the blocks below this one are being or will be removed (0: nothing scheduled) + cs::Sequence pruneTarget() const; // return unique id of database if at least one unique block has written, otherwise (only genesis block) 0 uint64_t uuid() const; @@ -303,6 +307,8 @@ public signals: cs::StopReadingBlocksSignal stopReadingBlocksEvent; cs::SuccessQSSignal successfullQuickStartEvent; + // a periodic checkpoint of this block was saved (live), with its state digest + cs::Signal checkpointSavedEvent; public slots: diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index 072c772e2..b2dfffaf4 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -1,6 +1,7 @@ #ifndef NODE_HPP #define NODE_HPP +#include #include #include #include @@ -26,6 +27,12 @@ class Transport; +namespace cs { +class SnapshotProvider; +class SnapshotService; +struct SnapshotNodeState; +} + namespace cs { class SolverCore; } @@ -425,6 +432,9 @@ public slots: private: // snapshot messages dropped: malformed, or not expected by this node uint64_t snapshotMessagesDropped_ = 0; + // the provider's view of this node: last block, round, synced, trusted + void snapshotNodeState(cs::SnapshotNodeState& state) const; + void logSnapshotSummary(); bool init(); void initPoolSynchronizer(); @@ -497,6 +507,11 @@ public slots: std::unique_ptr api_; #endif + // snapshots served to p2p neighbours ([snapshot] serve) + std::unique_ptr snapshotProvider_; + std::unique_ptr snapshotService_; + std::chrono::steady_clock::time_point snapshotSummaryAt_{}; + uint32_t startPacketRequestPoint_ = 0; // ms timeout diff --git a/csnode/include/csnode/snapshot_builder.hpp b/csnode/include/csnode/snapshot_builder.hpp index 84b6d8831..86bfda068 100644 --- a/csnode/include/csnode/snapshot_builder.hpp +++ b/csnode/include/csnode/snapshot_builder.hpp @@ -31,7 +31,7 @@ struct SnapshotBuildInput { std::function blockBytes; std::function&)> contracts; std::string nodeVersion; - uint64_t chunkSize = 4ull << 20; + uint64_t chunkSize = 512ull << 10; // one p2p piece (design 4.3) const std::atomic* stop = nullptr; // checked between blocks: true aborts the build }; diff --git a/csnode/include/csnode/snapshot_provider.hpp b/csnode/include/csnode/snapshot_provider.hpp new file mode 100644 index 000000000..5555f8122 --- /dev/null +++ b/csnode/include/csnode/snapshot_provider.hpp @@ -0,0 +1,125 @@ +#ifndef SNAPSHOT_PROVIDER_HPP +#define SNAPSHOT_PROVIDER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace cs { + +// what the node is doing when a snapshot request arrives +struct SnapshotNodeState { + cs::Sequence lastSeq = 0; + cs::RoundNumber round = 0; + bool synced = false; // design section 5.2 + bool trusted = false; // a confidant or the writer of the current round: half the budget +}; + +// Answers the snapshot requests of p2p neighbours from the newest snapshot this node built +// (docs/specs/2026-10-25-snapshot-p2p-design.md, section 5). Used on the node's processor thread; +// load() and unload() come from the snapshot service's thread. It keeps no state per requester +// beyond a table of the peers served in the last 30 s (at most 64), and every reply is bounded +// by a global token bucket and a per-peer one. +class SnapshotProvider { +public: + using Clock = std::function; + + struct Settings { + bool serve = false; + size_t rateKBps = 2000; // all peers together, KB/s; 0: unlimited + size_t maxPeers = 8; // peers served within 30 s + }; + + using NodeState = SnapshotNodeState; + + static constexpr size_t kMaxPeers = 64; + static constexpr auto kPeerWindow = std::chrono::seconds(30); + static constexpr auto kPieceWindow = std::chrono::milliseconds(100); + static constexpr size_t kPiecesPerWindow = 2; + static constexpr auto kManifestInterval = std::chrono::seconds(5); + // manifest replies to keys that are not downloading: peer keys cost nothing, so the per-key + // limit alone does not bound them; a peer downloading now is not held up by them + static constexpr double kManifestsPerSecond = 20; + static constexpr double kManifestBurst = 8; + static constexpr uint64_t kBurstBytes = 4ull << 20; + static constexpr uint32_t kMaxRetryMs = 5000; + + explicit SnapshotProvider(Clock clock = std::chrono::steady_clock::now); + ~SnapshotProvider(); + + // dir = snapshot/: its snapshot.json must parse, verify, name S, and its files must have the + // declared sizes; on failure the previous snapshot stays loaded + bool load(const std::filesystem::path& dir, std::string* error = nullptr); + void unload(); + std::optional loadedSeq() const; + + // nullopt: no answer (the peer asked again too soon, or the budget is spent; it asks again) + std::optional onManifestRequest(const cs::PublicKey& sender, const Settings& settings, + const NodeState& state); + snapshot_p2p::PieceReply onPieceRequest(const cs::PublicKey& sender, const snapshot_p2p::PieceRequest& request, + const Settings& settings, const NodeState& state); + + // a line about the traffic since the last call, empty when there was none + std::string takeSummary(); + // a peer that just received the last byte of a whole snapshot: once per peer and snapshot + std::function onPeerCompleted; + +private: + struct Loaded; + + struct Bucket { + double tokens = 0; + std::chrono::steady_clock::time_point at{}; + // refills at rate bytes/s up to burst; true and the cost taken when there is enough + bool take(double cost, double rate, double burst, std::chrono::steady_clock::time_point now, uint32_t& retryMs); + }; + + struct Peer { + std::chrono::steady_clock::time_point lastSeen{}; + std::chrono::steady_clock::time_point lastPiece{}; + std::chrono::steady_clock::time_point firstPiece{}; + std::chrono::steady_clock::time_point windowStart{}; + size_t piecesInWindow = 0; + std::chrono::steady_clock::time_point lastManifest{}; + bool manifestSent = false; + Bucket bucket; + bool bucketStarted = false; + cs::Sequence seq = 0; + uint64_t bytes = 0; // of seq + std::vector chunksSeen; // of seq, over all files in manifest order: re-requests do not count + bool completed = false; + }; + + Peer& peer(const cs::PublicKey& key, std::chrono::steady_clock::time_point now); + size_t activePeers(std::chrono::steady_clock::time_point now, const cs::PublicKey& except) const; + + Clock clock_; + mutable std::mutex loadedLock_; + std::shared_ptr loaded_; + + // processor thread only + std::map peers_; + Bucket global_; + bool globalStarted_ = false; + Bucket manifests_; + bool manifestsStarted_ = false; + uint64_t piecesServed_ = 0; + uint64_t bytesServed_ = 0; + uint64_t busyAnswers_ = 0; + uint64_t manifestsServed_ = 0; +}; + +} // namespace cs + +#endif // SNAPSHOT_PROVIDER_HPP diff --git a/csnode/include/csnode/snapshot_service.hpp b/csnode/include/csnode/snapshot_service.hpp new file mode 100644 index 000000000..71ac84629 --- /dev/null +++ b/csnode/include/csnode/snapshot_service.hpp @@ -0,0 +1,59 @@ +#ifndef SNAPSHOT_SERVICE_HPP +#define SNAPSHOT_SERVICE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +class BlockChain; + +namespace cs { + +// Snapshots served by this node to its p2p neighbours: every 10 s it reads [snapshot] from the +// configuration; while serve is on, it builds the snapshot of each new periodic checkpoint into +// snapshot/ at low priority (and of the latest one when serving starts) and hands the newest complete +// one to the provider, which answers the requests; serve off unloads it. Whether the node is synced +// is the provider's question, asked per request. +class SnapshotService { +public: + SnapshotService(BlockChain& blockchain, SnapshotProvider& provider, const cs::PrivateKey& key, std::string nodeVersion); + ~SnapshotService(); + + void start(); + void stop(); + // slot of BlockChain::checkpointSavedEvent: a new periodic checkpoint to build from + void onCheckpointSaved(cs::Sequence sequence); + +private: + void run(); + void buildLatest(); + // the newest snapshot/ with a snapshot.json, into the provider + void loadNewest(); + + BlockChain& blockchain_; + cs::PrivateKey key_; + std::string nodeVersion_; + std::filesystem::path root_ = "snapshot"; + SnapshotProvider& provider_; + bool serving_ = false; + std::thread thread_; + std::atomic stop_{false}; + std::atomic buildPending_{true}; + static constexpr int kMaxBuildAttempts = 5; + std::atomic failedBuilds_{0}; // reset by onCheckpointSaved on the thread that stores blocks + std::chrono::steady_clock::time_point retryAt_{}; + std::mutex m_; + std::condition_variable cv_; +}; + +} // namespace cs + +#endif // SNAPSHOT_SERVICE_HPP diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index d1daa1183..49304db0e 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1053,6 +1053,7 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { if (serializationManPtr_->save(pool.sequence(), head, digest)) { cslog() << cs::stateDigestLine(pool.sequence(), digest.digest, digest.wallets) << " checkpoint"; serializationManPtr_->pruneCheckpoints(sto.checkpointKeep); + emit checkpointSavedEvent(pool.sequence()); lastCheckpointWallClock_ = now; if (byCount) { schedulePruning(pool.sequence()); @@ -1064,6 +1065,16 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { } } +bool BlockChain::blockBytes(cs::Sequence sequence, cs::Bytes& out) const { + std::lock_guard lock(dbLock_); + return storage_.pool_bytes(sequence, out); +} + +cs::Sequence BlockChain::pruneTarget() const { + std::lock_guard lock(dbLock_); + return pruneUpTo_; +} + cs::CheckpointDigest BlockChain::checkpointDigest() const { cs::CheckpointDigest result; const auto& wallets = multiWallets(); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index de2ef8c84..b1ee9c4e7 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -259,6 +262,29 @@ bool Node::init() { api_->run(); #endif // NODE_API + // snapshots served to p2p neighbours: [snapshot] serve, read again every 10 s (off by default) + { + std::string version; + std::ifstream versionFile("VERSION"); + if (!versionFile || !std::getline(versionFile, version) || version.empty() || version.size() > 64) { + version = "4.2." + std::to_string(Config::getNodeVersion()) + "." + std::to_string(Config::getMinorNodeVersion()); + } + for (auto& c : version) { + if (c < 0x20 || c >= 0x7f || c == '"' || c == '\\') { + c = '_'; + } + } + snapshotProvider_ = std::make_unique(); + snapshotSummaryAt_ = std::chrono::steady_clock::now(); + snapshotProvider_->onPeerCompleted = [](const cs::PublicKey& key, cs::Sequence seq, uint64_t bytes, std::chrono::seconds took) { + cslog() << "SNAPSHOT: served block " << seq << " to " << EncodeBase58(key.data(), key.data() + key.size()) << ", " + << bytes / (1u << 20) << " MB in " << took.count() << " s"; + }; + snapshotService_ = std::make_unique(blockChain_, *snapshotProvider_, solver_->getPrivateKey(), version); + cs::Connector::connect(&blockChain_.checkpointSavedEvent, snapshotService_.get(), &cs::SnapshotService::onCheckpointSaved); + snapshotService_->start(); + } + if (!transport_->isGood()) { return false; } @@ -358,6 +384,11 @@ void Node::destroy() { api_->stop(); cswarning() << "[API STOPPED]"; + if (snapshotService_) { + snapshotService_->stop(); + cswarning() << "[SNAPSHOT SERVICE STOPPED]"; + } + solver_->finish(); cswarning() << "[SOLVER STOPPED]"; @@ -2162,17 +2193,53 @@ void logSnapshotDrop(uint64_t& dropped, const char* what, const cs::PublicKey& s } } // namespace +void Node::snapshotNodeState(cs::SnapshotNodeState& state) const { + state.lastSeq = blockChain_.getLastSeq(); + state.round = cs::Conveyer::instance().currentRoundNumber(); + // design 5.2: not syncing, not in a bootstrap round, at most 3 blocks behind a known round + state.synced = state.round != 0 && (!poolSynchronizer_ || !poolSynchronizer_->isSyncroStarted()) && !isBootstrapRound_ && + state.lastSeq + 3 >= state.round; + state.trusted = myLevel_ != Level::Normal; +} + +namespace { +cs::SnapshotProvider::Settings snapshotProviderSettings() { + const auto& cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + cs::SnapshotProvider::Settings settings; + settings.serve = cfg.serve; + settings.rateKBps = cfg.serveRateKbps; + settings.maxPeers = cfg.serveMaxPeers; + return settings; +} +} // namespace + +void Node::logSnapshotSummary() { + const auto now = std::chrono::steady_clock::now(); + if (now - snapshotSummaryAt_ < std::chrono::hours(1)) { + return; + } + snapshotSummaryAt_ = now; + const auto summary = snapshotProvider_->takeSummary(); + if (!summary.empty()) { + cslog() << "SNAPSHOT: in the last hour " << summary; + } +} + void Node::getSnapshotManifestRequest(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { cs::snapshot_p2p::ManifestRequest request; if (!cs::snapshot_p2p::decode(data, size, request)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed manifest request", sender); } - // this build does not serve snapshots yet - cs::snapshot_p2p::ManifestReply reply; - reply.status = cs::snapshot_p2p::ManifestStatus::NotServing; - reply.lastSeq = blockChain_.getLastSeq(); - reply.round = cs::Conveyer::instance().currentRoundNumber(); - sendSnapshotMessage(sender, MsgTypes::SnapshotManifestReply, cs::snapshot_p2p::encode(reply)); + if (!snapshotProvider_) { + return; + } + cs::SnapshotNodeState state; + snapshotNodeState(state); + const auto reply = snapshotProvider_->onManifestRequest(sender, snapshotProviderSettings(), state); + if (reply) { + sendSnapshotMessage(sender, MsgTypes::SnapshotManifestReply, cs::snapshot_p2p::encode(*reply)); + } + logSnapshotSummary(); } void Node::getSnapshotManifestReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { @@ -2189,12 +2256,14 @@ void Node::getSnapshotPieceRequest(const uint8_t* data, const size_t size, const if (!cs::snapshot_p2p::decode(data, size, request)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed piece request", sender); } - cs::snapshot_p2p::PieceReply reply; - reply.status = cs::snapshot_p2p::PieceStatus::NotServing; - reply.seq = request.seq; - reply.name = request.name; - reply.chunk = request.chunk; + if (!snapshotProvider_) { + return; + } + cs::SnapshotNodeState state; + snapshotNodeState(state); + const auto reply = snapshotProvider_->onPieceRequest(sender, request, snapshotProviderSettings(), state); sendSnapshotMessage(sender, MsgTypes::SnapshotPieceReply, cs::snapshot_p2p::encode(reply)); + logSnapshotSummary(); } void Node::getSnapshotPieceReply(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index f9fd80fbc..5b1938ea6 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -175,15 +175,6 @@ std::string utcNow() { return buf; } -bool readFile(const std::filesystem::path& path, std::vector& out) { - std::ifstream in(path, std::ios::binary); - if (!in) { - return false; - } - out.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); - return !in.bad(); -} - bool stopped(const SnapshotBuildInput& in) { return in.stop && in.stop->load(); } @@ -332,14 +323,28 @@ SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::fi if (!std::filesystem::exists(src, ec)) { continue; // tokens.dat and apihandler.dat exist only in builds with the API } - std::vector data; - if (!readFile(src, data)) { + // in pieces: walletscache.dat is about 180 MB on mainnet, and the builder runs inside the node + std::ifstream in(src, std::ios::binary); + if (!in) { return failWith("cannot read " + src.string()); } SnapshotFile f; f.name = "qs-" + name + ".lz4"; Lz4Writer w(tmp / f.name, m.chunkSize); - w.write(data); + std::vector piece(snapshot_lz4::kBlockSize); + while (in) { + if (stopped(input)) { + return failWith("stopped"); + } + in.read(reinterpret_cast(piece.data()), static_cast(piece.size())); + const auto n = in.gcount(); + if (n > 0) { + w.write(piece.data(), static_cast(n)); + } + } + if (in.bad()) { + return failWith("cannot read " + src.string()); + } if (!w.finish(f)) { return failWith("cannot write " + f.name); } diff --git a/csnode/src/snapshot_provider.cpp b/csnode/src/snapshot_provider.cpp new file mode 100644 index 000000000..af3caa879 --- /dev/null +++ b/csnode/src/snapshot_provider.cpp @@ -0,0 +1,389 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; +using SteadyClock = std::chrono::steady_clock; + +bool fail(std::string* error, const std::string& text) { + if (error) { + *error = text; + } + return false; +} + +bool powerOfTwo(uint64_t v) { + return v != 0 && (v & (v - 1)) == 0; +} + +uint32_t msUntil(SteadyClock::time_point now, SteadyClock::time_point when) { + const auto ms = std::chrono::duration_cast(when - now).count(); + return static_cast(std::clamp(ms, 1, SnapshotProvider::kMaxRetryMs)); +} +} // namespace + +struct SnapshotProvider::Loaded { + struct File { + int fd = -1; + uint64_t size = 0; + uint64_t chunks = 0; + uint64_t firstChunk = 0; // index of its first chunk among all files, in manifest order + }; + + SnapshotManifest manifest; + std::string json; + std::map files; + uint64_t totalBytes = 0; + uint64_t totalChunks = 0; + + ~Loaded() { + for (auto& [name, f] : files) { + if (f.fd >= 0) { + ::close(f.fd); + } + } + } +}; + +bool SnapshotProvider::Bucket::take(double cost, double rate, double burst, SteadyClock::time_point now, uint32_t& retryMs) { + const double elapsed = std::chrono::duration(now - at).count(); + tokens = std::min(burst, tokens + std::max(0.0, elapsed) * rate); + at = now; + if (tokens >= cost) { + tokens -= cost; + return true; + } + const double ms = std::ceil((cost - tokens) / rate * 1000.0); + retryMs = static_cast(std::clamp(ms, 1.0, static_cast(kMaxRetryMs))); + return false; +} + +SnapshotProvider::SnapshotProvider(Clock clock) +: clock_(std::move(clock)) { +} + +SnapshotProvider::~SnapshotProvider() = default; + +bool SnapshotProvider::load(const fs::path& dir, std::string* error) { + cs::Sequence seq = 0; + try { + size_t used = 0; + const auto name = dir.filename().string(); + seq = std::stoull(name, &used); + if (used != name.size()) { + return fail(error, "not a snapshot folder: " + dir.string()); + } + } + catch (const std::exception&) { + return fail(error, "not a snapshot folder: " + dir.string()); + } + + std::error_code ec; + const auto jsonPath = dir / "snapshot.json"; + const auto jsonSize = fs::file_size(jsonPath, ec); + if (ec || jsonSize > snapshot_p2p::kMaxManifestJson) { + return fail(error, "no snapshot.json of at most 256 KB in " + dir.string()); + } + std::ifstream in(jsonPath, std::ios::binary); + std::string json((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + if (!in.good() && !in.eof()) { + return fail(error, "cannot read " + jsonPath.string()); + } + + std::string why; + auto manifest = SnapshotManifest::fromJson(json, &why); + if (!manifest) { + return fail(error, jsonPath.string() + ": " + why); + } + if (manifest->seq != seq) { + return fail(error, jsonPath.string() + " is the snapshot of block " + std::to_string(manifest->seq)); + } + if (!manifest->verify()) { + return fail(error, jsonPath.string() + ": bad signature"); + } + if (manifest->chunkSize > snapshot_p2p::kMaxPieceBytes || !powerOfTwo(manifest->chunkSize)) { + return fail(error, jsonPath.string() + ": chunk size " + std::to_string(manifest->chunkSize) + " cannot be served"); + } + + auto loaded = std::make_shared(); + for (const auto& f : manifest->files) { + // the path comes from the manifest this node built and verified, never from a request + const auto path = dir / f.name; + Loaded::File entry; + entry.fd = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (entry.fd < 0) { + return fail(error, "cannot open " + path.string()); + } + loaded->files[f.name] = entry; // closed with loaded from here on + struct stat st {}; + if (::fstat(entry.fd, &st) != 0 || static_cast(st.st_size) != f.size) { + return fail(error, path.string() + " does not have the size of the manifest"); + } + // pieces are read on the thread that handles consensus messages: keep the files in the page cache + ::posix_fadvise(entry.fd, 0, 0, POSIX_FADV_WILLNEED); + loaded->files[f.name].size = f.size; + loaded->files[f.name].chunks = f.chunks.size(); + loaded->files[f.name].firstChunk = loaded->totalChunks; + loaded->totalChunks += f.chunks.size(); + loaded->totalBytes += f.size; + } + loaded->manifest = std::move(*manifest); + loaded->json = std::move(json); + + std::lock_guard lock(loadedLock_); + loaded_ = std::move(loaded); + return true; +} + +void SnapshotProvider::unload() { + std::lock_guard lock(loadedLock_); + loaded_.reset(); +} + +std::optional SnapshotProvider::loadedSeq() const { + std::lock_guard lock(loadedLock_); + if (!loaded_) { + return std::nullopt; + } + return loaded_->manifest.seq; +} + +SnapshotProvider::Peer& SnapshotProvider::peer(const cs::PublicKey& key, SteadyClock::time_point now) { + auto it = peers_.find(key); + if (it == peers_.end()) { + if (peers_.size() >= kMaxPeers) { + // a peer downloading now goes last: new keys (free to make up) must not push it out + auto downloading = [&](const Peer& p) { return p.lastPiece != SteadyClock::time_point{} && now - p.lastPiece < kPeerWindow; }; + auto oldest = std::min_element(peers_.begin(), peers_.end(), [&](const auto& a, const auto& b) { + const bool da = downloading(a.second); + const bool db = downloading(b.second); + return da != db ? !da : a.second.lastSeen < b.second.lastSeen; + }); + peers_.erase(oldest); + } + it = peers_.emplace(key, Peer{}).first; + } + it->second.lastSeen = now; + return it->second; +} + +size_t SnapshotProvider::activePeers(SteadyClock::time_point now, const cs::PublicKey& except) const { + size_t n = 0; + for (const auto& [key, p] : peers_) { + if (key != except && p.lastPiece != SteadyClock::time_point{} && now - p.lastPiece < kPeerWindow) { + ++n; + } + } + return n; +} + +std::optional SnapshotProvider::onManifestRequest(const cs::PublicKey& sender, const Settings& settings, + const NodeState& state) { + using snapshot_p2p::ManifestStatus; + snapshot_p2p::ManifestReply reply; + reply.lastSeq = state.lastSeq; + reply.round = state.round; + reply.servingTrusted = state.trusted; + + std::shared_ptr loaded; + { + std::lock_guard lock(loadedLock_); + loaded = loaded_; + } + if (!settings.serve) { + reply.status = ManifestStatus::NotServing; + return reply; + } + if (!loaded) { + reply.status = ManifestStatus::Building; + return reply; + } + if (!state.synced) { + reply.status = ManifestStatus::NotSynced; + return reply; + } + + // the manifest is the large reply to a small request: once per peer every 5 s, paid for, and + // to keys that are not downloading at most 20 a second (bursts of 8) + const auto now = clock_(); + const auto known = peers_.find(sender); + const bool downloading = known != peers_.end() && known->second.lastPiece != SteadyClock::time_point{} && + now - known->second.lastPiece < kPeerWindow; + if (known != peers_.end() && known->second.manifestSent && now - known->second.lastManifest < kManifestInterval) { + return std::nullopt; + } + if (!downloading) { + if (!manifestsStarted_) { + manifests_ = Bucket{kManifestBurst, now}; + manifestsStarted_ = true; + } + uint32_t retry = 0; + if (!manifests_.take(1, kManifestsPerSecond, kManifestBurst, now, retry)) { + ++busyAnswers_; + return std::nullopt; + } + } + auto& p = peer(sender, now); + if (settings.rateKBps != 0) { + const double rate = settings.rateKBps * 1000.0 / (state.trusted ? 2 : 1); + const double burst = static_cast(kBurstBytes) / (state.trusted ? 2 : 1); + if (!globalStarted_) { + global_ = Bucket{burst, now}; + globalStarted_ = true; + } + uint32_t retry = 0; + if (!global_.take(static_cast(loaded->json.size()), rate, burst, now, retry)) { + ++busyAnswers_; + return std::nullopt; + } + } + p.manifestSent = true; + p.lastManifest = now; + ++manifestsServed_; + reply.status = ManifestStatus::Ready; + reply.manifestJson = loaded->json; + return reply; +} + +snapshot_p2p::PieceReply SnapshotProvider::onPieceRequest(const cs::PublicKey& sender, const snapshot_p2p::PieceRequest& request, + const Settings& settings, const NodeState& state) { + using snapshot_p2p::PieceStatus; + snapshot_p2p::PieceReply reply; + reply.seq = request.seq; + reply.name = request.name; + reply.chunk = request.chunk; + + std::shared_ptr loaded; + { + std::lock_guard lock(loadedLock_); + loaded = loaded_; + } + if (!settings.serve || !loaded) { + reply.status = PieceStatus::NotServing; + return reply; + } + const auto now = clock_(); + auto busy = [&](uint32_t retryMs) { + ++busyAnswers_; + reply.status = PieceStatus::Busy; + reply.retryMs = retryMs; + return reply; + }; + if (!state.synced) { + return busy(kMaxRetryMs); + } + const auto file = loaded->files.find(request.name); + if (request.seq != loaded->manifest.seq || file == loaded->files.end() || request.chunk >= file->second.chunks) { + reply.status = PieceStatus::Unknown; + return reply; + } + const uint64_t chunkSize = loaded->manifest.chunkSize; + const uint64_t offset = uint64_t(request.chunk) * chunkSize; + const auto length = static_cast(std::min(chunkSize, file->second.size - offset)); + + auto& p = peer(sender, now); + const bool active = p.lastPiece != SteadyClock::time_point{} && now - p.lastPiece < kPeerWindow; + if (!active && activePeers(now, sender) >= std::max(settings.maxPeers, 1)) { + return busy(kMaxRetryMs); + } + if (now - p.windowStart >= kPieceWindow) { + p.windowStart = now; + p.piecesInWindow = 0; + } + if (p.piecesInWindow >= kPiecesPerWindow) { + return busy(msUntil(now, p.windowStart + kPieceWindow)); + } + if (settings.rateKBps != 0) { + const double rate = settings.rateKBps * 1000.0 / (state.trusted ? 2 : 1); + const double burst = static_cast(kBurstBytes) / (state.trusted ? 2 : 1); + if (!globalStarted_) { + global_ = Bucket{burst, now}; + globalStarted_ = true; + } + if (!p.bucketStarted) { + p.bucket = Bucket{burst / 2, now}; + p.bucketStarted = true; + } + // both buckets are checked before either is charged + Bucket peerTry = p.bucket; + Bucket globalTry = global_; + uint32_t retryPeer = 0; + uint32_t retryGlobal = 0; + const bool peerOk = peerTry.take(static_cast(length), rate / 2, burst / 2, now, retryPeer); + const bool globalOk = globalTry.take(static_cast(length), rate, burst, now, retryGlobal); + if (!peerOk || !globalOk) { + return busy(std::max(peerOk ? 0 : retryPeer, globalOk ? 0 : retryGlobal)); + } + p.bucket = peerTry; + global_ = globalTry; + } + + reply.data.resize(length); + size_t done = 0; + while (done < length) { + const auto n = ::pread(file->second.fd, reply.data.data() + done, length - done, static_cast(offset + done)); + if (n <= 0) { + reply.data.clear(); + reply.status = PieceStatus::NotServing; + return reply; + } + done += static_cast(n); + } + + ++p.piecesInWindow; + p.lastPiece = now; + if (p.seq != loaded->manifest.seq) { + p.seq = loaded->manifest.seq; + p.bytes = 0; + p.chunksSeen.assign(static_cast(loaded->totalChunks), false); + p.completed = false; + p.firstPiece = now; + } + const size_t index = static_cast(file->second.firstChunk + request.chunk); + if (index < p.chunksSeen.size() && !p.chunksSeen[index]) { + p.chunksSeen[index] = true; + p.bytes += length; + } + ++piecesServed_; + bytesServed_ += length; + if (!p.completed && p.bytes >= loaded->totalBytes) { + p.completed = true; + if (onPeerCompleted) { + onPeerCompleted(sender, p.seq, p.bytes, std::chrono::duration_cast(now - p.firstPiece)); + } + } + reply.status = PieceStatus::Ok; + return reply; +} + +std::string SnapshotProvider::takeSummary() { + if (piecesServed_ == 0 && manifestsServed_ == 0 && busyAnswers_ == 0) { + return {}; + } + const auto now = clock_(); + size_t recent = 0; + for (const auto& [key, p] : peers_) { + if (now - p.lastSeen < std::chrono::hours(1)) { + ++recent; + } + } + std::ostringstream out; + out << recent << " peers, " << manifestsServed_ << " manifests, " << piecesServed_ << " pieces, " << bytesServed_ / (1u << 20) + << " MiB, " << busyAnswers_ << " busy answers"; + piecesServed_ = bytesServed_ = busyAnswers_ = manifestsServed_ = 0; + return out.str(); +} + +} // namespace cs diff --git a/csnode/src/snapshot_service.cpp b/csnode/src/snapshot_service.cpp new file mode 100644 index 000000000..bcbbbd94c --- /dev/null +++ b/csnode/src/snapshot_service.cpp @@ -0,0 +1,203 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; +constexpr auto kPeriod = std::chrono::seconds(10); + +bool isDigits(const std::string& s) { + return !s.empty() && s.size() <= 12 && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; }); +} + +// the highest periodic checkpoint that records its state (qs/0, the shutdown one, is not periodic) +cs::Sequence latestCheckpoint() { + cs::Sequence best = 0; + std::error_code ec; + for (const auto& entry : fs::directory_iterator("qs", ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::exists(entry.path() / CheckpointDigest::kFileName, ec)) { + best = std::max(best, std::stoull(name)); + } + } + return best; +} +} // namespace + +SnapshotService::SnapshotService(BlockChain& blockchain, SnapshotProvider& provider, const cs::PrivateKey& key, std::string nodeVersion) +: blockchain_(blockchain), key_(key), nodeVersion_(std::move(nodeVersion)), provider_(provider) { +} + +SnapshotService::~SnapshotService() { + stop(); +} + +void SnapshotService::start() { + stop_ = false; + thread_ = std::thread([this]() { run(); }); +} + +void SnapshotService::stop() { + if (!thread_.joinable()) { + return; + } + { + std::lock_guard lock(m_); + stop_ = true; + } + cv_.notify_all(); + thread_.join(); + provider_.unload(); +} + +void SnapshotService::onCheckpointSaved(cs::Sequence sequence) { + (void)sequence; + { + // under the lock: the service thread cannot miss it between its check and its wait + std::lock_guard lock(m_); + failedBuilds_ = 0; + buildPending_ = true; + } + cv_.notify_all(); +} + +void SnapshotService::run() { + // the builder works on this thread: below the node's other work + ::setpriority(PRIO_PROCESS, static_cast(::syscall(SYS_gettid)), 10); + while (!stop_) { + const bool serve = cs::ConfigHolder::instance().config()->getSnapshotSettings().serve; // the observer may swap the config + if (serve != serving_) { + serving_ = serve; + if (serve) { + loadNewest(); + buildPending_ = true; + } + else { + provider_.unload(); + cslog() << "SNAPSHOT: not serving ([snapshot] serve=off)"; + } + } + if (serve && retryAt_ != std::chrono::steady_clock::time_point{} && std::chrono::steady_clock::now() >= retryAt_) { + retryAt_ = {}; + buildPending_ = true; + } + if (serve && buildPending_.exchange(false)) { + buildLatest(); + } + std::unique_lock lock(m_); + cv_.wait_for(lock, kPeriod, [this]() { return stop_.load() || (serving_ && buildPending_.load()); }); + } +} + +void SnapshotService::buildLatest() { + const cs::Sequence S = latestCheckpoint(); + std::error_code ec; + if (S == 0 || fs::exists(root_ / std::to_string(S) / "snapshot.json", ec)) { + return; // nothing new: no checkpoint with a digest yet, or already built + } + const auto cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + SnapshotBuildInput in; + in.seq = S; + in.checkpointDir = fs::path("qs") / std::to_string(S); + // pruning may be removing the lowest blocks right now (it follows the same checkpoint): start above it + in.firstStored = std::max(blockchain_.getFirstStoredSequence(), blockchain_.pruneTarget()); + in.snapshotBlocks = cfg.snapshotBlocks; + in.nodeVersion = nodeVersion_; + in.stop = &stop_; + in.blockBytes = [this](cs::Sequence seq, cs::Bytes& out) { + return blockchain_.blockBytes(seq, out) && !out.empty(); + }; + // no contract records: the contracts database holds the latest state of each contract, later than + // S, while the signed snapshot is the state at S; the checkpoint carries the states at S + // (smartcontracts.dat) and the node fetches any it lacks from the confidants + fs::create_directories(root_, ec); + // the node's own database shares the disk: a build that could fill it is not started + uint64_t estimate = 512ull << 20; + for (const auto& entry : fs::directory_iterator(in.checkpointDir, ec)) { + estimate += entry.is_regular_file(ec) ? entry.file_size(ec) / 4 : 0; + } + const auto space = fs::space(root_, ec); + if (!ec && space.available < 2 * estimate) { + cswarning() << "SNAPSHOT: not enough free disk for the snapshot of checkpoint " << S << " (" << space.available / (1u << 20) + << " MB free, about " << estimate / (1u << 20) << " MB needed twice)"; + return; + } + cslog() << "SNAPSHOT: building the snapshot of checkpoint " << S; + const auto started = std::chrono::steady_clock::now(); + const auto r = buildSnapshot(in, root_, key_); + if (!r.ok) { + // tried again a few times a minute apart (the block range may have been pruned under it) + const bool retry = !stop_ && ++failedBuilds_ < kMaxBuildAttempts; + cswarning() << "SNAPSHOT: no snapshot of checkpoint " << S << ": " << r.error << (retry ? ", trying again in a minute" : ""); + if (retry) { + retryAt_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + } + return; + } + failedBuilds_ = 0; + uint64_t bytes = 0; + for (const auto& f : r.manifest.files) { + bytes += f.size; + } + cslog() << "SNAPSHOT: built block " << S << ", " << r.manifest.files.size() << " files, " << bytes / (1u << 20) << " MB, state digest " + << r.manifest.stateDigest << " in " + << std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count() << " s"; + loadNewest(); +} + +void SnapshotService::loadNewest() { + std::error_code ec; + cs::Sequence best = 0; + for (const auto& entry : fs::directory_iterator(root_, ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::is_regular_file(entry.path() / "snapshot.json", ec)) { + best = std::max(best, std::stoull(name)); + } + } + if (best == 0) { + return; + } + if (provider_.loadedSeq() == best) { + return; + } + std::string error; + const auto dir = root_ / std::to_string(best); + if (!provider_.load(dir, &error)) { + // a snapshot of this node's own builder (it has snapshot.json): removed, so that the next + // round builds it again while its checkpoint is kept + const bool retry = ++failedBuilds_ < kMaxBuildAttempts; + cswarning() << "SNAPSHOT: " << dir.string() << " is not served: " << error << "; removed" + << (retry ? ", to be built again in a minute" : ""); + fs::remove_all(dir, ec); + if (retry) { + retryAt_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + } + return; + } + uint64_t bytes = 0; + size_t files = 0; + for (const auto& entry : fs::directory_iterator(dir, ec)) { + if (entry.is_regular_file(ec) && entry.path().filename() != "snapshot.json") { + bytes += entry.file_size(ec); + ++files; + } + } + cslog() << "SNAPSHOT: serving block " << best << " (" << files << " files, " << bytes / (1u << 20) << " MB)"; +} + +} // namespace cs diff --git a/docs/CHANGES.md b/docs/CHANGES.md index dd80f3bdc..bd38f7018 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -36,6 +36,8 @@ lines under the new release. |---|---| | #99 | config: the `[snapshot]` section takes the keys of p2p snapshot sharing (`serve`, `serve_rate_kbps`, `serve_max_peers`, `snapshot_blocks` from 1, `fetch`, `fetch_wait`, `fetch_fallback`, `min_sources` 2, `sources` as node keys, `sources_file`, `fetch_rate_kbps`); the HTTP service keys are gone and an unknown key is logged and ignored; nothing reads them yet | | #100 | net: four p2p messages for snapshot manifests and pieces (types 40-43) with bounded decoders; this build answers requests as not serving and sends none | +| #101 | net: `host_data` announces snapshot serving (bit 24, `[snapshot] serve=on` at start) and the community version (bits 32-47, `MINOR_NODE_VERSION`, now 13); `known_peers.json` at a clean stop lists the known peers with what they announce | +| #102 | snapshot: a node with `[snapshot] serve=on` builds the snapshot of each periodic checkpoint and serves it to its p2p neighbours, within `serve_rate_kbps` (KB/s), `serve_max_peers` and per-peer limits, at half budget while trusted; snapshots use 512 KiB chunks and no contracts file | ## community.12 — 2026-10-24 diff --git a/packaging/mainnet/config.ini b/packaging/mainnet/config.ini index 25c8f946f..fdd066ea5 100644 --- a/packaging/mainnet/config.ini +++ b/packaging/mainnet/config.ini @@ -67,9 +67,10 @@ reject_contract_consensus=true alarm_invalid_block=true big_bang=true -; Snapshots shared with the p2p neighbours (not active yet; the defaults are shown). +; Snapshots shared with the p2p neighbours (serve works; fetching at a first start comes later). Defaults shown. ;[snapshot] ;serve=off +; serve_rate_kbps and fetch_rate_kbps are in KB/s (2000 = 2 MB/s) ;serve_rate_kbps=2000 ;serve_max_peers=8 ;snapshot_blocks=101000 diff --git a/tests/csnode/snapshot_provider_tests.cpp b/tests/csnode/snapshot_provider_tests.cpp new file mode 100644 index 000000000..28dba0801 --- /dev/null +++ b/tests/csnode/snapshot_provider_tests.cpp @@ -0,0 +1,377 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; +using namespace std::chrono_literals; +using cs::snapshot_p2p::ManifestStatus; +using cs::snapshot_p2p::PieceStatus; + +void putU16(std::ofstream& f, uint16_t v) { + f.write(reinterpret_cast(&v), sizeof(v)); +} + +// a snapshot of block 200 built by buildSnapshot, with 64 KiB chunks and a checkpoint file that +// does not compress, so that it spans several chunks +struct Fixture { + fs::path root; + cs::PrivateKey key; + cs::PublicKey pub; + fs::path dir; + std::chrono::steady_clock::time_point now = std::chrono::steady_clock::time_point{} + 1000h; + cs::SnapshotProvider provider{[this]() { return now; }}; + + explicit Fixture(const char* name) : root(fs::temp_directory_path() / name) { + fs::remove_all(root); + fs::create_directories(root); + EXPECT_TRUE(cscrypto::cryptoInit()); + key = cscrypto::generateKeyPair(pub); + + std::map blocks; + cs::Bytes head; + csdb::PoolHash prev; + for (cs::Sequence s = 0; s <= 200; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(100 + s % 50, static_cast('a' + s % 26))); + EXPECT_TRUE(p.compose()); + blocks[s] = p.to_binary(); + head = p.hash().to_binary(); + prev = p.hash(); + } + const auto qs = root / "qs" / "200"; + fs::create_directories(qs); + { + std::ofstream f(qs / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t s = 200; + f.write(reinterpret_cast(&magic), sizeof(magic)); + f.write(reinterpret_cast(&s), sizeof(s)); + putU16(f, static_cast(head.size())); + f.write(reinterpret_cast(head.data()), static_cast(head.size())); + putU16(f, 0); + } + { + std::mt19937 rng(7); + std::string noise(300000, '\0'); + for (auto& c : noise) { + c = static_cast(rng()); + } + std::ofstream(qs / "walletscache.dat", std::ios::binary) << noise; + } + std::ofstream(qs / "blockchain.dat", std::ios::binary) << "settings"; + cs::CheckpointDigest d; + d.digest.fill(7); + d.settings.fill(9); + d.wallets = 42; + EXPECT_TRUE(d.write(qs / cs::CheckpointDigest::kFileName)); + + cs::SnapshotBuildInput in; + in.seq = 200; + in.checkpointDir = qs; + in.snapshotBlocks = 150; + in.nodeVersion = "test"; + in.chunkSize = 64 << 10; + in.blockBytes = [&blocks](cs::Sequence s, cs::Bytes& out) { + auto it = blocks.find(s); + if (it == blocks.end()) return false; + out = it->second; + return true; + }; + const auto r = cs::buildSnapshot(in, root / "snapshot", key); + EXPECT_TRUE(r.ok) << r.error; + dir = r.dir; + } + ~Fixture() { + std::error_code ec; + fs::remove_all(root, ec); + } + + cs::SnapshotManifest manifest() const { + std::ifstream f(dir / "snapshot.json", std::ios::binary); + return *cs::SnapshotManifest::fromJson(std::string(std::istreambuf_iterator(f), std::istreambuf_iterator())); + } +}; + +cs::PublicKey peerKey(uint8_t n) { + cs::PublicKey k{}; + k.fill(n); + return k; +} + +cs::SnapshotProvider::Settings serving(size_t rateKBps = 0, size_t maxPeers = 8) { + cs::SnapshotProvider::Settings s; + s.serve = true; + s.rateKBps = rateKBps; + s.maxPeers = maxPeers; + return s; +} + +cs::SnapshotNodeState synced(bool trusted = false) { + cs::SnapshotNodeState s; + s.lastSeq = 1000; + s.round = 1001; + s.synced = true; + s.trusted = trusted; + return s; +} + +cs::snapshot_p2p::PieceRequest piece(const std::string& name, uint32_t chunk, cs::Sequence seq = 200) { + cs::snapshot_p2p::PieceRequest r; + r.seq = seq; + r.name = name; + r.chunk = chunk; + return r; +} +} // namespace + +TEST(SnapshotProvider, ManifestStatuses) { + Fixture fx("snapshot_provider_status"); + const auto a = peerKey(1); + + auto off = fx.provider.onManifestRequest(a, cs::SnapshotProvider::Settings{}, synced()); + ASSERT_TRUE(off); + EXPECT_EQ(off->status, ManifestStatus::NotServing); + EXPECT_EQ(off->lastSeq, 1000u); + + auto building = fx.provider.onManifestRequest(a, serving(), synced()); + ASSERT_TRUE(building); + EXPECT_EQ(building->status, ManifestStatus::Building); + + ASSERT_TRUE(fx.provider.load(fx.dir)); + EXPECT_EQ(fx.provider.loadedSeq(), 200u); + + cs::SnapshotNodeState behind = synced(); + behind.synced = false; + auto notSynced = fx.provider.onManifestRequest(a, serving(), behind); + ASSERT_TRUE(notSynced); + EXPECT_EQ(notSynced->status, ManifestStatus::NotSynced); + + auto ready = fx.provider.onManifestRequest(a, serving(), synced(true)); + ASSERT_TRUE(ready); + EXPECT_EQ(ready->status, ManifestStatus::Ready); + EXPECT_TRUE(ready->servingTrusted); + const auto parsed = cs::SnapshotManifest::fromJson(ready->manifestJson); + ASSERT_TRUE(parsed); + EXPECT_TRUE(parsed->verify()); + EXPECT_EQ(parsed->seq, 200u); + + // once per peer every 5 s + EXPECT_FALSE(fx.provider.onManifestRequest(a, serving(), synced())); + EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(2), serving(), synced())); + fx.now += 5s; + EXPECT_TRUE(fx.provider.onManifestRequest(a, serving(), synced())); + + fx.provider.unload(); + EXPECT_FALSE(fx.provider.loadedSeq()); +} + +TEST(SnapshotProvider, EveryChunkMatchesTheManifest) { + Fixture fx("snapshot_provider_chunks"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto m = fx.manifest(); + bool sawShortLast = false; + bool sawSeveralChunks = false; + uint8_t n = 1; + for (const auto& f : m.files) { + sawSeveralChunks = sawSeveralChunks || f.chunks.size() > 2; + for (uint32_t c = 0; c < f.chunks.size(); ++c) { + fx.now += 1s; // well inside every limit + const auto r = fx.provider.onPieceRequest(peerKey(n), piece(f.name, c), serving(), synced()); + ASSERT_EQ(r.status, PieceStatus::Ok) << f.name << " " << c; + EXPECT_EQ(cs::sha256Hex(r.data.data(), r.data.size()), f.chunks[c]) << f.name << " " << c; + if (c + 1 == f.chunks.size() && r.data.size() < m.chunkSize) { + sawShortLast = true; + } + EXPECT_EQ(r.name, f.name); + EXPECT_EQ(r.chunk, c); + } + } + EXPECT_TRUE(sawSeveralChunks); + EXPECT_TRUE(sawShortLast); +} + +TEST(SnapshotProvider, UnknownRequests) { + Fixture fx("snapshot_provider_unknown"); + const auto a = peerKey(1); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0), serving(), synced()).status, PieceStatus::NotServing); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto m = fx.manifest(); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0, 199), serving(), synced()).status, PieceStatus::Unknown); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("contracts.bin.lz4", 0), serving(), synced()).status, PieceStatus::Unknown); + const auto chunks = static_cast(m.file("blocks.bin.lz4")->chunks.size()); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", chunks), serving(), synced()).status, PieceStatus::Unknown); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0), cs::SnapshotProvider::Settings{}, synced()).status, + PieceStatus::NotServing); + cs::SnapshotNodeState behind = synced(); + behind.synced = false; + const auto r = fx.provider.onPieceRequest(a, piece("blocks.bin.lz4", 0), serving(), behind); + EXPECT_EQ(r.status, PieceStatus::Busy); + EXPECT_EQ(r.retryMs, cs::SnapshotProvider::kMaxRetryMs); +} + +TEST(SnapshotProvider, TwoPiecesPerPeerEvery100ms) { + Fixture fx("snapshot_provider_window"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto a = peerKey(1); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 0), serving(), synced()).status, PieceStatus::Ok); + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 1), serving(), synced()).status, PieceStatus::Ok); + fx.now += 40ms; + const auto third = fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 2), serving(), synced()); + EXPECT_EQ(third.status, PieceStatus::Busy); + EXPECT_GT(third.retryMs, 0u); + EXPECT_LE(third.retryMs, 60u); + // another peer is not held up + EXPECT_EQ(fx.provider.onPieceRequest(peerKey(2), piece("qs-walletscache.dat.lz4", 2), serving(), synced()).status, PieceStatus::Ok); + fx.now += 60ms; + EXPECT_EQ(fx.provider.onPieceRequest(a, piece("qs-walletscache.dat.lz4", 2), serving(), synced()).status, PieceStatus::Ok); +} + +namespace { +// pieces of 64 KiB a peer gets, two every 100 ms, before the first busy answer +size_t piecesUntilBusy(Fixture& fx, const cs::PublicKey& peer, size_t rateKBps, bool trusted, uint32_t* retryMs = nullptr) { + for (size_t n = 0; n < 1000; ++n) { + if (n % 2 == 0) { + fx.now += 100ms; + } + const auto r = fx.provider.onPieceRequest(peer, piece("qs-walletscache.dat.lz4", 0), serving(rateKBps), synced(trusted)); + if (r.status == PieceStatus::Busy) { + if (retryMs) { + *retryMs = r.retryMs; + } + return n; + } + EXPECT_EQ(r.status, PieceStatus::Ok); + } + return 1000; +} +} // namespace + +TEST(SnapshotProvider, PerPeerAndGlobalBudget) { + Fixture fx("snapshot_provider_budget"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + // 100 KB/s: a peer may take half (50 KB/s) after a burst of 2 MiB, 32 pieces of 64 KiB + uint32_t retry = 0; + const auto first = piecesUntilBusy(fx, peerKey(1), 100, false, &retry); + EXPECT_GE(first, 32u); + EXPECT_LE(first, 36u); + EXPECT_GT(retry, 0u); + EXPECT_LE(retry, cs::SnapshotProvider::kMaxRetryMs); + // a second peer still gets its share of what the global burst has left (4 MiB - 2 MiB) + const auto second = piecesUntilBusy(fx, peerKey(2), 100, false); + EXPECT_GE(second, 28u); + // and then the global bucket is empty for everyone + EXPECT_LE(piecesUntilBusy(fx, peerKey(3), 100, false), 3u); +} + +TEST(SnapshotProvider, TrustedHalvesTheBudget) { + Fixture fx("snapshot_provider_trusted"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto n = piecesUntilBusy(fx, peerKey(1), 100, true); + EXPECT_GE(n, 16u); + EXPECT_LE(n, 18u); +} + +TEST(SnapshotProvider, MaxPeers) { + Fixture fx("snapshot_provider_maxpeers"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + EXPECT_EQ(fx.provider.onPieceRequest(peerKey(1), piece("blocks.bin.lz4", 0), serving(0, 1), synced()).status, PieceStatus::Ok); + const auto r = fx.provider.onPieceRequest(peerKey(2), piece("blocks.bin.lz4", 0), serving(0, 1), synced()); + EXPECT_EQ(r.status, PieceStatus::Busy); + EXPECT_EQ(r.retryMs, cs::SnapshotProvider::kMaxRetryMs); + fx.now += 31s; + EXPECT_EQ(fx.provider.onPieceRequest(peerKey(2), piece("blocks.bin.lz4", 0), serving(0, 1), synced()).status, PieceStatus::Ok); +} + +TEST(SnapshotProvider, ManyKeysDoNotStarveADownloader) { + Fixture fx("snapshot_provider_manifests"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto downloader = peerKey(200); + EXPECT_TRUE(fx.provider.onManifestRequest(downloader, serving(), synced())); + EXPECT_EQ(fx.provider.onPieceRequest(downloader, piece("blocks.bin.lz4", 0), serving(), synced()).status, PieceStatus::Ok); + // made-up keys get a burst of 8 (the downloader took one), then 20 a second + size_t answered = 0; + for (uint8_t i = 1; i <= 40; ++i) { + answered += fx.provider.onManifestRequest(peerKey(i), serving(), synced()) ? 1 : 0; + } + EXPECT_EQ(answered, 7u); + fx.now += 1s; + answered = 0; + for (uint8_t i = 41; i <= 80; ++i) { + answered += fx.provider.onManifestRequest(peerKey(i), serving(), synced()) ? 1 : 0; + } + EXPECT_EQ(answered, 8u); + // the downloader is answered after its 5 s whatever the others did + fx.now += 4s; + EXPECT_TRUE(fx.provider.onManifestRequest(downloader, serving(), synced())); +} + +TEST(SnapshotProvider, NewKeysDoNotEvictADownloader) { + Fixture fx("snapshot_provider_table"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + const auto downloader = peerKey(200); + const auto idle = peerKey(201); + EXPECT_TRUE(fx.provider.onManifestRequest(downloader, serving(), synced())); + EXPECT_EQ(fx.provider.onPieceRequest(downloader, piece("blocks.bin.lz4", 0), serving(), synced()).status, PieceStatus::Ok); + fx.now += 50ms; + EXPECT_TRUE(fx.provider.onManifestRequest(idle, serving(), synced())); + // 63 made-up keys fill the table to 65 entries + for (uint8_t i = 1; i < cs::SnapshotProvider::kMaxPeers; ++i) { + fx.now += 50ms; // within 20 a second + EXPECT_TRUE(fx.provider.onManifestRequest(peerKey(i), serving(), synced())); + } + // the downloader is still known (its 5 s are not over), the idle key was the one pushed out + fx.now += 50ms; + EXPECT_FALSE(fx.provider.onManifestRequest(downloader, serving(), synced())); + fx.now += 50ms; + EXPECT_TRUE(fx.provider.onManifestRequest(idle, serving(), synced())); +} + +TEST(SnapshotProvider, CompletedPeerAndSummary) { + Fixture fx("snapshot_provider_complete"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + int completed = 0; + fx.provider.onPeerCompleted = [&](const cs::PublicKey&, cs::Sequence seq, uint64_t, std::chrono::seconds) { + EXPECT_EQ(seq, 200u); + ++completed; + }; + const auto files = fx.manifest().files; + // the same chunk again and again does not make a whole snapshot + for (int i = 0; i < 100; ++i) { + fx.now += 1s; + ASSERT_EQ(fx.provider.onPieceRequest(peerKey(1), piece(files.front().name, 0), serving(), synced()).status, PieceStatus::Ok); + } + EXPECT_EQ(completed, 0); + for (const auto& f : files) { + for (uint32_t c = 0; c < f.chunks.size(); ++c) { + fx.now += 1s; + ASSERT_EQ(fx.provider.onPieceRequest(peerKey(1), piece(f.name, c), serving(), synced()).status, PieceStatus::Ok); + } + } + EXPECT_EQ(completed, 1); + EXPECT_NE(fx.provider.takeSummary().find("1 peers"), std::string::npos); + EXPECT_TRUE(fx.provider.takeSummary().empty()); +} + +TEST(SnapshotProvider, ATamperedFileIsNotLoaded) { + Fixture fx("snapshot_provider_tampered"); + ASSERT_TRUE(fx.provider.load(fx.dir)); + std::ofstream(fx.dir / "blocks.bin.lz4", std::ios::binary | std::ios::app) << "x"; + std::string error; + EXPECT_FALSE(fx.provider.load(fx.dir, &error)); + EXPECT_NE(error.find("size"), std::string::npos) << error; + EXPECT_EQ(fx.provider.loadedSeq(), 200u); // the one loaded before stays + EXPECT_FALSE(fx.provider.load(fx.root / "snapshot" / "not-a-number")); +}