From 8d167a0f5764d16d5176e34c2205c993b04315f3 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 07:50:33 +0100 Subject: [PATCH 1/9] csnode: snapshots served by the node over HTTP A node with [snapshot] serve_port set serves the snapshots it builds: - SnapshotServer (Boost.Beast, header-only): GET and HEAD of /snapshot/latest.json and /snapshot//, one Range of at most 16 MiB, 8 KB of headers, 15 s idle and 30 s write timeouts, at most serve_max_connections (503 beyond), a bandwidth cap shared by all connections (serve_rate_kbps, kilobytes a second); only names of the loaded manifest are served, never a path from the request. It answers 503 while the node is more than 3 rounds behind the network or has no snapshot built: only synced nodes serve. latest.json carries the node's last block (synced_to, outside the signed line). - SnapshotService, owned by Node: every 10 s it reads [snapshot] and starts, stops (port 0) or moves the server, applying the limits while it runs; while serving it builds the snapshot of each new periodic checkpoint (checkpointSavedEvent) and of the latest one when it starts, at low priority, into snapshot/, and serves it once complete. - The builder reads checkpoint files in 4 MiB pieces; Storage and BlockChain give the contract records to it. Off by default (serve_port=0). Step 4 of snapshots served by community nodes. Co-Authored-By: Claude Opus 5.5 --- csdb/include/csdb/storage.hpp | 6 + csdb/src/storage.cpp | 8 + csnode/CMakeLists.txt | 4 + csnode/include/csnode/blockchain.hpp | 4 + csnode/include/csnode/node.hpp | 5 + csnode/include/csnode/snapshot_server.hpp | 68 +++ csnode/include/csnode/snapshot_service.hpp | 51 +++ csnode/src/blockchain.cpp | 5 + csnode/src/node.cpp | 24 ++ csnode/src/snapshot_builder.cpp | 22 +- csnode/src/snapshot_server.cpp | 469 +++++++++++++++++++++ csnode/src/snapshot_service.cpp | 140 ++++++ tests/csnode/snapshot_server_tests.cpp | 233 ++++++++++ 13 files changed, 1031 insertions(+), 8 deletions(-) create mode 100644 csnode/include/csnode/snapshot_server.hpp create mode 100644 csnode/include/csnode/snapshot_service.hpp create mode 100644 csnode/src/snapshot_server.cpp create mode 100644 csnode/src/snapshot_service.cpp create mode 100644 tests/csnode/snapshot_server_tests.cpp diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 3eca33681..ab34d6546 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -364,6 +364,12 @@ class Storage final { */ cs::Sequence first_sequence() const; + /** + * Every contract record, in the engine's key order; fn returns false to stop early. For a + * snapshot of the node's state. + */ + bool forEachContractData(const std::function& fn) const; + /** * Returns the space of pruned pools to the file system. */ diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index c4b66a62b..925197918 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -1350,6 +1350,14 @@ bool Storage::pool_prune_batch(const std::vector& hashes) { return true; } +bool Storage::forEachContractData(const std::function& fn) const { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + return d->db->forEachContractData(fn); +} + cs::Sequence Storage::first_sequence() const { if (!isOpen()) { d->set_last_error(NotOpen); diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index e0c94800d..fba8762f1 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -72,6 +72,8 @@ add_library(csnode include/csnode/snapshot_manifest.hpp include/csnode/snapshot_builder.hpp include/csnode/snapshot_importer.hpp + include/csnode/snapshot_server.hpp + include/csnode/snapshot_service.hpp include/csnode/blockverifier.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -116,6 +118,8 @@ add_library(csnode src/snapshot_manifest.cpp src/snapshot_builder.cpp src/snapshot_importer.cpp + src/snapshot_server.cpp + src/snapshot_service.cpp src/blockverifier.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 4495eb6d7..e409a5cdc 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -95,6 +95,8 @@ class BlockChain { // pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were // removed once a later periodic checkpoint made them unnecessary cs::Sequence getFirstStoredSequence() const; + // every contract record of the storage, for a snapshot of the node's state + bool forEachContractData(const std::function& fn) const; // return unique id of database if at least one unique block has written, otherwise (only genesis block) 0 uint64_t uuid() const; @@ -303,6 +305,8 @@ public signals: cs::StopReadingBlocksSignal stopReadingBlocksEvent; cs::SuccessQSSignal successfullQuickStartEvent; + // a periodic checkpoint of this block was saved (live), with its state digest + cs::Signal checkpointSavedEvent; public slots: diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index 7136d6dca..b7ed59ef3 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -26,6 +26,10 @@ class Transport; +namespace cs { +class SnapshotService; +} + namespace cs { class SolverCore; } @@ -485,6 +489,7 @@ public slots: #ifdef NODE_API std::unique_ptr api_; + std::unique_ptr snapshotService_; #endif uint32_t startPacketRequestPoint_ = 0; diff --git a/csnode/include/csnode/snapshot_server.hpp b/csnode/include/csnode/snapshot_server.hpp new file mode 100644 index 000000000..4385614c5 --- /dev/null +++ b/csnode/include/csnode/snapshot_server.hpp @@ -0,0 +1,68 @@ +#ifndef SNAPSHOT_SERVER_HPP +#define SNAPSHOT_SERVER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace cs { + +// The read-only HTTP service of the snapshots this node built (docs/specs/2026-10-24-snapshot-serving-design.md, +// section 5): GET and HEAD of /snapshot/latest.json and /snapshot//, one Range of at most +// 16 MiB, at most maxConnections connections, a bandwidth cap shared by all of them. It answers 503 +// while the node is not synced with the network or no snapshot is built (only synced nodes serve). +// Files are looked up by the names of the manifest it loaded, never by a path from the request. +class SnapshotServer { +public: + struct Settings { + std::string bind; // empty: every interface + uint16_t port = 0; // 0: stopped + size_t maxConnections = 8; + size_t rateKBps = 0; // kilobytes a second, all connections together; 0: unlimited + bool operator==(const Settings& o) const { + return bind == o.bind && port == o.port && maxConnections == o.maxConnections && rateKBps == o.rateKBps; + } + }; + // the node's last block and the current round: synced when the block is within 3 rounds + using SyncedTo = std::function()>; + + static constexpr uint64_t kMaxRange = 16ull << 20; + static constexpr size_t kMaxRequestsPerConnection = 64; + static constexpr int kIdleTimeoutSec = 15; + static constexpr int kWriteTimeoutSec = 30; + static constexpr cs::Sequence kSyncedRounds = 3; + + SnapshotServer(std::filesystem::path root, SyncedTo syncedTo); + ~SnapshotServer(); + + // starts, stops (port 0), moves to another address or applies new limits, while it runs + void apply(const Settings& settings); + // the newest complete snapshot under root (one holding a snapshot.json that verifies) + void reload(); + void stop(); + + uint16_t port() const { + return boundPort_.load(); + } + bool running() const { + return boundPort_.load() != 0; + } + +private: + struct Impl; + std::unique_ptr impl_; + std::atomic boundPort_{0}; +}; + +} // namespace cs + +#endif // SNAPSHOT_SERVER_HPP diff --git a/csnode/include/csnode/snapshot_service.hpp b/csnode/include/csnode/snapshot_service.hpp new file mode 100644 index 000000000..00368dfc0 --- /dev/null +++ b/csnode/include/csnode/snapshot_service.hpp @@ -0,0 +1,51 @@ +#ifndef SNAPSHOT_SERVICE_HPP +#define SNAPSHOT_SERVICE_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +class BlockChain; + +namespace cs { + +// Snapshots served by this node, in the node: every 10 s it reads [snapshot] from the configuration +// and starts, stops or moves the HTTP service (serve_port 0: off); while it serves, it builds the +// snapshot of each new periodic checkpoint into snapshot/ at low priority (and of the latest one +// when it starts serving), and serves it once complete. Only a node synced with the network serves. +class SnapshotService { +public: + SnapshotService(BlockChain& blockchain, const cs::PrivateKey& key, std::string nodeVersion); + ~SnapshotService(); + + void start(); + void stop(); + // slot of BlockChain::checkpointSavedEvent: a new periodic checkpoint to build from + void onCheckpointSaved(cs::Sequence sequence); + +private: + void run(); + void buildLatest(); + + BlockChain& blockchain_; + cs::PrivateKey key_; + std::string nodeVersion_; + std::filesystem::path root_ = "snapshot"; + std::unique_ptr server_; + std::thread thread_; + std::atomic stop_{false}; + std::atomic buildPending_{true}; + std::mutex m_; + std::condition_variable cv_; +}; + +} // namespace cs + +#endif // SNAPSHOT_SERVICE_HPP diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index d1daa1183..bacf9f4e1 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1053,6 +1053,7 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { if (serializationManPtr_->save(pool.sequence(), head, digest)) { cslog() << cs::stateDigestLine(pool.sequence(), digest.digest, digest.wallets) << " checkpoint"; serializationManPtr_->pruneCheckpoints(sto.checkpointKeep); + emit checkpointSavedEvent(pool.sequence()); lastCheckpointWallClock_ = now; if (byCount) { schedulePruning(pool.sequence()); @@ -1064,6 +1065,10 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { } } +bool BlockChain::forEachContractData(const std::function& fn) const { + return storage_.forEachContractData(fn); +} + cs::CheckpointDigest BlockChain::checkpointDigest() const { cs::CheckpointDigest result; const auto& wallets = multiWallets(); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 62d210f80..aeaef7367 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -40,7 +40,9 @@ #include +#include #include +#include #include namespace { @@ -258,6 +260,23 @@ bool Node::init() { api_->run(); #endif // NODE_API + // snapshots served by this node: [snapshot] serve_port, read again every 10 s (off by default) + { + std::string version; + std::ifstream versionFile("VERSION"); + if (!versionFile || !std::getline(versionFile, version) || version.empty() || version.size() > 64) { + version = "4.2." + std::to_string(Config::getNodeVersion()); + } + for (auto& c : version) { + if (c < 0x20 || c >= 0x7f || c == '"' || c == '\\') { + c = '_'; + } + } + snapshotService_ = std::make_unique(blockChain_, solver_->getPrivateKey(), version); + cs::Connector::connect(&blockChain_.checkpointSavedEvent, snapshotService_.get(), &cs::SnapshotService::onCheckpointSaved); + snapshotService_->start(); + } + if (!transport_->isGood()) { return false; } @@ -357,6 +376,11 @@ void Node::destroy() { api_->stop(); cswarning() << "[API STOPPED]"; + if (snapshotService_) { + snapshotService_->stop(); + cswarning() << "[SNAPSHOT SERVICE STOPPED]"; + } + solver_->finish(); cswarning() << "[SOLVER STOPPED]"; diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index f9fd80fbc..c4f17a781 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -175,11 +175,6 @@ std::string utcNow() { return buf; } -bool readFile(const std::filesystem::path& path, std::vector& out) { - std::ifstream in(path, std::ios::binary); - if (!in) { - return false; - } out.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); return !in.bad(); } @@ -332,14 +327,25 @@ SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::fi if (!std::filesystem::exists(src, ec)) { continue; // tokens.dat and apihandler.dat exist only in builds with the API } - std::vector data; - if (!readFile(src, data)) { + // in pieces: walletscache.dat is about 180 MB on mainnet, and the builder runs inside the node + std::ifstream in(src, std::ios::binary); + if (!in) { return failWith("cannot read " + src.string()); } SnapshotFile f; f.name = "qs-" + name + ".lz4"; Lz4Writer w(tmp / f.name, m.chunkSize); - w.write(data); + std::vector piece(snapshot_lz4::kBlockSize); + while (in) { + in.read(reinterpret_cast(piece.data()), static_cast(piece.size())); + const auto n = in.gcount(); + if (n > 0) { + w.write(piece.data(), static_cast(n)); + } + } + if (in.bad()) { + return failWith("cannot read " + src.string()); + } if (!w.finish(f)) { return failWith("cannot write " + f.name); } diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp new file mode 100644 index 000000000..45f18e89c --- /dev/null +++ b/csnode/src/snapshot_server.cpp @@ -0,0 +1,469 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +namespace cs { + +namespace { +namespace asio = boost::asio; +namespace http = boost::beast::http; +using tcp = asio::ip::tcp; +namespace fs = std::filesystem; + +void setTimeout(tcp::socket& socket, int option, int seconds) { + timeval tv{}; + tv.tv_sec = seconds; + ::setsockopt(socket.native_handle(), SOL_SOCKET, option, &tv, sizeof(tv)); +} + +bool isDigits(const std::string& s) { + return !s.empty() && s.size() <= 12 && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; }); +} + +// "bytes=a-b", "bytes=a-" or "bytes=-n" for a file of size; false for anything else (two ranges too) +bool parseRange(const std::string& header, uint64_t size, uint64_t& first, uint64_t& last) { + const std::string prefix = "bytes="; + if (header.compare(0, prefix.size(), prefix) != 0) { + return false; + } + const std::string spec = header.substr(prefix.size()); + if (spec.find(',') != std::string::npos) { + return false; + } + const auto dash = spec.find('-'); + if (dash == std::string::npos) { + return false; + } + const std::string a = spec.substr(0, dash), b = spec.substr(dash + 1); + if (size == 0) { + return false; + } + if (a.empty()) { + if (!isDigits(b)) return false; + const uint64_t n = std::stoull(b); + if (n == 0) return false; + first = n >= size ? 0 : size - n; + last = size - 1; + return true; + } + if (!isDigits(a) || (!b.empty() && !isDigits(b))) { + return false; + } + first = std::stoull(a); + last = b.empty() ? size - 1 : std::min(std::stoull(b), size - 1); + return first < size && first <= last; +} + +// the bandwidth cap of all connections together +class TokenBucket { +public: + void setRate(size_t kBps) { + std::lock_guard lock(m_); + rate_ = static_cast(kBps) * 1000.0; + tokens_ = std::min(tokens_, rate_); + } + // waits until n bytes may be sent + void take(size_t n) { + for (;;) { + std::unique_lock lock(m_); + if (rate_ <= 0) { + return; + } + const auto now = std::chrono::steady_clock::now(); + tokens_ = std::min(rate_, tokens_ + rate_ * std::chrono::duration(now - last_).count()); + last_ = now; + if (tokens_ >= static_cast(n)) { + tokens_ -= static_cast(n); + return; + } + const double wait = (static_cast(n) - tokens_) / rate_; + lock.unlock(); + std::this_thread::sleep_for(std::chrono::duration(std::min(wait, 0.2))); + } + } + +private: + std::mutex m_; + double rate_ = 0; + double tokens_ = 0; + std::chrono::steady_clock::time_point last_ = std::chrono::steady_clock::now(); +}; + +struct Loaded { + SnapshotManifest manifest; + std::string json; // as built, with synced_to inserted per request + fs::path dir; +}; +} // namespace + +struct SnapshotServer::Impl { + fs::path root; + SyncedTo syncedTo; + Settings settings; + std::mutex settingsLock; + + std::shared_ptr loaded; + std::mutex loadedLock; + + TokenBucket bucket; + std::atomic maxConnections{8}; + std::atomic active{0}; + std::set> sockets; + std::mutex socketsLock; + + asio::io_context io; + std::unique_ptr acceptor; + std::thread acceptThread; + std::atomic stopping{false}; + + std::shared_ptr current() { + std::lock_guard lock(loadedLock); + return loaded; + } + + bool synced() const { + if (!syncedTo) { + return true; + } + const auto [lastBlock, round] = syncedTo(); + return lastBlock > 0 && round <= lastBlock + kSyncedRounds; + } + + template + void finish(http::response& res, bool keepAlive) { + res.set(http::field::server, "credits-node"); + res.keep_alive(keepAlive); + res.prepare_payload(); + } + + // returns false when the connection must end + bool handle(tcp::socket& socket, const http::request& req, uint64_t& served) { + const bool keepAlive = req.keep_alive(); + auto simple = [&](http::status status, const std::string& text, bool retry = false) { + http::response res{status, req.version()}; + res.set(http::field::content_type, "text/plain"); + if (retry) res.set(http::field::retry_after, "30"); + res.body() = text + "\n"; + finish(res, keepAlive); + boost::system::error_code ec; + http::write(socket, res, ec); + return !ec && keepAlive; + }; + if (req.method() != http::verb::get && req.method() != http::verb::head) { + return simple(http::status::method_not_allowed, "GET or HEAD only"); + } + const std::string target(req.target()); + const std::string prefix = "/snapshot/"; + if (target.compare(0, prefix.size(), prefix) != 0) { + return simple(http::status::not_found, "not found"); + } + const auto state = current(); + if (!synced()) { + return simple(http::status::service_unavailable, "this node is not synced with the network", true); + } + if (!state) { + return simple(http::status::service_unavailable, "no snapshot built yet", true); + } + const std::string rest = target.substr(prefix.size()); + const bool head = req.method() == http::verb::head; + if (rest == "latest.json") { + const auto [lastBlock, round] = syncedTo ? syncedTo() : std::pair{0, 0}; + (void)round; + std::string body = state->json; + // informative, outside the signed line: how far this node is + const auto brace = body.find('{'); + body.insert(brace + 1, "\n \"synced_to\": " + std::to_string(lastBlock) + ","); + http::response res{http::status::ok, req.version()}; + res.set(http::field::content_type, "application/json"); + res.body() = head ? std::string{} : body; + finish(res, keepAlive); + if (head) res.content_length(body.size()); + boost::system::error_code ec; + http::write(socket, res, ec); + return !ec && keepAlive; + } + const auto slash = rest.find('/'); + if (slash == std::string::npos || rest.substr(0, slash) != std::to_string(state->manifest.seq)) { + return simple(http::status::not_found, "not found"); + } + const auto* file = state->manifest.file(rest.substr(slash + 1)); + if (!file) { + return simple(http::status::not_found, "not found"); + } + const fs::path path = state->dir / file->name; + std::ifstream in(path, std::ios::binary); + if (!in) { + return simple(http::status::not_found, "not found"); + } + const uint64_t size = file->size; + uint64_t first = 0, last = size == 0 ? 0 : size - 1; + bool partial = false; + const auto range = req.find(http::field::range); + if (range != req.end()) { + if (!parseRange(std::string(range->value()), size, first, last) || last - first + 1 > kMaxRange) { + http::response res{http::status::range_not_satisfiable, req.version()}; + res.set(http::field::content_range, "bytes */" + std::to_string(size)); + res.body() = "one range of at most 16 MiB\n"; + finish(res, keepAlive); + boost::system::error_code ec; + http::write(socket, res, ec); + return !ec && keepAlive; + } + partial = true; + } + const uint64_t length = size == 0 ? 0 : last - first + 1; + http::response res{partial ? http::status::partial_content : http::status::ok, req.version()}; + res.set(http::field::server, "credits-node"); + res.set(http::field::content_type, "application/octet-stream"); + res.set(http::field::accept_ranges, "bytes"); + if (partial) { + res.set(http::field::content_range, "bytes " + std::to_string(first) + "-" + std::to_string(last) + "/" + std::to_string(size)); + } + res.content_length(length); + res.keep_alive(keepAlive); + boost::system::error_code ec; + http::response_serializer sr{res}; + http::write_header(socket, sr, ec); + if (ec) { + return false; + } + if (head || length == 0) { + return keepAlive; + } + in.seekg(static_cast(first)); + std::vector buffer(64 << 10); + uint64_t left = length; + while (left > 0 && !stopping) { + const size_t n = static_cast(std::min(left, buffer.size())); + in.read(buffer.data(), static_cast(n)); + if (static_cast(in.gcount()) != n) { + return false; + } + bucket.take(n); + asio::write(socket, asio::buffer(buffer.data(), n), ec); + if (ec) { + return false; + } + left -= n; + served += n; + } + return left == 0 && keepAlive; + } + + void session(std::shared_ptr socket) { + std::string peer; + { + boost::system::error_code ec; + peer = socket->remote_endpoint(ec).address().to_string(); + } + setTimeout(*socket, SO_RCVTIMEO, kIdleTimeoutSec); + setTimeout(*socket, SO_SNDTIMEO, kWriteTimeoutSec); + uint64_t served = 0; + boost::beast::flat_buffer buffer; + for (size_t i = 0; i < kMaxRequestsPerConnection && !stopping; ++i) { + http::request_parser parser; + parser.header_limit(8 * 1024); + boost::system::error_code ec; + http::read(*socket, buffer, parser, ec); + if (ec) { + break; + } + if (!handle(*socket, parser.get(), served)) { + break; + } + } + boost::system::error_code ec; + socket->shutdown(tcp::socket::shutdown_both, ec); + socket->close(ec); + if (served >= (1u << 20)) { + const auto state = current(); + cslog() << "SNAPSHOT: served " << served / (1u << 20) << " MB of block " << (state ? state->manifest.seq : 0) << " to " << peer; + } + { + std::lock_guard lock(socketsLock); + sockets.erase(socket); + } + --active; + } + + void refuse(tcp::socket& socket) { + setTimeout(socket, SO_SNDTIMEO, 5); + http::response res{http::status::service_unavailable, 11}; + res.set(http::field::retry_after, "5"); + res.body() = "too many connections\n"; + finish(res, false); + boost::system::error_code ec; + http::write(socket, res, ec); + socket.shutdown(tcp::socket::shutdown_both, ec); + socket.close(ec); + } + + void acceptLoop() { + while (!stopping) { + auto socket = std::make_shared(io); + boost::system::error_code ec; + acceptor->accept(*socket, ec); + if (ec == asio::error::would_block || ec == asio::error::try_again) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + if (ec) { + if (stopping) break; + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + socket->non_blocking(false, ec); + if (active.load() >= maxConnections.load()) { + refuse(*socket); + continue; + } + ++active; + { + std::lock_guard lock(socketsLock); + sockets.insert(socket); + } + std::thread([this, socket]() { session(socket); }).detach(); + } + } + + void stopListening() { + stopping = true; + if (acceptThread.joinable()) { + acceptThread.join(); + } + if (acceptor) { + boost::system::error_code ec; + acceptor->close(ec); + acceptor.reset(); + } + { + std::lock_guard lock(socketsLock); + for (const auto& s : sockets) { + boost::system::error_code ec; + s->shutdown(tcp::socket::shutdown_both, ec); + } + } + for (int i = 0; i < 100 && active.load() > 0; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + stopping = false; + } + + // returns the bound port, 0 when the address cannot be used + uint16_t listen(const Settings& s) { + boost::system::error_code ec; + const auto address = s.bind.empty() ? asio::ip::address(asio::ip::address_v6::any()) : asio::ip::make_address(s.bind, ec); + if (ec) { + cserror() << "SNAPSHOT: serve_bind " << s.bind << " is not an address"; + return 0; + } + acceptor = std::make_unique(io); + const tcp::endpoint endpoint(address, s.port); + acceptor->open(endpoint.protocol(), ec); + if (!ec && address.is_v6()) { + acceptor->set_option(asio::ip::v6_only(false), ec); + ec.clear(); + } + if (!ec) acceptor->set_option(tcp::acceptor::reuse_address(true), ec); + if (!ec) acceptor->bind(endpoint, ec); + if (!ec) acceptor->listen(asio::socket_base::max_listen_connections, ec); + if (!ec) acceptor->non_blocking(true, ec); + if (ec) { + cserror() << "SNAPSHOT: cannot listen on " << (s.bind.empty() ? "port " : s.bind + ":") << s.port << ": " << ec.message(); + acceptor.reset(); + return 0; + } + const uint16_t bound = acceptor->local_endpoint(ec).port(); + acceptThread = std::thread([this]() { acceptLoop(); }); + return bound; + } +}; + +SnapshotServer::SnapshotServer(fs::path root, SyncedTo syncedTo) : impl_(std::make_unique()) { + impl_->root = std::move(root); + impl_->syncedTo = std::move(syncedTo); +} + +SnapshotServer::~SnapshotServer() { + stop(); +} + +void SnapshotServer::apply(const Settings& settings) { + std::lock_guard lock(impl_->settingsLock); + impl_->bucket.setRate(settings.rateKBps); + impl_->maxConnections = std::max(1, settings.maxConnections); + const bool listening = boundPort_.load() != 0; + const bool sameAddress = listening && settings.bind == impl_->settings.bind && settings.port == impl_->settings.port; + impl_->settings = settings; + if (sameAddress) { + return; + } + if (listening) { + impl_->stopListening(); + cslog() << "SNAPSHOT: service stopped"; + boundPort_ = 0; + } + if (settings.port != 0) { + const uint16_t bound = impl_->listen(settings); + boundPort_ = bound; + if (bound != 0) { + cslog() << "SNAPSHOT: serving on " << (settings.bind.empty() ? std::string("port ") : settings.bind + ":") << bound; + } + } +} + +void SnapshotServer::reload() { + std::error_code ec; + cs::Sequence best = 0; + for (const auto& entry : fs::directory_iterator(impl_->root, ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::is_regular_file(entry.path() / "snapshot.json", ec)) { + best = std::max(best, std::stoull(name)); + } + } + std::shared_ptr next; + if (best != 0) { + const fs::path dir = impl_->root / std::to_string(best); + std::ifstream in(dir / "snapshot.json", std::ios::binary); + std::string json((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::string error; + auto manifest = SnapshotManifest::fromJson(json, &error); + if (manifest && manifest->seq == best) { + auto l = std::make_shared(); + l->manifest = std::move(*manifest); + l->json = std::move(json); + l->dir = dir; + next = l; + } + else { + cswarning() << "SNAPSHOT: " << (dir / "snapshot.json").string() << " is not served: " << error; + } + } + std::lock_guard lock(impl_->loadedLock); + impl_->loaded = next; +} + +void SnapshotServer::stop() { + std::lock_guard lock(impl_->settingsLock); + if (boundPort_.load() != 0) { + impl_->stopListening(); + boundPort_ = 0; + cslog() << "SNAPSHOT: service stopped"; + } +} + +} // namespace cs diff --git a/csnode/src/snapshot_service.cpp b/csnode/src/snapshot_service.cpp new file mode 100644 index 000000000..0d36bfa83 --- /dev/null +++ b/csnode/src/snapshot_service.cpp @@ -0,0 +1,140 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; +constexpr auto kPeriod = std::chrono::seconds(10); + +bool isDigits(const std::string& s) { + return !s.empty() && s.size() <= 12 && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; }); +} + +// the highest periodic checkpoint that records its state (qs/0, the shutdown one, is not periodic) +cs::Sequence latestCheckpoint() { + cs::Sequence best = 0; + std::error_code ec; + for (const auto& entry : fs::directory_iterator("qs", ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::exists(entry.path() / CheckpointDigest::kFileName, ec)) { + best = std::max(best, std::stoull(name)); + } + } + return best; +} +} // namespace + +SnapshotService::SnapshotService(BlockChain& blockchain, const cs::PrivateKey& key, std::string nodeVersion) +: blockchain_(blockchain), key_(key), nodeVersion_(std::move(nodeVersion)) { + server_ = std::make_unique(root_, [this]() { + return std::make_pair(blockchain_.getLastSeq(), static_cast(cs::Conveyer::instance().currentRoundNumber())); + }); +} + +SnapshotService::~SnapshotService() { + stop(); +} + +void SnapshotService::start() { + stop_ = false; + thread_ = std::thread([this]() { run(); }); +} + +void SnapshotService::stop() { + if (!thread_.joinable()) { + return; + } + { + std::lock_guard lock(m_); + stop_ = true; + } + cv_.notify_all(); + thread_.join(); + server_->stop(); +} + +void SnapshotService::onCheckpointSaved(cs::Sequence sequence) { + (void)sequence; + buildPending_ = true; + cv_.notify_all(); +} + +void SnapshotService::run() { + // the builder works on this thread: below the node's other work + ::setpriority(PRIO_PROCESS, static_cast(::syscall(SYS_gettid)), 10); + server_->reload(); + while (!stop_) { + const auto& cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + SnapshotServer::Settings settings; + settings.bind = cfg.serveBind; + settings.port = cfg.servePort; + settings.maxConnections = cfg.serveMaxConnections; + settings.rateKBps = cfg.serveRateKbps; + server_->apply(settings); + if (settings.port != 0 && buildPending_.exchange(false)) { + buildLatest(); + } + std::unique_lock lock(m_); + cv_.wait_for(lock, kPeriod, [this]() { return stop_.load(); }); + } +} + +void SnapshotService::buildLatest() { + const cs::Sequence S = latestCheckpoint(); + std::error_code ec; + if (S == 0 || fs::exists(root_ / std::to_string(S) / "snapshot.json", ec)) { + return; // nothing new: no checkpoint with a digest yet, or already built + } + const auto& cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + SnapshotBuildInput in; + in.seq = S; + in.checkpointDir = fs::path("qs") / std::to_string(S); + in.firstStored = blockchain_.getFirstStoredSequence(); + in.snapshotBlocks = cfg.snapshotBlocks; + in.nodeVersion = nodeVersion_; + in.stop = &stop_; + in.blockBytes = [this](cs::Sequence seq, cs::Bytes& out) { + const csdb::Pool pool = blockchain_.loadBlock(seq); + if (!pool.is_valid()) { + return false; + } + out = pool.to_binary(); + return !out.empty(); + }; + in.contracts = [this](const std::function& visit) { + return blockchain_.forEachContractData(visit); + }; + fs::create_directories(root_, ec); + cslog() << "SNAPSHOT: building the snapshot of checkpoint " << S; + const auto started = std::chrono::steady_clock::now(); + const auto r = buildSnapshot(in, root_, key_); + if (!r.ok) { + cswarning() << "SNAPSHOT: no snapshot of checkpoint " << S << ": " << r.error; + return; + } + uint64_t bytes = 0; + for (const auto& f : r.manifest.files) { + bytes += f.size; + } + cslog() << "SNAPSHOT: built block " << S << ", " << r.manifest.files.size() << " files, " << bytes / (1u << 20) << " MB, state digest " + << r.manifest.stateDigest << " in " + << std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count() << " s"; + server_->reload(); +} + +} // namespace cs diff --git a/tests/csnode/snapshot_server_tests.cpp b/tests/csnode/snapshot_server_tests.cpp new file mode 100644 index 000000000..5f3fe26cc --- /dev/null +++ b/tests/csnode/snapshot_server_tests.cpp @@ -0,0 +1,233 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; +namespace asio = boost::asio; +namespace http = boost::beast::http; +using tcp = asio::ip::tcp; + +uint16_t freePort() { + asio::io_context io; + tcp::acceptor a(io, tcp::endpoint(asio::ip::address_v4::loopback(), 0)); + return a.local_endpoint().port(); +} + +struct Reply { + unsigned status = 0; + std::string body; + std::string contentRange; + std::string retryAfter; +}; + +Reply request(uint16_t port, http::verb verb, const std::string& target, const std::string& range = {}) { + asio::io_context io; + tcp::socket socket(io); + socket.connect(tcp::endpoint(asio::ip::address_v4::loopback(), port)); + http::request req{verb, target, 11}; + req.set(http::field::host, "localhost"); + req.keep_alive(false); + if (!range.empty()) req.set(http::field::range, range); + http::write(socket, req); + boost::beast::flat_buffer buffer; + http::response_parser parser; + parser.body_limit(64u << 20); + if (verb == http::verb::head) parser.skip(true); + http::read(socket, buffer, parser); + Reply r; + r.status = parser.get().result_int(); + r.body = parser.get().body(); + r.contentRange = std::string(parser.get()[http::field::content_range]); + r.retryAfter = std::string(parser.get()[http::field::retry_after]); + return r; +} + +std::string readAll(const fs::path& p) { + std::ifstream f(p, std::ios::binary); + return std::string(std::istreambuf_iterator(f), {}); +} + +// a snapshot built by the builder from a small chain, served from root/ +struct Fixture { + fs::path root; + cs::SnapshotBuildResult built; + std::atomic synced{true}; + std::unique_ptr server; + uint16_t port = 0; + + explicit Fixture(const char* name) : root(fs::temp_directory_path() / name) { + fs::remove_all(root); + EXPECT_TRUE(cscrypto::cryptoInit()); + cs::PublicKey pub; + const auto key = cscrypto::generateKeyPair(pub); + std::map blocks; + csdb::PoolHash prev; + cs::Bytes head; + std::mt19937 random(42); + for (cs::Sequence s = 0; s <= 120; ++s) { + csdb::Pool p(prev, s); + std::string payload(3000, ' '); + for (auto& c : payload) c = static_cast('a' + random() % 26); // about 360 KB that lz4 cannot shrink much + p.add_user_field(1, payload); + EXPECT_TRUE(p.compose()); + blocks[s] = p.to_binary(); + prev = p.hash(); + head = p.hash().to_binary(); + } + const fs::path qs = root / "qs" / "120"; + fs::create_directories(qs); + { + std::ofstream f(qs / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t seq = 120; + const uint16_t hl = static_cast(head.size()), pl = 0; + f.write(reinterpret_cast(&magic), 4); + f.write(reinterpret_cast(&seq), 8); + f.write(reinterpret_cast(&hl), 2); + f.write(reinterpret_cast(head.data()), hl); + f.write(reinterpret_cast(&pl), 2); + } + cs::CheckpointDigest d; + d.wallets = 1; + EXPECT_TRUE(d.write(qs / cs::CheckpointDigest::kFileName)); + cs::SnapshotBuildInput in; + in.seq = 120; + in.checkpointDir = qs; + in.snapshotBlocks = 120; + in.chunkSize = 64 << 10; + in.blockBytes = [blocks](cs::Sequence s, cs::Bytes& out) { out = blocks.at(s); return true; }; + built = cs::buildSnapshot(in, root / "snapshot", key); + EXPECT_TRUE(built.ok) << built.error; + server = std::make_unique(root / "snapshot", [this]() { + return synced ? std::make_pair(cs::Sequence(1000), cs::Sequence(1001)) : std::make_pair(cs::Sequence(10), cs::Sequence(1000)); + }); + server->reload(); + port = freePort(); + cs::SnapshotServer::Settings s; + s.bind = "127.0.0.1"; + s.port = port; + server->apply(s); + EXPECT_TRUE(server->running()); + } + ~Fixture() { + server->stop(); + std::error_code ec; + fs::remove_all(root, ec); + } + std::string blocksPath() const { + return "/snapshot/120/blocks.bin.lz4"; + } +}; +} // namespace + +TEST(SnapshotServer, ServesTheManifestAndFiles) { + Fixture fx("snapshot_server_files"); + auto r = request(fx.port, http::verb::get, "/snapshot/latest.json"); + ASSERT_EQ(r.status, 200u); + EXPECT_NE(r.body.find("\"synced_to\": 1000"), std::string::npos); + const auto manifest = cs::SnapshotManifest::fromJson(r.body); + ASSERT_TRUE(manifest.has_value()); // synced_to is outside the signed line + EXPECT_EQ(manifest->seq, 120u); + + const auto whole = readAll(fx.built.dir / "blocks.bin.lz4"); + r = request(fx.port, http::verb::get, fx.blocksPath()); + ASSERT_EQ(r.status, 200u); + EXPECT_EQ(r.body, whole); + + r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-99"); + ASSERT_EQ(r.status, 206u); + EXPECT_EQ(r.body, whole.substr(0, 100)); + EXPECT_EQ(r.contentRange, "bytes 0-99/" + std::to_string(whole.size())); + r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=-10"); + EXPECT_EQ(r.body, whole.substr(whole.size() - 10)); + r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=100-"); + EXPECT_EQ(r.body, whole.substr(100)); + + EXPECT_EQ(request(fx.port, http::verb::head, fx.blocksPath()).status, 200u); +} + +TEST(SnapshotServer, RefusesWhatItDoesNotServe) { + Fixture fx("snapshot_server_refuse"); + const auto size = fs::file_size(fx.built.dir / "blocks.bin.lz4"); + EXPECT_EQ(request(fx.port, http::verb::get, fx.blocksPath(), "bytes=" + std::to_string(size) + "-").status, 416u); + EXPECT_EQ(request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-1,5-6").status, 416u); + EXPECT_EQ(request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-" + std::to_string((16u << 20) + 1)).status, 206u); // clipped to the file + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/119/blocks.bin.lz4").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/120/../../qs/120/head.bin").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/120/NodePrivate.txt").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::get, "/").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::post, "/snapshot/latest.json").status, 405u); +} + +TEST(SnapshotServer, OnlyASyncedNodeServes) { + Fixture fx("snapshot_server_synced"); + fx.synced = false; + const auto r = request(fx.port, http::verb::get, "/snapshot/latest.json"); + EXPECT_EQ(r.status, 503u); + EXPECT_EQ(r.retryAfter, "30"); + fx.synced = true; + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 200u); +} + +TEST(SnapshotServer, NothingBuiltIsUnavailable) { + Fixture fx("snapshot_server_empty"); + fs::remove_all(fx.root / "snapshot"); + fx.server->reload(); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); +} + +TEST(SnapshotServer, ConnectionCapAndRateAndMove) { + Fixture fx("snapshot_server_limits"); + cs::SnapshotServer::Settings s; + s.bind = "127.0.0.1"; + s.port = fx.port; + s.maxConnections = 1; + fx.server->apply(s); + // one connection held open: the next one is refused with 503 + asio::io_context io; + tcp::socket held(io); + held.connect(tcp::endpoint(asio::ip::address_v4::loopback(), fx.port)); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); + held.close(); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // 100 KB/s: 300 KB take about 3 s (the bucket starts empty) + s.maxConnections = 8; + s.rateKBps = 100; + fx.server->apply(s); + const auto t0 = std::chrono::steady_clock::now(); + const auto r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-299999"); + const double secs = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + ASSERT_EQ(r.status, 206u); + EXPECT_GT(secs, 2.0); + EXPECT_LT(secs, 6.0); + + // moved to another port while running, then stopped by port 0 + s.rateKBps = 0; + s.port = freePort(); + fx.server->apply(s); + EXPECT_EQ(fx.server->port(), s.port); + EXPECT_EQ(request(s.port, http::verb::get, "/snapshot/latest.json").status, 200u); + s.port = 0; + fx.server->apply(s); + EXPECT_FALSE(fx.server->running()); +} From b18471b20491966bfb3a396fd65a762951e889c2 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 07:53:23 +0100 Subject: [PATCH 2/9] snapshot server and builder: security review fixes - A session keeps the server's state alive (shared ownership) and the bandwidth wait gives up when the server stops; stopping waits for every session to end, so nothing outlives what it uses. - Every request's headers must arrive within 10 s and a body must flow at 32 KB/s at least (more time under the bandwidth cap): the accept thread closes connections past their deadline, so a client sending byte by byte holds no slot. At most 2 connections per address. - Node-built snapshots carry no contract records: the contracts database holds each contract's latest state, later than the checkpoint, while the signed snapshot is the state at the checkpoint, which carries the contract states at its block; a node fetches any it lacks. Co-Authored-By: Claude Opus 5.5 --- client/src/snapshot_export.cpp | 5 +- csnode/include/csnode/snapshot_server.hpp | 2 +- csnode/src/snapshot_server.cpp | 87 +++++++++++++++++++---- csnode/src/snapshot_service.cpp | 6 +- tests/csnode/snapshot_server_tests.cpp | 14 ++++ 5 files changed, 93 insertions(+), 21 deletions(-) diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index 7d6a2f6ab..a79e337a9 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -123,9 +123,8 @@ int run(const Config& config, const std::string& outDir) { in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) { return db->get(static_cast(seq), &out); }; - in.contracts = [&db](const std::function& visit) { - return db->forEachContractData(visit); - }; + // no contract records: the contracts database holds each contract's latest state, later than the + // checkpoint; the checkpoint carries the states at its block say() << "checkpoint " << S << ", blocks from " << std::max(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0) << ", into " << outDir << std::endl; diff --git a/csnode/include/csnode/snapshot_server.hpp b/csnode/include/csnode/snapshot_server.hpp index 4385614c5..aaca5d745 100644 --- a/csnode/include/csnode/snapshot_server.hpp +++ b/csnode/include/csnode/snapshot_server.hpp @@ -59,7 +59,7 @@ class SnapshotServer { private: struct Impl; - std::unique_ptr impl_; + std::shared_ptr impl_; std::atomic boundPort_{0}; }; diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp index 45f18e89c..cf7038d24 100644 --- a/csnode/src/snapshot_server.cpp +++ b/csnode/src/snapshot_server.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -76,19 +77,22 @@ class TokenBucket { rate_ = static_cast(kBps) * 1000.0; tokens_ = std::min(tokens_, rate_); } - // waits until n bytes may be sent - void take(size_t n) { + // waits until n bytes may be sent; false when the server stops meanwhile + bool take(size_t n, const std::atomic& stopping) { for (;;) { + if (stopping) { + return false; + } std::unique_lock lock(m_); if (rate_ <= 0) { - return; + return true; } const auto now = std::chrono::steady_clock::now(); tokens_ = std::min(rate_, tokens_ + rate_ * std::chrono::duration(now - last_).count()); last_ = now; if (tokens_ >= static_cast(n)) { tokens_ -= static_cast(n); - return; + return true; } const double wait = (static_cast(n) - tokens_) / rate_; lock.unlock(); @@ -110,7 +114,11 @@ struct Loaded { }; } // namespace -struct SnapshotServer::Impl { +struct SnapshotServer::Impl : std::enable_shared_from_this { + static constexpr int kHeaderDeadlineSec = 10; // a whole request line and headers + static constexpr uint64_t kMinWriteBytesPerSec = 32 << 10; + static constexpr size_t kMaxPerAddress = 2; + fs::path root; SyncedTo syncedTo; Settings settings; @@ -120,11 +128,37 @@ struct SnapshotServer::Impl { std::mutex loadedLock; TokenBucket bucket; + std::atomic bucketRateKBps{0}; std::atomic maxConnections{8}; std::atomic active{0}; - std::set> sockets; + // every open connection, its peer address and the time its current step must end by: the accept + // thread closes the ones past it (a client that sends its headers byte by byte holds no slot) + struct Conn { + std::string address; + std::chrono::steady_clock::time_point deadline; + }; + std::map, Conn> sockets; std::mutex socketsLock; + void setDeadline(const std::shared_ptr& socket, std::chrono::seconds in) { + std::lock_guard lock(socketsLock); + const auto it = sockets.find(socket); + if (it != sockets.end()) { + it->second.deadline = std::chrono::steady_clock::now() + in; + } + } + + void closeExpired() { + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(socketsLock); + for (auto& [socket, conn] : sockets) { + if (now > conn.deadline) { + boost::system::error_code ec; + socket->shutdown(tcp::socket::shutdown_both, ec); + } + } + } + asio::io_context io; std::unique_ptr acceptor; std::thread acceptThread; @@ -151,7 +185,8 @@ struct SnapshotServer::Impl { } // returns false when the connection must end - bool handle(tcp::socket& socket, const http::request& req, uint64_t& served) { + bool handle(const std::shared_ptr& socketPtr, const http::request& req, uint64_t& served) { + tcp::socket& socket = *socketPtr; const bool keepAlive = req.keep_alive(); auto simple = [&](http::status status, const std::string& text, bool retry = false) { http::response res{status, req.version()}; @@ -244,6 +279,9 @@ struct SnapshotServer::Impl { if (head || length == 0) { return keepAlive; } + // the body must flow at a minimum rate (the cap included): a stalled reader holds no slot + setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec + length / kMinWriteBytesPerSec + + (bucketRateKBps.load() ? length / (bucketRateKBps.load() * 1000) : 0))); in.seekg(static_cast(first)); std::vector buffer(64 << 10); uint64_t left = length; @@ -253,7 +291,9 @@ struct SnapshotServer::Impl { if (static_cast(in.gcount()) != n) { return false; } - bucket.take(n); + if (!bucket.take(n, stopping)) { + return false; + } asio::write(socket, asio::buffer(buffer.data(), n), ec); if (ec) { return false; @@ -277,12 +317,13 @@ struct SnapshotServer::Impl { for (size_t i = 0; i < kMaxRequestsPerConnection && !stopping; ++i) { http::request_parser parser; parser.header_limit(8 * 1024); + setDeadline(socket, std::chrono::seconds(i == 0 ? kHeaderDeadlineSec : kIdleTimeoutSec)); boost::system::error_code ec; http::read(*socket, buffer, parser, ec); if (ec) { break; } - if (!handle(*socket, parser.get(), served)) { + if (!handle(socket, parser.get(), served)) { break; } } @@ -318,6 +359,7 @@ struct SnapshotServer::Impl { boost::system::error_code ec; acceptor->accept(*socket, ec); if (ec == asio::error::would_block || ec == asio::error::try_again) { + closeExpired(); std::this_thread::sleep_for(std::chrono::milliseconds(50)); continue; } @@ -331,12 +373,27 @@ struct SnapshotServer::Impl { refuse(*socket); continue; } - ++active; + std::string address; + { + boost::system::error_code aec; + address = socket->remote_endpoint(aec).address().to_string(); + } + bool tooMany = false; { std::lock_guard lock(socketsLock); - sockets.insert(socket); + const auto same = std::count_if(sockets.begin(), sockets.end(), [&](const auto& kv) { return kv.second.address == address; }); + tooMany = static_cast(same) >= kMaxPerAddress; + if (!tooMany) { + sockets.emplace(socket, Conn{address, std::chrono::steady_clock::now() + std::chrono::seconds(kHeaderDeadlineSec)}); + } + } + if (tooMany) { + refuse(*socket); // outside the lock: it writes, up to its own timeout + continue; } - std::thread([this, socket]() { session(socket); }).detach(); + ++active; + // the session keeps the server's state alive until it ends, whatever happens to the server + std::thread([self = shared_from_this(), socket]() { self->session(socket); }).detach(); } } @@ -357,7 +414,8 @@ struct SnapshotServer::Impl { s->shutdown(tcp::socket::shutdown_both, ec); } } - for (int i = 0; i < 100 && active.load() > 0; ++i) { + // the sessions see their sockets shut and stopping set: they end within their current write + while (active.load() > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } stopping = false; @@ -393,7 +451,7 @@ struct SnapshotServer::Impl { } }; -SnapshotServer::SnapshotServer(fs::path root, SyncedTo syncedTo) : impl_(std::make_unique()) { +SnapshotServer::SnapshotServer(fs::path root, SyncedTo syncedTo) : impl_(std::make_shared()) { impl_->root = std::move(root); impl_->syncedTo = std::move(syncedTo); } @@ -405,6 +463,7 @@ SnapshotServer::~SnapshotServer() { void SnapshotServer::apply(const Settings& settings) { std::lock_guard lock(impl_->settingsLock); impl_->bucket.setRate(settings.rateKBps); + impl_->bucketRateKBps = settings.rateKBps; impl_->maxConnections = std::max(1, settings.maxConnections); const bool listening = boundPort_.load() != 0; const bool sameAddress = listening && settings.bind == impl_->settings.bind && settings.port == impl_->settings.port; diff --git a/csnode/src/snapshot_service.cpp b/csnode/src/snapshot_service.cpp index 0d36bfa83..b21cdfe90 100644 --- a/csnode/src/snapshot_service.cpp +++ b/csnode/src/snapshot_service.cpp @@ -116,9 +116,9 @@ void SnapshotService::buildLatest() { out = pool.to_binary(); return !out.empty(); }; - in.contracts = [this](const std::function& visit) { - return blockchain_.forEachContractData(visit); - }; + // no contract records: the contracts database holds the latest state of each contract, later than + // S, while the signed snapshot is the state at S; the checkpoint carries the states at S + // (smartcontracts.dat) and the node fetches any it lacks from the confidants fs::create_directories(root_, ec); cslog() << "SNAPSHOT: building the snapshot of checkpoint " << S; const auto started = std::chrono::steady_clock::now(); diff --git a/tests/csnode/snapshot_server_tests.cpp b/tests/csnode/snapshot_server_tests.cpp index 5f3fe26cc..2e3ee25c0 100644 --- a/tests/csnode/snapshot_server_tests.cpp +++ b/tests/csnode/snapshot_server_tests.cpp @@ -231,3 +231,17 @@ TEST(SnapshotServer, ConnectionCapAndRateAndMove) { fx.server->apply(s); EXPECT_FALSE(fx.server->running()); } + +TEST(SnapshotServer, SlowHeadersAreCutOffAndAddressesCapped) { + Fixture fx("snapshot_server_slow"); + // two connections from one address that send nothing: a third is refused, and the two are closed + // once the header deadline passes, after which the address is served again + asio::io_context io; + tcp::socket a(io), b(io); + a.connect(tcp::endpoint(asio::ip::address_v4::loopback(), fx.port)); + b.connect(tcp::endpoint(asio::ip::address_v4::loopback(), fx.port)); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); + std::this_thread::sleep_for(std::chrono::seconds(12)); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 200u); +} From 0b0d2e7a8d7e30a177359f60311f9745a2c00c74 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 07:55:28 +0100 Subject: [PATCH 3/9] snapshot server: deadline sweep on every pass, address groups, rolling write deadline - Expired connections are closed on every pass of the accept loop, also while connections keep arriving. - The per-address cap counts IPv4 addresses (IPv4-mapped ones too) and IPv6 /64 networks. - Every piece written renews a 30 s deadline, instead of one deadline for the whole body: a reader that stops reading is closed within it. Co-Authored-By: Claude Opus 5.5 --- csnode/src/snapshot_server.cpp | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp index cf7038d24..dd43d85e1 100644 --- a/csnode/src/snapshot_server.cpp +++ b/csnode/src/snapshot_server.cpp @@ -25,6 +25,21 @@ namespace http = boost::beast::http; using tcp = asio::ip::tcp; namespace fs = std::filesystem; +// the unit the per-address cap counts: an IPv4 address (also when written as IPv4-mapped IPv6), or +// the /64 network of an IPv6 address, which one host usually holds whole +std::string peerGroup(const asio::ip::address& address) { + if (address.is_v6()) { + const auto v6 = address.to_v6(); + if (v6.is_v4_mapped()) { + return asio::ip::make_address_v4(asio::ip::v4_mapped, v6).to_string(); + } + auto bytes = v6.to_bytes(); + std::fill(bytes.begin() + 8, bytes.end(), 0); + return asio::ip::address_v6(bytes).to_string() + "/64"; + } + return address.to_string(); +} + void setTimeout(tcp::socket& socket, int option, int seconds) { timeval tv{}; tv.tv_sec = seconds; @@ -116,7 +131,6 @@ struct Loaded { struct SnapshotServer::Impl : std::enable_shared_from_this { static constexpr int kHeaderDeadlineSec = 10; // a whole request line and headers - static constexpr uint64_t kMinWriteBytesPerSec = 32 << 10; static constexpr size_t kMaxPerAddress = 2; fs::path root; @@ -128,7 +142,6 @@ struct SnapshotServer::Impl : std::enable_shared_from_this std::mutex loadedLock; TokenBucket bucket; - std::atomic bucketRateKBps{0}; std::atomic maxConnections{8}; std::atomic active{0}; // every open connection, its peer address and the time its current step must end by: the accept @@ -279,9 +292,8 @@ struct SnapshotServer::Impl : std::enable_shared_from_this if (head || length == 0) { return keepAlive; } - // the body must flow at a minimum rate (the cap included): a stalled reader holds no slot - setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec + length / kMinWriteBytesPerSec + - (bucketRateKBps.load() ? length / (bucketRateKBps.load() * 1000) : 0))); + // a stalled reader holds no slot: every piece written renews a short deadline + setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); in.seekg(static_cast(first)); std::vector buffer(64 << 10); uint64_t left = length; @@ -298,6 +310,7 @@ struct SnapshotServer::Impl : std::enable_shared_from_this if (ec) { return false; } + setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); left -= n; served += n; } @@ -355,11 +368,11 @@ struct SnapshotServer::Impl : std::enable_shared_from_this void acceptLoop() { while (!stopping) { + closeExpired(); // on every pass, whether connections are waiting or not auto socket = std::make_shared(io); boost::system::error_code ec; acceptor->accept(*socket, ec); if (ec == asio::error::would_block || ec == asio::error::try_again) { - closeExpired(); std::this_thread::sleep_for(std::chrono::milliseconds(50)); continue; } @@ -376,7 +389,7 @@ struct SnapshotServer::Impl : std::enable_shared_from_this std::string address; { boost::system::error_code aec; - address = socket->remote_endpoint(aec).address().to_string(); + address = peerGroup(socket->remote_endpoint(aec).address()); } bool tooMany = false; { @@ -463,7 +476,6 @@ SnapshotServer::~SnapshotServer() { void SnapshotServer::apply(const Settings& settings) { std::lock_guard lock(impl_->settingsLock); impl_->bucket.setRate(settings.rateKBps); - impl_->bucketRateKBps = settings.rateKBps; impl_->maxConnections = std::max(1, settings.maxConnections); const bool listening = boundPort_.load() != 0; const bool sameAddress = listening && settings.bind == impl_->settings.bind && settings.port == impl_->settings.port; From 46c13cd60cd183196aba735bf354e035d624ea5b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 07:57:42 +0100 Subject: [PATCH 4/9] snapshot server: minimum average write rate and a maximum connection time A reader whose average rate, over the time spent writing to it (the bandwidth cap's waits not counted), falls below 32 KB/s after 10 s of writing is dropped; a connection takes no new request after 30 minutes. Co-Authored-By: Claude Opus 5.5 --- csnode/src/snapshot_server.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp index dd43d85e1..f45a1a169 100644 --- a/csnode/src/snapshot_server.cpp +++ b/csnode/src/snapshot_server.cpp @@ -132,6 +132,11 @@ struct Loaded { struct SnapshotServer::Impl : std::enable_shared_from_this { static constexpr int kHeaderDeadlineSec = 10; // a whole request line and headers static constexpr size_t kMaxPerAddress = 2; + // a reader slower than this on average, over the time spent writing to it (the bandwidth cap's + // waits not counted), is dropped once it has been written to for kMinRateAfterSec + static constexpr double kMinBytesPerSec = 32 * 1024; + static constexpr double kMinRateAfterSec = 10; + static constexpr int kMaxConnectionSec = 30 * 60; fs::path root; SyncedTo syncedTo; @@ -297,6 +302,8 @@ struct SnapshotServer::Impl : std::enable_shared_from_this in.seekg(static_cast(first)); std::vector buffer(64 << 10); uint64_t left = length; + double writeSeconds = 0; + uint64_t written = 0; while (left > 0 && !stopping) { const size_t n = static_cast(std::min(left, buffer.size())); in.read(buffer.data(), static_cast(n)); @@ -306,10 +313,16 @@ struct SnapshotServer::Impl : std::enable_shared_from_this if (!bucket.take(n, stopping)) { return false; } + const auto t0 = std::chrono::steady_clock::now(); asio::write(socket, asio::buffer(buffer.data(), n), ec); if (ec) { return false; } + writeSeconds += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + written += n; + if (writeSeconds >= kMinRateAfterSec && static_cast(written) / writeSeconds < kMinBytesPerSec) { + return false; // the reader is too slow to keep a slot + } setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); left -= n; served += n; @@ -327,7 +340,8 @@ struct SnapshotServer::Impl : std::enable_shared_from_this setTimeout(*socket, SO_SNDTIMEO, kWriteTimeoutSec); uint64_t served = 0; boost::beast::flat_buffer buffer; - for (size_t i = 0; i < kMaxRequestsPerConnection && !stopping; ++i) { + const auto connectionEnd = std::chrono::steady_clock::now() + std::chrono::seconds(kMaxConnectionSec); + for (size_t i = 0; i < kMaxRequestsPerConnection && !stopping && std::chrono::steady_clock::now() < connectionEnd; ++i) { http::request_parser parser; parser.header_limit(8 * 1024); setDeadline(socket, std::chrono::seconds(i == 0 ? kHeaderDeadlineSec : kIdleTimeoutSec)); From 8fd71114c2c2530f0033117eec2ecbf5b6005a97 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 07:59:10 +0100 Subject: [PATCH 5/9] snapshot server: rate floor and time limit per connection, not per request The minimum average rate and the 30-minute limit are counted across all requests of a connection, and the limit also ends a body being written. Co-Authored-By: Claude Opus 5.5 --- csnode/src/snapshot_server.cpp | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp index f45a1a169..409943366 100644 --- a/csnode/src/snapshot_server.cpp +++ b/csnode/src/snapshot_server.cpp @@ -203,7 +203,16 @@ struct SnapshotServer::Impl : std::enable_shared_from_this } // returns false when the connection must end - bool handle(const std::shared_ptr& socketPtr, const http::request& req, uint64_t& served) { + // what one connection has used, across its requests: the minimum rate and the time limit apply to + // the connection, not to each request + struct Usage { + uint64_t served = 0; + double writeSeconds = 0; + std::chrono::steady_clock::time_point end; + }; + + bool handle(const std::shared_ptr& socketPtr, const http::request& req, Usage& usage) { + uint64_t& served = usage.served; tcp::socket& socket = *socketPtr; const bool keepAlive = req.keep_alive(); auto simple = [&](http::status status, const std::string& text, bool retry = false) { @@ -302,9 +311,10 @@ struct SnapshotServer::Impl : std::enable_shared_from_this in.seekg(static_cast(first)); std::vector buffer(64 << 10); uint64_t left = length; - double writeSeconds = 0; - uint64_t written = 0; while (left > 0 && !stopping) { + if (std::chrono::steady_clock::now() >= usage.end) { + return false; // the connection's time is up, even in the middle of a body + } const size_t n = static_cast(std::min(left, buffer.size())); in.read(buffer.data(), static_cast(n)); if (static_cast(in.gcount()) != n) { @@ -318,9 +328,8 @@ struct SnapshotServer::Impl : std::enable_shared_from_this if (ec) { return false; } - writeSeconds += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); - written += n; - if (writeSeconds >= kMinRateAfterSec && static_cast(written) / writeSeconds < kMinBytesPerSec) { + usage.writeSeconds += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + if (usage.writeSeconds >= kMinRateAfterSec && static_cast(usage.served + n) / usage.writeSeconds < kMinBytesPerSec) { return false; // the reader is too slow to keep a slot } setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); @@ -338,10 +347,11 @@ struct SnapshotServer::Impl : std::enable_shared_from_this } setTimeout(*socket, SO_RCVTIMEO, kIdleTimeoutSec); setTimeout(*socket, SO_SNDTIMEO, kWriteTimeoutSec); - uint64_t served = 0; + Usage usage; + usage.end = std::chrono::steady_clock::now() + std::chrono::seconds(kMaxConnectionSec); + uint64_t& served = usage.served; boost::beast::flat_buffer buffer; - const auto connectionEnd = std::chrono::steady_clock::now() + std::chrono::seconds(kMaxConnectionSec); - for (size_t i = 0; i < kMaxRequestsPerConnection && !stopping && std::chrono::steady_clock::now() < connectionEnd; ++i) { + for (size_t i = 0; i < kMaxRequestsPerConnection && !stopping && std::chrono::steady_clock::now() < usage.end; ++i) { http::request_parser parser; parser.header_limit(8 * 1024); setDeadline(socket, std::chrono::seconds(i == 0 ? kHeaderDeadlineSec : kIdleTimeoutSec)); @@ -350,7 +360,7 @@ struct SnapshotServer::Impl : std::enable_shared_from_this if (ec) { break; } - if (!handle(socket, parser.get(), served)) { + if (!handle(socket, parser.get(), usage)) { break; } } From 92975ecb025e140e1e1b04d0aaf55a7551c33605 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 08:04:04 +0100 Subject: [PATCH 6/9] snapshot builder: remove the rest of the whole-file reader Co-Authored-By: Claude Opus 5.5 --- csnode/src/snapshot_builder.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index c4f17a781..faa08d90c 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -175,10 +175,6 @@ std::string utcNow() { return buf; } - out.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); - return !in.bad(); -} - bool stopped(const SnapshotBuildInput& in) { return in.stop && in.stop->load(); } From a728ac77a4b17fe4bcdb2e4e8b1b57bd780b1098 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 08:07:11 +0100 Subject: [PATCH 7/9] snapshot server: shut the sockets of the connection map down on stop Co-Authored-By: Claude Opus 5.5 --- csnode/src/snapshot_server.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp index 409943366..3bbb63c7e 100644 --- a/csnode/src/snapshot_server.cpp +++ b/csnode/src/snapshot_server.cpp @@ -446,9 +446,9 @@ struct SnapshotServer::Impl : std::enable_shared_from_this } { std::lock_guard lock(socketsLock); - for (const auto& s : sockets) { + for (const auto& [socket, conn] : sockets) { boost::system::error_code ec; - s->shutdown(tcp::socket::shutdown_both, ec); + socket->shutdown(tcp::socket::shutdown_both, ec); } } // the sessions see their sockets shut and stopping set: they end within their current write From a35a5322bc22c03eea37e630bdf5acbbfecb7d4b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 08:16:28 +0100 Subject: [PATCH 8/9] ci(testnet): a node serves its snapshot over HTTP (scenario snapshot-serve) SNAPSHOT_SERVE_NODE gives that node [snapshot] serve_port; at the end it must have built the snapshot of a periodic checkpoint and serve it: the manifest is fetched, the first piece of the blocks file has the declared hash, and a path outside the snapshot is answered 404. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 3 ++ ci/testnet/run.sh | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index a71c11778..91a0b201c 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -277,6 +277,9 @@ jobs: - name: snapshot-join-rocksdb-full env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,rocksdb PRUNE_NODE=6 JOIN_FULL=1 SNAPSHOT_AT=2300 TARGET_SEQ=3300 fixed: true + # a pruned node serves the snapshots it builds over HTTP (snapshot serving, step 4) + - name: snapshot-serve + env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_SERVE_NODE=6 TARGET_SEQ=2600 - name: snapshot-join-format1 env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_FORMAT=1 TARGET_SEQ=3100 # node 5 from full to pruned, node 6 from pruned to full, after node 6 pruned once diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index a0a00bb1f..666dfe9c8 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -67,6 +67,8 @@ # snapshot with --import-snapshot into its own backend and joins; with SNAPSHOT_CORRUPT=1 the # blocks file is truncated first, so the restore must fail and that node syncs from block 0. # SNAPSHOT_FORMAT=1: packaging/docker/snapshot-make.sh and snapshot-restore.sh instead +# SNAPSHOT_SERVE_NODE (default 0 = off): that node gets [snapshot] serve_port (BASE_PORT + 700 + i): +# at the end it must have built the snapshot of a periodic checkpoint and serve it over HTTP # STORAGE_SWITCH_AT (default 0 = off, needs PRUNE_NODE=6 and DB_BACKEND=lmdb): at that sequence # node 5 is restarted as a pruned node and node 6 as a full one # CONVERT_AT (default 0 = off, needs DB_BACKEND=berkeleydb): at that sequence n4 is converted to @@ -198,6 +200,7 @@ JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} SNAPSHOT_FORMAT=${SNAPSHOT_FORMAT:-2} +SNAPSHOT_SERVE_NODE=${SNAPSHOT_SERVE_NODE:-0} SNAPSHOT_CORRUPT=${SNAPSHOT_CORRUPT:-0} STORAGE_SWITCH_AT=${STORAGE_SWITCH_AT:-0} CONVERT_AT=${CONVERT_AT:-0} @@ -330,6 +333,15 @@ PROPS chmod +x "$WORK/n$i/executor.sh" } +# the [snapshot] section of node i: SNAPSHOT_SERVE_NODE serves the snapshots it builds over HTTP +snapshot_section() { + local i=$1 + if [ "$i" -eq "$SNAPSHOT_SERVE_NODE" ]; then + printf "\n[snapshot]\nserve_port=%s\nserve_bind=127.0.0.1\n" "$((BASE_PORT + 700 + i))" + fi + return 0 +} + # the [storage] section of node i: pruning, backend and checkpoint frequency storage_section() { local i=$1 backend="" lines="" @@ -384,6 +396,7 @@ $(api_settings "$i") ajax_port=0 diag_port=0 $(storage_section "$i") +$(snapshot_section "$i") [Core] Filter="%Severity% >= info" @@ -1155,6 +1168,48 @@ for i in $(seq 1 "$ALL_NODES"); do n=$(cat "$WORK/n$i"/*.log | grep -ac "but its caches already hold block" || true) if [ "${n:-0}" -gt 0 ]; then echo "n$i loaded $n checkpoint(s) one block behind their caches"; fi done +if [ "$SNAPSHOT_SERVE_NODE" -gt 0 ]; then + # the node built the snapshot of a periodic checkpoint and serves it: the manifest verifies, and a + # piece of the blocks file has the hash the manifest declares for it + serve_port=$((BASE_PORT + 700 + SNAPSHOT_SERVE_NODE)) + deadline=$(( $(date +%s) + 300 )) + until grep -aq 'SNAPSHOT: built block' "$WORK/n$SNAPSHOT_SERVE_NODE/node.log" || [ "$(date +%s)" -ge "$deadline" ]; do sleep 5; done + grep -a 'SNAPSHOT: ' "$WORK/n$SNAPSHOT_SERVE_NODE/node.log" | tail -4 | sed 's/^.*SNAPSHOT/serve test: SNAPSHOT/' + python3 - "$serve_port" <<'PY' || exit 1 +import hashlib, json, sys, time, urllib.request +port = int(sys.argv[1]) +base = f"http://127.0.0.1:{port}/snapshot" +for attempt in range(30): + try: + with urllib.request.urlopen(base + "/latest.json", timeout=10) as r: + manifest = json.load(r) + break + except Exception as e: # 503 while the node catches up or builds + last = e + time.sleep(10) +else: + print(f"FAIL: no manifest from the serving node: {last}") + sys.exit(1) +f = next(x for x in manifest["files"] if x["name"] == "blocks.bin.lz4") +n = min(manifest["chunk_size"], f["size"]) +req = urllib.request.Request(f"{base}/{manifest['seq']}/blocks.bin.lz4", headers={"Range": f"bytes=0-{n - 1}"}) +with urllib.request.urlopen(req, timeout=30) as r: + piece = r.read() + status = r.status +if status != 206 or hashlib.sha256(piece).hexdigest() != f["chunks"][0]: + print(f"FAIL: the first piece of blocks.bin.lz4 does not match the manifest (status {status}, {len(piece)} bytes)") + sys.exit(1) +try: + urllib.request.urlopen(f"{base}/{manifest['seq']}/../../config.ini", timeout=10) + print("FAIL: a path outside the snapshot was served") + sys.exit(1) +except urllib.error.HTTPError as e: + if e.code != 404: + print(f"FAIL: unexpected status {e.code} for a path outside the snapshot") + sys.exit(1) +print(f"serve test: block {manifest['seq']} served by node at {manifest.get('synced_to')}, {len(manifest['files'])} files, first piece verified") +PY +fi if [ "$SNAPSHOT_AT" -gt 0 ]; then if [ "$SNAPSHOT_CORRUPT" = 1 ]; then grep -aq 'SLOW START' "$WORK/n$JOIN_NODE"/node.log || { echo "FAIL: n$JOIN_NODE did not sync from block 0"; exit 1; } From 9b11168edc9fa774ca8d9a0556277876f23c3568 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 08:27:24 +0100 Subject: [PATCH 9/9] snapshot server and service: review fixes - The bandwidth cap lets a piece larger than a second's worth through once the bucket is out of debt (a cap below 65 KB/s hung every body), and its wait ends with the connection's time; the write deadline starts after that wait. The bucket starts empty when the rate is set. - A node that has not heard the network (round 0) is not synced. - Without serve_bind, a host without IPv6 listens on every IPv4 interface; a listen failure is logged once per setting. - The build starts above the blocks pruning is removing, reads stored bytes directly (no parsing, no pool cache), checks the free disk, is tried again a minute later up to 5 times, and stops between pieces of a checkpoint file. - HEAD error replies carry no body; the settings are copied, not held. - Tests: round 0 answers 503; a 50 KB/s cap completes a 100 KB range. Co-Authored-By: Claude Opus 5.5 --- csdb/include/csdb/storage.hpp | 5 +++ csdb/src/storage.cpp | 8 ++++ csnode/include/csnode/blockchain.hpp | 4 ++ csnode/include/csnode/snapshot_service.hpp | 4 ++ csnode/src/blockchain.cpp | 10 +++++ csnode/src/snapshot_builder.cpp | 3 ++ csnode/src/snapshot_server.cpp | 46 +++++++++++++++------- csnode/src/snapshot_service.cpp | 38 +++++++++++++----- tests/csnode/snapshot_server_tests.cpp | 15 ++++++- 9 files changed, 107 insertions(+), 26 deletions(-) diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index ab34d6546..808c8dc82 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -370,6 +370,11 @@ class Storage final { */ bool forEachContractData(const std::function& fn) const; + /** + * The stored bytes of block sequence, as the database holds them (no parsing, no pool cache). + */ + bool pool_bytes(cs::Sequence sequence, cs::Bytes& out) const; + /** * Returns the space of pruned pools to the file system. */ diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index 925197918..c94787c2c 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -1350,6 +1350,14 @@ bool Storage::pool_prune_batch(const std::vector& hashes) { return true; } +bool Storage::pool_bytes(cs::Sequence sequence, cs::Bytes& out) const { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + return d->db->get(static_cast(sequence), &out); +} + bool Storage::forEachContractData(const std::function& fn) const { if (!isOpen()) { d->set_last_error(NotOpen); diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index e409a5cdc..2f0bd3c98 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -95,6 +95,10 @@ class BlockChain { // pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were // removed once a later periodic checkpoint made them unnecessary cs::Sequence getFirstStoredSequence() const; + // the stored bytes of a block, under the storage lock, for a snapshot: no parsing, no pool cache + bool blockBytes(cs::Sequence sequence, cs::Bytes& out) const; + // pruned storage: the blocks below this one are being or will be removed (0: nothing scheduled) + cs::Sequence pruneTarget() const; // every contract record of the storage, for a snapshot of the node's state bool forEachContractData(const std::function& fn) const; diff --git a/csnode/include/csnode/snapshot_service.hpp b/csnode/include/csnode/snapshot_service.hpp index 00368dfc0..03ca57989 100644 --- a/csnode/include/csnode/snapshot_service.hpp +++ b/csnode/include/csnode/snapshot_service.hpp @@ -2,6 +2,7 @@ #define SNAPSHOT_SERVICE_HPP #include +#include #include #include #include @@ -42,6 +43,9 @@ class SnapshotService { std::thread thread_; std::atomic stop_{false}; std::atomic buildPending_{true}; + static constexpr int kMaxBuildAttempts = 5; + int failedBuilds_ = 0; + std::chrono::steady_clock::time_point retryAt_{}; std::mutex m_; std::condition_variable cv_; }; diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index bacf9f4e1..8f6414a00 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1065,6 +1065,16 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { } } +bool BlockChain::blockBytes(cs::Sequence sequence, cs::Bytes& out) const { + std::lock_guard lock(dbLock_); + return storage_.pool_bytes(sequence, out); +} + +cs::Sequence BlockChain::pruneTarget() const { + std::lock_guard lock(dbLock_); + return pruneUpTo_; +} + bool BlockChain::forEachContractData(const std::function& fn) const { return storage_.forEachContractData(fn); } diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index faa08d90c..5b1938ea6 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -333,6 +333,9 @@ SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::fi Lz4Writer w(tmp / f.name, m.chunkSize); std::vector piece(snapshot_lz4::kBlockSize); while (in) { + if (stopped(input)) { + return failWith("stopped"); + } in.read(reinterpret_cast(piece.data()), static_cast(piece.size())); const auto n = in.gcount(); if (n > 0) { diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp index 3bbb63c7e..44e09a7e5 100644 --- a/csnode/src/snapshot_server.cpp +++ b/csnode/src/snapshot_server.cpp @@ -90,12 +90,14 @@ class TokenBucket { void setRate(size_t kBps) { std::lock_guard lock(m_); rate_ = static_cast(kBps) * 1000.0; - tokens_ = std::min(tokens_, rate_); + tokens_ = 0; + last_ = std::chrono::steady_clock::now(); } - // waits until n bytes may be sent; false when the server stops meanwhile - bool take(size_t n, const std::atomic& stopping) { + // waits until n bytes may be sent; a piece larger than a second's worth is let through as soon as + // the bucket is not in debt, and leaves it in debt. false when the server stops or the deadline passes + bool take(size_t n, const std::atomic& stopping, std::chrono::steady_clock::time_point deadline) { for (;;) { - if (stopping) { + if (stopping || std::chrono::steady_clock::now() >= deadline) { return false; } std::unique_lock lock(m_); @@ -105,11 +107,12 @@ class TokenBucket { const auto now = std::chrono::steady_clock::now(); tokens_ = std::min(rate_, tokens_ + rate_ * std::chrono::duration(now - last_).count()); last_ = now; - if (tokens_ >= static_cast(n)) { + const double need = std::min(static_cast(n), rate_); + if (tokens_ >= need) { tokens_ -= static_cast(n); return true; } - const double wait = (static_cast(n) - tokens_) / rate_; + const double wait = (need - tokens_) / rate_; lock.unlock(); std::this_thread::sleep_for(std::chrono::duration(std::min(wait, 0.2))); } @@ -181,6 +184,7 @@ struct SnapshotServer::Impl : std::enable_shared_from_this std::unique_ptr acceptor; std::thread acceptThread; std::atomic stopping{false}; + Settings lastFailure; std::shared_ptr current() { std::lock_guard lock(loadedLock); @@ -192,7 +196,8 @@ struct SnapshotServer::Impl : std::enable_shared_from_this return true; } const auto [lastBlock, round] = syncedTo(); - return lastBlock > 0 && round <= lastBlock + kSyncedRounds; + // round 0: the node has not heard the network yet (a restart, a firewall), not synced + return lastBlock > 0 && round > 0 && round <= lastBlock + kSyncedRounds; } template @@ -219,11 +224,12 @@ struct SnapshotServer::Impl : std::enable_shared_from_this http::response res{status, req.version()}; res.set(http::field::content_type, "text/plain"); if (retry) res.set(http::field::retry_after, "30"); - res.body() = text + "\n"; - finish(res, keepAlive); + const bool head = req.method() == http::verb::head; + res.body() = head ? std::string{} : text + "\n"; + finish(res, keepAlive && !head); boost::system::error_code ec; http::write(socket, res, ec); - return !ec && keepAlive; + return !ec && keepAlive && !head; }; if (req.method() != http::verb::get && req.method() != http::verb::head) { return simple(http::status::method_not_allowed, "GET or HEAD only"); @@ -320,9 +326,11 @@ struct SnapshotServer::Impl : std::enable_shared_from_this if (static_cast(in.gcount()) != n) { return false; } - if (!bucket.take(n, stopping)) { + if (!bucket.take(n, stopping, usage.end)) { return false; } + // the wait for the bandwidth cap is not the reader's: the write deadline starts after it + setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); const auto t0 = std::chrono::steady_clock::now(); asio::write(socket, asio::buffer(buffer.data(), n), ec); if (ec) { @@ -467,9 +475,15 @@ struct SnapshotServer::Impl : std::enable_shared_from_this return 0; } acceptor = std::make_unique(io); - const tcp::endpoint endpoint(address, s.port); + tcp::endpoint endpoint(address, s.port); acceptor->open(endpoint.protocol(), ec); - if (!ec && address.is_v6()) { + if (ec && s.bind.empty()) { + // a host without IPv6 (common in containers): every IPv4 interface + ec.clear(); + endpoint = tcp::endpoint(asio::ip::address_v4::any(), s.port); + acceptor->open(endpoint.protocol(), ec); + } + if (!ec && endpoint.address().is_v6()) { acceptor->set_option(asio::ip::v6_only(false), ec); ec.clear(); } @@ -478,10 +492,14 @@ struct SnapshotServer::Impl : std::enable_shared_from_this if (!ec) acceptor->listen(asio::socket_base::max_listen_connections, ec); if (!ec) acceptor->non_blocking(true, ec); if (ec) { - cserror() << "SNAPSHOT: cannot listen on " << (s.bind.empty() ? "port " : s.bind + ":") << s.port << ": " << ec.message(); + if (!(lastFailure == s)) { // once per setting, not every 10 s + cserror() << "SNAPSHOT: cannot listen on " << (s.bind.empty() ? "port " : s.bind + ":") << s.port << ": " << ec.message(); + lastFailure = s; + } acceptor.reset(); return 0; } + lastFailure = Settings{}; const uint16_t bound = acceptor->local_endpoint(ec).port(); acceptThread = std::thread([this]() { acceptLoop(); }); return bound; diff --git a/csnode/src/snapshot_service.cpp b/csnode/src/snapshot_service.cpp index b21cdfe90..c1099823f 100644 --- a/csnode/src/snapshot_service.cpp +++ b/csnode/src/snapshot_service.cpp @@ -70,6 +70,7 @@ void SnapshotService::stop() { void SnapshotService::onCheckpointSaved(cs::Sequence sequence) { (void)sequence; + failedBuilds_ = 0; buildPending_ = true; cv_.notify_all(); } @@ -79,13 +80,17 @@ void SnapshotService::run() { ::setpriority(PRIO_PROCESS, static_cast(::syscall(SYS_gettid)), 10); server_->reload(); while (!stop_) { - const auto& cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + const auto cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); // a copy: the observer may swap the config SnapshotServer::Settings settings; settings.bind = cfg.serveBind; settings.port = cfg.servePort; settings.maxConnections = cfg.serveMaxConnections; settings.rateKBps = cfg.serveRateKbps; server_->apply(settings); + if (settings.port != 0 && retryAt_ != std::chrono::steady_clock::time_point{} && std::chrono::steady_clock::now() >= retryAt_) { + retryAt_ = {}; + buildPending_ = true; + } if (settings.port != 0 && buildPending_.exchange(false)) { buildLatest(); } @@ -100,33 +105,46 @@ void SnapshotService::buildLatest() { if (S == 0 || fs::exists(root_ / std::to_string(S) / "snapshot.json", ec)) { return; // nothing new: no checkpoint with a digest yet, or already built } - const auto& cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + const auto cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); SnapshotBuildInput in; in.seq = S; in.checkpointDir = fs::path("qs") / std::to_string(S); - in.firstStored = blockchain_.getFirstStoredSequence(); + // pruning may be removing the lowest blocks right now (it follows the same checkpoint): start above it + in.firstStored = std::max(blockchain_.getFirstStoredSequence(), blockchain_.pruneTarget()); in.snapshotBlocks = cfg.snapshotBlocks; in.nodeVersion = nodeVersion_; in.stop = &stop_; in.blockBytes = [this](cs::Sequence seq, cs::Bytes& out) { - const csdb::Pool pool = blockchain_.loadBlock(seq); - if (!pool.is_valid()) { - return false; - } - out = pool.to_binary(); - return !out.empty(); + return blockchain_.blockBytes(seq, out) && !out.empty(); }; // no contract records: the contracts database holds the latest state of each contract, later than // S, while the signed snapshot is the state at S; the checkpoint carries the states at S // (smartcontracts.dat) and the node fetches any it lacks from the confidants fs::create_directories(root_, ec); + // the node's own database shares the disk: a build that could fill it is not started + uint64_t estimate = 512ull << 20; + for (const auto& entry : fs::directory_iterator(in.checkpointDir, ec)) { + estimate += entry.is_regular_file(ec) ? entry.file_size(ec) / 4 : 0; + } + const auto space = fs::space(root_, ec); + if (!ec && space.available < 2 * estimate) { + cswarning() << "SNAPSHOT: not enough free disk for the snapshot of checkpoint " << S << " (" << space.available / (1u << 20) + << " MB free, about " << estimate / (1u << 20) << " MB needed twice)"; + return; + } cslog() << "SNAPSHOT: building the snapshot of checkpoint " << S; const auto started = std::chrono::steady_clock::now(); const auto r = buildSnapshot(in, root_, key_); if (!r.ok) { - cswarning() << "SNAPSHOT: no snapshot of checkpoint " << S << ": " << r.error; + // tried again a few times a minute apart (the block range may have been pruned under it) + const bool retry = !stop_ && ++failedBuilds_ < kMaxBuildAttempts; + cswarning() << "SNAPSHOT: no snapshot of checkpoint " << S << ": " << r.error << (retry ? ", trying again in a minute" : ""); + if (retry) { + retryAt_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + } return; } + failedBuilds_ = 0; uint64_t bytes = 0; for (const auto& f : r.manifest.files) { bytes += f.size; diff --git a/tests/csnode/snapshot_server_tests.cpp b/tests/csnode/snapshot_server_tests.cpp index 2e3ee25c0..1469d1b2a 100644 --- a/tests/csnode/snapshot_server_tests.cpp +++ b/tests/csnode/snapshot_server_tests.cpp @@ -70,6 +70,7 @@ struct Fixture { fs::path root; cs::SnapshotBuildResult built; std::atomic synced{true}; + std::atomic roundZero{false}; std::unique_ptr server; uint16_t port = 0; @@ -117,6 +118,7 @@ struct Fixture { built = cs::buildSnapshot(in, root / "snapshot", key); EXPECT_TRUE(built.ok) << built.error; server = std::make_unique(root / "snapshot", [this]() { + if (roundZero) return std::make_pair(cs::Sequence(1000), cs::Sequence(0)); // has not heard the network return synced ? std::make_pair(cs::Sequence(1000), cs::Sequence(1001)) : std::make_pair(cs::Sequence(10), cs::Sequence(1000)); }); server->reload(); @@ -185,6 +187,8 @@ TEST(SnapshotServer, OnlyASyncedNodeServes) { EXPECT_EQ(r.retryAfter, "30"); fx.synced = true; EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 200u); + fx.roundZero = true; + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); } TEST(SnapshotServer, NothingBuiltIsUnavailable) { @@ -210,7 +214,7 @@ TEST(SnapshotServer, ConnectionCapAndRateAndMove) { held.close(); std::this_thread::sleep_for(std::chrono::milliseconds(300)); - // 100 KB/s: 300 KB take about 3 s (the bucket starts empty) + // 100 KB/s: 300 KB take about 3 s (the bucket starts empty when the rate is set) s.maxConnections = 8; s.rateKBps = 100; fx.server->apply(s); @@ -218,9 +222,16 @@ TEST(SnapshotServer, ConnectionCapAndRateAndMove) { const auto r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-299999"); const double secs = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); ASSERT_EQ(r.status, 206u); - EXPECT_GT(secs, 2.0); + EXPECT_GT(secs, 1.8); EXPECT_LT(secs, 6.0); + // a cap below one piece (64 KB) a second still lets the body through, slowly + s.rateKBps = 50; + fx.server->apply(s); + const auto slow = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-99999"); + EXPECT_EQ(slow.status, 206u); + EXPECT_EQ(slow.body.size(), 100000u); + // moved to another port while running, then stopped by port 0 s.rateKBps = 0; s.port = freePort();