diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index a71c11778..91a0b201c 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -277,6 +277,9 @@ jobs: - name: snapshot-join-rocksdb-full env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,rocksdb PRUNE_NODE=6 JOIN_FULL=1 SNAPSHOT_AT=2300 TARGET_SEQ=3300 fixed: true + # a pruned node serves the snapshots it builds over HTTP (snapshot serving, step 4) + - name: snapshot-serve + env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_SERVE_NODE=6 TARGET_SEQ=2600 - name: snapshot-join-format1 env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_FORMAT=1 TARGET_SEQ=3100 # node 5 from full to pruned, node 6 from pruned to full, after node 6 pruned once diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index a0a00bb1f..666dfe9c8 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -67,6 +67,8 @@ # snapshot with --import-snapshot into its own backend and joins; with SNAPSHOT_CORRUPT=1 the # blocks file is truncated first, so the restore must fail and that node syncs from block 0. # SNAPSHOT_FORMAT=1: packaging/docker/snapshot-make.sh and snapshot-restore.sh instead +# SNAPSHOT_SERVE_NODE (default 0 = off): that node gets [snapshot] serve_port (BASE_PORT + 700 + i): +# at the end it must have built the snapshot of a periodic checkpoint and serve it over HTTP # STORAGE_SWITCH_AT (default 0 = off, needs PRUNE_NODE=6 and DB_BACKEND=lmdb): at that sequence # node 5 is restarted as a pruned node and node 6 as a full one # CONVERT_AT (default 0 = off, needs DB_BACKEND=berkeleydb): at that sequence n4 is converted to @@ -198,6 +200,7 @@ JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} SNAPSHOT_FORMAT=${SNAPSHOT_FORMAT:-2} +SNAPSHOT_SERVE_NODE=${SNAPSHOT_SERVE_NODE:-0} SNAPSHOT_CORRUPT=${SNAPSHOT_CORRUPT:-0} STORAGE_SWITCH_AT=${STORAGE_SWITCH_AT:-0} CONVERT_AT=${CONVERT_AT:-0} @@ -330,6 +333,15 @@ PROPS chmod +x "$WORK/n$i/executor.sh" } +# the [snapshot] section of node i: SNAPSHOT_SERVE_NODE serves the snapshots it builds over HTTP +snapshot_section() { + local i=$1 + if [ "$i" -eq "$SNAPSHOT_SERVE_NODE" ]; then + printf "\n[snapshot]\nserve_port=%s\nserve_bind=127.0.0.1\n" "$((BASE_PORT + 700 + i))" + fi + return 0 +} + # the [storage] section of node i: pruning, backend and checkpoint frequency storage_section() { local i=$1 backend="" lines="" @@ -384,6 +396,7 @@ $(api_settings "$i") ajax_port=0 diag_port=0 $(storage_section "$i") +$(snapshot_section "$i") [Core] Filter="%Severity% >= info" @@ -1155,6 +1168,48 @@ for i in $(seq 1 "$ALL_NODES"); do n=$(cat "$WORK/n$i"/*.log | grep -ac "but its caches already hold block" || true) if [ "${n:-0}" -gt 0 ]; then echo "n$i loaded $n checkpoint(s) one block behind their caches"; fi done +if [ "$SNAPSHOT_SERVE_NODE" -gt 0 ]; then + # the node built the snapshot of a periodic checkpoint and serves it: the manifest verifies, and a + # piece of the blocks file has the hash the manifest declares for it + serve_port=$((BASE_PORT + 700 + SNAPSHOT_SERVE_NODE)) + deadline=$(( $(date +%s) + 300 )) + until grep -aq 'SNAPSHOT: built block' "$WORK/n$SNAPSHOT_SERVE_NODE/node.log" || [ "$(date +%s)" -ge "$deadline" ]; do sleep 5; done + grep -a 'SNAPSHOT: ' "$WORK/n$SNAPSHOT_SERVE_NODE/node.log" | tail -4 | sed 's/^.*SNAPSHOT/serve test: SNAPSHOT/' + python3 - "$serve_port" <<'PY' || exit 1 +import hashlib, json, sys, time, urllib.request +port = int(sys.argv[1]) +base = f"http://127.0.0.1:{port}/snapshot" +for attempt in range(30): + try: + with urllib.request.urlopen(base + "/latest.json", timeout=10) as r: + manifest = json.load(r) + break + except Exception as e: # 503 while the node catches up or builds + last = e + time.sleep(10) +else: + print(f"FAIL: no manifest from the serving node: {last}") + sys.exit(1) +f = next(x for x in manifest["files"] if x["name"] == "blocks.bin.lz4") +n = min(manifest["chunk_size"], f["size"]) +req = urllib.request.Request(f"{base}/{manifest['seq']}/blocks.bin.lz4", headers={"Range": f"bytes=0-{n - 1}"}) +with urllib.request.urlopen(req, timeout=30) as r: + piece = r.read() + status = r.status +if status != 206 or hashlib.sha256(piece).hexdigest() != f["chunks"][0]: + print(f"FAIL: the first piece of blocks.bin.lz4 does not match the manifest (status {status}, {len(piece)} bytes)") + sys.exit(1) +try: + urllib.request.urlopen(f"{base}/{manifest['seq']}/../../config.ini", timeout=10) + print("FAIL: a path outside the snapshot was served") + sys.exit(1) +except urllib.error.HTTPError as e: + if e.code != 404: + print(f"FAIL: unexpected status {e.code} for a path outside the snapshot") + sys.exit(1) +print(f"serve test: block {manifest['seq']} served by node at {manifest.get('synced_to')}, {len(manifest['files'])} files, first piece verified") +PY +fi if [ "$SNAPSHOT_AT" -gt 0 ]; then if [ "$SNAPSHOT_CORRUPT" = 1 ]; then grep -aq 'SLOW START' "$WORK/n$JOIN_NODE"/node.log || { echo "FAIL: n$JOIN_NODE did not sync from block 0"; exit 1; } diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index 7d6a2f6ab..a79e337a9 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -123,9 +123,8 @@ int run(const Config& config, const std::string& outDir) { in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) { return db->get(static_cast(seq), &out); }; - in.contracts = [&db](const std::function& visit) { - return db->forEachContractData(visit); - }; + // no contract records: the contracts database holds each contract's latest state, later than the + // checkpoint; the checkpoint carries the states at its block say() << "checkpoint " << S << ", blocks from " << std::max(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0) << ", into " << outDir << std::endl; diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 3eca33681..808c8dc82 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -364,6 +364,17 @@ class Storage final { */ cs::Sequence first_sequence() const; + /** + * Every contract record, in the engine's key order; fn returns false to stop early. For a + * snapshot of the node's state. + */ + bool forEachContractData(const std::function& fn) const; + + /** + * The stored bytes of block sequence, as the database holds them (no parsing, no pool cache). + */ + bool pool_bytes(cs::Sequence sequence, cs::Bytes& out) const; + /** * Returns the space of pruned pools to the file system. */ diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index c4b66a62b..c94787c2c 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -1350,6 +1350,22 @@ bool Storage::pool_prune_batch(const std::vector& hashes) { return true; } +bool Storage::pool_bytes(cs::Sequence sequence, cs::Bytes& out) const { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + return d->db->get(static_cast(sequence), &out); +} + +bool Storage::forEachContractData(const std::function& fn) const { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + return d->db->forEachContractData(fn); +} + cs::Sequence Storage::first_sequence() const { if (!isOpen()) { d->set_last_error(NotOpen); diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index e0c94800d..fba8762f1 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -72,6 +72,8 @@ add_library(csnode include/csnode/snapshot_manifest.hpp include/csnode/snapshot_builder.hpp include/csnode/snapshot_importer.hpp + include/csnode/snapshot_server.hpp + include/csnode/snapshot_service.hpp include/csnode/blockverifier.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -116,6 +118,8 @@ add_library(csnode src/snapshot_manifest.cpp src/snapshot_builder.cpp src/snapshot_importer.cpp + src/snapshot_server.cpp + src/snapshot_service.cpp src/blockverifier.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 4495eb6d7..2f0bd3c98 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -95,6 +95,12 @@ class BlockChain { // pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were // removed once a later periodic checkpoint made them unnecessary cs::Sequence getFirstStoredSequence() const; + // the stored bytes of a block, under the storage lock, for a snapshot: no parsing, no pool cache + bool blockBytes(cs::Sequence sequence, cs::Bytes& out) const; + // pruned storage: the blocks below this one are being or will be removed (0: nothing scheduled) + cs::Sequence pruneTarget() const; + // every contract record of the storage, for a snapshot of the node's state + bool forEachContractData(const std::function& fn) const; // return unique id of database if at least one unique block has written, otherwise (only genesis block) 0 uint64_t uuid() const; @@ -303,6 +309,8 @@ public signals: cs::StopReadingBlocksSignal stopReadingBlocksEvent; cs::SuccessQSSignal successfullQuickStartEvent; + // a periodic checkpoint of this block was saved (live), with its state digest + cs::Signal checkpointSavedEvent; public slots: diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index 7136d6dca..b7ed59ef3 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -26,6 +26,10 @@ class Transport; +namespace cs { +class SnapshotService; +} + namespace cs { class SolverCore; } @@ -485,6 +489,7 @@ public slots: #ifdef NODE_API std::unique_ptr api_; + std::unique_ptr snapshotService_; #endif uint32_t startPacketRequestPoint_ = 0; diff --git a/csnode/include/csnode/snapshot_server.hpp b/csnode/include/csnode/snapshot_server.hpp new file mode 100644 index 000000000..aaca5d745 --- /dev/null +++ b/csnode/include/csnode/snapshot_server.hpp @@ -0,0 +1,68 @@ +#ifndef SNAPSHOT_SERVER_HPP +#define SNAPSHOT_SERVER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace cs { + +// The read-only HTTP service of the snapshots this node built (docs/specs/2026-10-24-snapshot-serving-design.md, +// section 5): GET and HEAD of /snapshot/latest.json and /snapshot//, one Range of at most +// 16 MiB, at most maxConnections connections, a bandwidth cap shared by all of them. It answers 503 +// while the node is not synced with the network or no snapshot is built (only synced nodes serve). +// Files are looked up by the names of the manifest it loaded, never by a path from the request. +class SnapshotServer { +public: + struct Settings { + std::string bind; // empty: every interface + uint16_t port = 0; // 0: stopped + size_t maxConnections = 8; + size_t rateKBps = 0; // kilobytes a second, all connections together; 0: unlimited + bool operator==(const Settings& o) const { + return bind == o.bind && port == o.port && maxConnections == o.maxConnections && rateKBps == o.rateKBps; + } + }; + // the node's last block and the current round: synced when the block is within 3 rounds + using SyncedTo = std::function()>; + + static constexpr uint64_t kMaxRange = 16ull << 20; + static constexpr size_t kMaxRequestsPerConnection = 64; + static constexpr int kIdleTimeoutSec = 15; + static constexpr int kWriteTimeoutSec = 30; + static constexpr cs::Sequence kSyncedRounds = 3; + + SnapshotServer(std::filesystem::path root, SyncedTo syncedTo); + ~SnapshotServer(); + + // starts, stops (port 0), moves to another address or applies new limits, while it runs + void apply(const Settings& settings); + // the newest complete snapshot under root (one holding a snapshot.json that verifies) + void reload(); + void stop(); + + uint16_t port() const { + return boundPort_.load(); + } + bool running() const { + return boundPort_.load() != 0; + } + +private: + struct Impl; + std::shared_ptr impl_; + std::atomic boundPort_{0}; +}; + +} // namespace cs + +#endif // SNAPSHOT_SERVER_HPP diff --git a/csnode/include/csnode/snapshot_service.hpp b/csnode/include/csnode/snapshot_service.hpp new file mode 100644 index 000000000..03ca57989 --- /dev/null +++ b/csnode/include/csnode/snapshot_service.hpp @@ -0,0 +1,55 @@ +#ifndef SNAPSHOT_SERVICE_HPP +#define SNAPSHOT_SERVICE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +class BlockChain; + +namespace cs { + +// Snapshots served by this node, in the node: every 10 s it reads [snapshot] from the configuration +// and starts, stops or moves the HTTP service (serve_port 0: off); while it serves, it builds the +// snapshot of each new periodic checkpoint into snapshot/ at low priority (and of the latest one +// when it starts serving), and serves it once complete. Only a node synced with the network serves. +class SnapshotService { +public: + SnapshotService(BlockChain& blockchain, const cs::PrivateKey& key, std::string nodeVersion); + ~SnapshotService(); + + void start(); + void stop(); + // slot of BlockChain::checkpointSavedEvent: a new periodic checkpoint to build from + void onCheckpointSaved(cs::Sequence sequence); + +private: + void run(); + void buildLatest(); + + BlockChain& blockchain_; + cs::PrivateKey key_; + std::string nodeVersion_; + std::filesystem::path root_ = "snapshot"; + std::unique_ptr server_; + std::thread thread_; + std::atomic stop_{false}; + std::atomic buildPending_{true}; + static constexpr int kMaxBuildAttempts = 5; + int failedBuilds_ = 0; + std::chrono::steady_clock::time_point retryAt_{}; + std::mutex m_; + std::condition_variable cv_; +}; + +} // namespace cs + +#endif // SNAPSHOT_SERVICE_HPP diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index d1daa1183..8f6414a00 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -1053,6 +1053,7 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { if (serializationManPtr_->save(pool.sequence(), head, digest)) { cslog() << cs::stateDigestLine(pool.sequence(), digest.digest, digest.wallets) << " checkpoint"; serializationManPtr_->pruneCheckpoints(sto.checkpointKeep); + emit checkpointSavedEvent(pool.sequence()); lastCheckpointWallClock_ = now; if (byCount) { schedulePruning(pool.sequence()); @@ -1064,6 +1065,20 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) { } } +bool BlockChain::blockBytes(cs::Sequence sequence, cs::Bytes& out) const { + std::lock_guard lock(dbLock_); + return storage_.pool_bytes(sequence, out); +} + +cs::Sequence BlockChain::pruneTarget() const { + std::lock_guard lock(dbLock_); + return pruneUpTo_; +} + +bool BlockChain::forEachContractData(const std::function& fn) const { + return storage_.forEachContractData(fn); +} + cs::CheckpointDigest BlockChain::checkpointDigest() const { cs::CheckpointDigest result; const auto& wallets = multiWallets(); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 62d210f80..aeaef7367 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -40,7 +40,9 @@ #include +#include #include +#include #include namespace { @@ -258,6 +260,23 @@ bool Node::init() { api_->run(); #endif // NODE_API + // snapshots served by this node: [snapshot] serve_port, read again every 10 s (off by default) + { + std::string version; + std::ifstream versionFile("VERSION"); + if (!versionFile || !std::getline(versionFile, version) || version.empty() || version.size() > 64) { + version = "4.2." + std::to_string(Config::getNodeVersion()); + } + for (auto& c : version) { + if (c < 0x20 || c >= 0x7f || c == '"' || c == '\\') { + c = '_'; + } + } + snapshotService_ = std::make_unique(blockChain_, solver_->getPrivateKey(), version); + cs::Connector::connect(&blockChain_.checkpointSavedEvent, snapshotService_.get(), &cs::SnapshotService::onCheckpointSaved); + snapshotService_->start(); + } + if (!transport_->isGood()) { return false; } @@ -357,6 +376,11 @@ void Node::destroy() { api_->stop(); cswarning() << "[API STOPPED]"; + if (snapshotService_) { + snapshotService_->stop(); + cswarning() << "[SNAPSHOT SERVICE STOPPED]"; + } + solver_->finish(); cswarning() << "[SOLVER STOPPED]"; diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index f9fd80fbc..5b1938ea6 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -175,15 +175,6 @@ std::string utcNow() { return buf; } -bool readFile(const std::filesystem::path& path, std::vector& out) { - std::ifstream in(path, std::ios::binary); - if (!in) { - return false; - } - out.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); - return !in.bad(); -} - bool stopped(const SnapshotBuildInput& in) { return in.stop && in.stop->load(); } @@ -332,14 +323,28 @@ SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::fi if (!std::filesystem::exists(src, ec)) { continue; // tokens.dat and apihandler.dat exist only in builds with the API } - std::vector data; - if (!readFile(src, data)) { + // in pieces: walletscache.dat is about 180 MB on mainnet, and the builder runs inside the node + std::ifstream in(src, std::ios::binary); + if (!in) { return failWith("cannot read " + src.string()); } SnapshotFile f; f.name = "qs-" + name + ".lz4"; Lz4Writer w(tmp / f.name, m.chunkSize); - w.write(data); + std::vector piece(snapshot_lz4::kBlockSize); + while (in) { + if (stopped(input)) { + return failWith("stopped"); + } + in.read(reinterpret_cast(piece.data()), static_cast(piece.size())); + const auto n = in.gcount(); + if (n > 0) { + w.write(piece.data(), static_cast(n)); + } + } + if (in.bad()) { + return failWith("cannot read " + src.string()); + } if (!w.finish(f)) { return failWith("cannot write " + f.name); } diff --git a/csnode/src/snapshot_server.cpp b/csnode/src/snapshot_server.cpp new file mode 100644 index 000000000..44e09a7e5 --- /dev/null +++ b/csnode/src/snapshot_server.cpp @@ -0,0 +1,582 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +namespace cs { + +namespace { +namespace asio = boost::asio; +namespace http = boost::beast::http; +using tcp = asio::ip::tcp; +namespace fs = std::filesystem; + +// the unit the per-address cap counts: an IPv4 address (also when written as IPv4-mapped IPv6), or +// the /64 network of an IPv6 address, which one host usually holds whole +std::string peerGroup(const asio::ip::address& address) { + if (address.is_v6()) { + const auto v6 = address.to_v6(); + if (v6.is_v4_mapped()) { + return asio::ip::make_address_v4(asio::ip::v4_mapped, v6).to_string(); + } + auto bytes = v6.to_bytes(); + std::fill(bytes.begin() + 8, bytes.end(), 0); + return asio::ip::address_v6(bytes).to_string() + "/64"; + } + return address.to_string(); +} + +void setTimeout(tcp::socket& socket, int option, int seconds) { + timeval tv{}; + tv.tv_sec = seconds; + ::setsockopt(socket.native_handle(), SOL_SOCKET, option, &tv, sizeof(tv)); +} + +bool isDigits(const std::string& s) { + return !s.empty() && s.size() <= 12 && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; }); +} + +// "bytes=a-b", "bytes=a-" or "bytes=-n" for a file of size; false for anything else (two ranges too) +bool parseRange(const std::string& header, uint64_t size, uint64_t& first, uint64_t& last) { + const std::string prefix = "bytes="; + if (header.compare(0, prefix.size(), prefix) != 0) { + return false; + } + const std::string spec = header.substr(prefix.size()); + if (spec.find(',') != std::string::npos) { + return false; + } + const auto dash = spec.find('-'); + if (dash == std::string::npos) { + return false; + } + const std::string a = spec.substr(0, dash), b = spec.substr(dash + 1); + if (size == 0) { + return false; + } + if (a.empty()) { + if (!isDigits(b)) return false; + const uint64_t n = std::stoull(b); + if (n == 0) return false; + first = n >= size ? 0 : size - n; + last = size - 1; + return true; + } + if (!isDigits(a) || (!b.empty() && !isDigits(b))) { + return false; + } + first = std::stoull(a); + last = b.empty() ? size - 1 : std::min(std::stoull(b), size - 1); + return first < size && first <= last; +} + +// the bandwidth cap of all connections together +class TokenBucket { +public: + void setRate(size_t kBps) { + std::lock_guard lock(m_); + rate_ = static_cast(kBps) * 1000.0; + tokens_ = 0; + last_ = std::chrono::steady_clock::now(); + } + // waits until n bytes may be sent; a piece larger than a second's worth is let through as soon as + // the bucket is not in debt, and leaves it in debt. false when the server stops or the deadline passes + bool take(size_t n, const std::atomic& stopping, std::chrono::steady_clock::time_point deadline) { + for (;;) { + if (stopping || std::chrono::steady_clock::now() >= deadline) { + return false; + } + std::unique_lock lock(m_); + if (rate_ <= 0) { + return true; + } + const auto now = std::chrono::steady_clock::now(); + tokens_ = std::min(rate_, tokens_ + rate_ * std::chrono::duration(now - last_).count()); + last_ = now; + const double need = std::min(static_cast(n), rate_); + if (tokens_ >= need) { + tokens_ -= static_cast(n); + return true; + } + const double wait = (need - tokens_) / rate_; + lock.unlock(); + std::this_thread::sleep_for(std::chrono::duration(std::min(wait, 0.2))); + } + } + +private: + std::mutex m_; + double rate_ = 0; + double tokens_ = 0; + std::chrono::steady_clock::time_point last_ = std::chrono::steady_clock::now(); +}; + +struct Loaded { + SnapshotManifest manifest; + std::string json; // as built, with synced_to inserted per request + fs::path dir; +}; +} // namespace + +struct SnapshotServer::Impl : std::enable_shared_from_this { + static constexpr int kHeaderDeadlineSec = 10; // a whole request line and headers + static constexpr size_t kMaxPerAddress = 2; + // a reader slower than this on average, over the time spent writing to it (the bandwidth cap's + // waits not counted), is dropped once it has been written to for kMinRateAfterSec + static constexpr double kMinBytesPerSec = 32 * 1024; + static constexpr double kMinRateAfterSec = 10; + static constexpr int kMaxConnectionSec = 30 * 60; + + fs::path root; + SyncedTo syncedTo; + Settings settings; + std::mutex settingsLock; + + std::shared_ptr loaded; + std::mutex loadedLock; + + TokenBucket bucket; + std::atomic maxConnections{8}; + std::atomic active{0}; + // every open connection, its peer address and the time its current step must end by: the accept + // thread closes the ones past it (a client that sends its headers byte by byte holds no slot) + struct Conn { + std::string address; + std::chrono::steady_clock::time_point deadline; + }; + std::map, Conn> sockets; + std::mutex socketsLock; + + void setDeadline(const std::shared_ptr& socket, std::chrono::seconds in) { + std::lock_guard lock(socketsLock); + const auto it = sockets.find(socket); + if (it != sockets.end()) { + it->second.deadline = std::chrono::steady_clock::now() + in; + } + } + + void closeExpired() { + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(socketsLock); + for (auto& [socket, conn] : sockets) { + if (now > conn.deadline) { + boost::system::error_code ec; + socket->shutdown(tcp::socket::shutdown_both, ec); + } + } + } + + asio::io_context io; + std::unique_ptr acceptor; + std::thread acceptThread; + std::atomic stopping{false}; + Settings lastFailure; + + std::shared_ptr current() { + std::lock_guard lock(loadedLock); + return loaded; + } + + bool synced() const { + if (!syncedTo) { + return true; + } + const auto [lastBlock, round] = syncedTo(); + // round 0: the node has not heard the network yet (a restart, a firewall), not synced + return lastBlock > 0 && round > 0 && round <= lastBlock + kSyncedRounds; + } + + template + void finish(http::response& res, bool keepAlive) { + res.set(http::field::server, "credits-node"); + res.keep_alive(keepAlive); + res.prepare_payload(); + } + + // returns false when the connection must end + // what one connection has used, across its requests: the minimum rate and the time limit apply to + // the connection, not to each request + struct Usage { + uint64_t served = 0; + double writeSeconds = 0; + std::chrono::steady_clock::time_point end; + }; + + bool handle(const std::shared_ptr& socketPtr, const http::request& req, Usage& usage) { + uint64_t& served = usage.served; + tcp::socket& socket = *socketPtr; + const bool keepAlive = req.keep_alive(); + auto simple = [&](http::status status, const std::string& text, bool retry = false) { + http::response res{status, req.version()}; + res.set(http::field::content_type, "text/plain"); + if (retry) res.set(http::field::retry_after, "30"); + const bool head = req.method() == http::verb::head; + res.body() = head ? std::string{} : text + "\n"; + finish(res, keepAlive && !head); + boost::system::error_code ec; + http::write(socket, res, ec); + return !ec && keepAlive && !head; + }; + if (req.method() != http::verb::get && req.method() != http::verb::head) { + return simple(http::status::method_not_allowed, "GET or HEAD only"); + } + const std::string target(req.target()); + const std::string prefix = "/snapshot/"; + if (target.compare(0, prefix.size(), prefix) != 0) { + return simple(http::status::not_found, "not found"); + } + const auto state = current(); + if (!synced()) { + return simple(http::status::service_unavailable, "this node is not synced with the network", true); + } + if (!state) { + return simple(http::status::service_unavailable, "no snapshot built yet", true); + } + const std::string rest = target.substr(prefix.size()); + const bool head = req.method() == http::verb::head; + if (rest == "latest.json") { + const auto [lastBlock, round] = syncedTo ? syncedTo() : std::pair{0, 0}; + (void)round; + std::string body = state->json; + // informative, outside the signed line: how far this node is + const auto brace = body.find('{'); + body.insert(brace + 1, "\n \"synced_to\": " + std::to_string(lastBlock) + ","); + http::response res{http::status::ok, req.version()}; + res.set(http::field::content_type, "application/json"); + res.body() = head ? std::string{} : body; + finish(res, keepAlive); + if (head) res.content_length(body.size()); + boost::system::error_code ec; + http::write(socket, res, ec); + return !ec && keepAlive; + } + const auto slash = rest.find('/'); + if (slash == std::string::npos || rest.substr(0, slash) != std::to_string(state->manifest.seq)) { + return simple(http::status::not_found, "not found"); + } + const auto* file = state->manifest.file(rest.substr(slash + 1)); + if (!file) { + return simple(http::status::not_found, "not found"); + } + const fs::path path = state->dir / file->name; + std::ifstream in(path, std::ios::binary); + if (!in) { + return simple(http::status::not_found, "not found"); + } + const uint64_t size = file->size; + uint64_t first = 0, last = size == 0 ? 0 : size - 1; + bool partial = false; + const auto range = req.find(http::field::range); + if (range != req.end()) { + if (!parseRange(std::string(range->value()), size, first, last) || last - first + 1 > kMaxRange) { + http::response res{http::status::range_not_satisfiable, req.version()}; + res.set(http::field::content_range, "bytes */" + std::to_string(size)); + res.body() = "one range of at most 16 MiB\n"; + finish(res, keepAlive); + boost::system::error_code ec; + http::write(socket, res, ec); + return !ec && keepAlive; + } + partial = true; + } + const uint64_t length = size == 0 ? 0 : last - first + 1; + http::response res{partial ? http::status::partial_content : http::status::ok, req.version()}; + res.set(http::field::server, "credits-node"); + res.set(http::field::content_type, "application/octet-stream"); + res.set(http::field::accept_ranges, "bytes"); + if (partial) { + res.set(http::field::content_range, "bytes " + std::to_string(first) + "-" + std::to_string(last) + "/" + std::to_string(size)); + } + res.content_length(length); + res.keep_alive(keepAlive); + boost::system::error_code ec; + http::response_serializer sr{res}; + http::write_header(socket, sr, ec); + if (ec) { + return false; + } + if (head || length == 0) { + return keepAlive; + } + // a stalled reader holds no slot: every piece written renews a short deadline + setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); + in.seekg(static_cast(first)); + std::vector buffer(64 << 10); + uint64_t left = length; + while (left > 0 && !stopping) { + if (std::chrono::steady_clock::now() >= usage.end) { + return false; // the connection's time is up, even in the middle of a body + } + const size_t n = static_cast(std::min(left, buffer.size())); + in.read(buffer.data(), static_cast(n)); + if (static_cast(in.gcount()) != n) { + return false; + } + if (!bucket.take(n, stopping, usage.end)) { + return false; + } + // the wait for the bandwidth cap is not the reader's: the write deadline starts after it + setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); + const auto t0 = std::chrono::steady_clock::now(); + asio::write(socket, asio::buffer(buffer.data(), n), ec); + if (ec) { + return false; + } + usage.writeSeconds += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + if (usage.writeSeconds >= kMinRateAfterSec && static_cast(usage.served + n) / usage.writeSeconds < kMinBytesPerSec) { + return false; // the reader is too slow to keep a slot + } + setDeadline(socketPtr, std::chrono::seconds(kWriteTimeoutSec)); + left -= n; + served += n; + } + return left == 0 && keepAlive; + } + + void session(std::shared_ptr socket) { + std::string peer; + { + boost::system::error_code ec; + peer = socket->remote_endpoint(ec).address().to_string(); + } + setTimeout(*socket, SO_RCVTIMEO, kIdleTimeoutSec); + setTimeout(*socket, SO_SNDTIMEO, kWriteTimeoutSec); + Usage usage; + usage.end = std::chrono::steady_clock::now() + std::chrono::seconds(kMaxConnectionSec); + uint64_t& served = usage.served; + boost::beast::flat_buffer buffer; + for (size_t i = 0; i < kMaxRequestsPerConnection && !stopping && std::chrono::steady_clock::now() < usage.end; ++i) { + http::request_parser parser; + parser.header_limit(8 * 1024); + setDeadline(socket, std::chrono::seconds(i == 0 ? kHeaderDeadlineSec : kIdleTimeoutSec)); + boost::system::error_code ec; + http::read(*socket, buffer, parser, ec); + if (ec) { + break; + } + if (!handle(socket, parser.get(), usage)) { + break; + } + } + boost::system::error_code ec; + socket->shutdown(tcp::socket::shutdown_both, ec); + socket->close(ec); + if (served >= (1u << 20)) { + const auto state = current(); + cslog() << "SNAPSHOT: served " << served / (1u << 20) << " MB of block " << (state ? state->manifest.seq : 0) << " to " << peer; + } + { + std::lock_guard lock(socketsLock); + sockets.erase(socket); + } + --active; + } + + void refuse(tcp::socket& socket) { + setTimeout(socket, SO_SNDTIMEO, 5); + http::response res{http::status::service_unavailable, 11}; + res.set(http::field::retry_after, "5"); + res.body() = "too many connections\n"; + finish(res, false); + boost::system::error_code ec; + http::write(socket, res, ec); + socket.shutdown(tcp::socket::shutdown_both, ec); + socket.close(ec); + } + + void acceptLoop() { + while (!stopping) { + closeExpired(); // on every pass, whether connections are waiting or not + auto socket = std::make_shared(io); + boost::system::error_code ec; + acceptor->accept(*socket, ec); + if (ec == asio::error::would_block || ec == asio::error::try_again) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + if (ec) { + if (stopping) break; + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + socket->non_blocking(false, ec); + if (active.load() >= maxConnections.load()) { + refuse(*socket); + continue; + } + std::string address; + { + boost::system::error_code aec; + address = peerGroup(socket->remote_endpoint(aec).address()); + } + bool tooMany = false; + { + std::lock_guard lock(socketsLock); + const auto same = std::count_if(sockets.begin(), sockets.end(), [&](const auto& kv) { return kv.second.address == address; }); + tooMany = static_cast(same) >= kMaxPerAddress; + if (!tooMany) { + sockets.emplace(socket, Conn{address, std::chrono::steady_clock::now() + std::chrono::seconds(kHeaderDeadlineSec)}); + } + } + if (tooMany) { + refuse(*socket); // outside the lock: it writes, up to its own timeout + continue; + } + ++active; + // the session keeps the server's state alive until it ends, whatever happens to the server + std::thread([self = shared_from_this(), socket]() { self->session(socket); }).detach(); + } + } + + void stopListening() { + stopping = true; + if (acceptThread.joinable()) { + acceptThread.join(); + } + if (acceptor) { + boost::system::error_code ec; + acceptor->close(ec); + acceptor.reset(); + } + { + std::lock_guard lock(socketsLock); + for (const auto& [socket, conn] : sockets) { + boost::system::error_code ec; + socket->shutdown(tcp::socket::shutdown_both, ec); + } + } + // the sessions see their sockets shut and stopping set: they end within their current write + while (active.load() > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + stopping = false; + } + + // returns the bound port, 0 when the address cannot be used + uint16_t listen(const Settings& s) { + boost::system::error_code ec; + const auto address = s.bind.empty() ? asio::ip::address(asio::ip::address_v6::any()) : asio::ip::make_address(s.bind, ec); + if (ec) { + cserror() << "SNAPSHOT: serve_bind " << s.bind << " is not an address"; + return 0; + } + acceptor = std::make_unique(io); + tcp::endpoint endpoint(address, s.port); + acceptor->open(endpoint.protocol(), ec); + if (ec && s.bind.empty()) { + // a host without IPv6 (common in containers): every IPv4 interface + ec.clear(); + endpoint = tcp::endpoint(asio::ip::address_v4::any(), s.port); + acceptor->open(endpoint.protocol(), ec); + } + if (!ec && endpoint.address().is_v6()) { + acceptor->set_option(asio::ip::v6_only(false), ec); + ec.clear(); + } + if (!ec) acceptor->set_option(tcp::acceptor::reuse_address(true), ec); + if (!ec) acceptor->bind(endpoint, ec); + if (!ec) acceptor->listen(asio::socket_base::max_listen_connections, ec); + if (!ec) acceptor->non_blocking(true, ec); + if (ec) { + if (!(lastFailure == s)) { // once per setting, not every 10 s + cserror() << "SNAPSHOT: cannot listen on " << (s.bind.empty() ? "port " : s.bind + ":") << s.port << ": " << ec.message(); + lastFailure = s; + } + acceptor.reset(); + return 0; + } + lastFailure = Settings{}; + const uint16_t bound = acceptor->local_endpoint(ec).port(); + acceptThread = std::thread([this]() { acceptLoop(); }); + return bound; + } +}; + +SnapshotServer::SnapshotServer(fs::path root, SyncedTo syncedTo) : impl_(std::make_shared()) { + impl_->root = std::move(root); + impl_->syncedTo = std::move(syncedTo); +} + +SnapshotServer::~SnapshotServer() { + stop(); +} + +void SnapshotServer::apply(const Settings& settings) { + std::lock_guard lock(impl_->settingsLock); + impl_->bucket.setRate(settings.rateKBps); + impl_->maxConnections = std::max(1, settings.maxConnections); + const bool listening = boundPort_.load() != 0; + const bool sameAddress = listening && settings.bind == impl_->settings.bind && settings.port == impl_->settings.port; + impl_->settings = settings; + if (sameAddress) { + return; + } + if (listening) { + impl_->stopListening(); + cslog() << "SNAPSHOT: service stopped"; + boundPort_ = 0; + } + if (settings.port != 0) { + const uint16_t bound = impl_->listen(settings); + boundPort_ = bound; + if (bound != 0) { + cslog() << "SNAPSHOT: serving on " << (settings.bind.empty() ? std::string("port ") : settings.bind + ":") << bound; + } + } +} + +void SnapshotServer::reload() { + std::error_code ec; + cs::Sequence best = 0; + for (const auto& entry : fs::directory_iterator(impl_->root, ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::is_regular_file(entry.path() / "snapshot.json", ec)) { + best = std::max(best, std::stoull(name)); + } + } + std::shared_ptr next; + if (best != 0) { + const fs::path dir = impl_->root / std::to_string(best); + std::ifstream in(dir / "snapshot.json", std::ios::binary); + std::string json((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::string error; + auto manifest = SnapshotManifest::fromJson(json, &error); + if (manifest && manifest->seq == best) { + auto l = std::make_shared(); + l->manifest = std::move(*manifest); + l->json = std::move(json); + l->dir = dir; + next = l; + } + else { + cswarning() << "SNAPSHOT: " << (dir / "snapshot.json").string() << " is not served: " << error; + } + } + std::lock_guard lock(impl_->loadedLock); + impl_->loaded = next; +} + +void SnapshotServer::stop() { + std::lock_guard lock(impl_->settingsLock); + if (boundPort_.load() != 0) { + impl_->stopListening(); + boundPort_ = 0; + cslog() << "SNAPSHOT: service stopped"; + } +} + +} // namespace cs diff --git a/csnode/src/snapshot_service.cpp b/csnode/src/snapshot_service.cpp new file mode 100644 index 000000000..c1099823f --- /dev/null +++ b/csnode/src/snapshot_service.cpp @@ -0,0 +1,158 @@ +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; +constexpr auto kPeriod = std::chrono::seconds(10); + +bool isDigits(const std::string& s) { + return !s.empty() && s.size() <= 12 && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; }); +} + +// the highest periodic checkpoint that records its state (qs/0, the shutdown one, is not periodic) +cs::Sequence latestCheckpoint() { + cs::Sequence best = 0; + std::error_code ec; + for (const auto& entry : fs::directory_iterator("qs", ec)) { + const auto name = entry.path().filename().string(); + if (isDigits(name) && fs::exists(entry.path() / CheckpointDigest::kFileName, ec)) { + best = std::max(best, std::stoull(name)); + } + } + return best; +} +} // namespace + +SnapshotService::SnapshotService(BlockChain& blockchain, const cs::PrivateKey& key, std::string nodeVersion) +: blockchain_(blockchain), key_(key), nodeVersion_(std::move(nodeVersion)) { + server_ = std::make_unique(root_, [this]() { + return std::make_pair(blockchain_.getLastSeq(), static_cast(cs::Conveyer::instance().currentRoundNumber())); + }); +} + +SnapshotService::~SnapshotService() { + stop(); +} + +void SnapshotService::start() { + stop_ = false; + thread_ = std::thread([this]() { run(); }); +} + +void SnapshotService::stop() { + if (!thread_.joinable()) { + return; + } + { + std::lock_guard lock(m_); + stop_ = true; + } + cv_.notify_all(); + thread_.join(); + server_->stop(); +} + +void SnapshotService::onCheckpointSaved(cs::Sequence sequence) { + (void)sequence; + failedBuilds_ = 0; + buildPending_ = true; + cv_.notify_all(); +} + +void SnapshotService::run() { + // the builder works on this thread: below the node's other work + ::setpriority(PRIO_PROCESS, static_cast(::syscall(SYS_gettid)), 10); + server_->reload(); + while (!stop_) { + const auto cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); // a copy: the observer may swap the config + SnapshotServer::Settings settings; + settings.bind = cfg.serveBind; + settings.port = cfg.servePort; + settings.maxConnections = cfg.serveMaxConnections; + settings.rateKBps = cfg.serveRateKbps; + server_->apply(settings); + if (settings.port != 0 && retryAt_ != std::chrono::steady_clock::time_point{} && std::chrono::steady_clock::now() >= retryAt_) { + retryAt_ = {}; + buildPending_ = true; + } + if (settings.port != 0 && buildPending_.exchange(false)) { + buildLatest(); + } + std::unique_lock lock(m_); + cv_.wait_for(lock, kPeriod, [this]() { return stop_.load(); }); + } +} + +void SnapshotService::buildLatest() { + const cs::Sequence S = latestCheckpoint(); + std::error_code ec; + if (S == 0 || fs::exists(root_ / std::to_string(S) / "snapshot.json", ec)) { + return; // nothing new: no checkpoint with a digest yet, or already built + } + const auto cfg = cs::ConfigHolder::instance().config()->getSnapshotSettings(); + SnapshotBuildInput in; + in.seq = S; + in.checkpointDir = fs::path("qs") / std::to_string(S); + // pruning may be removing the lowest blocks right now (it follows the same checkpoint): start above it + in.firstStored = std::max(blockchain_.getFirstStoredSequence(), blockchain_.pruneTarget()); + in.snapshotBlocks = cfg.snapshotBlocks; + in.nodeVersion = nodeVersion_; + in.stop = &stop_; + in.blockBytes = [this](cs::Sequence seq, cs::Bytes& out) { + return blockchain_.blockBytes(seq, out) && !out.empty(); + }; + // no contract records: the contracts database holds the latest state of each contract, later than + // S, while the signed snapshot is the state at S; the checkpoint carries the states at S + // (smartcontracts.dat) and the node fetches any it lacks from the confidants + fs::create_directories(root_, ec); + // the node's own database shares the disk: a build that could fill it is not started + uint64_t estimate = 512ull << 20; + for (const auto& entry : fs::directory_iterator(in.checkpointDir, ec)) { + estimate += entry.is_regular_file(ec) ? entry.file_size(ec) / 4 : 0; + } + const auto space = fs::space(root_, ec); + if (!ec && space.available < 2 * estimate) { + cswarning() << "SNAPSHOT: not enough free disk for the snapshot of checkpoint " << S << " (" << space.available / (1u << 20) + << " MB free, about " << estimate / (1u << 20) << " MB needed twice)"; + return; + } + cslog() << "SNAPSHOT: building the snapshot of checkpoint " << S; + const auto started = std::chrono::steady_clock::now(); + const auto r = buildSnapshot(in, root_, key_); + if (!r.ok) { + // tried again a few times a minute apart (the block range may have been pruned under it) + const bool retry = !stop_ && ++failedBuilds_ < kMaxBuildAttempts; + cswarning() << "SNAPSHOT: no snapshot of checkpoint " << S << ": " << r.error << (retry ? ", trying again in a minute" : ""); + if (retry) { + retryAt_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + } + return; + } + failedBuilds_ = 0; + uint64_t bytes = 0; + for (const auto& f : r.manifest.files) { + bytes += f.size; + } + cslog() << "SNAPSHOT: built block " << S << ", " << r.manifest.files.size() << " files, " << bytes / (1u << 20) << " MB, state digest " + << r.manifest.stateDigest << " in " + << std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count() << " s"; + server_->reload(); +} + +} // namespace cs diff --git a/tests/csnode/snapshot_server_tests.cpp b/tests/csnode/snapshot_server_tests.cpp new file mode 100644 index 000000000..1469d1b2a --- /dev/null +++ b/tests/csnode/snapshot_server_tests.cpp @@ -0,0 +1,258 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; +namespace asio = boost::asio; +namespace http = boost::beast::http; +using tcp = asio::ip::tcp; + +uint16_t freePort() { + asio::io_context io; + tcp::acceptor a(io, tcp::endpoint(asio::ip::address_v4::loopback(), 0)); + return a.local_endpoint().port(); +} + +struct Reply { + unsigned status = 0; + std::string body; + std::string contentRange; + std::string retryAfter; +}; + +Reply request(uint16_t port, http::verb verb, const std::string& target, const std::string& range = {}) { + asio::io_context io; + tcp::socket socket(io); + socket.connect(tcp::endpoint(asio::ip::address_v4::loopback(), port)); + http::request req{verb, target, 11}; + req.set(http::field::host, "localhost"); + req.keep_alive(false); + if (!range.empty()) req.set(http::field::range, range); + http::write(socket, req); + boost::beast::flat_buffer buffer; + http::response_parser parser; + parser.body_limit(64u << 20); + if (verb == http::verb::head) parser.skip(true); + http::read(socket, buffer, parser); + Reply r; + r.status = parser.get().result_int(); + r.body = parser.get().body(); + r.contentRange = std::string(parser.get()[http::field::content_range]); + r.retryAfter = std::string(parser.get()[http::field::retry_after]); + return r; +} + +std::string readAll(const fs::path& p) { + std::ifstream f(p, std::ios::binary); + return std::string(std::istreambuf_iterator(f), {}); +} + +// a snapshot built by the builder from a small chain, served from root/ +struct Fixture { + fs::path root; + cs::SnapshotBuildResult built; + std::atomic synced{true}; + std::atomic roundZero{false}; + std::unique_ptr server; + uint16_t port = 0; + + explicit Fixture(const char* name) : root(fs::temp_directory_path() / name) { + fs::remove_all(root); + EXPECT_TRUE(cscrypto::cryptoInit()); + cs::PublicKey pub; + const auto key = cscrypto::generateKeyPair(pub); + std::map blocks; + csdb::PoolHash prev; + cs::Bytes head; + std::mt19937 random(42); + for (cs::Sequence s = 0; s <= 120; ++s) { + csdb::Pool p(prev, s); + std::string payload(3000, ' '); + for (auto& c : payload) c = static_cast('a' + random() % 26); // about 360 KB that lz4 cannot shrink much + p.add_user_field(1, payload); + EXPECT_TRUE(p.compose()); + blocks[s] = p.to_binary(); + prev = p.hash(); + head = p.hash().to_binary(); + } + const fs::path qs = root / "qs" / "120"; + fs::create_directories(qs); + { + std::ofstream f(qs / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t seq = 120; + const uint16_t hl = static_cast(head.size()), pl = 0; + f.write(reinterpret_cast(&magic), 4); + f.write(reinterpret_cast(&seq), 8); + f.write(reinterpret_cast(&hl), 2); + f.write(reinterpret_cast(head.data()), hl); + f.write(reinterpret_cast(&pl), 2); + } + cs::CheckpointDigest d; + d.wallets = 1; + EXPECT_TRUE(d.write(qs / cs::CheckpointDigest::kFileName)); + cs::SnapshotBuildInput in; + in.seq = 120; + in.checkpointDir = qs; + in.snapshotBlocks = 120; + in.chunkSize = 64 << 10; + in.blockBytes = [blocks](cs::Sequence s, cs::Bytes& out) { out = blocks.at(s); return true; }; + built = cs::buildSnapshot(in, root / "snapshot", key); + EXPECT_TRUE(built.ok) << built.error; + server = std::make_unique(root / "snapshot", [this]() { + if (roundZero) return std::make_pair(cs::Sequence(1000), cs::Sequence(0)); // has not heard the network + return synced ? std::make_pair(cs::Sequence(1000), cs::Sequence(1001)) : std::make_pair(cs::Sequence(10), cs::Sequence(1000)); + }); + server->reload(); + port = freePort(); + cs::SnapshotServer::Settings s; + s.bind = "127.0.0.1"; + s.port = port; + server->apply(s); + EXPECT_TRUE(server->running()); + } + ~Fixture() { + server->stop(); + std::error_code ec; + fs::remove_all(root, ec); + } + std::string blocksPath() const { + return "/snapshot/120/blocks.bin.lz4"; + } +}; +} // namespace + +TEST(SnapshotServer, ServesTheManifestAndFiles) { + Fixture fx("snapshot_server_files"); + auto r = request(fx.port, http::verb::get, "/snapshot/latest.json"); + ASSERT_EQ(r.status, 200u); + EXPECT_NE(r.body.find("\"synced_to\": 1000"), std::string::npos); + const auto manifest = cs::SnapshotManifest::fromJson(r.body); + ASSERT_TRUE(manifest.has_value()); // synced_to is outside the signed line + EXPECT_EQ(manifest->seq, 120u); + + const auto whole = readAll(fx.built.dir / "blocks.bin.lz4"); + r = request(fx.port, http::verb::get, fx.blocksPath()); + ASSERT_EQ(r.status, 200u); + EXPECT_EQ(r.body, whole); + + r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-99"); + ASSERT_EQ(r.status, 206u); + EXPECT_EQ(r.body, whole.substr(0, 100)); + EXPECT_EQ(r.contentRange, "bytes 0-99/" + std::to_string(whole.size())); + r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=-10"); + EXPECT_EQ(r.body, whole.substr(whole.size() - 10)); + r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=100-"); + EXPECT_EQ(r.body, whole.substr(100)); + + EXPECT_EQ(request(fx.port, http::verb::head, fx.blocksPath()).status, 200u); +} + +TEST(SnapshotServer, RefusesWhatItDoesNotServe) { + Fixture fx("snapshot_server_refuse"); + const auto size = fs::file_size(fx.built.dir / "blocks.bin.lz4"); + EXPECT_EQ(request(fx.port, http::verb::get, fx.blocksPath(), "bytes=" + std::to_string(size) + "-").status, 416u); + EXPECT_EQ(request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-1,5-6").status, 416u); + EXPECT_EQ(request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-" + std::to_string((16u << 20) + 1)).status, 206u); // clipped to the file + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/119/blocks.bin.lz4").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/120/../../qs/120/head.bin").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/120/NodePrivate.txt").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::get, "/").status, 404u); + EXPECT_EQ(request(fx.port, http::verb::post, "/snapshot/latest.json").status, 405u); +} + +TEST(SnapshotServer, OnlyASyncedNodeServes) { + Fixture fx("snapshot_server_synced"); + fx.synced = false; + const auto r = request(fx.port, http::verb::get, "/snapshot/latest.json"); + EXPECT_EQ(r.status, 503u); + EXPECT_EQ(r.retryAfter, "30"); + fx.synced = true; + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 200u); + fx.roundZero = true; + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); +} + +TEST(SnapshotServer, NothingBuiltIsUnavailable) { + Fixture fx("snapshot_server_empty"); + fs::remove_all(fx.root / "snapshot"); + fx.server->reload(); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); +} + +TEST(SnapshotServer, ConnectionCapAndRateAndMove) { + Fixture fx("snapshot_server_limits"); + cs::SnapshotServer::Settings s; + s.bind = "127.0.0.1"; + s.port = fx.port; + s.maxConnections = 1; + fx.server->apply(s); + // one connection held open: the next one is refused with 503 + asio::io_context io; + tcp::socket held(io); + held.connect(tcp::endpoint(asio::ip::address_v4::loopback(), fx.port)); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); + held.close(); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // 100 KB/s: 300 KB take about 3 s (the bucket starts empty when the rate is set) + s.maxConnections = 8; + s.rateKBps = 100; + fx.server->apply(s); + const auto t0 = std::chrono::steady_clock::now(); + const auto r = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-299999"); + const double secs = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + ASSERT_EQ(r.status, 206u); + EXPECT_GT(secs, 1.8); + EXPECT_LT(secs, 6.0); + + // a cap below one piece (64 KB) a second still lets the body through, slowly + s.rateKBps = 50; + fx.server->apply(s); + const auto slow = request(fx.port, http::verb::get, fx.blocksPath(), "bytes=0-99999"); + EXPECT_EQ(slow.status, 206u); + EXPECT_EQ(slow.body.size(), 100000u); + + // moved to another port while running, then stopped by port 0 + s.rateKBps = 0; + s.port = freePort(); + fx.server->apply(s); + EXPECT_EQ(fx.server->port(), s.port); + EXPECT_EQ(request(s.port, http::verb::get, "/snapshot/latest.json").status, 200u); + s.port = 0; + fx.server->apply(s); + EXPECT_FALSE(fx.server->running()); +} + +TEST(SnapshotServer, SlowHeadersAreCutOffAndAddressesCapped) { + Fixture fx("snapshot_server_slow"); + // two connections from one address that send nothing: a third is refused, and the two are closed + // once the header deadline passes, after which the address is served again + asio::io_context io; + tcp::socket a(io), b(io); + a.connect(tcp::endpoint(asio::ip::address_v4::loopback(), fx.port)); + b.connect(tcp::endpoint(asio::ip::address_v4::loopback(), fx.port)); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 503u); + std::this_thread::sleep_for(std::chrono::seconds(12)); + EXPECT_EQ(request(fx.port, http::verb::get, "/snapshot/latest.json").status, 200u); +}