From f7a51a1c4282f4b047e8b25b1686106f5c467505 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 10:00:40 +0200 Subject: [PATCH 1/5] csnode, client: snapshot format 2, builder and --export-snapshot A snapshot of checkpoint S is built by the node from its own state: the files of qs/S, the blocks F..S read through the storage as a hash chain that must end at the checkpoint's head (F = S - snapshot_blocks, bounded by the first stored block), and the contract data. Each file is lz4 HC in independent 4 MiB blocks (no new library; within 10-30 % of zstd -3 on the mainnet files), hashed whole and per chunk. snapshot.json carries the block, head hash, state and settings digests, the files, and an Ed25519 signature of the node's key; fromJson accepts only a well-formed format-2 manifest within the limits and with known file names. --export-snapshot builds the snapshot of the latest periodic checkpoint that has state_digest.bin on a stopped node and exits. Step 3a of snapshots served by community nodes. Co-Authored-By: Claude Opus 5.5 --- client/CMakeLists.txt | 2 + client/src/main.cpp | 6 + client/src/snapshot_export.cpp | 109 +++++ client/src/snapshot_export.hpp | 13 + csnode/CMakeLists.txt | 4 + .../csnode/caches_serialization_manager.hpp | 4 + csnode/include/csnode/snapshot_builder.hpp | 57 +++ csnode/include/csnode/snapshot_manifest.hpp | 72 +++ csnode/src/caches_serialization_manager.cpp | 62 +-- csnode/src/snapshot_builder.cpp | 423 ++++++++++++++++++ csnode/src/snapshot_manifest.cpp | 261 +++++++++++ tests/csnode/snapshot_tests.cpp | 259 +++++++++++ 12 files changed, 1243 insertions(+), 29 deletions(-) create mode 100644 client/src/snapshot_export.cpp create mode 100644 client/src/snapshot_export.hpp create mode 100644 csnode/include/csnode/snapshot_builder.hpp create mode 100644 csnode/include/csnode/snapshot_manifest.hpp create mode 100644 csnode/src/snapshot_builder.cpp create mode 100644 csnode/src/snapshot_manifest.cpp create mode 100644 tests/csnode/snapshot_tests.cpp diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index ce6a13a44..f3f21cf61 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -48,6 +48,8 @@ add_executable(client src/main.cpp src/dbcheck.cpp src/dbcheck.hpp + src/snapshot_export.cpp + src/snapshot_export.hpp "${CMAKE_CURRENT_BINARY_DIR}/version.cpp" include/client/version.hpp ) diff --git a/client/src/main.cpp b/client/src/main.cpp index d4ce95e17..eb0e6670b 100644 --- a/client/src/main.cpp +++ b/client/src/main.cpp @@ -11,6 +11,7 @@ #include #include "dbcheck.hpp" +#include "snapshot_export.hpp" #include #include @@ -60,6 +61,7 @@ int main(int argc, char* argv[]) { ("convert-db", "convert an existing block database to the configured db_backend without asking") ("convert-only", "convert the block database to the configured db_backend, then exit") ("convert-batch", po::value()->default_value(100000), "blocks per conversion batch (1 to 1000000)") + ("export-snapshot", po::value(), "build the snapshot of the latest periodic checkpoint into the given folder, then exit (the node must be stopped)") ("config-file", po::value(), "path to configuration file (default: \"config.ini\")") ("public-key-file", po::value(), "path to public key file (default: \"NodePublic.txt\")") ("private-key-file", po::value(), "path to private key file (default: \"NodePrivate.txt\")") @@ -208,6 +210,10 @@ int main(int argc, char* argv[]) { } } + if (vm.count("export-snapshot") > 0) { + return snapshot_export::run(config, vm["export-snapshot"].as()); + } + if (vm.count(cmdline::argDumpKeys) > 0) { auto fName = vm[cmdline::argDumpKeys].as(); if (fName.size() > 0) { diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp new file mode 100644 index 000000000..4499699b5 --- /dev/null +++ b/client/src/snapshot_export.cpp @@ -0,0 +1,109 @@ +#include "snapshot_export.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace snapshot_export { + +namespace { +namespace fs = std::filesystem; + +std::ostream& say() { + return std::cout << "export-snapshot: "; +} + +// the highest periodic checkpoint that records its state (qs/0, the shutdown checkpoint, is not one) +cs::Sequence latestCheckpoint() { + cs::Sequence best = 0; + std::error_code ec; + for (const auto& entry : fs::directory_iterator("qs", ec)) { + const auto name = entry.path().filename().string(); + if (name.empty() || !std::all_of(name.begin(), name.end(), [](unsigned char c) { return std::isdigit(c) != 0; })) { + continue; + } + const cs::Sequence seq = std::stoull(name); + if (seq > best && fs::exists(entry.path() / cs::CheckpointDigest::kFileName, ec)) { + best = seq; + } + } + return best; +} + +std::string nodeVersion() { + std::ifstream f("VERSION"); + std::string v; + if (f && std::getline(f, v) && !v.empty() && v.size() <= 64) { + return v; + } + return std::to_string(Config::getNodeVersion()) + "." + std::to_string(Config::getMinorNodeVersion()) + " " + + std::string(client::Version::GIT_SHA1).substr(0, 8); +} +} // namespace + +int run(const Config& config, const std::string& outDir) { + const auto& storage = config.getStorageSettings(); + const std::string dbPath = config.getPathToDB(); + const auto S = latestCheckpoint(); + if (S == 0) { + say() << "no periodic checkpoint with a state digest in qs/ (checkpoints of earlier builds have none)" << std::endl; + return 1; + } + auto db = csdb::backend::open(storage.dbBackend, dbPath); + if (!db || !db->is_open()) { + say() << "cannot open the " << storage.dbBackend << " database in " << dbPath << std::endl; + return 1; + } + cs::Sequence first = 0; + { + auto it = db->new_iterator(); + it->seek_to_first(); + if (!it->is_valid()) { + say() << "the database holds no block" << std::endl; + return 1; + } + first = it->key(); + } + + cs::SnapshotBuildInput in; + in.seq = S; + in.checkpointDir = fs::path("qs") / std::to_string(S); + in.firstStored = first; + in.snapshotBlocks = config.getSnapshotSettings().snapshotBlocks; + in.nodeVersion = nodeVersion(); + in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) { + return db->get(static_cast(seq), &out); + }; + in.contracts = [&db](const std::function& visit) { + return db->forEachContractData(visit); + }; + + say() << "checkpoint " << S << ", blocks from " << std::max(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0) + << ", into " << outDir << std::endl; + std::error_code ec; + fs::create_directories(outDir, ec); + const auto r = cs::buildSnapshot(in, outDir, config.getMyPrivateKey()); + if (!r.ok) { + say() << "failed: " << r.error << std::endl; + return 1; + } + uint64_t bytes = 0; + for (const auto& f : r.manifest.files) { + bytes += f.size; + } + say() << "built " << r.dir.string() << ": block " << r.manifest.seq << ", state digest " << r.manifest.stateDigest + << ", " << r.manifest.wallets << " wallets, " << r.manifest.files.size() << " files, " << bytes / (1 << 20) << " MB" << std::endl; + return 0; +} + +} // namespace snapshot_export diff --git a/client/src/snapshot_export.hpp b/client/src/snapshot_export.hpp new file mode 100644 index 000000000..4850440ec --- /dev/null +++ b/client/src/snapshot_export.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include + +class Config; + +namespace snapshot_export { + +// --export-snapshot : builds the snapshot of the latest periodic checkpoint (qs/ with a +// state_digest.bin) of this stopped node into /, signed with the node's key. 0 on success. +int run(const Config& config, const std::string& outDir); + +} // namespace snapshot_export diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index 373b3df8e..0ef20c56c 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -69,6 +69,8 @@ add_library(csnode include/csnode/configholder.hpp include/csnode/multiwallets.hpp include/csnode/statedigest.hpp + include/csnode/snapshot_manifest.hpp + include/csnode/snapshot_builder.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp include/csnode/idatastream.hpp @@ -109,6 +111,8 @@ add_library(csnode src/configholder.cpp src/multiwallets.cpp src/statedigest.cpp + src/snapshot_manifest.cpp + src/snapshot_builder.cpp src/sendcachedata.cpp src/eventreport.cpp src/poolcache.cpp diff --git a/csnode/include/csnode/caches_serialization_manager.hpp b/csnode/include/csnode/caches_serialization_manager.hpp index b42bb7b00..31132eb8c 100644 --- a/csnode/include/csnode/caches_serialization_manager.hpp +++ b/csnode/include/csnode/caches_serialization_manager.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -35,6 +36,9 @@ struct CheckpointHead { // True when the checkpoint head is the block the chain holds at that sequence, or when this // cannot be checked (checkpoint without head, or hash not known locally). +// head.bin of a checkpoint directory; nullopt when missing or malformed +std::optional readCheckpointHead(const std::filesystem::path& dir); + bool isCheckpointOnChain(const CheckpointHead& head, const std::function& chainHashAt); class CachesSerializationManager { diff --git a/csnode/include/csnode/snapshot_builder.hpp b/csnode/include/csnode/snapshot_builder.hpp new file mode 100644 index 000000000..de0046964 --- /dev/null +++ b/csnode/include/csnode/snapshot_builder.hpp @@ -0,0 +1,57 @@ +#ifndef SNAPSHOT_BUILDER_HPP +#define SNAPSHOT_BUILDER_HPP + +#include +#include +#include +#include +#include + +#include +#include + +namespace cs { + +// Compressed files of a snapshot: independent lz4 (HC) blocks of at most 4 MiB of raw data, each +// "u32 raw_len, u32 compressed_len, bytes" (little-endian), so that a reader needs no frame library +// and memory stays bounded. +namespace snapshot_lz4 { +constexpr size_t kBlockSize = 4u << 20; +// calls sink with each decompressed block; false for a malformed file or when sink returns false +bool decompressFile(const std::filesystem::path& file, const std::function& sink); +} // namespace snapshot_lz4 + +// What a snapshot of checkpoint seq is built from: the checkpoint directory (qs/) and the +// node's storage, read through these functions (raw stored block bytes by sequence, contract data). +struct SnapshotBuildInput { + cs::Sequence seq = 0; + std::filesystem::path checkpointDir; + cs::Sequence firstStored = 0; + size_t snapshotBlocks = 101'000; + std::function blockBytes; + std::function&)> contracts; + std::string nodeVersion; + uint64_t chunkSize = 4ull << 20; + const std::atomic* stop = nullptr; // checked between blocks: true aborts the build +}; + +struct SnapshotBuildResult { + bool ok = false; + std::string error; + SnapshotManifest manifest; + std::filesystem::path dir; // / +}; + +// Builds // (snapshot.json and its files) through /.tmp, then removes +// the other built snapshots in outRoot. Nothing is left behind on failure. The blocks must form a +// hash chain ending at the checkpoint's head, and the checkpoint must have state_digest.bin. +SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::filesystem::path& outRoot, + const cs::PrivateKey& key); + +// the blocks.bin record stream: "u32 seq, u32 len, bytes" (little-endian) +bool forEachSnapshotBlock(const std::filesystem::path& blocksFile, + const std::function& visit); + +} // namespace cs + +#endif // SNAPSHOT_BUILDER_HPP diff --git a/csnode/include/csnode/snapshot_manifest.hpp b/csnode/include/csnode/snapshot_manifest.hpp new file mode 100644 index 000000000..c138ed10e --- /dev/null +++ b/csnode/include/csnode/snapshot_manifest.hpp @@ -0,0 +1,72 @@ +#ifndef SNAPSHOT_MANIFEST_HPP +#define SNAPSHOT_MANIFEST_HPP + +#include +#include +#include +#include +#include + +#include + +namespace cs { + +// snapshot.json of a format-2 snapshot (docs/specs/2026-10-24-snapshot-serving-design.md, section 2): +// the state of a checkpoint S (its block hash, state and settings digests), the files that carry it, +// each with its SHA-256 and the SHA-256 of every chunk, signed with the key of the node that built it. +struct SnapshotFile { + std::string name; // one of SnapshotManifest::isKnownFileName + uint64_t size = 0; // bytes of the file as served (compressed) + uint64_t rawSize = 0; // bytes once decompressed + std::string sha256; // hex, of the file as served + std::vector chunks; // hex SHA-256 of each chunkSize piece, the last one shorter +}; + +struct SnapshotManifest { + static constexpr uint32_t kFormat = 2; + static constexpr uint64_t kMaxChunkSize = 16ull << 20; + static constexpr uint64_t kMaxTotalSize = 8ull << 30; + static constexpr size_t kMaxFiles = 32; + static constexpr size_t kMaxJsonSize = 1 << 20; + + uint32_t format = kFormat; + cs::Sequence seq = 0; + std::string headHash; // hex, hash of block seq + std::string stateDigest; // hex, wallets digest of the checkpoint (state_digest.bin) + uint64_t wallets = 0; + std::string settingsDigest; // hex, consensus settings hash of the checkpoint (state_digest.bin) + cs::Sequence firstBlock = 0; // first block of blocks.bin + std::string created; // UTC, ISO 8601 + std::string node; // version of the node that built it + std::string nodeKey; // base58 public key of that node + uint64_t chunkSize = 4ull << 20; + std::string compression = "lz4"; + std::vector files; + std::string signature; // hex Ed25519 signature of signedLine() + + // the snapshot's files: blocks.bin.lz4, contracts.bin.lz4 and qs-.lz4 for every file of a + // checkpoint directory; anything else in a manifest rejects it + static bool isKnownFileName(const std::string& name); + static const std::vector& checkpointFileNames(); + + // SHA-256 of the lines "name size sha256\n" in manifest order + std::string filesHash() const; + std::string signedLine() const; + + void sign(const cs::PrivateKey& key); + // the signature is node_key's over signedLine(), recomputed from the fields + bool verify() const; + + std::string toJson() const; + // nullopt, with the reason in *error, for anything but a well-formed format-2 manifest within the limits + static std::optional fromJson(const std::string& json, std::string* error = nullptr); + + const SnapshotFile* file(const std::string& name) const; +}; + +// lower-case hex SHA-256 (libsodium) +std::string sha256Hex(const uint8_t* data, size_t size); + +} // namespace cs + +#endif // SNAPSHOT_MANIFEST_HPP diff --git a/csnode/src/caches_serialization_manager.cpp b/csnode/src/caches_serialization_manager.cpp index 58d67565f..7d36dec5d 100644 --- a/csnode/src/caches_serialization_manager.cpp +++ b/csnode/src/caches_serialization_manager.cpp @@ -263,35 +263,7 @@ struct CachesSerializationManager::Impl { } std::optional loadHead(const std::filesystem::path& dir) { - std::ifstream f(dir / kHeadFile, std::ios::binary); - if (!f.is_open()) return std::nullopt; - uint32_t magic = 0; - f.read(reinterpret_cast(&magic), sizeof(magic)); - if (!f || magic != kHeadMagic) { - cswarning() << "CachesSerializationManager: head.bin magic mismatch in " << dir; - return std::nullopt; - } - CheckpointHead head; - uint64_t seq = 0; - f.read(reinterpret_cast(&seq), sizeof(seq)); - head.sequence = static_cast(seq); - uint16_t hh = 0; - f.read(reinterpret_cast(&hh), sizeof(hh)); - if (hh) { - head.head_hash.resize(hh); - f.read(reinterpret_cast(head.head_hash.data()), hh); - } - uint16_t ph = 0; - f.read(reinterpret_cast(&ph), sizeof(ph)); - if (ph) { - head.prev_hash.resize(ph); - f.read(reinterpret_cast(head.prev_hash.data()), ph); - } - if (!f) { - cswarning() << "CachesSerializationManager: head.bin truncated in " << dir; - return std::nullopt; - } - return head; + return readCheckpointHead(dir); } std::vector divideHashes(const std::string& initial) const { @@ -639,6 +611,38 @@ bool CachesSerializationManager::save(size_t version, const CheckpointHead& head return true; } +std::optional readCheckpointHead(const std::filesystem::path& dir) { + std::ifstream f(dir / "head.bin", std::ios::binary); + if (!f.is_open()) return std::nullopt; + uint32_t magic = 0; + f.read(reinterpret_cast(&magic), sizeof(magic)); + if (!f || magic != 0x44414548u) { + cswarning() << "CachesSerializationManager: head.bin magic mismatch in " << dir; + return std::nullopt; + } + CheckpointHead head; + uint64_t seq = 0; + f.read(reinterpret_cast(&seq), sizeof(seq)); + head.sequence = static_cast(seq); + uint16_t hh = 0; + f.read(reinterpret_cast(&hh), sizeof(hh)); + if (hh) { + head.head_hash.resize(hh); + f.read(reinterpret_cast(head.head_hash.data()), hh); + } + uint16_t ph = 0; + f.read(reinterpret_cast(&ph), sizeof(ph)); + if (ph) { + head.prev_hash.resize(ph); + f.read(reinterpret_cast(head.prev_hash.data()), ph); + } + if (!f) { + cswarning() << "CachesSerializationManager: head.bin truncated in " << dir; + return std::nullopt; + } + return head; +} + bool isCheckpointOnChain(const CheckpointHead& head, const std::function& chainHashAt) { if (head.sequence == 0 || head.head_hash.empty()) { return true; diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp new file mode 100644 index 000000000..720fc8460 --- /dev/null +++ b/csnode/src/snapshot_builder.cpp @@ -0,0 +1,423 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace cs { + +namespace { +constexpr int kLz4Level = 9; // LZ4HC: 10-30 % above zstd -3 on the mainnet files, measured 2026-10-24 + +void putU32(std::vector& out, uint32_t v) { + for (int i = 0; i < 4; ++i) { + out.push_back(static_cast(v >> (8 * i))); + } +} + +uint32_t getU32(const uint8_t* p) { + return uint32_t(p[0]) | uint32_t(p[1]) << 8 | uint32_t(p[2]) << 16 | uint32_t(p[3]) << 24; +} + +std::string hexOf(const uint8_t* data, size_t size) { + static const char* digits = "0123456789abcdef"; + std::string out; + out.reserve(size * 2); + for (size_t i = 0; i < size; ++i) { + out.push_back(digits[data[i] >> 4]); + out.push_back(digits[data[i] & 15]); + } + return out; +} + +// writes the served (compressed) bytes of one snapshot file and hashes them: the whole file and +// every chunkSize piece +class HashingWriter { +public: + HashingWriter(const std::filesystem::path& path, uint64_t chunkSize) + : out_(path, std::ios::binary | std::ios::trunc), chunkSize_(chunkSize) { + crypto_hash_sha256_init(&file_); + crypto_hash_sha256_init(&chunk_); + } + + bool good() const { + return static_cast(out_); + } + + void write(const uint8_t* data, size_t size) { + out_.write(reinterpret_cast(data), static_cast(size)); + crypto_hash_sha256_update(&file_, data, size); + size_ += size; + while (size > 0) { + const size_t room = static_cast(chunkSize_ - inChunk_); + const size_t n = std::min(room, size); + crypto_hash_sha256_update(&chunk_, data, n); + inChunk_ += n; + data += n; + size -= n; + if (inChunk_ == chunkSize_) { + closeChunk(); + } + } + } + + bool finish(SnapshotFile& file) { + if (inChunk_ > 0) { + closeChunk(); + } + std::array h{}; + crypto_hash_sha256_final(&file_, h.data()); + file.sha256 = hexOf(h.data(), h.size()); + file.size = size_; + file.chunks = std::move(chunks_); + out_.flush(); + out_.close(); + return !out_.fail(); + } + +private: + void closeChunk() { + std::array h{}; + crypto_hash_sha256_final(&chunk_, h.data()); + chunks_.push_back(hexOf(h.data(), h.size())); + crypto_hash_sha256_init(&chunk_); + inChunk_ = 0; + } + + std::ofstream out_; + uint64_t chunkSize_; + uint64_t size_ = 0; + uint64_t inChunk_ = 0; + crypto_hash_sha256_state file_; + crypto_hash_sha256_state chunk_; + std::vector chunks_; +}; + +// raw data in, lz4 blocks out through a HashingWriter +class Lz4Writer { +public: + Lz4Writer(const std::filesystem::path& path, uint64_t chunkSize) : out_(path, chunkSize) { + raw_.reserve(snapshot_lz4::kBlockSize); + packed_.resize(static_cast(LZ4_compressBound(static_cast(snapshot_lz4::kBlockSize)))); + } + + bool good() const { + return out_.good(); + } + + void write(const uint8_t* data, size_t size) { + rawSize_ += size; + while (size > 0) { + const size_t n = std::min(snapshot_lz4::kBlockSize - raw_.size(), size); + raw_.insert(raw_.end(), data, data + n); + data += n; + size -= n; + if (raw_.size() == snapshot_lz4::kBlockSize) { + flushBlock(); + } + } + } + + void write(const std::vector& data) { + write(data.data(), data.size()); + } + + bool finish(SnapshotFile& file) { + if (!raw_.empty()) { + flushBlock(); + } + file.rawSize = rawSize_; + return out_.finish(file) && ok_; + } + +private: + void flushBlock() { + const int n = LZ4_compress_HC(reinterpret_cast(raw_.data()), reinterpret_cast(packed_.data()), + static_cast(raw_.size()), static_cast(packed_.size()), kLz4Level); + if (n <= 0) { + ok_ = false; + } + std::vector header; + putU32(header, static_cast(raw_.size())); + putU32(header, static_cast(std::max(n, 0))); + out_.write(header.data(), header.size()); + out_.write(packed_.data(), static_cast(std::max(n, 0))); + raw_.clear(); + } + + HashingWriter out_; + std::vector raw_; + std::vector packed_; + uint64_t rawSize_ = 0; + bool ok_ = true; +}; + +std::string utcNow() { + const std::time_t t = std::time(nullptr); + std::tm tm{}; + gmtime_r(&t, &tm); + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm); + return buf; +} + +bool readFile(const std::filesystem::path& path, std::vector& out) { + std::ifstream in(path, std::ios::binary); + if (!in) { + return false; + } + out.assign(std::istreambuf_iterator(in), std::istreambuf_iterator()); + return !in.bad(); +} + +bool stopped(const SnapshotBuildInput& in) { + return in.stop && in.stop->load(); +} +} // namespace + +namespace snapshot_lz4 { +bool decompressFile(const std::filesystem::path& file, const std::function& sink) { + std::ifstream in(file, std::ios::binary); + if (!in) { + return false; + } + std::vector packed; + std::vector raw(kBlockSize); + const int bound = LZ4_compressBound(static_cast(kBlockSize)); + for (;;) { + uint8_t header[8]; + in.read(reinterpret_cast(header), sizeof(header)); + if (in.gcount() == 0 && in.eof()) { + return true; + } + if (in.gcount() != sizeof(header)) { + return false; + } + const uint32_t rawLen = getU32(header); + const uint32_t packedLen = getU32(header + 4); + if (rawLen == 0 || rawLen > kBlockSize || packedLen == 0 || packedLen > static_cast(bound)) { + return false; + } + packed.resize(packedLen); + in.read(reinterpret_cast(packed.data()), packedLen); + if (static_cast(in.gcount()) != packedLen) { + return false; + } + const int n = LZ4_decompress_safe(reinterpret_cast(packed.data()), reinterpret_cast(raw.data()), + static_cast(packedLen), static_cast(rawLen)); + if (n != static_cast(rawLen) || !sink(raw.data(), rawLen)) { + return false; + } + } +} +} // namespace snapshot_lz4 + +bool forEachSnapshotBlock(const std::filesystem::path& blocksFile, const std::function& visit) { + // records may span lz4 blocks: keep the unparsed tail + std::vector pending; + bool ok = snapshot_lz4::decompressFile(blocksFile, [&](const uint8_t* data, size_t size) { + pending.insert(pending.end(), data, data + size); + size_t pos = 0; + while (pending.size() - pos >= 8) { + const uint32_t seq = getU32(pending.data() + pos); + const uint32_t len = getU32(pending.data() + pos + 4); + if (len == 0 || len > (64u << 20)) { + return false; + } + if (pending.size() - pos - 8 < len) { + break; + } + cs::Bytes bytes(pending.begin() + static_cast(pos + 8), pending.begin() + static_cast(pos + 8 + len)); + if (!visit(seq, std::move(bytes))) { + return false; + } + pos += 8 + len; + } + pending.erase(pending.begin(), pending.begin() + static_cast(pos)); + return true; + }); + return ok && pending.empty(); +} + +SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::filesystem::path& outRoot, const cs::PrivateKey& key) { + SnapshotBuildResult result; + const cs::Sequence S = input.seq; + const auto tmp = outRoot / (std::to_string(S) + ".tmp"); + const auto final = outRoot / std::to_string(S); + std::error_code ec; + + auto failWith = [&](const std::string& reason) { + std::filesystem::remove_all(tmp, ec); + result.ok = false; + result.error = reason; + return result; + }; + + if (S == 0 || !input.blockBytes) { + return failWith("no checkpoint"); + } + const auto digest = CheckpointDigest::read(input.checkpointDir / CheckpointDigest::kFileName); + if (!digest) { + return failWith("checkpoint " + std::to_string(S) + " has no state_digest.bin (saved by an earlier build)"); + } + const auto head = readCheckpointHead(input.checkpointDir); + if (!head || head->sequence != S || head->head_hash.empty()) { + return failWith("checkpoint " + std::to_string(S) + " has no head for that block"); + } + if (S > std::numeric_limits::max()) { + return failWith("block number above 32 bits"); + } + const cs::Sequence wanted = S > input.snapshotBlocks ? S - input.snapshotBlocks : 0; + const cs::Sequence F = std::max(wanted, input.firstStored); + if (F > S) { + return failWith("the checkpoint's block is not stored"); + } + + std::filesystem::remove_all(tmp, ec); + std::filesystem::create_directories(tmp, ec); + if (ec) { + return failWith("cannot create " + tmp.string() + ": " + ec.message()); + } + + SnapshotManifest& m = result.manifest; + m.seq = S; + m.headHash = hexOf(head->head_hash.data(), head->head_hash.size()); + m.stateDigest = hexOf(digest->digest.data(), digest->digest.size()); + m.wallets = digest->wallets; + m.settingsDigest = hexOf(digest->settings.data(), digest->settings.size()); + m.firstBlock = F; + m.created = utcNow(); + m.node = input.nodeVersion; + m.chunkSize = input.chunkSize; + + // 1. the checkpoint's files + for (const auto& name : SnapshotManifest::checkpointFileNames()) { + const auto src = input.checkpointDir / name; + if (!std::filesystem::exists(src, ec)) { + continue; // tokens.dat and apihandler.dat exist only in builds with the API + } + std::vector data; + if (!readFile(src, data)) { + return failWith("cannot read " + src.string()); + } + SnapshotFile f; + f.name = "qs-" + name + ".lz4"; + Lz4Writer w(tmp / f.name, m.chunkSize); + w.write(data); + if (!w.finish(f)) { + return failWith("cannot write " + f.name); + } + m.files.push_back(std::move(f)); + } + + // 2. the blocks F..S, a hash chain ending at the checkpoint's head + { + SnapshotFile f; + f.name = "blocks.bin.lz4"; + Lz4Writer w(tmp / f.name, m.chunkSize); + if (!w.good()) { + return failWith("cannot write blocks"); + } + cs::Bytes previousHash; + std::vector record; + for (cs::Sequence seq = F; seq <= S; ++seq) { + if (stopped(input)) { + return failWith("stopped"); + } + cs::Bytes bytes; + if (!input.blockBytes(seq, bytes) || bytes.empty()) { + return failWith("block " + std::to_string(seq) + " is not stored"); + } + const auto pool = csdb::Pool::from_binary(cs::Bytes(bytes)); + if (!pool.is_valid() || pool.sequence() != seq) { + return failWith("block " + std::to_string(seq) + " does not parse"); + } + const auto hash = pool.hash().to_binary(); + if (seq > F && pool.previous_hash().to_binary() != previousHash) { + return failWith("block " + std::to_string(seq) + " does not follow block " + std::to_string(seq - 1)); + } + if (seq == S && hash != head->head_hash) { + return failWith("block " + std::to_string(S) + " is not the checkpoint's head"); + } + previousHash = hash; + record.clear(); + putU32(record, static_cast(seq)); + putU32(record, static_cast(bytes.size())); + record.insert(record.end(), bytes.begin(), bytes.end()); + w.write(record); + } + if (!w.finish(f)) { + return failWith("cannot write blocks"); + } + m.files.push_back(std::move(f)); + } + + // 3. contract data, sorted by key + if (input.contracts) { + std::vector> records; + if (!input.contracts([&](const cs::Bytes& k, const cs::Bytes& v) { + records.emplace_back(k, v); + return !stopped(input); + })) { + return failWith(stopped(input) ? "stopped" : "cannot read the contract data"); + } + std::sort(records.begin(), records.end()); + SnapshotFile f; + f.name = "contracts.bin.lz4"; + Lz4Writer w(tmp / f.name, m.chunkSize); + std::vector record; + for (const auto& [k, v] : records) { + record.clear(); + putU32(record, static_cast(k.size())); + record.insert(record.end(), k.begin(), k.end()); + putU32(record, static_cast(v.size())); + record.insert(record.end(), v.begin(), v.end()); + w.write(record); + } + if (!w.finish(f)) { + return failWith("cannot write contracts"); + } + m.files.push_back(std::move(f)); + } + + // 4. manifest, signed; publish + m.sign(key); + { + std::ofstream out(tmp / "snapshot.json", std::ios::binary | std::ios::trunc); + out << m.toJson(); + out.flush(); + if (!out) { + return failWith("cannot write snapshot.json"); + } + } + std::filesystem::remove_all(final, ec); + std::filesystem::rename(tmp, final, ec); + if (ec) { + return failWith("cannot rename the snapshot: " + ec.message()); + } + for (const auto& entry : std::filesystem::directory_iterator(outRoot, ec)) { + const auto name = entry.path().filename().string(); + if (name != std::to_string(S) && !name.empty() && std::all_of(name.begin(), name.end(), [](unsigned char c) { return std::isdigit(c) != 0; })) { + std::filesystem::remove_all(entry.path(), ec); + } + } + result.ok = true; + result.dir = final; + return result; +} + +} // namespace cs diff --git a/csnode/src/snapshot_manifest.cpp b/csnode/src/snapshot_manifest.cpp new file mode 100644 index 000000000..15eccd66e --- /dev/null +++ b/csnode/src/snapshot_manifest.cpp @@ -0,0 +1,261 @@ +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +namespace cs { + +namespace { +const std::vector kCheckpointFiles = { + "walletscache.dat", "walletsids.dat", "smartcontracts.dat", "blockchain.dat", "roundstat.dat", + "tokens.dat", "apihandler.dat", "consensus_pending.dat", "head.bin", "quick_start_hashes.dat", + "schema_version", "sentinel.bin", "state_digest.bin"}; + +std::string toHex(const uint8_t* data, size_t size) { + static const char* digits = "0123456789abcdef"; + std::string out; + out.reserve(size * 2); + for (size_t i = 0; i < size; ++i) { + out.push_back(digits[data[i] >> 4]); + out.push_back(digits[data[i] & 15]); + } + return out; +} + +bool fromHex(const std::string& hex, cs::Bytes& out) { + if (hex.size() % 2 != 0) { + return false; + } + out.clear(); + out.reserve(hex.size() / 2); + auto nibble = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + for (size_t i = 0; i < hex.size(); i += 2) { + const int hi = nibble(hex[i]); + const int lo = nibble(hex[i + 1]); + if (hi < 0 || lo < 0) { + return false; + } + out.push_back(static_cast(hi << 4 | lo)); + } + return true; +} + +bool isHex(const std::string& s, size_t length) { + return s.size() == length && std::all_of(s.begin(), s.end(), [](char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + }); +} + +// values the manifest prints into JSON strings: no character that needs escaping +bool isPlain(const std::string& s, size_t maxLength) { + return s.size() <= maxLength && std::all_of(s.begin(), s.end(), [](char c) { + return c >= 0x20 && c < 0x7f && c != '"' && c != '\\'; + }); +} + +bool fail(std::string* error, const std::string& reason) { + if (error) { + *error = reason; + } + return false; +} +} // namespace + +std::string sha256Hex(const uint8_t* data, size_t size) { + std::array hash{}; + crypto_hash_sha256(hash.data(), data, size); + return toHex(hash.data(), hash.size()); +} + +const std::vector& SnapshotManifest::checkpointFileNames() { + return kCheckpointFiles; +} + +bool SnapshotManifest::isKnownFileName(const std::string& name) { + if (name == "blocks.bin.lz4" || name == "contracts.bin.lz4") { + return true; + } + for (const auto& f : kCheckpointFiles) { + if (name == "qs-" + f + ".lz4") { + return true; + } + } + return false; +} + +std::string SnapshotManifest::filesHash() const { + std::string lines; + for (const auto& f : files) { + lines += f.name + " " + std::to_string(f.size) + " " + f.sha256 + "\n"; + } + return sha256Hex(reinterpret_cast(lines.data()), lines.size()); +} + +std::string SnapshotManifest::signedLine() const { + return "credits-snapshot/2 " + std::to_string(seq) + " " + headHash + " " + stateDigest + " " + + std::to_string(wallets) + " " + settingsDigest + " " + std::to_string(firstBlock) + " " + filesHash(); +} + +void SnapshotManifest::sign(const cs::PrivateKey& key) { + const auto publicKey = cscrypto::getMatchingPublic(key); + nodeKey = EncodeBase58(cs::Bytes(publicKey.begin(), publicKey.end())); + const auto line = signedLine(); + const auto sig = cscrypto::generateSignature(key, reinterpret_cast(line.data()), line.size()); + signature = toHex(sig.data(), sig.size()); +} + +bool SnapshotManifest::verify() const { + cs::Bytes key; + if (!DecodeBase58(nodeKey, key) || key.size() != cscrypto::kPublicKeySize) { + return false; + } + cs::Bytes sig; + if (!isHex(signature, 2 * cscrypto::kSignatureSize) || !fromHex(signature, sig)) { + return false; + } + const auto line = signedLine(); + return cscrypto::verifySignature(sig.data(), key.data(), reinterpret_cast(line.data()), line.size()); +} + +const SnapshotFile* SnapshotManifest::file(const std::string& name) const { + for (const auto& f : files) { + if (f.name == name) { + return &f; + } + } + return nullptr; +} + +std::string SnapshotManifest::toJson() const { + std::ostringstream os; + os << "{\n"; + os << " \"format\": " << format << ",\n"; + os << " \"seq\": " << seq << ",\n"; + os << " \"head_hash\": \"" << headHash << "\",\n"; + os << " \"state_digest\": \"" << stateDigest << "\",\n"; + os << " \"wallets\": " << wallets << ",\n"; + os << " \"settings_digest\": \"" << settingsDigest << "\",\n"; + os << " \"first_block\": " << firstBlock << ",\n"; + os << " \"created\": \"" << created << "\",\n"; + os << " \"node\": \"" << node << "\",\n"; + os << " \"node_key\": \"" << nodeKey << "\",\n"; + os << " \"chunk_size\": " << chunkSize << ",\n"; + os << " \"compression\": \"" << compression << "\",\n"; + os << " \"files\": ["; + for (size_t i = 0; i < files.size(); ++i) { + const auto& f = files[i]; + os << (i ? ",\n" : "\n") << " { \"name\": \"" << f.name << "\", \"size\": " << f.size + << ", \"raw_size\": " << f.rawSize << ", \"sha256\": \"" << f.sha256 << "\", \"chunks\": ["; + for (size_t c = 0; c < f.chunks.size(); ++c) { + os << (c ? ", " : "") << "\"" << f.chunks[c] << "\""; + } + os << "] }"; + } + os << "\n ],\n"; + os << " \"signed\": \"" << signedLine() << "\",\n"; + os << " \"signature\": \"" << signature << "\"\n"; + os << "}\n"; + return os.str(); +} + +std::optional SnapshotManifest::fromJson(const std::string& json, std::string* error) { + if (json.size() > kMaxJsonSize) { + fail(error, "manifest larger than 1 MB"); + return std::nullopt; + } + boost::property_tree::ptree tree; + try { + std::istringstream in(json); + boost::property_tree::read_json(in, tree); + } + catch (const std::exception& e) { + fail(error, std::string("not JSON: ") + e.what()); + return std::nullopt; + } + + SnapshotManifest m; + try { + m.format = tree.get("format"); + m.seq = tree.get("seq"); + m.headHash = tree.get("head_hash"); + m.stateDigest = tree.get("state_digest"); + m.wallets = tree.get("wallets"); + m.settingsDigest = tree.get("settings_digest"); + m.firstBlock = tree.get("first_block"); + m.created = tree.get("created"); + m.node = tree.get("node"); + m.nodeKey = tree.get("node_key"); + m.chunkSize = tree.get("chunk_size"); + m.compression = tree.get("compression"); + m.signature = tree.get("signature"); + for (const auto& item : tree.get_child("files")) { + SnapshotFile f; + f.name = item.second.get("name"); + f.size = item.second.get("size"); + f.rawSize = item.second.get("raw_size"); + f.sha256 = item.second.get("sha256"); + for (const auto& chunk : item.second.get_child("chunks")) { + f.chunks.push_back(chunk.second.get_value()); + } + m.files.push_back(std::move(f)); + if (m.files.size() > kMaxFiles) { + fail(error, "more than 32 files"); + return std::nullopt; + } + } + } + catch (const std::exception& e) { + fail(error, std::string("missing or malformed field: ") + e.what()); + return std::nullopt; + } + + auto reject = [&](const std::string& reason) { + fail(error, reason); + return std::nullopt; + }; + if (m.format != kFormat) return reject("format is not 2"); + if (m.seq == 0 || m.firstBlock > m.seq) return reject("bad block range"); + if (!isHex(m.headHash, 64) || !isHex(m.stateDigest, 64) || !isHex(m.settingsDigest, 64)) return reject("bad hash or digest"); + if (!isHex(m.signature, 128)) return reject("bad signature"); + if (!isPlain(m.created, 32) || !isPlain(m.node, 64) || !isPlain(m.nodeKey, 64)) return reject("bad text field"); + if (m.compression != "lz4") return reject("unknown compression"); + if (m.chunkSize < (64u << 10) || m.chunkSize > kMaxChunkSize) return reject("chunk size out of range"); + if (m.files.empty()) return reject("no files"); + uint64_t total = 0; + for (const auto& f : m.files) { + if (!isKnownFileName(f.name)) return reject("unknown file " + f.name.substr(0, 64)); + if (std::count_if(m.files.begin(), m.files.end(), [&](const SnapshotFile& o) { return o.name == f.name; }) != 1) { + return reject("file listed twice"); + } + if (!isHex(f.sha256, 64)) return reject("bad file hash"); + if (f.size > kMaxTotalSize || f.rawSize > kMaxTotalSize) return reject("file too large"); + total += f.size; + if (total > kMaxTotalSize) return reject("snapshot larger than 8 GB"); + const uint64_t chunks = (f.size + m.chunkSize - 1) / m.chunkSize; + if (f.chunks.size() != chunks) return reject("chunk count does not match the file size"); + for (const auto& c : f.chunks) { + if (!isHex(c, 64)) return reject("bad chunk hash"); + } + } + if (!m.file("blocks.bin.lz4") || !m.file("qs-state_digest.bin.lz4") || !m.file("qs-head.bin.lz4")) { + return reject("blocks, head or state digest missing"); + } + if (tree.get("signed", "") != m.signedLine()) return reject("signed line does not match the fields"); + if (!m.verify()) return reject("signature does not verify"); + return m; +} + +} // namespace cs diff --git a/tests/csnode/snapshot_tests.cpp b/tests/csnode/snapshot_tests.cpp new file mode 100644 index 000000000..2ac534fe8 --- /dev/null +++ b/tests/csnode/snapshot_tests.cpp @@ -0,0 +1,259 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; + +struct Chain { + std::map blocks; + std::map hashes; +}; + +Chain makeChain(cs::Sequence count) { + Chain c; + csdb::PoolHash prev; + for (cs::Sequence s = 0; s < count; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(100 + s % 50, static_cast('a' + s % 26))); + EXPECT_TRUE(p.compose()); + c.blocks[s] = p.to_binary(); + c.hashes[s] = p.hash().to_binary(); + prev = p.hash(); + } + return c; +} + +void putU16(std::ofstream& f, uint16_t v) { + f.write(reinterpret_cast(&v), sizeof(v)); +} + +// a checkpoint directory as CachesSerializationManager writes it, with the files the builder reads +void makeCheckpoint(const fs::path& dir, cs::Sequence seq, const cs::Bytes& headHash, bool withDigest = true) { + fs::create_directories(dir); + { + std::ofstream f(dir / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t s = seq; + f.write(reinterpret_cast(&magic), sizeof(magic)); + f.write(reinterpret_cast(&s), sizeof(s)); + putU16(f, static_cast(headHash.size())); + f.write(reinterpret_cast(headHash.data()), static_cast(headHash.size())); + putU16(f, 0); + } + std::ofstream(dir / "walletscache.dat", std::ios::binary) << std::string(300000, 'w') << "end"; + std::ofstream(dir / "blockchain.dat", std::ios::binary) << "settings"; + if (withDigest) { + cs::CheckpointDigest d; + d.digest.fill(7); + d.settings.fill(9); + d.wallets = 42; + ASSERT_TRUE(d.write(dir / cs::CheckpointDigest::kFileName)); + } +} + +struct Fixture { + fs::path root; + Chain chain = makeChain(301); + cs::PrivateKey key; + cs::PublicKey pub; + std::map contracts = {{{1, 2}, {3}}, {{0, 9}, {4, 5}}}; + + explicit Fixture(const char* name) : root(fs::temp_directory_path() / name) { + fs::remove_all(root); + fs::create_directories(root); + key = cscrypto::generateKeyPair(pub); + } + ~Fixture() { + std::error_code ec; + fs::remove_all(root, ec); + } + + cs::SnapshotBuildInput input(cs::Sequence seq = 200) { + cs::SnapshotBuildInput in; + in.seq = seq; + in.checkpointDir = root / "qs" / std::to_string(seq); + in.firstStored = 0; + in.snapshotBlocks = 150; + in.nodeVersion = "test"; + in.chunkSize = 64 << 10; + in.blockBytes = [this](cs::Sequence s, cs::Bytes& out) { + auto it = chain.blocks.find(s); + if (it == chain.blocks.end()) return false; + out = it->second; + return true; + }; + in.contracts = [this](const std::function& visit) { + for (const auto& [k, v] : contracts) { + if (!visit(k, v)) return false; + } + return true; + }; + return in; + } +}; + +std::string readAll(const fs::path& p) { + std::ifstream f(p, std::ios::binary); + return std::string(std::istreambuf_iterator(f), std::istreambuf_iterator()); +} +} // namespace + +TEST(SnapshotBuilder, BuildsAVerifiableSnapshot) { + Fixture fx("snapshot_build_ok"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + fs::create_directories(fx.root / "out" / "100"); // an older snapshot: removed + const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); + ASSERT_TRUE(r.ok) << r.error; + EXPECT_FALSE(fs::exists(fx.root / "out" / "100")); + EXPECT_FALSE(fs::exists(fx.root / "out" / "200.tmp")); + + const auto parsed = cs::SnapshotManifest::fromJson(readAll(r.dir / "snapshot.json")); + ASSERT_TRUE(parsed.has_value()); + EXPECT_EQ(parsed->seq, 200u); + EXPECT_EQ(parsed->firstBlock, 50u); + EXPECT_EQ(parsed->wallets, 42u); + std::string sevens, nines; + for (int i = 0; i < 32; ++i) { + sevens += "07"; + nines += "09"; + } + EXPECT_EQ(parsed->stateDigest, sevens); + EXPECT_EQ(parsed->settingsDigest, nines); + EXPECT_TRUE(parsed->verify()); + + // every file and chunk hash matches what was written + for (const auto& f : parsed->files) { + const auto data = readAll(r.dir / f.name); + ASSERT_EQ(data.size(), f.size) << f.name; + EXPECT_EQ(cs::sha256Hex(reinterpret_cast(data.data()), data.size()), f.sha256) << f.name; + for (size_t c = 0; c < f.chunks.size(); ++c) { + const size_t off = c * parsed->chunkSize; + const size_t len = std::min(parsed->chunkSize, data.size() - off); + EXPECT_EQ(cs::sha256Hex(reinterpret_cast(data.data() + off), len), f.chunks[c]) << f.name << " chunk " << c; + } + } + + // blocks 50..200 in order, byte for byte + cs::Sequence expected = 50; + ASSERT_TRUE(cs::forEachSnapshotBlock(r.dir / "blocks.bin.lz4", [&](cs::Sequence s, cs::Bytes&& bytes) { + EXPECT_EQ(s, expected); + EXPECT_EQ(bytes, fx.chain.blocks[s]); + ++expected; + return true; + })); + EXPECT_EQ(expected, 201u); + + // a checkpoint file survives compression + std::string wallets; + ASSERT_TRUE(cs::snapshot_lz4::decompressFile(r.dir / "qs-walletscache.dat.lz4", [&](const uint8_t* d, size_t n) { + wallets.append(reinterpret_cast(d), n); + return true; + })); + EXPECT_EQ(wallets, readAll(fx.root / "qs" / "200" / "walletscache.dat")); + EXPECT_NE(parsed->file("contracts.bin.lz4"), nullptr); +} + +TEST(SnapshotBuilder, FirstBlockIsBoundedByTheStorage) { + Fixture fx("snapshot_build_first"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + auto in = fx.input(); + in.firstStored = 120; + const auto r = cs::buildSnapshot(in, fx.root / "out", fx.key); + ASSERT_TRUE(r.ok) << r.error; + EXPECT_EQ(r.manifest.firstBlock, 120u); +} + +TEST(SnapshotBuilder, BrokenChainBuildsNothing) { + Fixture fx("snapshot_build_chain"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + fx.chain.blocks[150] = fx.chain.blocks[149]; // block 150 replaced: it does not parse as 150 + const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); + EXPECT_FALSE(r.ok); + EXPECT_FALSE(fs::exists(fx.root / "out" / "200")); + EXPECT_FALSE(fs::exists(fx.root / "out" / "200.tmp")); +} + +TEST(SnapshotBuilder, AnotherHeadBuildsNothing) { + Fixture fx("snapshot_build_head"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[199]); + EXPECT_FALSE(cs::buildSnapshot(fx.input(), fx.root / "out", fx.key).ok); +} + +TEST(SnapshotBuilder, CheckpointWithoutDigestBuildsNothing) { + Fixture fx("snapshot_build_nodigest"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200], false); + const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); + EXPECT_FALSE(r.ok); + EXPECT_NE(r.error.find("state_digest"), std::string::npos); +} + +TEST(SnapshotBuilder, StopLeavesNothing) { + Fixture fx("snapshot_build_stop"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + std::atomic stop{true}; + auto in = fx.input(); + in.stop = &stop; + EXPECT_FALSE(cs::buildSnapshot(in, fx.root / "out", fx.key).ok); + EXPECT_FALSE(fs::exists(fx.root / "out" / "200.tmp")); +} + +TEST(SnapshotManifest, TamperedFieldsDoNotVerify) { + Fixture fx("snapshot_manifest_tamper"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); + ASSERT_TRUE(r.ok) << r.error; + const auto& m = r.manifest; + ASSERT_TRUE(m.verify()); + auto t = m; t.seq += 1; EXPECT_FALSE(t.verify()); + t = m; t.headHash[0] = t.headHash[0] == 'a' ? 'b' : 'a'; EXPECT_FALSE(t.verify()); + t = m; t.stateDigest[0] = t.stateDigest[0] == 'a' ? 'b' : 'a'; EXPECT_FALSE(t.verify()); + t = m; t.settingsDigest[0] = t.settingsDigest[0] == 'a' ? 'b' : 'a'; EXPECT_FALSE(t.verify()); + t = m; t.wallets += 1; EXPECT_FALSE(t.verify()); + t = m; t.firstBlock += 1; EXPECT_FALSE(t.verify()); + t = m; t.files[0].sha256[0] = t.files[0].sha256[0] == 'a' ? 'b' : 'a'; EXPECT_FALSE(t.verify()); + t = m; t.files[0].size += 1; EXPECT_FALSE(t.verify()); +} + +TEST(SnapshotManifest, JsonRoundTripAndRejections) { + Fixture fx("snapshot_manifest_json"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); + ASSERT_TRUE(r.ok) << r.error; + const auto json = r.manifest.toJson(); + std::string error; + const auto back = cs::SnapshotManifest::fromJson(json, &error); + ASSERT_TRUE(back.has_value()) << error; + EXPECT_EQ(back->toJson(), json); + + auto replaced = [&](const std::string& from, const std::string& to) { + auto j = json; + const auto at = j.find(from); + EXPECT_NE(at, std::string::npos) << from; + return j.replace(at, from.size(), to); + }; + EXPECT_FALSE(cs::SnapshotManifest::fromJson(replaced("\"format\": 2", "\"format\": 3")).has_value()); + EXPECT_FALSE(cs::SnapshotManifest::fromJson(replaced("\"blocks.bin.lz4\"", "\"../blocks.bin.lz4\"")).has_value()); + EXPECT_FALSE(cs::SnapshotManifest::fromJson(replaced("\"compression\": \"lz4\"", "\"compression\": \"zip\"")).has_value()); + EXPECT_FALSE(cs::SnapshotManifest::fromJson(replaced("\"chunk_size\": 65536", "\"chunk_size\": 1073741824")).has_value()); + EXPECT_FALSE(cs::SnapshotManifest::fromJson("not json").has_value()); + EXPECT_FALSE(cs::SnapshotManifest::fromJson(std::string(2 << 20, ' ')).has_value()); +} + +TEST(SnapshotManifest, OnlyKnownNames) { + EXPECT_TRUE(cs::SnapshotManifest::isKnownFileName("blocks.bin.lz4")); + EXPECT_TRUE(cs::SnapshotManifest::isKnownFileName("qs-walletscache.dat.lz4")); + EXPECT_FALSE(cs::SnapshotManifest::isKnownFileName("qs-../../NodePrivate.txt.lz4")); + EXPECT_FALSE(cs::SnapshotManifest::isKnownFileName("NodePrivate.txt")); + EXPECT_FALSE(cs::SnapshotManifest::isKnownFileName("qs-config.ini.lz4")); +} From 6f6243a22a46b09fd576184d6c4fd8f329820f22 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 10:15:59 +0200 Subject: [PATCH 2/5] ci(testnet): the snapshot test also builds a format-2 snapshot with --export-snapshot Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 1c671d2c9..5e827baca 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -672,6 +672,26 @@ while :; do echo "snapshot test: n$PRUNE_NODE stopped, head $head_digest" "$REPO/packaging/docker/snapshot-make.sh" "$WORK/n$PRUNE_NODE" "$WORK/snapshot" "$MDB_COPY" local \ | sed 's/^/snapshot test: /' + # format 2: the node builds the snapshot of its latest periodic checkpoint itself; the + # manifest must declare the digest the checkpoint logged when it was saved + rc=0 + (cd "$WORK/n$PRUNE_NODE" && exec "$BIN" --db-path db --config-file config.ini --export-snapshot "$WORK/snapshot2") \ + > "$WORK/export.log" 2>&1 || rc=$? + grep -a 'export-snapshot:' "$WORK/export.log" | sed 's/^/snapshot test: /' + [ "$rc" = 0 ] || { tail -20 "$WORK/export.log"; echo "FAIL: --export-snapshot exited $rc"; exit 1; } + python3 - "$WORK/snapshot2" "$WORK/n$PRUNE_NODE/node.log" <<'PY' || exit 1 +import json, os, re, sys +root, log = sys.argv[1:] +dirs = [d for d in os.listdir(root) if d.isdigit()] +assert len(dirs) == 1, dirs +m = json.load(open(os.path.join(root, dirs[0], "snapshot.json"))) +lines = re.findall(r"STATE DIGEST #(\d+) ([0-9A-Fa-f]{64}) wallets (\d+) checkpoint\b", open(log, errors="replace").read()) +saved = {int(s): (d.lower(), int(w)) for s, d, w in lines} +if saved.get(m["seq"]) != (m["state_digest"], m["wallets"]): + print(f"FAIL: format-2 snapshot of #{m['seq']} declares {m['state_digest']}, the checkpoint logged {saved.get(m['seq'])}") + sys.exit(1) +print(f"snapshot test: format-2 snapshot of #{m['seq']} ({m['first_block']}..{m['seq']}), digest as logged at the checkpoint, {len(m['files'])} files") +PY start_node "$PRUNE_NODE" if [ "$SNAPSHOT_CORRUPT" = 1 ]; then f=$(ls "$WORK"/snapshot/credits-pruned-*.tar.zst) From 997e5b9cc430b73b15db6cfa4635f745dc1ffd8d Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 11:09:00 +0200 Subject: [PATCH 3/5] snapshot builder and export: review fixes - The builder replaces or removes only folders it built (they hold snapshot.json) and refuses an existing folder that is not one, so a misdirected output (qs, db, ...) loses nothing. - --export-snapshot refuses next to a running node, when the folder holds no database of the configured backend, and when the output is the node's folder or inside qs, db or caches. - The VERSION text is reduced to plain characters before it goes into the manifest. - The signed line covers the chunk size and every chunk hash; fromJson refuses block numbers above 32 bits and nesting deeper than 8 before parsing. - The CI check reads only the lines a checkpoint save logs. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 2 +- client/src/dbcheck.cpp | 4 ++++ client/src/dbcheck.hpp | 3 +++ client/src/snapshot_export.cpp | 35 +++++++++++++++++++++++++++++ csnode/src/snapshot_builder.cpp | 19 +++++++++++++++- csnode/src/snapshot_manifest.cpp | 33 ++++++++++++++++++++++++--- tests/csnode/snapshot_tests.cpp | 38 ++++++++++++++++++++++++++++++++ 7 files changed, 129 insertions(+), 5 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 5e827baca..508a8b32d 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -685,7 +685,7 @@ root, log = sys.argv[1:] dirs = [d for d in os.listdir(root) if d.isdigit()] assert len(dirs) == 1, dirs m = json.load(open(os.path.join(root, dirs[0], "snapshot.json"))) -lines = re.findall(r"STATE DIGEST #(\d+) ([0-9A-Fa-f]{64}) wallets (\d+) checkpoint\b", open(log, errors="replace").read()) +lines = re.findall(r"STATE DIGEST #(\d+) ([0-9A-Fa-f]{64}) wallets (\d+) checkpoint$", open(log, errors="replace").read(), re.M) saved = {int(s): (d.lower(), int(w)) for s, d, w in lines} if saved.get(m["seq"]) != (m["state_digest"], m["wallets"]): print(f"FAIL: format-2 snapshot of #{m['seq']} declares {m['state_digest']}, the checkpoint logged {saved.get(m['seq'])}") diff --git a/client/src/dbcheck.cpp b/client/src/dbcheck.cpp index 6a047955e..679d2d617 100644 --- a/client/src/dbcheck.cpp +++ b/client/src/dbcheck.cpp @@ -300,6 +300,10 @@ int checkImpl(const std::string& dbPath, const std::string& configured, bool con } // namespace +bool lockedByAnotherProcess(const std::string& dbPath) { + return lockDatabase(dbPath) == Lock::Busy; +} + int check(const std::string& dbPath, const std::string& configured, bool convertFlag, bool convertOnly, uint32_t batch) { // No engine exception may escape: print it and stop with nothing changed. The signal handlers // are installed and restored inside convertNow, around csdb::convert::run only. diff --git a/client/src/dbcheck.hpp b/client/src/dbcheck.hpp index fb46801db..bb5397538 100644 --- a/client/src/dbcheck.hpp +++ b/client/src/dbcheck.hpp @@ -29,4 +29,7 @@ constexpr uint32_t kMaxBatch = 1000000; // outside 1..kMaxBatch). int check(const std::string& dbPath, const std::string& configured, bool convertFlag, bool convertOnly, uint32_t batch); +// true when another process (a running node) holds /.lock; this process takes the lock otherwise +bool lockedByAnotherProcess(const std::string& dbPath); + } // namespace dbcheck diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index 4499699b5..fcd094534 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -1,4 +1,5 @@ #include "snapshot_export.hpp" +#include "dbcheck.hpp" #include #include @@ -44,6 +45,12 @@ std::string nodeVersion() { std::ifstream f("VERSION"); std::string v; if (f && std::getline(f, v) && !v.empty() && v.size() <= 64) { + // printed into the manifest's JSON as it is: only plain characters + for (auto& c : v) { + if (c < 0x20 || c >= 0x7f || c == '"' || c == '\\') { + c = '_'; + } + } return v; } return std::to_string(Config::getNodeVersion()) + "." + std::to_string(Config::getMinorNodeVersion()) + " " + @@ -59,6 +66,34 @@ int run(const Config& config, const std::string& outDir) { say() << "no periodic checkpoint with a state digest in qs/ (checkpoints of earlier builds have none)" << std::endl; return 1; } + // the export reads the database files directly: never next to a running node (BerkeleyDB would + // even run recovery on its live environment), never into an empty folder or another backend's + if (dbcheck::lockedByAnotherProcess(dbPath)) { + say() << "a node is running on " << dbPath << ": stop it first" << std::endl; + return 1; + } + const auto found = csdb::backend::detect(dbPath, true); + if (std::find(found.begin(), found.end(), storage.dbBackend) == found.end()) { + say() << dbPath << " holds no " << storage.dbBackend << " database (the configured db_backend)" << std::endl; + return 1; + } + // the output must not be a folder of the node's data: the builder replaces numbered folders in it + { + std::error_code e1, e2; + const auto out = fs::weakly_canonical(outDir, e1); + const auto here = fs::weakly_canonical(fs::current_path(), e2); + if (e1 || out == here) { + say() << outDir << " is the node's folder, choose another one" << std::endl; + return 1; + } + for (const fs::path data : {fs::path("qs"), fs::path(dbPath), fs::path("caches")}) { + const auto d = fs::weakly_canonical(data, e2); + if (!e2 && std::mismatch(d.begin(), d.end(), out.begin(), out.end()).first == d.end()) { + say() << outDir << " is inside the node's data (" << data.string() << "), choose another folder" << std::endl; + return 1; + } + } + } auto db = csdb::backend::open(storage.dbBackend, dbPath); if (!db || !db->is_open()) { say() << "cannot open the " << storage.dbBackend << " database in " << dbPath << std::endl; diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index 720fc8460..54c77e470 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -287,6 +287,22 @@ SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::fi return failWith("the checkpoint's block is not stored"); } + // only folders this builder made are replaced or removed: a misdirected outRoot (qs, db, ...) + // must lose nothing + auto isBuiltSnapshot = [&](const std::filesystem::path& dir) { + return std::filesystem::is_regular_file(dir / "snapshot.json", ec); + }; + if (std::filesystem::exists(final, ec) && !isBuiltSnapshot(final)) { + return failWith(final.string() + " exists and is not a snapshot"); + } + // a build interrupted earlier left only snapshot files in .tmp + for (const auto& entry : std::filesystem::directory_iterator(tmp, ec)) { + const auto name = entry.path().filename().string(); + if (name != "snapshot.json" && !SnapshotManifest::isKnownFileName(name)) { + result.error = tmp.string() + " holds " + name + ", which is not a snapshot file"; + return result; + } + } std::filesystem::remove_all(tmp, ec); std::filesystem::create_directories(tmp, ec); if (ec) { @@ -411,7 +427,8 @@ SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::fi } for (const auto& entry : std::filesystem::directory_iterator(outRoot, ec)) { const auto name = entry.path().filename().string(); - if (name != std::to_string(S) && !name.empty() && std::all_of(name.begin(), name.end(), [](unsigned char c) { return std::isdigit(c) != 0; })) { + if (name != std::to_string(S) && !name.empty() && std::all_of(name.begin(), name.end(), [](unsigned char c) { return std::isdigit(c) != 0; }) && + isBuiltSnapshot(entry.path())) { std::filesystem::remove_all(entry.path(), ec); } } diff --git a/csnode/src/snapshot_manifest.cpp b/csnode/src/snapshot_manifest.cpp index 15eccd66e..10fc6431e 100644 --- a/csnode/src/snapshot_manifest.cpp +++ b/csnode/src/snapshot_manifest.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -98,15 +99,21 @@ bool SnapshotManifest::isKnownFileName(const std::string& name) { std::string SnapshotManifest::filesHash() const { std::string lines; + // every byte a client checks is covered: whole file and each chunk for (const auto& f : files) { - lines += f.name + " " + std::to_string(f.size) + " " + f.sha256 + "\n"; + lines += f.name + " " + std::to_string(f.size) + " " + std::to_string(f.rawSize) + " " + f.sha256; + for (const auto& c : f.chunks) { + lines += " " + c; + } + lines += "\n"; } return sha256Hex(reinterpret_cast(lines.data()), lines.size()); } std::string SnapshotManifest::signedLine() const { return "credits-snapshot/2 " + std::to_string(seq) + " " + headHash + " " + stateDigest + " " + - std::to_string(wallets) + " " + settingsDigest + " " + std::to_string(firstBlock) + " " + filesHash(); + std::to_string(wallets) + " " + settingsDigest + " " + std::to_string(firstBlock) + " " + + std::to_string(chunkSize) + " " + filesHash(); } void SnapshotManifest::sign(const cs::PrivateKey& key) { @@ -176,6 +183,26 @@ std::optional SnapshotManifest::fromJson(const std::string& js fail(error, "manifest larger than 1 MB"); return std::nullopt; } + // read_json recurses: a nesting depth this format never has is refused before parsing + { + int depth = 0; + bool inString = false; + for (size_t i = 0; i < json.size(); ++i) { + const char c = json[i]; + if (inString) { + if (c == '\\') ++i; + else if (c == '"') inString = false; + } + else if (c == '"') inString = true; + else if (c == '{' || c == '[') { + if (++depth > 8) { + fail(error, "nested too deep"); + return std::nullopt; + } + } + else if (c == '}' || c == ']') --depth; + } + } boost::property_tree::ptree tree; try { std::istringstream in(json); @@ -227,7 +254,7 @@ std::optional SnapshotManifest::fromJson(const std::string& js return std::nullopt; }; if (m.format != kFormat) return reject("format is not 2"); - if (m.seq == 0 || m.firstBlock > m.seq) return reject("bad block range"); + if (m.seq == 0 || m.firstBlock > m.seq || m.seq > std::numeric_limits::max()) return reject("bad block range"); if (!isHex(m.headHash, 64) || !isHex(m.stateDigest, 64) || !isHex(m.settingsDigest, 64)) return reject("bad hash or digest"); if (!isHex(m.signature, 128)) return reject("bad signature"); if (!isPlain(m.created, 32) || !isPlain(m.node, 64) || !isPlain(m.nodeKey, 64)) return reject("bad text field"); diff --git a/tests/csnode/snapshot_tests.cpp b/tests/csnode/snapshot_tests.cpp index 2ac534fe8..7a9b92118 100644 --- a/tests/csnode/snapshot_tests.cpp +++ b/tests/csnode/snapshot_tests.cpp @@ -257,3 +257,41 @@ TEST(SnapshotManifest, OnlyKnownNames) { EXPECT_FALSE(cs::SnapshotManifest::isKnownFileName("NodePrivate.txt")); EXPECT_FALSE(cs::SnapshotManifest::isKnownFileName("qs-config.ini.lz4")); } + +TEST(SnapshotBuilder, RemovesOnlySnapshotFolders) { + Fixture fx("snapshot_build_only_snapshots"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + fs::create_directories(fx.root / "out" / "100"); // a numbered folder that is not a snapshot + std::ofstream(fx.root / "out" / "100" / "walletscache.dat") << "x"; + fs::create_directories(fx.root / "out" / "150"); // an earlier snapshot + std::ofstream(fx.root / "out" / "150" / "snapshot.json") << "{}"; + const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); + ASSERT_TRUE(r.ok) << r.error; + EXPECT_TRUE(fs::exists(fx.root / "out" / "100" / "walletscache.dat")); + EXPECT_FALSE(fs::exists(fx.root / "out" / "150")); +} + +TEST(SnapshotBuilder, RefusesToReplaceAFolderThatIsNotASnapshot) { + Fixture fx("snapshot_build_refuse"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + // the output root is the checkpoints folder itself: qs/200 must survive + const auto r = cs::buildSnapshot(fx.input(), fx.root / "qs", fx.key); + EXPECT_FALSE(r.ok); + EXPECT_TRUE(fs::exists(fx.root / "qs" / "200" / "walletscache.dat")); + EXPECT_TRUE(fs::exists(fx.root / "qs" / "200" / "head.bin")); +} + +TEST(SnapshotManifest, DeepNestingAndLargeBlockNumbersRejected) { + EXPECT_FALSE(cs::SnapshotManifest::fromJson(std::string(1000, '[') + std::string(1000, ']')).has_value()); + Fixture fx("snapshot_manifest_seq"); + makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); + const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); + ASSERT_TRUE(r.ok) << r.error; + auto big = r.manifest; + big.seq = 1ull << 33; + big.sign(fx.key); + EXPECT_FALSE(cs::SnapshotManifest::fromJson(big.toJson()).has_value()); + auto chunks = r.manifest; + chunks.files[0].chunks[0][0] = chunks.files[0].chunks[0][0] == 'a' ? 'b' : 'a'; + EXPECT_FALSE(chunks.verify()); +} From e54d1f88b32e8df43e396c4f228c158ff7e52733 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 11:14:31 +0200 Subject: [PATCH 4/5] tests: an older snapshot folder holds snapshot.json Co-Authored-By: Claude Opus 5.5 --- tests/csnode/snapshot_tests.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/csnode/snapshot_tests.cpp b/tests/csnode/snapshot_tests.cpp index 7a9b92118..d8e36b6ca 100644 --- a/tests/csnode/snapshot_tests.cpp +++ b/tests/csnode/snapshot_tests.cpp @@ -113,6 +113,7 @@ TEST(SnapshotBuilder, BuildsAVerifiableSnapshot) { Fixture fx("snapshot_build_ok"); makeCheckpoint(fx.root / "qs" / "200", 200, fx.chain.hashes[200]); fs::create_directories(fx.root / "out" / "100"); // an older snapshot: removed + std::ofstream(fx.root / "out" / "100" / "snapshot.json") << "{}"; const auto r = cs::buildSnapshot(fx.input(), fx.root / "out", fx.key); ASSERT_TRUE(r.ok) << r.error; EXPECT_FALSE(fs::exists(fx.root / "out" / "100")); From 74c0663504c0860c8e6f2c0a425a0868429df69a Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 11:49:07 +0200 Subject: [PATCH 5/5] tests: initialise the crypto library before generating keys Co-Authored-By: Claude Opus 5.5 --- tests/csnode/snapshot_tests.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/csnode/snapshot_tests.cpp b/tests/csnode/snapshot_tests.cpp index d8e36b6ca..d0c8c18d5 100644 --- a/tests/csnode/snapshot_tests.cpp +++ b/tests/csnode/snapshot_tests.cpp @@ -72,6 +72,8 @@ struct Fixture { explicit Fixture(const char* name) : root(fs::temp_directory_path() / name) { fs::remove_all(root); fs::create_directories(root); + // keys live in sodium's guarded memory: the library must be initialised first + EXPECT_TRUE(cscrypto::cryptoInit()); key = cscrypto::generateKeyPair(pub); } ~Fixture() {