From c7787d91080e56ee9952b08c8a0d1570f465a5d7 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 12:36:44 +0200 Subject: [PATCH 01/10] csnode, client: snapshot importer and --import-snapshot A format-2 snapshot is restored into an empty node folder: every file must match the manifest (size, chunk and whole-file SHA-256); qs/ is written from the checkpoint files, whose head and state digest must be the manifest's; every block passes the BlockVerifier and the hash chain ending at the manifest's head before it is written to a new database of the configured backend (lmdb or rocksdb) with caches/hashdb and seqdb; then the contract data and snapshot.json. Any failure removes what the import created. The node then quick-starts from qs/ and rebuilds its transaction index. --import-snapshot restores from a local folder and exits 0, or 2 with the reason. CI: the snapshot test uses --export-snapshot and --import-snapshot (SNAPSHOT_FORMAT=1 keeps the scripts); new scenarios export from a pruned BerkeleyDB node into a pruned LMDB one and from a pruned RocksDB node into a full RocksDB one. Step 3c of snapshots served by community nodes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 10 + ci/testnet/run.sh | 42 ++- client/src/main.cpp | 6 + client/src/snapshot_export.cpp | 33 +++ client/src/snapshot_export.hpp | 5 + csnode/CMakeLists.txt | 2 + csnode/include/csnode/snapshot_importer.hpp | 32 +++ csnode/src/snapshot_importer.cpp | 295 ++++++++++++++++++++ tests/csnode/snapshot_import_tests.cpp | 279 ++++++++++++++++++ 9 files changed, 692 insertions(+), 12 deletions(-) create mode 100644 csnode/include/csnode/snapshot_importer.hpp create mode 100644 csnode/src/snapshot_importer.cpp create mode 100644 tests/csnode/snapshot_import_tests.cpp diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 094d542cf..5f1cf6cd3 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -269,6 +269,16 @@ jobs: env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3100 - name: snapshot-corrupt env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_CORRUPT=1 TARGET_SEQ=3100 + # format 2 across block databases: exported by a pruned BerkeleyDB node into a pruned LMDB one, + # and by a pruned RocksDB node into a full RocksDB one; format 1 kept for the previous image + - name: snapshot-join-from-berkeleydb + env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,lmdb:pruned PRUNE_NODE=4 SNAPSHOT_AT=2300 TARGET_SEQ=3300 + fixed: true + - name: snapshot-join-rocksdb-full + env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,rocksdb PRUNE_NODE=6 JOIN_FULL=1 SNAPSHOT_AT=2300 TARGET_SEQ=3300 + fixed: true + - name: snapshot-join-format1 + env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_FORMAT=1 TARGET_SEQ=3100 # node 5 from full to pruned, node 6 from pruned to full, after node 6 pruned once - name: storage-switch env: DB_BACKEND=lmdb PRUNE_NODE=6 STORAGE_SWITCH_AT=2300 TARGET_SEQ=3300 diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 508a8b32d..2ecb04986 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -59,9 +59,10 @@ # precedence over DB_BACKEND, and every pruned node must prune # SNAPSHOT_AT (default 0 = off, needs PRUNE_NODE): at that sequence the pruned node is stopped, # started once without peers to log the digest of its head, packaged with -# packaging/docker/snapshot-make.sh and started again; one more node (NODES + 1) restores the -# snapshot with snapshot-restore.sh and joins; with SNAPSHOT_CORRUPT=1 the archive is truncated -# first, so the restore must fail and that node syncs from block 0 +# --export-snapshot (format 2) and started again; one more node (NODES + 1) restores the +# snapshot with --import-snapshot into its own backend and joins; with SNAPSHOT_CORRUPT=1 the +# blocks file is truncated first, so the restore must fail and that node syncs from block 0. +# SNAPSHOT_FORMAT=1: packaging/docker/snapshot-make.sh and snapshot-restore.sh instead # STORAGE_SWITCH_AT (default 0 = off, needs PRUNE_NODE=6 and DB_BACKEND=lmdb): at that sequence # node 5 is restarted as a pruned node and node 6 as a full one # CONVERT_AT (default 0 = off, needs DB_BACKEND=berkeleydb): at that sequence n4 is converted to @@ -190,6 +191,7 @@ CRASH_ALL_AT=${CRASH_ALL_AT:-0} JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0} # the joining node is node NODES + 1; it is in nobody's hosts or trusted list SNAPSHOT_AT=${SNAPSHOT_AT:-0} +SNAPSHOT_FORMAT=${SNAPSHOT_FORMAT:-2} SNAPSHOT_CORRUPT=${SNAPSHOT_CORRUPT:-0} STORAGE_SWITCH_AT=${STORAGE_SWITCH_AT:-0} CONVERT_AT=${CONVERT_AT:-0} @@ -670,8 +672,10 @@ while :; do snapshot_done=1 head_digest=$(digest_offline "$PRUNE_NODE") || exit 1 echo "snapshot test: n$PRUNE_NODE stopped, head $head_digest" - "$REPO/packaging/docker/snapshot-make.sh" "$WORK/n$PRUNE_NODE" "$WORK/snapshot" "$MDB_COPY" local \ - | sed 's/^/snapshot test: /' + if [ "$SNAPSHOT_FORMAT" = 1 ]; then + "$REPO/packaging/docker/snapshot-make.sh" "$WORK/n$PRUNE_NODE" "$WORK/snapshot" "$MDB_COPY" local \ + | sed 's/^/snapshot test: /' + fi # format 2: the node builds the snapshot of its latest periodic checkpoint itself; the # manifest must declare the digest the checkpoint logged when it was saved rc=0 @@ -693,14 +697,28 @@ if saved.get(m["seq"]) != (m["state_digest"], m["wallets"]): print(f"snapshot test: format-2 snapshot of #{m['seq']} ({m['first_block']}..{m['seq']}), digest as logged at the checkpoint, {len(m['files'])} files") PY start_node "$PRUNE_NODE" - if [ "$SNAPSHOT_CORRUPT" = 1 ]; then - f=$(ls "$WORK"/snapshot/credits-pruned-*.tar.zst) - head -c $(( $(stat -c %s "$f") / 2 )) "$f" > "$f.cut" && mv "$f.cut" "$f" - echo "snapshot test: archive truncated" - fi rc=0 - "$REPO/packaging/docker/snapshot-restore.sh" "$WORK/n$JOIN_NODE" "$WORK/snapshot/latest.json" \ - > "$WORK/restore.log" 2>&1 || rc=$? + if [ "$SNAPSHOT_FORMAT" = 1 ]; then + if [ "$SNAPSHOT_CORRUPT" = 1 ]; then + f=$(ls "$WORK"/snapshot/credits-pruned-*.tar.zst) + head -c $(( $(stat -c %s "$f") / 2 )) "$f" > "$f.cut" && mv "$f.cut" "$f" + echo "snapshot test: archive truncated" + fi + "$REPO/packaging/docker/snapshot-restore.sh" "$WORK/n$JOIN_NODE" "$WORK/snapshot/latest.json" \ + > "$WORK/restore.log" 2>&1 || rc=$? + else + manifest=$(ls "$WORK"/snapshot2/*/snapshot.json) + if [ "$SNAPSHOT_CORRUPT" = 1 ]; then + f="$(dirname "$manifest")/blocks.bin.lz4" + head -c $(( $(stat -c %s "$f") / 2 )) "$f" > "$f.cut" && mv "$f.cut" "$f" + echo "snapshot test: blocks.bin.lz4 truncated" + fi + # the joining node restores the snapshot itself into its configured backend + (cd "$WORK/n$JOIN_NODE" && exec "$BIN" --db-path db --config-file config.ini --import-snapshot "$manifest") \ + > "$WORK/restore.log" 2>&1 || rc=$? + grep -a 'import-snapshot:' "$WORK/restore.log" > "$WORK/restore.short" || true + mv "$WORK/restore.short" "$WORK/restore.log" + fi sed 's/^/snapshot test: /' "$WORK/restore.log" echo "snapshot test: restore exit $rc; n$JOIN_NODE joins at sequence $min" if [ "$SNAPSHOT_CORRUPT" = 1 ] && [ "$rc" != 2 ]; then echo "FAIL: a truncated snapshot was restored"; exit 1; fi diff --git a/client/src/main.cpp b/client/src/main.cpp index eb0e6670b..0aa0bd4b4 100644 --- a/client/src/main.cpp +++ b/client/src/main.cpp @@ -62,6 +62,7 @@ int main(int argc, char* argv[]) { ("convert-only", "convert the block database to the configured db_backend, then exit") ("convert-batch", po::value()->default_value(100000), "blocks per conversion batch (1 to 1000000)") ("export-snapshot", po::value(), "build the snapshot of the latest periodic checkpoint into the given folder, then exit (the node must be stopped)") + ("import-snapshot", po::value(), "restore the snapshot of the given snapshot.json into this node's empty database, then exit") ("config-file", po::value(), "path to configuration file (default: \"config.ini\")") ("public-key-file", po::value(), "path to public key file (default: \"NodePublic.txt\")") ("private-key-file", po::value(), "path to private key file (default: \"NodePrivate.txt\")") @@ -199,6 +200,11 @@ int main(int argc, char* argv[]) { } } + // no keys needed: the import writes only the database, the caches and the checkpoint + if (vm.count("import-snapshot") > 0) { + return snapshot_export::runImport(config, vm["import-snapshot"].as()); + } + if (vm.count(cmdline::argSeed) == 0) { if (!config.readKeys(vm)) { return EXIT_FAILURE; diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index fcd094534..9208b2fa9 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include namespace snapshot_export { @@ -141,4 +143,35 @@ int run(const Config& config, const std::string& outDir) { return 0; } +int runImport(const Config& config, const std::string& manifestFile) { + auto sayImport = []() -> std::ostream& { return std::cout << "import-snapshot: "; }; + const std::string dbPath = config.getPathToDB(); + if (dbcheck::lockedByAnotherProcess(dbPath)) { + sayImport() << "a node is running on " << dbPath << ": stop it first" << std::endl; + return 2; + } + std::ifstream in(manifestFile, std::ios::binary); + if (!in) { + sayImport() << "cannot read " << manifestFile << std::endl; + return 2; + } + std::string json((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::string error; + const auto manifest = cs::SnapshotManifest::fromJson(json, &error); + if (!manifest) { + sayImport() << "not restored: the manifest is refused: " << error << std::endl; + return 2; + } + const auto backend = config.getStorageSettings().dbBackend; + sayImport() << "block " << manifest->seq << " (blocks " << manifest->firstBlock << ".." << manifest->seq << "), state digest " + << manifest->stateDigest << ", signed by " << manifest->nodeKey << ", into a new " << backend << " database" << std::endl; + const auto r = cs::importSnapshot(fs::path(manifestFile).parent_path(), *manifest, fs::current_path(), dbPath, backend); + if (!r.ok) { + sayImport() << "not restored: " << r.error << (r.fileAtFault.empty() ? "" : " (" + r.fileAtFault + ")") << std::endl; + return 2; + } + sayImport() << "restored block " << manifest->seq << std::endl; + return 0; +} + } // namespace snapshot_export diff --git a/client/src/snapshot_export.hpp b/client/src/snapshot_export.hpp index 4850440ec..1d3faf971 100644 --- a/client/src/snapshot_export.hpp +++ b/client/src/snapshot_export.hpp @@ -10,4 +10,9 @@ namespace snapshot_export { // state_digest.bin) of this stopped node into /, signed with the node's key. 0 on success. int run(const Config& config, const std::string& outDir); +// --import-snapshot : restores the format-2 snapshot whose manifest and files are in +// that folder into this node's empty database (configured db_backend, lmdb or rocksdb). 0 on +// success, 2 when nothing was restored (the reason is printed), as snapshot-restore.sh. +int runImport(const Config& config, const std::string& manifestFile); + } // namespace snapshot_export diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index e5efa3b89..e0c94800d 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -71,6 +71,7 @@ add_library(csnode include/csnode/statedigest.hpp include/csnode/snapshot_manifest.hpp include/csnode/snapshot_builder.hpp + include/csnode/snapshot_importer.hpp include/csnode/blockverifier.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -114,6 +115,7 @@ add_library(csnode src/statedigest.cpp src/snapshot_manifest.cpp src/snapshot_builder.cpp + src/snapshot_importer.cpp src/blockverifier.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/snapshot_importer.hpp b/csnode/include/csnode/snapshot_importer.hpp new file mode 100644 index 000000000..d949da73a --- /dev/null +++ b/csnode/include/csnode/snapshot_importer.hpp @@ -0,0 +1,32 @@ +#ifndef SNAPSHOT_IMPORTER_HPP +#define SNAPSHOT_IMPORTER_HPP + +#include +#include +#include + +#include + +namespace cs { + +struct SnapshotImportResult { + bool ok = false; + std::string error; + std::string fileAtFault; // the snapshot file that failed, empty when the fault is not a file's +}; + +// Restores a format-2 snapshot (manifest m, its files in srcDir, all downloaded) into a node folder +// whose database is empty: the files must match the manifest (sizes, whole-file and chunk hashes); +// qs/ is written from the checkpoint files, whose head.bin and state_digest.bin must carry the +// manifest's block hash and digests; every block is checked with the BlockVerifier and as a hash +// chain ending at the manifest's head, then written to a new database of the given backend (lmdb or +// rocksdb) with caches/hashdb and caches/seqdb; then the contract data and nodeDir/snapshot.json. +// The node quick-starts from qs/ and rebuilds its transaction index. Any failure removes +// everything the import created. +SnapshotImportResult importSnapshot(const std::filesystem::path& srcDir, const SnapshotManifest& m, + const std::filesystem::path& nodeDir, const std::filesystem::path& dbPath, + const std::string& backend, const std::atomic* stop = nullptr); + +} // namespace cs + +#endif // SNAPSHOT_IMPORTER_HPP diff --git a/csnode/src/snapshot_importer.cpp b/csnode/src/snapshot_importer.cpp new file mode 100644 index 000000000..231dde9f7 --- /dev/null +++ b/csnode/src/snapshot_importer.cpp @@ -0,0 +1,295 @@ +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; + +constexpr uint64_t kMaxRawFile = 4ull << 30; +constexpr size_t kBatch = 1000; + +std::string hexOf(const uint8_t* data, size_t size) { + static const char* digits = "0123456789abcdef"; + std::string out; + out.reserve(size * 2); + for (size_t i = 0; i < size; ++i) { + out.push_back(digits[data[i] >> 4]); + out.push_back(digits[data[i] & 15]); + } + return out; +} + +uint32_t getU32(const uint8_t* p) { + return uint32_t(p[0]) | uint32_t(p[1]) << 8 | uint32_t(p[2]) << 16 | uint32_t(p[3]) << 24; +} + +// the file as downloaded: size, every chunk and the whole file as the manifest declares +bool fileMatches(const fs::path& path, const SnapshotFile& f, uint64_t chunkSize, std::string& why) { + std::error_code ec; + if (!fs::is_regular_file(path, ec) || fs::file_size(path, ec) != f.size) { + why = "missing or of another size"; + return false; + } + std::ifstream in(path, std::ios::binary); + crypto_hash_sha256_state whole; + crypto_hash_sha256_init(&whole); + std::vector buf(static_cast(chunkSize)); + size_t chunk = 0; + for (uint64_t done = 0; done < f.size; ++chunk) { + const size_t n = static_cast(std::min(chunkSize, f.size - done)); + in.read(reinterpret_cast(buf.data()), static_cast(n)); + if (static_cast(in.gcount()) != n || chunk >= f.chunks.size() || sha256Hex(buf.data(), n) != f.chunks[chunk]) { + why = "chunk " + std::to_string(chunk) + " does not match"; + return false; + } + crypto_hash_sha256_update(&whole, buf.data(), n); + done += n; + } + std::array h{}; + crypto_hash_sha256_final(&whole, h.data()); + if (chunk != f.chunks.size() || hexOf(h.data(), h.size()) != f.sha256) { + why = "the file hash does not match"; + return false; + } + return true; +} + +bool stopped(const std::atomic* stop) { + return stop && stop->load(); +} +} // namespace + +SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManifest& m, const fs::path& nodeDir, + const fs::path& dbPath, const std::string& backend, const std::atomic* stop) { + SnapshotImportResult result; + std::error_code ec; + const cs::Sequence S = m.seq; + const fs::path qsFinal = nodeDir / "qs" / std::to_string(S); + const fs::path qsTmp = nodeDir / "qs" / (std::to_string(S) + ".import"); + const fs::path caches = nodeDir / "caches"; + bool createdDb = false, createdQs = false, createdCaches = false; + + auto failWith = [&](const std::string& reason, const std::string& file = std::string{}) { + fs::remove_all(qsTmp, ec); + if (createdQs) fs::remove_all(qsFinal, ec); + if (createdDb) { + // only the database files this import created: db/ may hold the node's lock file + for (const auto& f : csdb::backend::files(backend, dbPath.string())) { + fs::remove_all(f, ec); + } + fs::remove(dbPath / csdb::backend::kMarkerFile, ec); + } + if (createdCaches) { + fs::remove_all(caches / "hashdb", ec); + fs::remove_all(caches / "seqdb", ec); + } + result.ok = false; + result.error = reason; + result.fileAtFault = file; + return result; + }; + + if (backend != "lmdb" && backend != "rocksdb") { + return failWith("a snapshot starts above block 0, which a " + backend + " database cannot hold: use lmdb or rocksdb"); + } + if (!csdb::backend::detect(dbPath.string(), true).empty()) { + return failWith(dbPath.string() + " already holds a database"); + } + if (fs::exists(qsFinal, ec)) { + return failWith(qsFinal.string() + " already exists"); + } + if (!m.verify()) { + return failWith("the manifest's signature does not verify"); + } + + // 1. every file as the manifest declares it + for (const auto& f : m.files) { + std::string why; + if (!SnapshotManifest::isKnownFileName(f.name)) { + return failWith("unknown file", f.name); + } + if (!fileMatches(srcDir / f.name, f, m.chunkSize, why)) { + return failWith(f.name + ": " + why, f.name); + } + if (f.rawSize > kMaxRawFile) { + return failWith(f.name + " is larger than 4 GB once decompressed", f.name); + } + } + + // 2. the checkpoint, whose head and digests must be the manifest's + fs::remove_all(qsTmp, ec); + fs::create_directories(qsTmp, ec); + if (ec) { + return failWith("cannot create " + qsTmp.string() + ": " + ec.message()); + } + for (const auto& name : SnapshotManifest::checkpointFileNames()) { + const auto* f = m.file("qs-" + name + ".lz4"); + if (!f) { + continue; + } + std::ofstream out(qsTmp / name, std::ios::binary | std::ios::trunc); + uint64_t written = 0; + const bool ok = snapshot_lz4::decompressFile(srcDir / f->name, [&](const uint8_t* data, size_t size) { + written += size; + if (written > f->rawSize) { + return false; + } + out.write(reinterpret_cast(data), static_cast(size)); + return static_cast(out); + }); + out.close(); + if (!ok || written != f->rawSize || !out) { + return failWith(f->name + " does not decompress to its declared size", f->name); + } + } + const auto head = readCheckpointHead(qsTmp); + if (!head || head->sequence != S || hexOf(head->head_hash.data(), head->head_hash.size()) != m.headHash) { + return failWith("the checkpoint's head is not the manifest's block", "qs-head.bin.lz4"); + } + const auto digest = CheckpointDigest::read(qsTmp / CheckpointDigest::kFileName); + if (!digest || hexOf(digest->digest.data(), digest->digest.size()) != m.stateDigest || digest->wallets != m.wallets || + hexOf(digest->settings.data(), digest->settings.size()) != m.settingsDigest) { + return failWith("the checkpoint's state digest is not the manifest's", "qs-state_digest.bin.lz4"); + } + + // 3. the blocks: verified, chained to the manifest's head, written with their hashes + if (fs::exists(caches / "hashdb", ec) || fs::exists(caches / "seqdb", ec)) { + return failWith(caches.string() + " already holds block hash databases"); + } + createdDb = true; + createdCaches = true; + { + auto db = csdb::backend::open(backend, dbPath.string()); + if (!db || !db->is_open()) { + return failWith("cannot create the " + backend + " database in " + dbPath.string()); + } + fs::create_directories(caches, ec); + BlockHashes hashes(caches.string()); + std::vector batch; + cs::Sequence expected = m.firstBlock; + csdb::PoolHash previous; + std::string reason; + const bool read = forEachSnapshotBlock(srcDir / "blocks.bin.lz4", [&](cs::Sequence seq, cs::Bytes&& bytes) { + if (stopped(stop)) { + reason = "stopped"; + return false; + } + if (seq != expected || seq > S) { + reason = "block " + std::to_string(seq) + " where " + std::to_string(expected) + " was expected"; + return false; + } + const auto pool = csdb::Pool::from_binary(cs::Bytes(bytes)); + std::string why; + if (!BlockVerifier::verify(pool, seq, csdb::PoolHash{}, why)) { + reason = "block " + std::to_string(seq) + " " + why; + return false; + } + // anchored: each block is the one the next block names, and the last is the agreed head + if (seq > m.firstBlock && pool.previous_hash() != previous) { + reason = "block " + std::to_string(seq) + " does not follow block " + std::to_string(seq - 1); + return false; + } + const auto hash = pool.hash(); + if (seq == S && hexOf(hash.to_binary().data(), hash.to_binary().size()) != m.headHash) { + reason = "block " + std::to_string(S) + " is not the manifest's head"; + return false; + } + if (!hashes.update(pool)) { + reason = "cannot write the block hash databases"; + return false; + } + batch.push_back(csdb::Database::PendingWrite{hash.to_binary(), static_cast(seq), std::move(bytes)}); + if (batch.size() >= kBatch && !db->put_batch(batch)) { + reason = "cannot write blocks to the database"; + return false; + } + if (batch.size() >= kBatch) { + batch.clear(); + } + previous = hash; + ++expected; + return true; + }); + if (!read || expected != S + 1) { + hashes.close(); + return failWith(reason.empty() ? "blocks.bin.lz4 is truncated or malformed" : reason, "blocks.bin.lz4"); + } + if (!batch.empty() && !db->put_batch(batch)) { + hashes.close(); + return failWith("cannot write blocks to the database"); + } + + // 4. contract data + if (const auto* f = m.file("contracts.bin.lz4")) { + std::vector pending; + bool good = true; + const bool ok = snapshot_lz4::decompressFile(srcDir / f->name, [&](const uint8_t* data, size_t size) { + pending.insert(pending.end(), data, data + size); + size_t pos = 0; + for (;;) { + if (pending.size() - pos < 4) break; + const uint32_t klen = getU32(pending.data() + pos); + if (pending.size() - pos < 8ull + klen) break; + const uint32_t vlen = getU32(pending.data() + pos + 4 + klen); + if (pending.size() - pos < 8ull + klen + vlen) break; + const cs::Bytes key(pending.begin() + static_cast(pos + 4), pending.begin() + static_cast(pos + 4 + klen)); + const cs::Bytes value(pending.begin() + static_cast(pos + 8 + klen), + pending.begin() + static_cast(pos + 8 + klen + vlen)); + if (!db->updateContractData(key, value)) { + good = false; + return false; + } + pos += 8ull + klen + vlen; + } + pending.erase(pending.begin(), pending.begin() + static_cast(pos)); + return true; + }); + if (!ok || !good || !pending.empty()) { + hashes.close(); + return failWith("contracts.bin.lz4 does not restore", f->name); + } + } + if (!db->flush()) { + hashes.close(); + return failWith("cannot flush the database"); + } + hashes.close(); + } + if (!csdb::backend::writeMarker(dbPath.string(), backend)) { + return failWith("cannot write the database marker"); + } + + // 5. publish the checkpoint and the manifest + fs::rename(qsTmp, qsFinal, ec); + if (ec) { + return failWith("cannot move the checkpoint into place: " + ec.message()); + } + createdQs = true; + { + std::ofstream out(nodeDir / "snapshot.json", std::ios::binary | std::ios::trunc); + out << m.toJson(); + out.flush(); + if (!out) { + return failWith("cannot write snapshot.json"); + } + } + result.ok = true; + return result; +} + +} // namespace cs diff --git a/tests/csnode/snapshot_import_tests.cpp b/tests/csnode/snapshot_import_tests.cpp new file mode 100644 index 000000000..1249a5dce --- /dev/null +++ b/tests/csnode/snapshot_import_tests.cpp @@ -0,0 +1,279 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; + +struct SignedChain { + std::map blocks; + std::map hashes; +}; + +cs::Hash toHash(const csdb::PoolHash& h) { + cs::Hash out{}; + const auto b = h.to_binary(); + std::copy(b.begin(), b.end(), out.begin()); + return out; +} + +// blocks 0..count-1 linked by hash, each (but 0) signed by its three confidants +SignedChain makeSignedChain(cs::Sequence count) { + EXPECT_TRUE(cscrypto::cryptoInit()); + std::vector pub(3); + std::vector priv; + for (auto& p : pub) { + priv.push_back(cscrypto::generateKeyPair(p)); + } + SignedChain c; + csdb::PoolHash prev; + for (cs::Sequence s = 0; s < count; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(64 + s % 30, static_cast('a' + s % 26))); + if (s > 0) { + p.set_confidants(pub); + p.add_number_trusted(3); + p.add_real_trusted(0b111); + auto probe = p.clone(); + EXPECT_TRUE(probe.compose()); + const auto hash = toHash(probe.hash()); + std::vector sigs; + for (const auto& k : priv) { + sigs.push_back(cscrypto::generateSignature(k, hash.data(), hash.size())); + } + p.set_signatures(sigs); + } + EXPECT_TRUE(p.compose()); + c.blocks[s] = p.to_binary(); + c.hashes[s] = p.hash().to_binary(); + prev = p.hash(); + } + return c; +} + +void putU16(std::ofstream& f, uint16_t v) { + f.write(reinterpret_cast(&v), sizeof(v)); +} + +void makeCheckpoint(const fs::path& dir, cs::Sequence seq, const cs::Bytes& headHash) { + fs::create_directories(dir); + std::ofstream f(dir / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t s = seq; + f.write(reinterpret_cast(&magic), sizeof(magic)); + f.write(reinterpret_cast(&s), sizeof(s)); + putU16(f, static_cast(headHash.size())); + f.write(reinterpret_cast(headHash.data()), static_cast(headHash.size())); + putU16(f, 0); + f.close(); + std::ofstream(dir / "walletscache.dat", std::ios::binary) << std::string(100000, 'w'); + cs::CheckpointDigest d; + d.digest.fill(3); + d.settings.fill(4); + d.wallets = 11; + EXPECT_TRUE(d.write(dir / cs::CheckpointDigest::kFileName)); +} + +struct Fixture { + fs::path root; + SignedChain chain = makeSignedChain(161); + cs::PrivateKey key; + cs::PublicKey pub; + std::map contracts = {{{1, 2, 3}, {7, 7}}, {{9}, {}}}; + + explicit Fixture(const char* name) : root(fs::temp_directory_path() / name) { + fs::remove_all(root); + fs::create_directories(root / "source"); + fs::create_directories(root / "target"); + key = cscrypto::generateKeyPair(pub); + makeCheckpoint(root / "source" / "qs" / "150", 150, chain.hashes[150]); + } + ~Fixture() { + std::error_code ec; + fs::remove_all(root, ec); + } + + cs::SnapshotBuildResult build() { + cs::SnapshotBuildInput in; + in.seq = 150; + in.checkpointDir = root / "source" / "qs" / "150"; + in.snapshotBlocks = 100; + in.nodeVersion = "test"; + in.chunkSize = 64 << 10; + in.blockBytes = [this](cs::Sequence s, cs::Bytes& out) { + out = chain.blocks.at(s); + return true; + }; + in.contracts = [this](const std::function& visit) { + for (const auto& [k, v] : contracts) { + if (!visit(k, v)) return false; + } + return true; + }; + return cs::buildSnapshot(in, root / "out", key); + } + + cs::SnapshotImportResult import(const cs::SnapshotBuildResult& r, const std::string& backend = "lmdb") { + return cs::importSnapshot(r.dir, r.manifest, root / "target", root / "target" / "db", backend); + } + + // nothing the import creates is left after a failure + void expectNothingLeft() { + const auto t = root / "target"; + EXPECT_TRUE(csdb::backend::detect((t / "db").string(), false).empty()); + EXPECT_FALSE(fs::exists(t / "qs" / "150")); + EXPECT_FALSE(fs::exists(t / "qs" / "150.import")); + EXPECT_FALSE(fs::exists(t / "caches" / "hashdb")); + EXPECT_FALSE(fs::exists(t / "caches" / "seqdb")); + EXPECT_FALSE(fs::exists(t / "snapshot.json")); + } +}; + +void roundTrip(const std::string& backend) { + Fixture fx(("snapshot_import_" + backend).c_str()); + const auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + const auto r = fx.import(built, backend); + ASSERT_TRUE(r.ok) << r.error << " " << r.fileAtFault; + + const auto t = fx.root / "target"; + EXPECT_EQ(csdb::backend::readMarker((t / "db").string()), backend); + auto db = csdb::backend::open(backend, (t / "db").string()); + ASSERT_TRUE(db && db->is_open()); + for (cs::Sequence s = 50; s <= 150; ++s) { + cs::Bytes bytes; + ASSERT_TRUE(db->get(static_cast(s), &bytes)) << s; + EXPECT_EQ(bytes, fx.chain.blocks[s]) << s; + } + EXPECT_FALSE(db->get(static_cast(49))); + for (const auto& [k, v] : fx.contracts) { + cs::Bytes value; + EXPECT_TRUE(db->getContractData(k, value)); + EXPECT_EQ(value, v); + } + db.reset(); + + cs::BlockHashes hashes((t / "caches").string()); + EXPECT_EQ(hashes.find(150).to_binary(), fx.chain.hashes[150]); + EXPECT_EQ(hashes.find(csdb::PoolHash::from_binary(cs::Bytes(fx.chain.hashes[50]))), 50u); + hashes.close(); + + EXPECT_TRUE(fs::exists(t / "qs" / "150" / "head.bin")); + std::ifstream a(fx.root / "source" / "qs" / "150" / "walletscache.dat", std::ios::binary); + std::ifstream b(t / "qs" / "150" / "walletscache.dat", std::ios::binary); + EXPECT_EQ(std::string(std::istreambuf_iterator(a), {}), std::string(std::istreambuf_iterator(b), {})); + EXPECT_TRUE(fs::exists(t / "snapshot.json")); +} +} // namespace + +TEST(SnapshotImport, RoundTripLmdb) { + roundTrip("lmdb"); +} + +TEST(SnapshotImport, RoundTripRocksDb) { + roundTrip("rocksdb"); +} + +TEST(SnapshotImport, BerkeleyDbRefused) { + Fixture fx("snapshot_import_bdb"); + const auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + EXPECT_FALSE(fx.import(built, "berkeleydb").ok); + fx.expectNothingLeft(); +} + +TEST(SnapshotImport, AlteredSignatureRejectedNothingLeft) { + Fixture fx("snapshot_import_sig"); + auto& bytes = fx.chain.blocks[120]; + const auto pool = csdb::Pool::from_binary(cs::Bytes(bytes)); + bytes[pool.hashingLength() + sizeof(size_t)] ^= 1; // first byte of the first signature + const auto built = fx.build(); // the builder checks the chain, not signatures + ASSERT_TRUE(built.ok) << built.error; + const auto r = fx.import(built); + EXPECT_FALSE(r.ok); + EXPECT_EQ(r.fileAtFault, "blocks.bin.lz4"); + EXPECT_NE(r.error.find("120"), std::string::npos) << r.error; + fx.expectNothingLeft(); +} + +TEST(SnapshotImport, ManifestDigestOtherThanTheCheckpointsRejected) { + Fixture fx("snapshot_import_digest"); + auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + built.manifest.stateDigest = std::string(64, 'f'); + built.manifest.sign(fx.key); // a validly signed manifest that lies about the state + const auto r = fx.import(built); + EXPECT_FALSE(r.ok); + EXPECT_EQ(r.fileAtFault, "qs-state_digest.bin.lz4"); + fx.expectNothingLeft(); +} + +TEST(SnapshotImport, ManifestHeadOtherThanTheChainsRejected) { + Fixture fx("snapshot_import_head"); + auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + built.manifest.headHash = std::string(64, 'e'); + built.manifest.sign(fx.key); + EXPECT_FALSE(fx.import(built).ok); + fx.expectNothingLeft(); +} + +TEST(SnapshotImport, TruncatedOrAlteredFileRejected) { + Fixture fx("snapshot_import_trunc"); + const auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + const auto blocks = built.dir / "blocks.bin.lz4"; + const auto size = fs::file_size(blocks); + fs::resize_file(blocks, size - 10); + auto r = fx.import(built); + EXPECT_FALSE(r.ok); + EXPECT_EQ(r.fileAtFault, "blocks.bin.lz4"); + fx.expectNothingLeft(); + + fs::resize_file(blocks, size); // right size, wrong bytes + r = fx.import(built); + EXPECT_FALSE(r.ok); + fx.expectNothingLeft(); +} + +TEST(SnapshotImport, OversizedRawFileRejected) { + Fixture fx("snapshot_import_raw"); + auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + for (auto& f : built.manifest.files) { + if (f.name == "qs-walletscache.dat.lz4") { + f.rawSize = 5ull << 30; + } + } + built.manifest.sign(fx.key); + EXPECT_FALSE(fx.import(built).ok); + fx.expectNothingLeft(); +} + +TEST(SnapshotImport, ExistingDatabaseRefused) { + Fixture fx("snapshot_import_existing"); + const auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + ASSERT_TRUE(fx.import(built).ok); + fs::remove_all(fx.root / "target" / "qs"); + const auto again = fx.import(built); + EXPECT_FALSE(again.ok); + // the first import is untouched + auto db = csdb::backend::open("lmdb", (fx.root / "target" / "db").string()); + cs::Bytes bytes; + EXPECT_TRUE(db && db->get(150u, &bytes)); +} From 881deed9a46426900644cb622562343ddd734f55 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 12:39:33 +0200 Subject: [PATCH 02/10] snapshot importer: confirmations chained, decompression capped Security review of the importer: - every block's round confirmations must verify against the previous block's confidants (the node's own rule for a stored block), so a forged range needs the keys of every round's confidants; - blocks.bin and contracts.bin are capped at their declared raw_size and a contract key or value at 64 MB before anything is buffered; - the header states what the import proves and what anchors it (the agreement of sources, then the network, and the quick-start check). Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_builder.hpp | 6 ++-- csnode/include/csnode/snapshot_importer.hpp | 6 ++++ csnode/src/snapshot_builder.cpp | 8 ++++- csnode/src/snapshot_importer.cpp | 27 ++++++++++++++-- tests/csnode/snapshot_import_tests.cpp | 36 +++++++++++++++++++++ 5 files changed, 78 insertions(+), 5 deletions(-) diff --git a/csnode/include/csnode/snapshot_builder.hpp b/csnode/include/csnode/snapshot_builder.hpp index de0046964..84b6d8831 100644 --- a/csnode/include/csnode/snapshot_builder.hpp +++ b/csnode/include/csnode/snapshot_builder.hpp @@ -48,9 +48,11 @@ struct SnapshotBuildResult { SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::filesystem::path& outRoot, const cs::PrivateKey& key); -// the blocks.bin record stream: "u32 seq, u32 len, bytes" (little-endian) +// the blocks.bin record stream: "u32 seq, u32 len, bytes" (little-endian); false beyond maxRawBytes +// decompressed bytes (a manifest's raw_size), for a record above 64 MB, or a truncated stream bool forEachSnapshotBlock(const std::filesystem::path& blocksFile, - const std::function& visit); + const std::function& visit, + uint64_t maxRawBytes = UINT64_MAX); } // namespace cs diff --git a/csnode/include/csnode/snapshot_importer.hpp b/csnode/include/csnode/snapshot_importer.hpp index d949da73a..d6c2c2333 100644 --- a/csnode/include/csnode/snapshot_importer.hpp +++ b/csnode/include/csnode/snapshot_importer.hpp @@ -23,6 +23,12 @@ struct SnapshotImportResult { // rocksdb) with caches/hashdb and caches/seqdb; then the contract data and nodeDir/snapshot.json. // The node quick-starts from qs/ and rebuilds its transaction index. Any failure removes // everything the import created. +// Trust: this proves that the snapshot is consistent with itself (signed manifest, a hash chain of +// blocks signed by their confidants and confirmed by the previous round's, ending at the declared +// head, a checkpoint whose head and digests are the declared ones). It does not prove that the +// head is the network's: the caller anchors it, by the agreement of independent sources on +// (seq, head_hash, state_digest) before calling, and the node by continuing from that block with +// the network; the node also recomputes the state from the loaded caches at its quick start. SnapshotImportResult importSnapshot(const std::filesystem::path& srcDir, const SnapshotManifest& m, const std::filesystem::path& nodeDir, const std::filesystem::path& dbPath, const std::string& backend, const std::atomic* stop = nullptr); diff --git a/csnode/src/snapshot_builder.cpp b/csnode/src/snapshot_builder.cpp index 54c77e470..f9fd80fbc 100644 --- a/csnode/src/snapshot_builder.cpp +++ b/csnode/src/snapshot_builder.cpp @@ -226,10 +226,16 @@ bool decompressFile(const std::filesystem::path& file, const std::function& visit) { +bool forEachSnapshotBlock(const std::filesystem::path& blocksFile, const std::function& visit, + uint64_t maxRawBytes) { // records may span lz4 blocks: keep the unparsed tail std::vector pending; + uint64_t total = 0; bool ok = snapshot_lz4::decompressFile(blocksFile, [&](const uint8_t* data, size_t size) { + total += size; + if (total > maxRawBytes) { + return false; + } pending.insert(pending.end(), data, data + size); size_t pos = 0; while (pending.size() - pos >= 8) { diff --git a/csnode/src/snapshot_importer.cpp b/csnode/src/snapshot_importer.cpp index 231dde9f7..04ace41a0 100644 --- a/csnode/src/snapshot_importer.cpp +++ b/csnode/src/snapshot_importer.cpp @@ -22,6 +22,7 @@ namespace fs = std::filesystem; constexpr uint64_t kMaxRawFile = 4ull << 30; constexpr size_t kBatch = 1000; +constexpr uint32_t kMaxContractField = 64u << 20; std::string hexOf(const uint8_t* data, size_t size) { static const char* digits = "0123456789abcdef"; @@ -183,7 +184,9 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife std::vector batch; cs::Sequence expected = m.firstBlock; csdb::PoolHash previous; + cs::PublicKeys previousConfidants; std::string reason; + const auto* blocksFile = m.file("blocks.bin.lz4"); const bool read = forEachSnapshotBlock(srcDir / "blocks.bin.lz4", [&](cs::Sequence seq, cs::Bytes&& bytes) { if (stopped(stop)) { reason = "stopped"; @@ -204,6 +207,13 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife reason = "block " + std::to_string(seq) + " does not follow block " + std::to_string(seq - 1); return false; } + // and the confidants of each round confirmed the next round's, as the node checks when + // it stores a block: a forged range needs the keys of every round's confidants + if (seq > m.firstBlock && !BlockVerifier::verifyConfirmations(pool, previousConfidants, why)) { + reason = "block " + std::to_string(seq) + " " + why; + return false; + } + previousConfidants = pool.confidants(); const auto hash = pool.hash(); if (seq == S && hexOf(hash.to_binary().data(), hash.to_binary().size()) != m.headHash) { reason = "block " + std::to_string(S) + " is not the manifest's head"; @@ -224,7 +234,7 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife previous = hash; ++expected; return true; - }); + }, blocksFile ? blocksFile->rawSize : 0); if (!read || expected != S + 1) { hashes.close(); return failWith(reason.empty() ? "blocks.bin.lz4 is truncated or malformed" : reason, "blocks.bin.lz4"); @@ -238,14 +248,27 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife if (const auto* f = m.file("contracts.bin.lz4")) { std::vector pending; bool good = true; + uint64_t total = 0; const bool ok = snapshot_lz4::decompressFile(srcDir / f->name, [&](const uint8_t* data, size_t size) { + total += size; + if (total > f->rawSize) { + return false; + } pending.insert(pending.end(), data, data + size); size_t pos = 0; for (;;) { if (pending.size() - pos < 4) break; const uint32_t klen = getU32(pending.data() + pos); + if (klen > kMaxContractField) { + good = false; + return false; + } if (pending.size() - pos < 8ull + klen) break; const uint32_t vlen = getU32(pending.data() + pos + 4 + klen); + if (vlen > kMaxContractField) { + good = false; + return false; + } if (pending.size() - pos < 8ull + klen + vlen) break; const cs::Bytes key(pending.begin() + static_cast(pos + 4), pending.begin() + static_cast(pos + 4 + klen)); const cs::Bytes value(pending.begin() + static_cast(pos + 8 + klen), @@ -259,7 +282,7 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife pending.erase(pending.begin(), pending.begin() + static_cast(pos)); return true; }); - if (!ok || !good || !pending.empty()) { + if (!ok || !good || !pending.empty() || total != f->rawSize) { hashes.close(); return failWith("contracts.bin.lz4 does not restore", f->name); } diff --git a/tests/csnode/snapshot_import_tests.cpp b/tests/csnode/snapshot_import_tests.cpp index 1249a5dce..98f6d609c 100644 --- a/tests/csnode/snapshot_import_tests.cpp +++ b/tests/csnode/snapshot_import_tests.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,21 @@ SignedChain makeSignedChain(cs::Sequence count) { p.set_confidants(pub); p.add_number_trusted(3); p.add_real_trusted(0b111); + if (s > 1) { + // the previous round's confidants (the same keys here) confirm this round's + cs::Bytes trustedToHash; + cs::ODataStream stream(trustedToHash); + stream << s; + stream << pub; + const cs::Hash h = cscrypto::calculateHash(trustedToHash.data(), trustedToHash.size()); + std::vector confirmations; + for (const auto& k : priv) { + confirmations.push_back(cscrypto::generateSignature(k, h.data(), h.size())); + } + p.add_number_confirmations(3); + p.add_confirmation_mask(0b111); + p.add_round_confirmations(confirmations); + } auto probe = p.clone(); EXPECT_TRUE(probe.compose()); const auto hash = toHash(probe.hash()); @@ -277,3 +293,23 @@ TEST(SnapshotImport, ExistingDatabaseRefused) { cs::Bytes bytes; EXPECT_TRUE(db && db->get(150u, &bytes)); } + +TEST(SnapshotImport, BlockWithoutTheRoundsConfirmationsRejected) { + Fixture fx("snapshot_import_confirmations"); + // block 100 rebuilt without confirmations: still signed by its confidants, still in the chain + // for the builder, but no longer confirmed by the round before it + const auto original = csdb::Pool::from_binary(cs::Bytes(fx.chain.blocks[100])); + csdb::Pool p(original.previous_hash(), 100); + p.add_user_field(1, original.user_field(1).value()); + p.set_confidants(original.confidants()); + p.add_number_trusted(3); + p.add_real_trusted(0b111); + ASSERT_TRUE(p.compose()); + fx.chain.blocks[100] = p.to_binary(); // the chain is broken at 101 now: the import must stop at 100 or 101 + const auto built = fx.build(); + if (built.ok) { + EXPECT_FALSE(fx.import(built).ok); + fx.expectNothingLeft(); + } +} + From 69c778d6cdf3ddbe4904bc65224dd40c90ab385d Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 12:39:47 +0200 Subject: [PATCH 03/10] tests: name the rebuilt-block case for what it covers Co-Authored-By: Claude Opus 5.5 --- tests/csnode/snapshot_import_tests.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/csnode/snapshot_import_tests.cpp b/tests/csnode/snapshot_import_tests.cpp index 98f6d609c..b0037211d 100644 --- a/tests/csnode/snapshot_import_tests.cpp +++ b/tests/csnode/snapshot_import_tests.cpp @@ -294,10 +294,9 @@ TEST(SnapshotImport, ExistingDatabaseRefused) { EXPECT_TRUE(db && db->get(150u, &bytes)); } -TEST(SnapshotImport, BlockWithoutTheRoundsConfirmationsRejected) { +TEST(SnapshotImport, RebuiltBlockWithoutSignaturesOrConfirmationsRejected) { Fixture fx("snapshot_import_confirmations"); - // block 100 rebuilt without confirmations: still signed by its confidants, still in the chain - // for the builder, but no longer confirmed by the round before it + // block 100 rebuilt by someone without the confidants' keys: no signatures, no confirmations const auto original = csdb::Pool::from_binary(cs::Bytes(fx.chain.blocks[100])); csdb::Pool p(original.previous_hash(), 100); p.add_user_field(1, original.user_field(1).value()); @@ -305,7 +304,7 @@ TEST(SnapshotImport, BlockWithoutTheRoundsConfirmationsRejected) { p.add_number_trusted(3); p.add_real_trusted(0b111); ASSERT_TRUE(p.compose()); - fx.chain.blocks[100] = p.to_binary(); // the chain is broken at 101 now: the import must stop at 100 or 101 + fx.chain.blocks[100] = p.to_binary(); // its hash changes too: block 101 no longer follows it const auto built = fx.build(); if (built.ok) { EXPECT_FALSE(fx.import(built).ok); From ea97cb80ff7b8b336d0e5c50095f18fb88ab0ae5 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 12:47:34 +0200 Subject: [PATCH 04/10] ci(scale): a scenario input passes run.sh variables, so the large networks can run the snapshot test Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scale.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index c21d6cbc0..2db3e5607 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -31,6 +31,9 @@ on: timeout_min: description: 'Minutes each network has to reach its target sequence' default: '60' + scenario: + description: 'More run.sh variables as NAME=value pairs, as a robustness scenario (e.g. PRUNE_NODE=6 SNAPSHOT_AT=2300 TARGET_SEQ=3300)' + default: '' permissions: contents: read @@ -210,10 +213,17 @@ jobs: FUND: '1' LOAD_SECONDS: '300' WORK: ${{ github.workspace }}/testnet-run + SCENARIO: ${{ inputs.scenario || '' }} run: | chmod +x build/node IFS=, read -r CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE <<< "$CONTRACT_LOAD" export CONTRACT_LOAD_SECONDS CONTRACT_WRITE_RATE CONTRACT_READ_RATE + # NAME=value pairs only, as the robustness scenarios + for kv in $SCENARIO; do + echo "$kv" | grep -qE '^[A-Z_]+=[a-z0-9,:]+$' || { echo "bad scenario entry $kv"; exit 1; } + export "$kv" + done + [ -z "$SCENARIO" ] || echo "scenario: $SCENARIO" ci/testnet/run.sh build/node 2>&1 | tee testnet-run.log exit "${PIPESTATUS[0]}" From fc3a2ea23372b9c885944f1aa7a1a50ab1fcea72 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 13:38:07 +0200 Subject: [PATCH 05/10] export-snapshot: recognise the database from its files, without opening the engine A pruned RocksDB node whose blocks were still in the write-ahead log was not recognised: detection opened the engine a second time in the process to look for blocks. Co-Authored-By: Claude Opus 5.5 --- client/src/snapshot_export.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index 9208b2fa9..61a4b7e7c 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -74,7 +74,9 @@ int run(const Config& config, const std::string& outDir) { say() << "a node is running on " << dbPath << ": stop it first" << std::endl; return 1; } - const auto found = csdb::backend::detect(dbPath, true); + // from the files alone: the start-up check has already decided the folder holds the configured + // backend, and opening an engine twice in one process can fail on its own lock + const auto found = csdb::backend::detect(dbPath, false); if (std::find(found.begin(), found.end(), storage.dbBackend) == found.end()) { say() << dbPath << " holds no " << storage.dbBackend << " database (the configured db_backend)" << std::endl; return 1; From 4641fc25529ff3d5485fc5805b93da72c10f77f8 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 14:21:35 +0200 Subject: [PATCH 06/10] csdb: say why a RocksDB database does not open Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 2 +- csdb/src/storage_backend.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 2ecb04986..b2079d416 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -681,7 +681,7 @@ while :; do rc=0 (cd "$WORK/n$PRUNE_NODE" && exec "$BIN" --db-path db --config-file config.ini --export-snapshot "$WORK/snapshot2") \ > "$WORK/export.log" 2>&1 || rc=$? - grep -a 'export-snapshot:' "$WORK/export.log" | sed 's/^/snapshot test: /' + grep -a 'export-snapshot:\|rocksdb: cannot' "$WORK/export.log" | sed 's/^/snapshot test: /' [ "$rc" = 0 ] || { tail -20 "$WORK/export.log"; echo "FAIL: --export-snapshot exited $rc"; exit 1; } python3 - "$WORK/snapshot2" "$WORK/n$PRUNE_NODE/node.log" <<'PY' || exit 1 import json, os, re, sys diff --git a/csdb/src/storage_backend.cpp b/csdb/src/storage_backend.cpp index 1bf2f1e2f..be316c320 100644 --- a/csdb/src/storage_backend.cpp +++ b/csdb/src/storage_backend.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -120,7 +121,11 @@ std::shared_ptr open(const std::string& name, const std::string& dir) #if defined(CSDB_USE_ROCKSDB) if (name == "rocksdb") { auto db = std::make_shared(); - return db->open(dir) ? db : nullptr; + if (db->open(dir)) { + return db; + } + std::cerr << "rocksdb: cannot open " << dir << ": " << db->last_error_message() << std::endl; + return nullptr; } #endif #if !defined(CSDB_USE_ROCKSDB) || defined(CSDB_USE_BERKELEYDB) From 57d5bd622047c63d2e811f9bb6e3aa383c6a8c8b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 14:54:56 +0200 Subject: [PATCH 07/10] ci(testnet): show who holds the database before the export Co-Authored-By: Claude Opus 5.5 --- ci/testnet/run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index b2079d416..fe5956df6 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -678,6 +678,9 @@ while :; do fi # format 2: the node builds the snapshot of its latest periodic checkpoint itself; the # manifest must declare the digest the checkpoint logged when it was saved + # who still holds the database files (RocksDB's LOCK allows one process) + { fuser -v "$WORK/n$PRUNE_NODE"/db/LOCK "$WORK/n$PRUNE_NODE"/db/lmdb/lock.mdb 2>&1 || true; } | sed 's/^/snapshot test: holder: /' + ps -eo pid,ppid,etimes,args | grep -a "n$PRUNE_NODE\b\|contract-executor" | grep -v grep | sed 's/^/snapshot test: process: /' || true rc=0 (cd "$WORK/n$PRUNE_NODE" && exec "$BIN" --db-path db --config-file config.ini --export-snapshot "$WORK/snapshot2") \ > "$WORK/export.log" 2>&1 || rc=$? From 760d27337d69d20158c6cb95a0bf2a5233346e2b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 15:21:45 +0200 Subject: [PATCH 08/10] ci: digest_offline waits for the node to stop and for the start-up digest run.sh's digest_offline runs in a subshell, where wait cannot wait for the node (not its child): the node was still stopping while the offline start and the export opened its database, which RocksDB's lock refuses. Both scripts now wait for the start-up line, not the digest a quick start logs for the checkpoint it loaded, before stopping the node. Co-Authored-By: Claude Opus 5.5 --- ci/snapshot/digest_offline.sh | 3 ++- ci/testnet/run.sh | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ci/snapshot/digest_offline.sh b/ci/snapshot/digest_offline.sh index baea95dee..04536c289 100755 --- a/ci/snapshot/digest_offline.sh +++ b/ci/snapshot/digest_offline.sh @@ -16,7 +16,8 @@ awk ' { print }' config.ini > config-offline.ini # the node's console output of this run (the packaged config.ini logs info to the console): a count # of digest lines in log/ can go down when the start rotates the logs and removes the oldest file -digests() { grep -a 'STATE DIGEST #' offline.out 2>/dev/null || true; } +# the start-up line only: a quick start also logs the digest of the checkpoint it loaded +digests() { grep -aE 'STATE DIGEST #[0-9]+ [0-9A-Fa-f]+ wallets [0-9]+ *$' offline.out 2>/dev/null || true; } ./node --db-path db --config-file config-offline.ini offline.out 2>&1 & pid=$! deadline=$(( $(date +%s) + timeout )) diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index fe5956df6..ae4243638 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -474,7 +474,9 @@ seq_of() { # STATE DIGEST is the digest of its head, and stop it again digest_offline() { local i=$1 dir="$WORK/n$1" before - kill -TERM "${PIDS[$i]}"; wait "${PIDS[$i]}" 2>/dev/null || true + # this runs in a subshell ($(...)): wait cannot wait for the node, which is not its child + kill -TERM "${PIDS[$i]}" + for _ in $(seq 1 180); do kill -0 "${PIDS[$i]}" 2>/dev/null || break; sleep 1; done pkill -f "cs.testnet.node=n$i " 2>/dev/null || true # an empty hosts file stops the node ("No hosts specified"): one host nobody listens on echo "127.0.0.1:9 11111111111111111111111111111111" > "$dir/hosts-none.txt" @@ -483,18 +485,22 @@ digest_offline() { /^hosts_filename=/ { print "hosts_filename=hosts-none.txt"; next } section == "[host_input]" && /^port=/ { print "port=" port; next } { print }' "$dir/config.ini" > "$dir/config-offline.ini" - before=$(grep -ac 'STATE DIGEST #' "$dir/node.log" || true) + # the start-up line only ("... wallets " at the end), not the checkpoint lines + local startup='STATE DIGEST #[0-9]+ [0-9A-Fa-f]+ wallets [0-9]+ *$' + before=$(grep -acE "$startup" "$dir/node.log" || true) (cd "$dir" && exec "$BIN" --db-path db --config-file config-offline.ini > stdout.log 2>&1) & local pid=$! deadline=$(( $(date +%s) + 300 )) - while [ "$(grep -ac 'STATE DIGEST #' "$dir/node.log" || true)" -le "${before:-0}" ] && [ "$(date +%s)" -lt "$deadline" ]; do + while [ "$(grep -acE "$startup" "$dir/node.log" || true)" -le "${before:-0}" ] && [ "$(date +%s)" -lt "$deadline" ]; do sleep 2 done kill -TERM "$pid"; wait "$pid" 2>/dev/null || true - if [ "$(grep -ac 'STATE DIGEST #' "$dir/node.log" || true)" -le "${before:-0}" ]; then + for _ in $(seq 1 180); do kill -0 "$pid" 2>/dev/null || break; sleep 1; done + pkill -f "[c]ontract-executor.*n$i/" 2>/dev/null || true + if [ "$(grep -acE "$startup" "$dir/node.log" || true)" -le "${before:-0}" ]; then echo "FAIL: n$i logged no start-up digest without peers" >&2 return 1 fi - grep -a 'STATE DIGEST #' "$dir/node.log" | tail -1 | sed 's/^.*STATE DIGEST/STATE DIGEST/' + grep -aE "$startup" "$dir/node.log" | tail -1 | sed 's/^.*STATE DIGEST/STATE DIGEST/' } # the lowest sequence among the nodes of the restart test From 03260f3f65468c9255d4db7c0b384dd76548ede1 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 16:39:31 +0200 Subject: [PATCH 09/10] snapshot importer: review fixes - The checkpoint is written to qs/.tmp, which the node's checkpoint manager removes at its start: qs/.import was read as version S. - The database and the hash databases are closed before a failed import removes their files (RocksDB writes on close); a hash database failure fails the import and its rebuild marker is removed too. - The decompressed checkpoint files, the qs folder, snapshot.json and the node folder are synced to disk before the import reports success. - --import-snapshot creates db/ first, so that it holds db/.lock while it writes. - Test: a failed RocksDB import leaves no table or CURRENT file. Co-Authored-By: Claude Opus 5.5 --- client/src/snapshot_export.cpp | 6 ++++ csnode/src/snapshot_importer.cpp | 41 +++++++++++++++++++++----- tests/csnode/snapshot_import_tests.cpp | 19 +++++++++++- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/client/src/snapshot_export.cpp b/client/src/snapshot_export.cpp index 61a4b7e7c..7d6a2f6ab 100644 --- a/client/src/snapshot_export.cpp +++ b/client/src/snapshot_export.cpp @@ -148,6 +148,12 @@ int run(const Config& config, const std::string& outDir) { int runImport(const Config& config, const std::string& manifestFile) { auto sayImport = []() -> std::ostream& { return std::cout << "import-snapshot: "; }; const std::string dbPath = config.getPathToDB(); + // db/ first, so that the import holds db/.lock while it writes: a node started in this folder + // meanwhile finds it in use + { + std::error_code ec; + fs::create_directories(dbPath, ec); + } if (dbcheck::lockedByAnotherProcess(dbPath)) { sayImport() << "a node is running on " << dbPath << ": stop it first" << std::endl; return 2; diff --git a/csnode/src/snapshot_importer.cpp b/csnode/src/snapshot_importer.cpp index 04ace41a0..bdee4c4e6 100644 --- a/csnode/src/snapshot_importer.cpp +++ b/csnode/src/snapshot_importer.cpp @@ -4,7 +4,9 @@ #include #include +#include #include +#include #include #include @@ -70,6 +72,15 @@ bool fileMatches(const fs::path& path, const SnapshotFile& f, uint64_t chunkSize return true; } +// on disk before the next step relies on it: a power cut must not leave a truncated checkpoint +void syncPath(const fs::path& path, bool directory) { + const int fd = ::open(path.c_str(), directory ? O_RDONLY | O_DIRECTORY : O_RDONLY); + if (fd >= 0) { + ::fsync(fd); + ::close(fd); + } +} + bool stopped(const std::atomic* stop) { return stop && stop->load(); } @@ -81,7 +92,9 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife std::error_code ec; const cs::Sequence S = m.seq; const fs::path qsFinal = nodeDir / "qs" / std::to_string(S); - const fs::path qsTmp = nodeDir / "qs" / (std::to_string(S) + ".import"); + // ".tmp": the node's checkpoint manager removes such a leftover at its start and never takes it + // for a checkpoint + const fs::path qsTmp = nodeDir / "qs" / (std::to_string(S) + ".tmp"); const fs::path caches = nodeDir / "caches"; bool createdDb = false, createdQs = false, createdCaches = false; @@ -98,6 +111,7 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife if (createdCaches) { fs::remove_all(caches / "hashdb", ec); fs::remove_all(caches / "seqdb", ec); + fs::remove(caches / "blockhashes.rebuild", ec); } result.ok = false; result.error = reason; @@ -154,6 +168,7 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife return static_cast(out); }); out.close(); + syncPath(qsTmp / name, false); if (!ok || written != f->rawSize || !out) { return failWith(f->name + " does not decompress to its declared size", f->name); } @@ -174,10 +189,12 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife } createdDb = true; createdCaches = true; - { + // the database and the hash databases are closed when this returns: on failure their files are + // removed only then (RocksDB writes on close) + const auto [dbError, dbErrorFile] = [&]() -> std::pair { auto db = csdb::backend::open(backend, dbPath.string()); if (!db || !db->is_open()) { - return failWith("cannot create the " + backend + " database in " + dbPath.string()); + return {"cannot create the " + backend + " database in " + dbPath.string(), std::string{}}; } fs::create_directories(caches, ec); BlockHashes hashes(caches.string()); @@ -237,11 +254,11 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife }, blocksFile ? blocksFile->rawSize : 0); if (!read || expected != S + 1) { hashes.close(); - return failWith(reason.empty() ? "blocks.bin.lz4 is truncated or malformed" : reason, "blocks.bin.lz4"); + return {reason.empty() ? "blocks.bin.lz4 is truncated or malformed" : reason, "blocks.bin.lz4"}; } if (!batch.empty() && !db->put_batch(batch)) { hashes.close(); - return failWith("cannot write blocks to the database"); + return {"cannot write blocks to the database", std::string{}}; } // 4. contract data @@ -284,14 +301,21 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife }); if (!ok || !good || !pending.empty() || total != f->rawSize) { hashes.close(); - return failWith("contracts.bin.lz4 does not restore", f->name); + return {"contracts.bin.lz4 does not restore", f->name}; } } if (!db->flush()) { hashes.close(); - return failWith("cannot flush the database"); + return {"cannot flush the database", std::string{}}; } hashes.close(); + if (hashes.rebuildRequested()) { + return {"cannot write the block hash databases", std::string{}}; + } + return {}; + }(); + if (!dbError.empty()) { + return failWith(dbError, dbErrorFile); } if (!csdb::backend::writeMarker(dbPath.string(), backend)) { return failWith("cannot write the database marker"); @@ -303,6 +327,7 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife return failWith("cannot move the checkpoint into place: " + ec.message()); } createdQs = true; + syncPath(qsTmp.parent_path(), true); { std::ofstream out(nodeDir / "snapshot.json", std::ios::binary | std::ios::trunc); out << m.toJson(); @@ -311,6 +336,8 @@ SnapshotImportResult importSnapshot(const fs::path& srcDir, const SnapshotManife return failWith("cannot write snapshot.json"); } } + syncPath(nodeDir / "snapshot.json", false); + syncPath(nodeDir, true); result.ok = true; return result; } diff --git a/tests/csnode/snapshot_import_tests.cpp b/tests/csnode/snapshot_import_tests.cpp index b0037211d..7e6d27ddb 100644 --- a/tests/csnode/snapshot_import_tests.cpp +++ b/tests/csnode/snapshot_import_tests.cpp @@ -152,7 +152,7 @@ struct Fixture { const auto t = root / "target"; EXPECT_TRUE(csdb::backend::detect((t / "db").string(), false).empty()); EXPECT_FALSE(fs::exists(t / "qs" / "150")); - EXPECT_FALSE(fs::exists(t / "qs" / "150.import")); + EXPECT_FALSE(fs::exists(t / "qs" / "150.tmp")); EXPECT_FALSE(fs::exists(t / "caches" / "hashdb")); EXPECT_FALSE(fs::exists(t / "caches" / "seqdb")); EXPECT_FALSE(fs::exists(t / "snapshot.json")); @@ -312,3 +312,20 @@ TEST(SnapshotImport, RebuiltBlockWithoutSignaturesOrConfirmationsRejected) { } } +TEST(SnapshotImport, FailedRocksDbImportLeavesNoFiles) { + Fixture fx("snapshot_import_rocksdb_fail"); + auto& bytes = fx.chain.blocks[120]; + const auto pool = csdb::Pool::from_binary(cs::Bytes(bytes)); + bytes[pool.hashingLength() + sizeof(size_t)] ^= 1; + const auto built = fx.build(); + ASSERT_TRUE(built.ok) << built.error; + EXPECT_FALSE(fx.import(built, "rocksdb").ok); + fx.expectNothingLeft(); + // RocksDB writes on close: nothing may appear after the cleanup + std::error_code ec; + for (const auto& e : fs::directory_iterator(fx.root / "target" / "db", ec)) { + EXPECT_NE(e.path().extension(), ".sst") << e.path(); + EXPECT_NE(e.path().filename(), "CURRENT"); + } +} + From 92b6e2765b5bbb81814a7e83b11d6e3a4c6b93c3 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 24 Oct 2026 16:39:44 +0200 Subject: [PATCH 10/10] docs: CHANGES.md, #92 Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 9fb7283bb..5a6ad9bdc 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -38,6 +38,7 @@ lines under the new release. | #88 | client: `[snapshot]` section of `config.ini` for snapshots served by community nodes (no behaviour yet) | | #90 | csnode: `BlockVerifier` for blocks that come from outside the consensus (hash, signatures against the block's confidants, round confirmations), for the snapshot import and backward sync | | #89 | csnode, client: snapshot format 2 built by the node from its own checkpoint and blocks on any backend (lz4, per-chunk SHA-256, manifest signed with the node key); `--export-snapshot ` | +| #92 | csnode, client: snapshot importer and `--import-snapshot`: a format-2 snapshot is checked file by file and block by block (signatures, round confirmations, hash chain to the declared head, checkpoint digest) and written into a new LMDB or RocksDB database; the CI snapshot test uses export and import; digest_offline waits for the node to stop | ## community.11 — 2026-10-24