Skip to content
20 changes: 20 additions & 0 deletions ci/testnet/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,26 @@ while :; do
echo "snapshot test: n$PRUNE_NODE stopped, head $head_digest"
"$REPO/packaging/docker/snapshot-make.sh" "$WORK/n$PRUNE_NODE" "$WORK/snapshot" "$MDB_COPY" local \
| sed 's/^/snapshot test: /'
# format 2: the node builds the snapshot of its latest periodic checkpoint itself; the
# manifest must declare the digest the checkpoint logged when it was saved
rc=0
(cd "$WORK/n$PRUNE_NODE" && exec "$BIN" --db-path db --config-file config.ini --export-snapshot "$WORK/snapshot2") \
> "$WORK/export.log" 2>&1 || rc=$?
grep -a 'export-snapshot:' "$WORK/export.log" | sed 's/^/snapshot test: /'
[ "$rc" = 0 ] || { tail -20 "$WORK/export.log"; echo "FAIL: --export-snapshot exited $rc"; exit 1; }
python3 - "$WORK/snapshot2" "$WORK/n$PRUNE_NODE/node.log" <<'PY' || exit 1
import json, os, re, sys
root, log = sys.argv[1:]
dirs = [d for d in os.listdir(root) if d.isdigit()]
assert len(dirs) == 1, dirs
m = json.load(open(os.path.join(root, dirs[0], "snapshot.json")))
lines = re.findall(r"STATE DIGEST #(\d+) ([0-9A-Fa-f]{64}) wallets (\d+) checkpoint$", open(log, errors="replace").read(), re.M)
saved = {int(s): (d.lower(), int(w)) for s, d, w in lines}
if saved.get(m["seq"]) != (m["state_digest"], m["wallets"]):
print(f"FAIL: format-2 snapshot of #{m['seq']} declares {m['state_digest']}, the checkpoint logged {saved.get(m['seq'])}")
sys.exit(1)
print(f"snapshot test: format-2 snapshot of #{m['seq']} ({m['first_block']}..{m['seq']}), digest as logged at the checkpoint, {len(m['files'])} files")
PY
start_node "$PRUNE_NODE"
if [ "$SNAPSHOT_CORRUPT" = 1 ]; then
f=$(ls "$WORK"/snapshot/credits-pruned-*.tar.zst)
Expand Down
2 changes: 2 additions & 0 deletions client/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ add_executable(client
src/main.cpp
src/dbcheck.cpp
src/dbcheck.hpp
src/snapshot_export.cpp
src/snapshot_export.hpp
"${CMAKE_CURRENT_BINARY_DIR}/version.cpp" include/client/version.hpp
)

Expand Down
4 changes: 4 additions & 0 deletions client/src/dbcheck.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,10 @@ int checkImpl(const std::string& dbPath, const std::string& configured, bool con

} // namespace

bool lockedByAnotherProcess(const std::string& dbPath) {
return lockDatabase(dbPath) == Lock::Busy;
}

int check(const std::string& dbPath, const std::string& configured, bool convertFlag, bool convertOnly, uint32_t batch) {
// No engine exception may escape: print it and stop with nothing changed. The signal handlers
// are installed and restored inside convertNow, around csdb::convert::run only.
Expand Down
3 changes: 3 additions & 0 deletions client/src/dbcheck.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,7 @@ constexpr uint32_t kMaxBatch = 1000000;
// outside 1..kMaxBatch).
int check(const std::string& dbPath, const std::string& configured, bool convertFlag, bool convertOnly, uint32_t batch);

// true when another process (a running node) holds <db>/.lock; this process takes the lock otherwise
bool lockedByAnotherProcess(const std::string& dbPath);

} // namespace dbcheck
6 changes: 6 additions & 0 deletions client/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#include <params.hpp>
#include "dbcheck.hpp"
#include "snapshot_export.hpp"
#include <version.hpp>

#include <sys/types.h>
Expand Down Expand Up @@ -60,6 +61,7 @@ int main(int argc, char* argv[]) {
("convert-db", "convert an existing block database to the configured db_backend without asking")
("convert-only", "convert the block database to the configured db_backend, then exit")
("convert-batch", po::value<uint32_t>()->default_value(100000), "blocks per conversion batch (1 to 1000000)")
("export-snapshot", po::value<std::string>(), "build the snapshot of the latest periodic checkpoint into the given folder, then exit (the node must be stopped)")
("config-file", po::value<std::string>(), "path to configuration file (default: \"config.ini\")")
("public-key-file", po::value<std::string>(), "path to public key file (default: \"NodePublic.txt\")")
("private-key-file", po::value<std::string>(), "path to private key file (default: \"NodePrivate.txt\")")
Expand Down Expand Up @@ -208,6 +210,10 @@ int main(int argc, char* argv[]) {
}
}

if (vm.count("export-snapshot") > 0) {
return snapshot_export::run(config, vm["export-snapshot"].as<std::string>());
}

if (vm.count(cmdline::argDumpKeys) > 0) {
auto fName = vm[cmdline::argDumpKeys].as<std::string>();
if (fName.size() > 0) {
Expand Down
144 changes: 144 additions & 0 deletions client/src/snapshot_export.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
#include "snapshot_export.hpp"
#include "dbcheck.hpp"

#include <algorithm>
#include <cctype>
#include <filesystem>
#include <fstream>
#include <iostream>

#include <config.hpp>
#include <version.hpp>

#include <csdb/database.hpp>
#include <csdb/storage_backend.hpp>
#include <csnode/snapshot_builder.hpp>
#include <csnode/statedigest.hpp>

namespace snapshot_export {

namespace {
namespace fs = std::filesystem;

std::ostream& say() {
return std::cout << "export-snapshot: ";
}

// the highest periodic checkpoint that records its state (qs/0, the shutdown checkpoint, is not one)
cs::Sequence latestCheckpoint() {
cs::Sequence best = 0;
std::error_code ec;
for (const auto& entry : fs::directory_iterator("qs", ec)) {
const auto name = entry.path().filename().string();
if (name.empty() || !std::all_of(name.begin(), name.end(), [](unsigned char c) { return std::isdigit(c) != 0; })) {
continue;
}
const cs::Sequence seq = std::stoull(name);
if (seq > best && fs::exists(entry.path() / cs::CheckpointDigest::kFileName, ec)) {
best = seq;
}
}
return best;
}

std::string nodeVersion() {
std::ifstream f("VERSION");
std::string v;
if (f && std::getline(f, v) && !v.empty() && v.size() <= 64) {
// printed into the manifest's JSON as it is: only plain characters
for (auto& c : v) {
if (c < 0x20 || c >= 0x7f || c == '"' || c == '\\') {
c = '_';
}
}
return v;
}
return std::to_string(Config::getNodeVersion()) + "." + std::to_string(Config::getMinorNodeVersion()) + " " +
std::string(client::Version::GIT_SHA1).substr(0, 8);
}
} // namespace

int run(const Config& config, const std::string& outDir) {
const auto& storage = config.getStorageSettings();
const std::string dbPath = config.getPathToDB();
const auto S = latestCheckpoint();
if (S == 0) {
say() << "no periodic checkpoint with a state digest in qs/ (checkpoints of earlier builds have none)" << std::endl;
return 1;
}
// the export reads the database files directly: never next to a running node (BerkeleyDB would
// even run recovery on its live environment), never into an empty folder or another backend's
if (dbcheck::lockedByAnotherProcess(dbPath)) {
say() << "a node is running on " << dbPath << ": stop it first" << std::endl;
return 1;
}
const auto found = csdb::backend::detect(dbPath, true);
if (std::find(found.begin(), found.end(), storage.dbBackend) == found.end()) {
say() << dbPath << " holds no " << storage.dbBackend << " database (the configured db_backend)" << std::endl;
return 1;
}
// the output must not be a folder of the node's data: the builder replaces numbered folders in it
{
std::error_code e1, e2;
const auto out = fs::weakly_canonical(outDir, e1);
const auto here = fs::weakly_canonical(fs::current_path(), e2);
if (e1 || out == here) {
say() << outDir << " is the node's folder, choose another one" << std::endl;
return 1;
}
for (const fs::path data : {fs::path("qs"), fs::path(dbPath), fs::path("caches")}) {
const auto d = fs::weakly_canonical(data, e2);
if (!e2 && std::mismatch(d.begin(), d.end(), out.begin(), out.end()).first == d.end()) {
say() << outDir << " is inside the node's data (" << data.string() << "), choose another folder" << std::endl;
return 1;
}
}
}
auto db = csdb::backend::open(storage.dbBackend, dbPath);
if (!db || !db->is_open()) {
say() << "cannot open the " << storage.dbBackend << " database in " << dbPath << std::endl;
return 1;
}
cs::Sequence first = 0;
{
auto it = db->new_iterator();
it->seek_to_first();
if (!it->is_valid()) {
say() << "the database holds no block" << std::endl;
return 1;
}
first = it->key();
}

cs::SnapshotBuildInput in;
in.seq = S;
in.checkpointDir = fs::path("qs") / std::to_string(S);
in.firstStored = first;
in.snapshotBlocks = config.getSnapshotSettings().snapshotBlocks;
in.nodeVersion = nodeVersion();
in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) {
return db->get(static_cast<uint32_t>(seq), &out);
};
in.contracts = [&db](const std::function<bool(const cs::Bytes&, const cs::Bytes&)>& visit) {
return db->forEachContractData(visit);
};

say() << "checkpoint " << S << ", blocks from " << std::max<cs::Sequence>(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0)
<< ", into " << outDir << std::endl;
std::error_code ec;
fs::create_directories(outDir, ec);
const auto r = cs::buildSnapshot(in, outDir, config.getMyPrivateKey());
if (!r.ok) {
say() << "failed: " << r.error << std::endl;
return 1;
}
uint64_t bytes = 0;
for (const auto& f : r.manifest.files) {
bytes += f.size;
}
say() << "built " << r.dir.string() << ": block " << r.manifest.seq << ", state digest " << r.manifest.stateDigest
<< ", " << r.manifest.wallets << " wallets, " << r.manifest.files.size() << " files, " << bytes / (1 << 20) << " MB" << std::endl;
return 0;
}

} // namespace snapshot_export
13 changes: 13 additions & 0 deletions client/src/snapshot_export.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#pragma once

#include <string>

class Config;

namespace snapshot_export {

// --export-snapshot <dir>: builds the snapshot of the latest periodic checkpoint (qs/<seq> with a
// state_digest.bin) of this stopped node into <dir>/<seq>, signed with the node's key. 0 on success.
int run(const Config& config, const std::string& outDir);

} // namespace snapshot_export
4 changes: 4 additions & 0 deletions csnode/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ add_library(csnode
include/csnode/configholder.hpp
include/csnode/multiwallets.hpp
include/csnode/statedigest.hpp
include/csnode/snapshot_manifest.hpp
include/csnode/snapshot_builder.hpp
include/csnode/blockverifier.hpp
include/csnode/sendcachedata.hpp
include/csnode/eventreport.hpp
Expand Down Expand Up @@ -110,6 +112,8 @@ add_library(csnode
src/configholder.cpp
src/multiwallets.cpp
src/statedigest.cpp
src/snapshot_manifest.cpp
src/snapshot_builder.cpp
src/blockverifier.cpp
src/sendcachedata.cpp
src/eventreport.cpp
Expand Down
4 changes: 4 additions & 0 deletions csnode/include/csnode/caches_serialization_manager.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#pragma once

#include <filesystem>
#include <functional>
#include <memory>
#include <optional>
Expand Down Expand Up @@ -35,6 +36,9 @@ struct CheckpointHead {

// True when the checkpoint head is the block the chain holds at that sequence, or when this
// cannot be checked (checkpoint without head, or hash not known locally).
// head.bin of a checkpoint directory; nullopt when missing or malformed
std::optional<CheckpointHead> readCheckpointHead(const std::filesystem::path& dir);

bool isCheckpointOnChain(const CheckpointHead& head, const std::function<cs::Bytes(cs::Sequence)>& chainHashAt);

class CachesSerializationManager {
Expand Down
57 changes: 57 additions & 0 deletions csnode/include/csnode/snapshot_builder.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#ifndef SNAPSHOT_BUILDER_HPP
#define SNAPSHOT_BUILDER_HPP

#include <atomic>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <string>

#include <csnode/snapshot_manifest.hpp>
#include <lib/system/common.hpp>

namespace cs {

// Compressed files of a snapshot: independent lz4 (HC) blocks of at most 4 MiB of raw data, each
// "u32 raw_len, u32 compressed_len, bytes" (little-endian), so that a reader needs no frame library
// and memory stays bounded.
namespace snapshot_lz4 {
constexpr size_t kBlockSize = 4u << 20;
// calls sink with each decompressed block; false for a malformed file or when sink returns false
bool decompressFile(const std::filesystem::path& file, const std::function<bool(const uint8_t*, size_t)>& sink);
} // namespace snapshot_lz4

// What a snapshot of checkpoint seq is built from: the checkpoint directory (qs/<seq>) and the
// node's storage, read through these functions (raw stored block bytes by sequence, contract data).
struct SnapshotBuildInput {
cs::Sequence seq = 0;
std::filesystem::path checkpointDir;
cs::Sequence firstStored = 0;
size_t snapshotBlocks = 101'000;
std::function<bool(cs::Sequence, cs::Bytes&)> blockBytes;
std::function<bool(const std::function<bool(const cs::Bytes&, const cs::Bytes&)>&)> contracts;
std::string nodeVersion;
uint64_t chunkSize = 4ull << 20;
const std::atomic<bool>* stop = nullptr; // checked between blocks: true aborts the build
};

struct SnapshotBuildResult {
bool ok = false;
std::string error;
SnapshotManifest manifest;
std::filesystem::path dir; // <outRoot>/<seq>
};

// Builds <outRoot>/<seq>/ (snapshot.json and its files) through <outRoot>/<seq>.tmp, then removes
// the other built snapshots in outRoot. Nothing is left behind on failure. The blocks must form a
// hash chain ending at the checkpoint's head, and the checkpoint must have state_digest.bin.
SnapshotBuildResult buildSnapshot(const SnapshotBuildInput& input, const std::filesystem::path& outRoot,
const cs::PrivateKey& key);

// the blocks.bin record stream: "u32 seq, u32 len, bytes" (little-endian)
bool forEachSnapshotBlock(const std::filesystem::path& blocksFile,
const std::function<bool(cs::Sequence, cs::Bytes&&)>& visit);

} // namespace cs

#endif // SNAPSHOT_BUILDER_HPP
Loading
Loading