Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,9 @@ jobs:
- name: snapshot-join-rocksdb-full
env: NODE_STORAGE=berkeleydb,lmdb,rocksdb,berkeleydb:pruned,lmdb:pruned,rocksdb:pruned,berkeleydb,rocksdb PRUNE_NODE=6 JOIN_FULL=1 SNAPSHOT_AT=2300 TARGET_SEQ=3300
fixed: true
# a pruned node serves the snapshots it builds over HTTP (snapshot serving, step 4)
- name: snapshot-serve
env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_SERVE_NODE=6 TARGET_SEQ=2600
- name: snapshot-join-format1
env: DB_BACKEND=lmdb PRUNE_NODE=6 SNAPSHOT_AT=2300 SNAPSHOT_FORMAT=1 TARGET_SEQ=3100
# node 5 from full to pruned, node 6 from pruned to full, after node 6 pruned once
Expand Down
55 changes: 55 additions & 0 deletions ci/testnet/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@
# snapshot with --import-snapshot into its own backend and joins; with SNAPSHOT_CORRUPT=1 the
# blocks file is truncated first, so the restore must fail and that node syncs from block 0.
# SNAPSHOT_FORMAT=1: packaging/docker/snapshot-make.sh and snapshot-restore.sh instead
# SNAPSHOT_SERVE_NODE (default 0 = off): that node gets [snapshot] serve_port (BASE_PORT + 700 + i):
# at the end it must have built the snapshot of a periodic checkpoint and serve it over HTTP
# STORAGE_SWITCH_AT (default 0 = off, needs PRUNE_NODE=6 and DB_BACKEND=lmdb): at that sequence
# node 5 is restarted as a pruned node and node 6 as a full one
# CONVERT_AT (default 0 = off, needs DB_BACKEND=berkeleydb): at that sequence n4 is converted to
Expand Down Expand Up @@ -198,6 +200,7 @@ JOIN_PRUNED_AT=${JOIN_PRUNED_AT:-0}
# the joining node is node NODES + 1; it is in nobody's hosts or trusted list
SNAPSHOT_AT=${SNAPSHOT_AT:-0}
SNAPSHOT_FORMAT=${SNAPSHOT_FORMAT:-2}
SNAPSHOT_SERVE_NODE=${SNAPSHOT_SERVE_NODE:-0}
SNAPSHOT_CORRUPT=${SNAPSHOT_CORRUPT:-0}
STORAGE_SWITCH_AT=${STORAGE_SWITCH_AT:-0}
CONVERT_AT=${CONVERT_AT:-0}
Expand Down Expand Up @@ -330,6 +333,15 @@ PROPS
chmod +x "$WORK/n$i/executor.sh"
}

# the [snapshot] section of node i: SNAPSHOT_SERVE_NODE serves the snapshots it builds over HTTP
snapshot_section() {
local i=$1
if [ "$i" -eq "$SNAPSHOT_SERVE_NODE" ]; then
printf "\n[snapshot]\nserve_port=%s\nserve_bind=127.0.0.1\n" "$((BASE_PORT + 700 + i))"
fi
return 0
}

# the [storage] section of node i: pruning, backend and checkpoint frequency
storage_section() {
local i=$1 backend="" lines=""
Expand Down Expand Up @@ -384,6 +396,7 @@ $(api_settings "$i")
ajax_port=0
diag_port=0
$(storage_section "$i")
$(snapshot_section "$i")

[Core]
Filter="%Severity% >= info"
Expand Down Expand Up @@ -1155,6 +1168,48 @@ for i in $(seq 1 "$ALL_NODES"); do
n=$(cat "$WORK/n$i"/*.log | grep -ac "but its caches already hold block" || true)
if [ "${n:-0}" -gt 0 ]; then echo "n$i loaded $n checkpoint(s) one block behind their caches"; fi
done
if [ "$SNAPSHOT_SERVE_NODE" -gt 0 ]; then
# the node built the snapshot of a periodic checkpoint and serves it: the manifest verifies, and a
# piece of the blocks file has the hash the manifest declares for it
serve_port=$((BASE_PORT + 700 + SNAPSHOT_SERVE_NODE))
deadline=$(( $(date +%s) + 300 ))
until grep -aq 'SNAPSHOT: built block' "$WORK/n$SNAPSHOT_SERVE_NODE/node.log" || [ "$(date +%s)" -ge "$deadline" ]; do sleep 5; done
grep -a 'SNAPSHOT: ' "$WORK/n$SNAPSHOT_SERVE_NODE/node.log" | tail -4 | sed 's/^.*SNAPSHOT/serve test: SNAPSHOT/'
python3 - "$serve_port" <<'PY' || exit 1
import hashlib, json, sys, time, urllib.request
port = int(sys.argv[1])
base = f"http://127.0.0.1:{port}/snapshot"
for attempt in range(30):
try:
with urllib.request.urlopen(base + "/latest.json", timeout=10) as r:
manifest = json.load(r)
break
except Exception as e: # 503 while the node catches up or builds
last = e
time.sleep(10)
else:
print(f"FAIL: no manifest from the serving node: {last}")
sys.exit(1)
f = next(x for x in manifest["files"] if x["name"] == "blocks.bin.lz4")
n = min(manifest["chunk_size"], f["size"])
req = urllib.request.Request(f"{base}/{manifest['seq']}/blocks.bin.lz4", headers={"Range": f"bytes=0-{n - 1}"})
with urllib.request.urlopen(req, timeout=30) as r:
piece = r.read()
status = r.status
if status != 206 or hashlib.sha256(piece).hexdigest() != f["chunks"][0]:
print(f"FAIL: the first piece of blocks.bin.lz4 does not match the manifest (status {status}, {len(piece)} bytes)")
sys.exit(1)
try:
urllib.request.urlopen(f"{base}/{manifest['seq']}/../../config.ini", timeout=10)
print("FAIL: a path outside the snapshot was served")
sys.exit(1)
except urllib.error.HTTPError as e:
if e.code != 404:
print(f"FAIL: unexpected status {e.code} for a path outside the snapshot")
sys.exit(1)
print(f"serve test: block {manifest['seq']} served by node at {manifest.get('synced_to')}, {len(manifest['files'])} files, first piece verified")
PY
fi
if [ "$SNAPSHOT_AT" -gt 0 ]; then
if [ "$SNAPSHOT_CORRUPT" = 1 ]; then
grep -aq 'SLOW START' "$WORK/n$JOIN_NODE"/node.log || { echo "FAIL: n$JOIN_NODE did not sync from block 0"; exit 1; }
Expand Down
5 changes: 2 additions & 3 deletions client/src/snapshot_export.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,8 @@ int run(const Config& config, const std::string& outDir) {
in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) {
return db->get(static_cast<uint32_t>(seq), &out);
};
in.contracts = [&db](const std::function<bool(const cs::Bytes&, const cs::Bytes&)>& visit) {
return db->forEachContractData(visit);
};
// no contract records: the contracts database holds each contract's latest state, later than the
// checkpoint; the checkpoint carries the states at its block

say() << "checkpoint " << S << ", blocks from " << std::max<cs::Sequence>(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0)
<< ", into " << outDir << std::endl;
Expand Down
11 changes: 11 additions & 0 deletions csdb/include/csdb/storage.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,17 @@ class Storage final {
*/
cs::Sequence first_sequence() const;

/**
* Every contract record, in the engine's key order; fn returns false to stop early. For a
* snapshot of the node's state.
*/
bool forEachContractData(const std::function<bool(const cs::Bytes& key, const cs::Bytes& value)>& fn) const;

/**
* The stored bytes of block sequence, as the database holds them (no parsing, no pool cache).
*/
bool pool_bytes(cs::Sequence sequence, cs::Bytes& out) const;

/**
* Returns the space of pruned pools to the file system.
*/
Expand Down
16 changes: 16 additions & 0 deletions csdb/src/storage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1350,6 +1350,22 @@ bool Storage::pool_prune_batch(const std::vector<PoolHash>& hashes) {
return true;
}

bool Storage::pool_bytes(cs::Sequence sequence, cs::Bytes& out) const {
if (!isOpen()) {
d->set_last_error(NotOpen);
return false;
}
return d->db->get(static_cast<uint32_t>(sequence), &out);
}

bool Storage::forEachContractData(const std::function<bool(const cs::Bytes& key, const cs::Bytes& value)>& fn) const {
if (!isOpen()) {
d->set_last_error(NotOpen);
return false;
}
return d->db->forEachContractData(fn);
}

cs::Sequence Storage::first_sequence() const {
if (!isOpen()) {
d->set_last_error(NotOpen);
Expand Down
4 changes: 4 additions & 0 deletions csnode/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ add_library(csnode
include/csnode/snapshot_manifest.hpp
include/csnode/snapshot_builder.hpp
include/csnode/snapshot_importer.hpp
include/csnode/snapshot_server.hpp
include/csnode/snapshot_service.hpp
include/csnode/blockverifier.hpp
include/csnode/sendcachedata.hpp
include/csnode/eventreport.hpp
Expand Down Expand Up @@ -116,6 +118,8 @@ add_library(csnode
src/snapshot_manifest.cpp
src/snapshot_builder.cpp
src/snapshot_importer.cpp
src/snapshot_server.cpp
src/snapshot_service.cpp
src/blockverifier.cpp
src/sendcachedata.cpp
src/eventreport.cpp
Expand Down
8 changes: 8 additions & 0 deletions csnode/include/csnode/blockchain.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,12 @@ class BlockChain {
// pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were
// removed once a later periodic checkpoint made them unnecessary
cs::Sequence getFirstStoredSequence() const;
// the stored bytes of a block, under the storage lock, for a snapshot: no parsing, no pool cache
bool blockBytes(cs::Sequence sequence, cs::Bytes& out) const;
// pruned storage: the blocks below this one are being or will be removed (0: nothing scheduled)
cs::Sequence pruneTarget() const;
// every contract record of the storage, for a snapshot of the node's state
bool forEachContractData(const std::function<bool(const cs::Bytes& key, const cs::Bytes& value)>& fn) const;

// return unique id of database if at least one unique block has written, otherwise (only genesis block) 0
uint64_t uuid() const;
Expand Down Expand Up @@ -303,6 +309,8 @@ public signals:

cs::StopReadingBlocksSignal stopReadingBlocksEvent;
cs::SuccessQSSignal successfullQuickStartEvent;
// a periodic checkpoint of this block was saved (live), with its state digest
cs::Signal<void(cs::Sequence)> checkpointSavedEvent;

public slots:

Expand Down
5 changes: 5 additions & 0 deletions csnode/include/csnode/node.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@

class Transport;

namespace cs {
class SnapshotService;
}

namespace cs {
class SolverCore;
}
Expand Down Expand Up @@ -485,6 +489,7 @@ public slots:

#ifdef NODE_API
std::unique_ptr<csconnector::connector> api_;
std::unique_ptr<cs::SnapshotService> snapshotService_;
#endif

uint32_t startPacketRequestPoint_ = 0;
Expand Down
68 changes: 68 additions & 0 deletions csnode/include/csnode/snapshot_server.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#ifndef SNAPSHOT_SERVER_HPP
#define SNAPSHOT_SERVER_HPP

#include <atomic>
#include <condition_variable>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <thread>

#include <csnode/snapshot_manifest.hpp>
#include <lib/system/common.hpp>

namespace cs {

// The read-only HTTP service of the snapshots this node built (docs/specs/2026-10-24-snapshot-serving-design.md,
// section 5): GET and HEAD of /snapshot/latest.json and /snapshot/<seq>/<name>, one Range of at most
// 16 MiB, at most maxConnections connections, a bandwidth cap shared by all of them. It answers 503
// while the node is not synced with the network or no snapshot is built (only synced nodes serve).
// Files are looked up by the names of the manifest it loaded, never by a path from the request.
class SnapshotServer {
public:
struct Settings {
std::string bind; // empty: every interface
uint16_t port = 0; // 0: stopped
size_t maxConnections = 8;
size_t rateKBps = 0; // kilobytes a second, all connections together; 0: unlimited
bool operator==(const Settings& o) const {
return bind == o.bind && port == o.port && maxConnections == o.maxConnections && rateKBps == o.rateKBps;
}
};
// the node's last block and the current round: synced when the block is within 3 rounds
using SyncedTo = std::function<std::pair<cs::Sequence, cs::Sequence>()>;

static constexpr uint64_t kMaxRange = 16ull << 20;
static constexpr size_t kMaxRequestsPerConnection = 64;
static constexpr int kIdleTimeoutSec = 15;
static constexpr int kWriteTimeoutSec = 30;
static constexpr cs::Sequence kSyncedRounds = 3;

SnapshotServer(std::filesystem::path root, SyncedTo syncedTo);
~SnapshotServer();

// starts, stops (port 0), moves to another address or applies new limits, while it runs
void apply(const Settings& settings);
// the newest complete snapshot under root (one holding a snapshot.json that verifies)
void reload();
void stop();

uint16_t port() const {
return boundPort_.load();
}
bool running() const {
return boundPort_.load() != 0;
}

private:
struct Impl;
std::shared_ptr<Impl> impl_;
std::atomic<uint16_t> boundPort_{0};
};

} // namespace cs

#endif // SNAPSHOT_SERVER_HPP
55 changes: 55 additions & 0 deletions csnode/include/csnode/snapshot_service.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#ifndef SNAPSHOT_SERVICE_HPP
#define SNAPSHOT_SERVICE_HPP

#include <atomic>
#include <chrono>
#include <condition_variable>
#include <filesystem>
#include <memory>
#include <mutex>
#include <string>
#include <thread>

#include <csnode/snapshot_server.hpp>
#include <lib/system/common.hpp>

class BlockChain;

namespace cs {

// Snapshots served by this node, in the node: every 10 s it reads [snapshot] from the configuration
// and starts, stops or moves the HTTP service (serve_port 0: off); while it serves, it builds the
// snapshot of each new periodic checkpoint into snapshot/ at low priority (and of the latest one
// when it starts serving), and serves it once complete. Only a node synced with the network serves.
class SnapshotService {
public:
SnapshotService(BlockChain& blockchain, const cs::PrivateKey& key, std::string nodeVersion);
~SnapshotService();

void start();
void stop();
// slot of BlockChain::checkpointSavedEvent: a new periodic checkpoint to build from
void onCheckpointSaved(cs::Sequence sequence);

private:
void run();
void buildLatest();

BlockChain& blockchain_;
cs::PrivateKey key_;
std::string nodeVersion_;
std::filesystem::path root_ = "snapshot";
std::unique_ptr<SnapshotServer> server_;
std::thread thread_;
std::atomic<bool> stop_{false};
std::atomic<bool> buildPending_{true};
static constexpr int kMaxBuildAttempts = 5;
int failedBuilds_ = 0;
std::chrono::steady_clock::time_point retryAt_{};
std::mutex m_;
std::condition_variable cv_;
};

} // namespace cs

#endif // SNAPSHOT_SERVICE_HPP
15 changes: 15 additions & 0 deletions csnode/src/blockchain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1053,6 +1053,7 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) {
if (serializationManPtr_->save(pool.sequence(), head, digest)) {
cslog() << cs::stateDigestLine(pool.sequence(), digest.digest, digest.wallets) << " checkpoint";
serializationManPtr_->pruneCheckpoints(sto.checkpointKeep);
emit checkpointSavedEvent(pool.sequence());
lastCheckpointWallClock_ = now;
if (byCount) {
schedulePruning(pool.sequence());
Expand All @@ -1064,6 +1065,20 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) {
}
}

bool BlockChain::blockBytes(cs::Sequence sequence, cs::Bytes& out) const {
std::lock_guard lock(dbLock_);
return storage_.pool_bytes(sequence, out);
}

cs::Sequence BlockChain::pruneTarget() const {
std::lock_guard lock(dbLock_);
return pruneUpTo_;
}

bool BlockChain::forEachContractData(const std::function<bool(const cs::Bytes& key, const cs::Bytes& value)>& fn) const {
return storage_.forEachContractData(fn);
}

cs::CheckpointDigest BlockChain::checkpointDigest() const {
cs::CheckpointDigest result;
const auto& wallets = multiWallets();
Expand Down
Loading
Loading