Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,14 @@ jobs:
# slow nodes: 5% of one CPU for two trusted nodes
- name: slow-nodes
env: SLOW_NODES=3,6 SLOW_CPU=5
# snapshot sharing announced in host_data: n3 and n5 serve, a stopped node lists them
# snapshot sharing: n3 (full) and n5 serve, build their snapshots and announce them in
# host_data; a stopped node lists them
- name: snapshot-capability
env: SNAPSHOT_SERVE_NODES=3,5
# a pruned LMDB node and a full RocksDB one serve, the pruned one while it prunes (odd nodes:
# the even ones run node_alt_ref in a mixed run)
- name: snapshot-p2p-serve
env: NODE_STORAGE=berkeleydb,berkeleydb,berkeleydb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb SNAPSHOT_SERVE_NODES=5,7 TARGET_SEQ=2600
# random datagrams to every node's p2p port
- name: garbage-datagrams
env: GARBAGE_AT=150 GARBAGE_RATE=300
Expand Down
26 changes: 23 additions & 3 deletions ci/testnet/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,10 @@
# snapshot with --import-snapshot into its own backend and joins; with SNAPSHOT_CORRUPT=1 the
# blocks file is truncated first, so the restore must fail and that node syncs from block 0.
# SNAPSHOT_FORMAT=1: packaging/docker/snapshot-make.sh and snapshot-restore.sh instead
# SNAPSHOT_SERVE_NODES (optional, comma separated): those nodes run with [snapshot] serve=on;
# at the end one other node of this build is stopped and its known_peers.json must list each
# of them with the snapshot capability, and no other node with it
# SNAPSHOT_SERVE_NODES (optional, comma separated): those nodes run with [snapshot] serve=on
# and a checkpoint every 1'000 blocks; each must build the snapshot of a checkpoint and serve
# it; at the end one other node of this build is stopped and its known_peers.json must list
# each of them with the snapshot capability, and no other node with it
# STORAGE_SWITCH_AT (default 0 = off, needs PRUNE_NODE=6 and DB_BACKEND=lmdb): at that sequence
# node 5 is restarted as a pruned node and node 6 as a full one
# CONVERT_AT (default 0 = off, needs DB_BACKEND=berkeleydb): at that sequence n4 is converted to
Expand Down Expand Up @@ -203,6 +204,12 @@ SNAPSHOT_AT=${SNAPSHOT_AT:-0}
SNAPSHOT_FORMAT=${SNAPSHOT_FORMAT:-2}
SNAPSHOT_CORRUPT=${SNAPSHOT_CORRUPT:-0}
SNAPSHOT_SERVE_NODES=${SNAPSHOT_SERVE_NODES:-}
if [ -n "$BIN_ALT" ]; then
for i in ${SNAPSHOT_SERVE_NODES//,/ }; do
# the even nodes run BIN_ALT, which may not serve snapshots
[ $((i % 2)) -eq 1 ] || { echo "FAIL: SNAPSHOT_SERVE_NODES names n$i, an even node, which runs BIN_ALT"; exit 1; }
done
fi
STORAGE_SWITCH_AT=${STORAGE_SWITCH_AT:-0}
CONVERT_AT=${CONVERT_AT:-0}
INDEX_LOSS_AT=${INDEX_LOSS_AT:-0}
Expand Down Expand Up @@ -355,6 +362,10 @@ storage_section() {
# that the kills hit the moments right after one
lines="checkpoint_every=1000\ncheckpoint_every_minutes=1\ncheckpoint_keep=3\n"
fi
if [[ ",$SNAPSHOT_SERVE_NODES," == *",$i,"* ]] && [[ "$lines" != *checkpoint_every=* ]]; then
# a serving node builds its snapshots from the periodic checkpoints
lines="${lines}checkpoint_every=1000\ncheckpoint_keep=2\n"
fi
[ -n "$backend" ] && lines="${lines}db_backend=$backend\n"
[ -n "$lines" ] && printf "\n[storage]\n$lines"
return 0
Expand Down Expand Up @@ -1316,6 +1327,15 @@ if [ "$LOW_STAKE_NODES" -gt 0 ]; then
echo "stake: none of the $LOW_STAKE_NODES nodes below the minimum stake was trusted after round $from"
fi
if [ -n "$SNAPSHOT_SERVE_NODES" ]; then
# every serving node built the snapshot of a checkpoint and serves it (the build follows the
# checkpoint at 1'000 on a thread of low priority: a minute at most)
for i in ${SNAPSHOT_SERVE_NODES//,/ }; do
for _ in $(seq 1 60); do grep -aq 'SNAPSHOT: serving block' "$WORK/n$i/node.log" && break; sleep 1; done
built=$({ grep -a 'SNAPSHOT: built block' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E 's/.*(built block [0-9]+[^,]*, [^,]*, [^,]*).*/\1/')
serving=$({ grep -a 'SNAPSHOT: serving block' "$WORK/n$i/node.log" || true; } | tail -1 | sed -E 's/.*SNAPSHOT: //')
[ -n "$built" ] && [ -n "$serving" ] || { echo "FAIL: n$i did not build and serve a snapshot"; { grep -a 'SNAPSHOT' "$WORK/n$i/node.log" || true; } | tail -5; exit 1; }
echo "snapshot: n$i $built; $serving"
done
# known_peers.json is written at a clean stop: stop one node of this build that does not serve
observer=0
for i in $(seq 1 "$NODES"); do
Expand Down
2 changes: 1 addition & 1 deletion client/config/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ struct StorageData {
// observer: every key but fetch* applies while the node runs.
struct SnapshotData {
bool serve = false; // build snapshots and answer snapshot requests
size_t serveRateKbps = 2000; // upload cap of all peers together; 0: unlimited
size_t serveRateKbps = 2000; // upload cap of all peers together, KB/s (2000: 2 MB/s); 0: unlimited
size_t serveMaxPeers = 8; // peers served within 30 s; more get busy
size_t snapshotBlocks = 101'000; // blocks before the checkpoint in a snapshot
std::string fetch = "auto"; // auto: fetch at a start on an empty database; off: never
Expand Down
5 changes: 2 additions & 3 deletions client/src/snapshot_export.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,8 @@ int run(const Config& config, const std::string& outDir) {
in.blockBytes = [&db](cs::Sequence seq, cs::Bytes& out) {
return db->get(static_cast<uint32_t>(seq), &out);
};
in.contracts = [&db](const std::function<bool(const cs::Bytes&, const cs::Bytes&)>& visit) {
return db->forEachContractData(visit);
};
// no contract records: the contracts database holds each contract's latest state, later than the
// checkpoint; the checkpoint carries the states at its block

say() << "checkpoint " << S << ", blocks from " << std::max<cs::Sequence>(first, S > in.snapshotBlocks ? S - in.snapshotBlocks : 0)
<< ", into " << outDir << std::endl;
Expand Down
5 changes: 5 additions & 0 deletions csdb/include/csdb/storage.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,11 @@ class Storage final {
*/
cs::Sequence first_sequence() const;

/**
* The stored bytes of block sequence, as the database holds them (no parsing, no pool cache).
*/
bool pool_bytes(cs::Sequence sequence, cs::Bytes& out) const;

/**
* Returns the space of pruned pools to the file system.
*/
Expand Down
8 changes: 8 additions & 0 deletions csdb/src/storage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1350,6 +1350,14 @@ bool Storage::pool_prune_batch(const std::vector<PoolHash>& hashes) {
return true;
}

bool Storage::pool_bytes(cs::Sequence sequence, cs::Bytes& out) const {
if (!isOpen()) {
d->set_last_error(NotOpen);
return false;
}
return d->db->get(static_cast<uint32_t>(sequence), &out);
}

cs::Sequence Storage::first_sequence() const {
if (!isOpen()) {
d->set_last_error(NotOpen);
Expand Down
4 changes: 4 additions & 0 deletions csnode/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ add_library(csnode
include/csnode/snapshot_builder.hpp
include/csnode/snapshot_importer.hpp
include/csnode/snapshot_messages.hpp
include/csnode/snapshot_provider.hpp
include/csnode/snapshot_service.hpp
include/csnode/blockverifier.hpp
include/csnode/sendcachedata.hpp
include/csnode/eventreport.hpp
Expand Down Expand Up @@ -118,6 +120,8 @@ add_library(csnode
src/snapshot_builder.cpp
src/snapshot_importer.cpp
src/snapshot_messages.cpp
src/snapshot_provider.cpp
src/snapshot_service.cpp
src/blockverifier.cpp
src/sendcachedata.cpp
src/eventreport.cpp
Expand Down
6 changes: 6 additions & 0 deletions csnode/include/csnode/blockchain.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ class BlockChain {
// pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were
// removed once a later periodic checkpoint made them unnecessary
cs::Sequence getFirstStoredSequence() const;
// the stored bytes of a block, under the storage lock, for a snapshot: no parsing, no pool cache
bool blockBytes(cs::Sequence sequence, cs::Bytes& out) const;
// pruned storage: the blocks below this one are being or will be removed (0: nothing scheduled)
cs::Sequence pruneTarget() const;

// return unique id of database if at least one unique block has written, otherwise (only genesis block) 0
uint64_t uuid() const;
Expand Down Expand Up @@ -303,6 +307,8 @@ public signals:

cs::StopReadingBlocksSignal stopReadingBlocksEvent;
cs::SuccessQSSignal successfullQuickStartEvent;
// a periodic checkpoint of this block was saved (live), with its state digest
cs::Signal<void(cs::Sequence)> checkpointSavedEvent;

public slots:

Expand Down
15 changes: 15 additions & 0 deletions csnode/include/csnode/node.hpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#ifndef NODE_HPP
#define NODE_HPP

#include <chrono>
#include <iostream>
#include <memory>
#include <string>
Expand All @@ -26,6 +27,12 @@

class Transport;

namespace cs {
class SnapshotProvider;
class SnapshotService;
struct SnapshotNodeState;
}

namespace cs {
class SolverCore;
}
Expand Down Expand Up @@ -425,6 +432,9 @@ public slots:
private:
// snapshot messages dropped: malformed, or not expected by this node
uint64_t snapshotMessagesDropped_ = 0;
// the provider's view of this node: last block, round, synced, trusted
void snapshotNodeState(cs::SnapshotNodeState& state) const;
void logSnapshotSummary();

bool init();
void initPoolSynchronizer();
Expand Down Expand Up @@ -497,6 +507,11 @@ public slots:
std::unique_ptr<csconnector::connector> api_;
#endif

// snapshots served to p2p neighbours ([snapshot] serve)
std::unique_ptr<cs::SnapshotProvider> snapshotProvider_;
std::unique_ptr<cs::SnapshotService> snapshotService_;
std::chrono::steady_clock::time_point snapshotSummaryAt_{};

uint32_t startPacketRequestPoint_ = 0;

// ms timeout
Expand Down
2 changes: 1 addition & 1 deletion csnode/include/csnode/snapshot_builder.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ struct SnapshotBuildInput {
std::function<bool(cs::Sequence, cs::Bytes&)> blockBytes;
std::function<bool(const std::function<bool(const cs::Bytes&, const cs::Bytes&)>&)> contracts;
std::string nodeVersion;
uint64_t chunkSize = 4ull << 20;
uint64_t chunkSize = 512ull << 10; // one p2p piece (design 4.3)
const std::atomic<bool>* stop = nullptr; // checked between blocks: true aborts the build
};

Expand Down
125 changes: 125 additions & 0 deletions csnode/include/csnode/snapshot_provider.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
#ifndef SNAPSHOT_PROVIDER_HPP
#define SNAPSHOT_PROVIDER_HPP

#include <chrono>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <vector>

#include <csnode/snapshot_manifest.hpp>
#include <csnode/snapshot_messages.hpp>
#include <lib/system/common.hpp>

namespace cs {

// what the node is doing when a snapshot request arrives
struct SnapshotNodeState {
cs::Sequence lastSeq = 0;
cs::RoundNumber round = 0;
bool synced = false; // design section 5.2
bool trusted = false; // a confidant or the writer of the current round: half the budget
};

// Answers the snapshot requests of p2p neighbours from the newest snapshot this node built
// (docs/specs/2026-10-25-snapshot-p2p-design.md, section 5). Used on the node's processor thread;
// load() and unload() come from the snapshot service's thread. It keeps no state per requester
// beyond a table of the peers served in the last 30 s (at most 64), and every reply is bounded
// by a global token bucket and a per-peer one.
class SnapshotProvider {
public:
using Clock = std::function<std::chrono::steady_clock::time_point()>;

struct Settings {
bool serve = false;
size_t rateKBps = 2000; // all peers together, KB/s; 0: unlimited
size_t maxPeers = 8; // peers served within 30 s
};

using NodeState = SnapshotNodeState;

static constexpr size_t kMaxPeers = 64;
static constexpr auto kPeerWindow = std::chrono::seconds(30);
static constexpr auto kPieceWindow = std::chrono::milliseconds(100);
static constexpr size_t kPiecesPerWindow = 2;
static constexpr auto kManifestInterval = std::chrono::seconds(5);
// manifest replies to keys that are not downloading: peer keys cost nothing, so the per-key
// limit alone does not bound them; a peer downloading now is not held up by them
static constexpr double kManifestsPerSecond = 20;
static constexpr double kManifestBurst = 8;
static constexpr uint64_t kBurstBytes = 4ull << 20;
static constexpr uint32_t kMaxRetryMs = 5000;

explicit SnapshotProvider(Clock clock = std::chrono::steady_clock::now);
~SnapshotProvider();

// dir = snapshot/<S>: its snapshot.json must parse, verify, name S, and its files must have the
// declared sizes; on failure the previous snapshot stays loaded
bool load(const std::filesystem::path& dir, std::string* error = nullptr);
void unload();
std::optional<cs::Sequence> loadedSeq() const;

// nullopt: no answer (the peer asked again too soon, or the budget is spent; it asks again)
std::optional<snapshot_p2p::ManifestReply> onManifestRequest(const cs::PublicKey& sender, const Settings& settings,
const NodeState& state);
snapshot_p2p::PieceReply onPieceRequest(const cs::PublicKey& sender, const snapshot_p2p::PieceRequest& request,
const Settings& settings, const NodeState& state);

// a line about the traffic since the last call, empty when there was none
std::string takeSummary();
// a peer that just received the last byte of a whole snapshot: once per peer and snapshot
std::function<void(const cs::PublicKey&, cs::Sequence, uint64_t bytes, std::chrono::seconds took)> onPeerCompleted;

private:
struct Loaded;

struct Bucket {
double tokens = 0;
std::chrono::steady_clock::time_point at{};
// refills at rate bytes/s up to burst; true and the cost taken when there is enough
bool take(double cost, double rate, double burst, std::chrono::steady_clock::time_point now, uint32_t& retryMs);
};

struct Peer {
std::chrono::steady_clock::time_point lastSeen{};
std::chrono::steady_clock::time_point lastPiece{};
std::chrono::steady_clock::time_point firstPiece{};
std::chrono::steady_clock::time_point windowStart{};
size_t piecesInWindow = 0;
std::chrono::steady_clock::time_point lastManifest{};
bool manifestSent = false;
Bucket bucket;
bool bucketStarted = false;
cs::Sequence seq = 0;
uint64_t bytes = 0; // of seq
std::vector<bool> chunksSeen; // of seq, over all files in manifest order: re-requests do not count
bool completed = false;
};

Peer& peer(const cs::PublicKey& key, std::chrono::steady_clock::time_point now);
size_t activePeers(std::chrono::steady_clock::time_point now, const cs::PublicKey& except) const;

Clock clock_;
mutable std::mutex loadedLock_;
std::shared_ptr<const Loaded> loaded_;

// processor thread only
std::map<cs::PublicKey, Peer> peers_;
Bucket global_;
bool globalStarted_ = false;
Bucket manifests_;
bool manifestsStarted_ = false;
uint64_t piecesServed_ = 0;
uint64_t bytesServed_ = 0;
uint64_t busyAnswers_ = 0;
uint64_t manifestsServed_ = 0;
};

} // namespace cs

#endif // SNAPSHOT_PROVIDER_HPP
59 changes: 59 additions & 0 deletions csnode/include/csnode/snapshot_service.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#ifndef SNAPSHOT_SERVICE_HPP
#define SNAPSHOT_SERVICE_HPP

#include <atomic>
#include <chrono>
#include <condition_variable>
#include <filesystem>
#include <memory>
#include <mutex>
#include <string>
#include <thread>

#include <csnode/snapshot_provider.hpp>
#include <lib/system/common.hpp>

class BlockChain;

namespace cs {

// Snapshots served by this node to its p2p neighbours: every 10 s it reads [snapshot] from the
// configuration; while serve is on, it builds the snapshot of each new periodic checkpoint into
// snapshot/ at low priority (and of the latest one when serving starts) and hands the newest complete
// one to the provider, which answers the requests; serve off unloads it. Whether the node is synced
// is the provider's question, asked per request.
class SnapshotService {
public:
SnapshotService(BlockChain& blockchain, SnapshotProvider& provider, const cs::PrivateKey& key, std::string nodeVersion);
~SnapshotService();

void start();
void stop();
// slot of BlockChain::checkpointSavedEvent: a new periodic checkpoint to build from
void onCheckpointSaved(cs::Sequence sequence);

private:
void run();
void buildLatest();
// the newest snapshot/<S> with a snapshot.json, into the provider
void loadNewest();

BlockChain& blockchain_;
cs::PrivateKey key_;
std::string nodeVersion_;
std::filesystem::path root_ = "snapshot";
SnapshotProvider& provider_;
bool serving_ = false;
std::thread thread_;
std::atomic<bool> stop_{false};
std::atomic<bool> buildPending_{true};
static constexpr int kMaxBuildAttempts = 5;
std::atomic<int> failedBuilds_{0}; // reset by onCheckpointSaved on the thread that stores blocks
std::chrono::steady_clock::time_point retryAt_{};
std::mutex m_;
std::condition_variable cv_;
};

} // namespace cs

#endif // SNAPSHOT_SERVICE_HPP
Loading
Loading