Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions csnode/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ add_library(csnode
include/csnode/snapshot_messages.hpp
include/csnode/snapshot_provider.hpp
include/csnode/snapshot_service.hpp
include/csnode/snapshot_agreement.hpp
include/csnode/snapshot_fetcher.hpp
include/csnode/blockverifier.hpp
include/csnode/sendcachedata.hpp
include/csnode/eventreport.hpp
Expand Down Expand Up @@ -122,6 +124,8 @@ add_library(csnode
src/snapshot_messages.cpp
src/snapshot_provider.cpp
src/snapshot_service.cpp
src/snapshot_agreement.cpp
src/snapshot_fetcher.cpp
src/blockverifier.cpp
src/sendcachedata.cpp
src/eventreport.cpp
Expand Down
3 changes: 3 additions & 0 deletions csnode/include/csnode/node.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
class Transport;

namespace cs {
class SnapshotFetcher;
class SnapshotProvider;
class SnapshotService;
struct SnapshotNodeState;
Expand Down Expand Up @@ -510,6 +511,8 @@ public slots:
// snapshots served to p2p neighbours ([snapshot] serve)
std::unique_ptr<cs::SnapshotProvider> snapshotProvider_;
std::unique_ptr<cs::SnapshotService> snapshotService_;
// snapshot fetched from p2p neighbours at a start on an empty database (constructed in bootstrap mode only)
std::unique_ptr<cs::SnapshotFetcher> snapshotFetcher_;
std::chrono::steady_clock::time_point snapshotSummaryAt_{};

uint32_t startPacketRequestPoint_ = 0;
Expand Down
70 changes: 70 additions & 0 deletions csnode/include/csnode/snapshot_agreement.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#ifndef SNAPSHOT_AGREEMENT_HPP
#define SNAPSHOT_AGREEMENT_HPP

#include <cstddef>
#include <optional>
#include <string>
#include <vector>

#include <csnode/snapshot_manifest.hpp>
#include <lib/system/common.hpp>

// Agreement of independent sources on a snapshot (docs/specs/2026-10-25-snapshot-p2p-design.md,
// section 6.3): pure functions over the manifests collected from the p2p neighbours.
namespace cs {

// a manifest as received from one source: its p2p key, its address in the routing table and
// whether the operator listed the key in [snapshot] sources
struct SourceManifest {
cs::PublicKey key{};
std::string address;
bool trusted = false;
SnapshotManifest manifest;
};

// The independence unit of an address: the /24 of an IPv4 address ("a.b.c"), the /48 of an
// IPv6 one ("2001:db8:1::/48"); an IPv4-mapped IPv6 address counts as IPv4. A port after the
// address (a.b.c.d:port, [v6]:port) is ignored. Empty for an address that does not parse.
std::string netPrefix(const std::string& address);
// the wider block around it: /16 for IPv4, /32 for IPv6; empty when the address does not parse
std::string netBlock(const std::string& address);

// sources declaring one (seq, head_hash, state_digest, settings_digest)
struct AgreementGroup {
cs::Sequence seq = 0;
std::string headHash;
std::string stateDigest;
std::string settingsDigest;
std::vector<size_t> sources; // indices into agree()'s input
std::vector<cs::PublicKey> keys; // their keys, same order
size_t independent = 0; // counting sources: one per /24 or /48, trusted keys once each
bool configured = false; // holds a key of [snapshot] sources
bool chosen = false;
std::string reason; // why it was not chosen
};

using GroupReport = AgreementGroup;

struct AgreementResult {
std::optional<AgreementGroup> chosen;
std::vector<AgreementGroup> groups; // every group, strongest first
size_t shortfall = 0; // counting sources the best group lacks (0 when chosen)
};

// Groups by (seq, head_hash, state_digest, settings_digest). Among the groups with at least
// minSources counting sources, the chosen one is, in this order: a group holding a key of
// [snapshot] sources (trusted), then the group with most counting sources, then the highest seq
// (only a tie-breaker: a later block never wins over more support, so that min_sources forged
// sources at a made-up later block do not beat a larger honest group; an honest network's newest
// snapshot gains sources as the nodes rebuild, and the fetcher asks again every 30 s). Two groups
// equal on all three are a conflict: nothing is chosen. Sources are counted by netPrefix(address),
// trusted ones by key; a source whose address is unknown or does not parse does not count; a key that
// appears twice counts once. Every group that is not chosen carries the reason.
AgreementResult agree(const std::vector<SourceManifest>& sources, size_t minSources);

// "block S hash H digest D : N sources (keys k1 k2 …)" with shortened hashes and keys
std::string describe(const AgreementGroup& group);

} // namespace cs

#endif // SNAPSHOT_AGREEMENT_HPP
269 changes: 269 additions & 0 deletions csnode/include/csnode/snapshot_fetcher.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,269 @@
#ifndef SNAPSHOT_FETCHER_HPP
#define SNAPSHOT_FETCHER_HPP

#include <cstdint>
#include <filesystem>
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <vector>

#include <config.hpp>
#include <csnode/snapshot_agreement.hpp>
#include <csnode/snapshot_manifest.hpp>
#include <csnode/snapshot_messages.hpp>
#include <lib/system/common.hpp>

namespace cs {

// Gets a snapshot from the p2p neighbours of a node that starts on an empty database
// (docs/specs/2026-10-25-snapshot-p2p-design.md, sections 6.2-6.5): brings the peers that serve
// snapshots into the neighbour table, collects their signed manifests, waits for the agreement of
// [snapshot] min_sources independent sources, then downloads the files of the agreed snapshot
// piece by piece into tmpDir, each piece checked against the manifest's chunk hash and each file
// against its SHA-256. Single-threaded: every call comes from the node's processor thread. The
// network, the routing table and the clock are reached only through Hooks.
class SnapshotFetcher {
public:
struct Peers {
std::vector<std::pair<cs::PublicKey, uint8_t>> known; // key, capabilities (net/hostdata.hpp)
std::set<cs::PublicKey> neighbours;
};

struct Hooks {
std::function<void(const cs::PublicKey&, const snapshot_p2p::ManifestRequest&)> sendManifestRequest;
std::function<void(const cs::PublicKey&, const snapshot_p2p::PieceRequest&)> sendPieceRequest;
std::function<void(const std::set<cs::PublicKey>&)> addNeighbours;
std::function<std::string(const cs::PublicKey&)> addressOf; // "" when unknown
std::function<uint64_t()> clock; // ms, steady; default: steady_clock
std::function<void(const std::string&)> onLog; // optional: every SNAPSHOT line, for tests
};

enum class Phase {
Discover, // no manifest asked yet
Collect, // manifests asked, waiting for agreement
Fetch, // downloading the agreed snapshot
Import, // every file downloaded and verified: result() is set
Done, // dry run: agreement reached or fetch_wait over, groups() has what was seen
Fallback // no snapshot: the node syncs from block 0
};

static constexpr uint64_t kManifestIntervalMs = 30'000;
static constexpr uint64_t kManifestRetryMs = 10'000; // a request without an answer: asked again sooner
static constexpr size_t kManifestRetries = 2; // ... this many times, then every 30 s
static constexpr uint64_t kSilentMs = kManifestIntervalMs + 5'000; // no answer this long: silent
static constexpr uint64_t kNeighbourRetryMs = 60'000;
static constexpr uint64_t kPieceTimeoutMs = 30'000; // plus the bytes outstanding to the source at 64 KB/s
static constexpr uint64_t kTimeoutBytesPerSecond = 64 << 10;
static constexpr uint64_t kRecentPieceMs = 60'000; // a source that delivered this recently is slow, not stalled
static constexpr size_t kMaxLate = 64; // timed-out pieces whose late reply is still taken
static constexpr uint64_t kMarkMs = 60'000;
static constexpr uint64_t kMinBusyMs = 500;
static constexpr uint64_t kMaxBusyMs = 10'000;
static constexpr uint64_t kBusyMarkMs = 60'000; // busy this long without a piece: marked
static constexpr uint64_t kNoProgressMs = 300'000; // a fetch without a new byte this long ends
static constexpr uint64_t kLogIntervalMs = 30'000;
static constexpr uint64_t kWaitLogIntervalMs = 300'000;
static constexpr uint64_t kStallMs = 120'000;
static constexpr size_t kPiecesPerSource = 2;
static constexpr size_t kPiecesTotal = 8;
static constexpr size_t kMaxSources = 64; // discovered sources; configured keys come on top
static constexpr size_t kMaxPerPrefix = 4; // discovered sources per /24 (IPv4) or /48 (IPv6)
static constexpr size_t kMaxPerBlock = 16; // discovered sources per /16 (IPv4) or /32 (IPv6)
static constexpr uint64_t kEvictedMs = 60'000; // an evicted key is not taken again before
static constexpr size_t kMaxEvicted = 1024;
static constexpr size_t kMaxAttempts = 3; // refused imports, and with fetch_fallback=sync failed
// fetches without a proven bad source: then Fallback
static constexpr size_t kMaxExcusedAborts = 10; // fetches ended by a proven bad source (fetch_fallback=sync)
static constexpr uint64_t kBootstrapCapMs = 30 * 60'000; // fetch_fallback=sync: Fallback after fetch_wait + this,
// plus the time the agreed snapshot takes to download:
// at fetch_rate_kbps when set, else at 64 KB/s for at most
static constexpr uint64_t kMaxCapExtensionMs = 2 * 3'600'000; // this (a manifest may declare gigabytes)
static constexpr uint64_t kHashBytesPerTick = 64ull << 20; // whole-file SHA-256 per tick
static constexpr size_t kMaxSourcesFileKeys = 64;

// cfg.fetchWait = 0: no wait, Fallback (or Done) at the first tick without agreement
SnapshotFetcher(const SnapshotData& cfg, std::filesystem::path tmpDir, Hooks hooks, bool dryRun = false);
~SnapshotFetcher();
SnapshotFetcher(const SnapshotFetcher&) = delete;
SnapshotFetcher& operator=(const SnapshotFetcher&) = delete;

void tick(const Peers& peers); // every 500 ms
void onManifestReply(const cs::PublicKey& sender, const snapshot_p2p::ManifestReply& reply);
void onPieceReply(const cs::PublicKey& sender, snapshot_p2p::PieceReply&& reply);

Phase phase() const {
return phase_;
}
// the manifest of the downloaded files (in tmpDir) and the agreed sources; set in Import
std::optional<std::pair<SnapshotManifest, std::vector<SourceManifest>>> result() const;
// the importer refused fileAtFault: that file's SHA-256 is not fetched again; back to Collect,
// or Fallback after kMaxAttempts failed imports
void onImportFailed(const std::string& fileAtFault);
std::vector<GroupReport> groups() const;
// no request is sent and no reply is taken after this
void stop();

// for the logs and the tests
size_t manifests() const;
size_t outstanding() const;
uint64_t dropped() const {
return dropped_;
}
uint64_t bytesDone() const;
uint64_t bytesTotal() const;

private:
struct Source {
bool configured = false; // [snapshot] sources or sources_file
bool trusted = false; // [snapshot] sources
bool serving = false; // announced capability bit 24
std::string prefix; // netPrefix and netBlock of the address, empty while unknown;
std::string block; // resolved again when a manifest arrives
bool everAnswered = false;
size_t unanswered = 0; // manifest requests since the last answer
uint64_t firstUnansweredAt = 0;
uint64_t discoveredAt = 0;
uint64_t addedAt = 0; // last addNeighbours
bool added = false;
uint64_t askedAt = 0;
bool everAsked = false;
bool awaiting = false; // a manifest request without a reply yet
bool loggedAsk = false;
std::optional<SnapshotManifest> manifest;
uint64_t manifestAt = 0;
std::string filesHash;
std::string address;
size_t marks = 0;
uint64_t markedUntil = 0;
bool banned = false; // for the rest of the bootstrap
uint64_t busyUntil = 0;
uint64_t busySince = 0; // first busy answer since its last piece; 0: none
size_t budget = kPiecesPerSource; // pieces it may have outstanding: 1 after a timeout of a source that delivers
size_t streak = 0; // pieces delivered in time in a row
bool delivered = false;
uint64_t lastPieceAt = 0;
size_t outstanding = 0;
};

struct InFlight {
cs::PublicKey source{};
uint64_t sentAt = 0;
};

struct HashState;

struct FileState {
SnapshotFile file;
std::vector<cs::PublicKey> sources; // agreed sources declaring this file with this SHA-256
std::vector<bool> have;
size_t missing = 0; // chunks not on disk yet
uint64_t doneBytes = 0;
std::map<uint32_t, InFlight> inFlight;
int fd = -1;
bool verifyPending = false;
bool verified = false;
std::shared_ptr<HashState> hashing; // whole-file SHA-256 in progress, kHashBytesPerTick per tick
uint64_t hashed = 0;
};

void log(const std::string& line, bool warning = false);
uint64_t now() const;
std::string keyText(const cs::PublicKey& key) const;
Source* findSource(const cs::PublicKey& key);
Source* addConfigured(const cs::PublicKey& key);
Source* addDiscovered(const cs::PublicKey& key, uint64_t t);
bool roomFor(const cs::PublicKey& key, const std::string& prefix, const std::string& block) const;
bool evictOne(uint64_t t);
bool silent(const Source& s, uint64_t t) const;
void evict(std::map<cs::PublicKey, Source>::iterator it, uint64_t t);
bool isProtected(const cs::PublicKey& key, const Source& s) const;
void loadSourcesFile();

void discover(const Peers& peers, uint64_t t);
void askManifests(const Peers& peers, uint64_t t);
bool tryAgree(uint64_t t);
void logWaiting(uint64_t t);
void logGroups(bool chosenOnly);
std::string seenText() const;
void checkWait(uint64_t t);

bool usable(const cs::PublicKey& key, const Source& s, uint64_t t) const;
size_t lateOnTheWay(const cs::PublicKey& key, uint64_t t) const;
uint64_t capMs() const;
bool everUsable(const Source& s) const;
bool startFetch(uint64_t t);
bool chooseTarget(uint64_t t);
void fill(uint64_t t);
void expire(uint64_t t);
void verifyFiles(uint64_t t);
void clearInFlight();
void releaseSource(const cs::PublicKey& key);
void mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why);
void backToCollect(const std::string& why);
void forgetLate(const cs::PublicKey& key);
void banDeclaring(const std::string& name, const std::string& sha256, const std::string& why);
void closeFile(FileState& f);
bool openFile(FileState& f, bool truncate);
uint64_t pieceLength(const FileState& f, uint32_t chunk) const;
void logProgress(uint64_t t, bool force);
bool takeTokens(uint64_t cost, uint64_t t);

SnapshotData cfg_;
std::filesystem::path tmpDir_;
Hooks hooks_;
bool dryRun_;

Phase phase_ = Phase::Discover;
bool stopped_ = false;
bool started_ = false;
uint64_t startedAt_ = 0;
std::map<cs::PublicKey, Source> sources_;
std::set<cs::PublicKey> neighbours_; // as of the last tick
std::map<cs::PublicKey, uint64_t> evicted_; // discovered keys let go, and when

AgreementResult agreement_;
std::vector<SourceManifest> agreedSources_;
std::optional<SnapshotManifest> target_;
std::vector<FileState> files_;
std::set<std::pair<std::string, std::string>> badFiles_; // (name, sha256) refused by the importer
size_t importFailures_ = 0; // imports refused (onImportFailed)
size_t fetchFailures_ = 0; // fetches that ended without a snapshot and without a misbehaving source
size_t excusedAborts_ = 0; // fetches that ended with a source banned for a proven fault
bool misbehaviour_ = false; // a source was banned for a proven fault (a hash against the signed manifest) in this fetch
// pieces that timed out, (file, chunk) -> the source asked: its late reply is still taken, and
// for one more timeout the piece still counts as on its way from that source
struct Late {
cs::PublicKey source{};
uint64_t sentAt = 0;
uint64_t expiredAt = 0;
uint64_t timeout = 0;
};
std::map<std::pair<std::string, uint32_t>, Late> late_;
uint64_t capExtensionMs_ = 0; // the bootstrap cap grows by the download time of the agreed snapshot
bool dirty_ = true; // the input of the agreement changed since it was computed
uint64_t lastByteAt_ = 0;
uint64_t lastSendAt_ = 0;
size_t rr_ = 0; // first source of the next fill

// fetch_rate_kbps on requested bytes: a request goes out when nothing is owed, then owes its size
double owed_ = 0;
uint64_t owedAt_ = 0;

uint64_t dropped_ = 0;
uint64_t lastLogAt_ = 0;
uint64_t lastWaitLogAt_ = 0;
bool fallbackLogged_ = false;
uint64_t lastProgressAt_ = 0;
uint64_t lastProgressBytes_ = 0;
int lastProgressDecile_ = -1;
uint64_t fetchStartedAt_ = 0;
};

} // namespace cs

#endif // SNAPSHOT_FETCHER_HPP
Loading
Loading