From 55765ce69da14ff0ee17f0429057404caf40f773 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 14:56:09 +0100 Subject: [PATCH 1/9] snapshot: p2p fetcher, agreement and download (P4) snapshot_agreement: netPrefix (/24, /48) and agree() over the collected manifests (design 6.3): groups by (seq, head) and digests, highest block first, trusted keys counted by key, shortfall for the waiting line. SnapshotFetcher: discovery of serving peers and configured keys, manifest requests every 30 s, agreement at each tick, download of the agreed snapshot's files piece by piece (2 per source, 8 in total, 30-s timeout, busy, marks, bad pieces, group switch, whole-file check, rate cap), fetch_wait with sync/wait, import failures, dry run, stop. Single-threaded, hooks for the network and the clock. Node forwards the two reply messages to snapshotFetcher_ when it exists; nothing constructs it yet (P5). Co-Authored-By: Claude Opus 5.5 --- csnode/CMakeLists.txt | 4 + csnode/include/csnode/node.hpp | 3 + csnode/include/csnode/snapshot_agreement.hpp | 63 ++ csnode/include/csnode/snapshot_fetcher.hpp | 206 ++++ csnode/src/node.cpp | 8 +- csnode/src/snapshot_agreement.cpp | 157 +++ csnode/src/snapshot_fetcher.cpp | 1008 ++++++++++++++++++ tests/csnode/snapshot_agreement_tests.cpp | 176 +++ tests/csnode/snapshot_fetcher_tests.cpp | 846 +++++++++++++++ 9 files changed, 2470 insertions(+), 1 deletion(-) create mode 100644 csnode/include/csnode/snapshot_agreement.hpp create mode 100644 csnode/include/csnode/snapshot_fetcher.hpp create mode 100644 csnode/src/snapshot_agreement.cpp create mode 100644 csnode/src/snapshot_fetcher.cpp create mode 100644 tests/csnode/snapshot_agreement_tests.cpp create mode 100644 tests/csnode/snapshot_fetcher_tests.cpp diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index 0d23c1f8f..a3bde1dc0 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -75,6 +75,8 @@ add_library(csnode include/csnode/snapshot_messages.hpp include/csnode/snapshot_provider.hpp include/csnode/snapshot_service.hpp + include/csnode/snapshot_agreement.hpp + include/csnode/snapshot_fetcher.hpp include/csnode/blockverifier.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -122,6 +124,8 @@ add_library(csnode src/snapshot_messages.cpp src/snapshot_provider.cpp src/snapshot_service.cpp + src/snapshot_agreement.cpp + src/snapshot_fetcher.cpp src/blockverifier.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index b2dfffaf4..cc5b1b6e3 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -28,6 +28,7 @@ class Transport; namespace cs { +class SnapshotFetcher; class SnapshotProvider; class SnapshotService; struct SnapshotNodeState; @@ -510,6 +511,8 @@ public slots: // snapshots served to p2p neighbours ([snapshot] serve) std::unique_ptr snapshotProvider_; std::unique_ptr snapshotService_; + // snapshot fetched from p2p neighbours at a start on an empty database (constructed in bootstrap mode only) + std::unique_ptr snapshotFetcher_; std::chrono::steady_clock::time_point snapshotSummaryAt_{}; uint32_t startPacketRequestPoint_ = 0; diff --git a/csnode/include/csnode/snapshot_agreement.hpp b/csnode/include/csnode/snapshot_agreement.hpp new file mode 100644 index 000000000..28741a8d7 --- /dev/null +++ b/csnode/include/csnode/snapshot_agreement.hpp @@ -0,0 +1,63 @@ +#ifndef SNAPSHOT_AGREEMENT_HPP +#define SNAPSHOT_AGREEMENT_HPP + +#include +#include +#include +#include + +#include +#include + +// Agreement of independent sources on a snapshot (docs/specs/2026-10-25-snapshot-p2p-design.md, +// section 6.3): pure functions over the manifests collected from the p2p neighbours. +namespace cs { + +// a manifest as received from one source: its p2p key, its address in the routing table and +// whether the operator listed the key in [snapshot] sources +struct SourceManifest { + cs::PublicKey key{}; + std::string address; + bool trusted = false; + SnapshotManifest manifest; +}; + +// The independence unit of an address: the /24 of an IPv4 address ("a.b.c"), the /48 of an +// IPv6 one ("2001:db8:1::/48"); an IPv4-mapped IPv6 address counts as IPv4. A port after the +// address (a.b.c.d:port, [v6]:port) is ignored. Empty for an address that does not parse. +std::string netPrefix(const std::string& address); + +// sources declaring one (seq, head_hash, state_digest, settings_digest) +struct AgreementGroup { + cs::Sequence seq = 0; + std::string headHash; + std::string stateDigest; + std::string settingsDigest; + std::vector sources; // indices into agree()'s input + std::vector keys; // their keys, same order + size_t independent = 0; // counting sources: one per /24 or /48, trusted keys once each + bool chosen = false; +}; + +using GroupReport = AgreementGroup; + +struct AgreementResult { + std::optional chosen; + std::vector groups; // every group, highest seq first, then most sources + size_t shortfall = 0; // counting sources the best group lacks (0 when chosen) +}; + +// Groups by (seq, head_hash), inside by (state_digest, settings_digest). Candidate blocks have at +// least minSources counting sources, highest seq first; in a candidate block the digest group with +// most counting sources is chosen if it reaches minSources and no other group of that block has as +// many (a tie is a conflict: that block is skipped). Sources are counted by netPrefix(address), +// trusted ones by key; sources whose address does not parse count together as one; a key that +// appears twice counts once. +AgreementResult agree(const std::vector& sources, size_t minSources); + +// "block S hash H digest D : N sources (keys k1 k2 …)" with shortened hashes and keys +std::string describe(const AgreementGroup& group); + +} // namespace cs + +#endif // SNAPSHOT_AGREEMENT_HPP diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp new file mode 100644 index 000000000..ccf6773f6 --- /dev/null +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -0,0 +1,206 @@ +#ifndef SNAPSHOT_FETCHER_HPP +#define SNAPSHOT_FETCHER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace cs { + +// Gets a snapshot from the p2p neighbours of a node that starts on an empty database +// (docs/specs/2026-10-25-snapshot-p2p-design.md, sections 6.2-6.5): brings the peers that serve +// snapshots into the neighbour table, collects their signed manifests, waits for the agreement of +// [snapshot] min_sources independent sources, then downloads the files of the agreed snapshot +// piece by piece into tmpDir, each piece checked against the manifest's chunk hash and each file +// against its SHA-256. Single-threaded: every call comes from the node's processor thread. The +// network, the routing table and the clock are reached only through Hooks. +class SnapshotFetcher { +public: + struct Peers { + std::vector> known; // key, capabilities (net/hostdata.hpp) + std::set neighbours; + }; + + struct Hooks { + std::function sendManifestRequest; + std::function sendPieceRequest; + std::function&)> addNeighbours; + std::function addressOf; // "" when unknown + std::function clock; // ms, steady; default: steady_clock + std::function onLog; // optional: every SNAPSHOT line, for tests + }; + + enum class Phase { + Discover, // no manifest asked yet + Collect, // manifests asked, waiting for agreement + Fetch, // downloading the agreed snapshot + Import, // every file downloaded and verified: result() is set + Done, // dry run: agreement reached or fetch_wait over, groups() has what was seen + Fallback // no snapshot: the node syncs from block 0 + }; + + static constexpr uint64_t kManifestIntervalMs = 30'000; + static constexpr uint64_t kNeighbourRetryMs = 60'000; + static constexpr uint64_t kPieceTimeoutMs = 30'000; + static constexpr uint64_t kMarkMs = 60'000; + static constexpr uint64_t kMaxBusyMs = 10'000; + static constexpr uint64_t kLogIntervalMs = 30'000; + static constexpr uint64_t kWaitLogIntervalMs = 300'000; + static constexpr uint64_t kStallMs = 120'000; + static constexpr size_t kPiecesPerSource = 2; + static constexpr size_t kPiecesTotal = 8; + static constexpr size_t kMaxSources = 64; + static constexpr size_t kMaxAttempts = 3; + static constexpr size_t kMaxSourcesFileKeys = 64; + + SnapshotFetcher(const SnapshotData& cfg, std::filesystem::path tmpDir, Hooks hooks, bool dryRun = false); + ~SnapshotFetcher(); + SnapshotFetcher(const SnapshotFetcher&) = delete; + SnapshotFetcher& operator=(const SnapshotFetcher&) = delete; + + void tick(const Peers& peers); // every 500 ms + void onManifestReply(const cs::PublicKey& sender, const snapshot_p2p::ManifestReply& reply); + void onPieceReply(const cs::PublicKey& sender, snapshot_p2p::PieceReply&& reply); + + Phase phase() const { + return phase_; + } + // the manifest of the downloaded files (in tmpDir) and the agreed sources; set in Import + std::optional>> result() const; + // the importer refused fileAtFault: that file's SHA-256 is not fetched again; back to Collect, + // or Fallback after kMaxAttempts failed imports + void onImportFailed(const std::string& fileAtFault); + std::vector groups() const; + // no request is sent and no reply is taken after this + void stop(); + + // for the logs and the tests + size_t manifests() const; + size_t outstanding() const; + uint64_t dropped() const { + return dropped_; + } + uint64_t bytesDone() const; + uint64_t bytesTotal() const; + +private: + struct Source { + bool configured = false; // [snapshot] sources or sources_file + bool trusted = false; // [snapshot] sources + bool serving = false; // announced capability bit 24 + uint64_t addedAt = 0; // last addNeighbours + bool added = false; + uint64_t askedAt = 0; + bool everAsked = false; + bool awaiting = false; // a manifest request without a reply yet + bool loggedAsk = false; + std::optional manifest; + uint64_t manifestAt = 0; + std::string filesHash; + std::string address; + size_t marks = 0; + uint64_t markedUntil = 0; + bool banned = false; // for the rest of the bootstrap + uint64_t busyUntil = 0; + size_t outstanding = 0; + }; + + struct InFlight { + cs::PublicKey source{}; + uint64_t sentAt = 0; + }; + + struct FileState { + SnapshotFile file; + std::vector sources; // agreed sources declaring this file with this SHA-256 + std::vector have; + size_t missing = 0; // chunks not on disk yet + uint64_t doneBytes = 0; + std::map inFlight; + int fd = -1; + bool verifyPending = false; + bool verified = false; + }; + + void log(const std::string& line, bool warning = false); + uint64_t now() const; + std::string keyText(const cs::PublicKey& key) const; + Source* findSource(const cs::PublicKey& key); + Source* addSource(const cs::PublicKey& key); + void loadSourcesFile(); + + void discover(const Peers& peers, uint64_t t); + void askManifests(const Peers& peers, uint64_t t); + bool tryAgree(uint64_t t); + void logWaiting(uint64_t t); + void logGroups(bool chosenOnly); + std::string seenText() const; + void checkWait(uint64_t t); + + bool usable(const Source& s, uint64_t t) const; + bool everUsable(const Source& s) const; + bool startFetch(uint64_t t); + bool chooseTarget(uint64_t t); + void fill(uint64_t t); + void expire(uint64_t t); + void verifyFiles(uint64_t t); + void clearInFlight(); + void releaseSource(const cs::PublicKey& key); + void mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why); + void backToCollect(const std::string& why); + void closeFile(FileState& f); + bool openFile(FileState& f, bool truncate); + uint64_t pieceLength(const FileState& f, uint32_t chunk) const; + void logProgress(uint64_t t, bool force); + bool takeTokens(uint64_t cost, uint64_t t); + + SnapshotData cfg_; + std::filesystem::path tmpDir_; + Hooks hooks_; + bool dryRun_; + + Phase phase_ = Phase::Discover; + bool stopped_ = false; + bool started_ = false; + uint64_t startedAt_ = 0; + std::map sources_; + std::set neighbours_; // as of the last tick + + AgreementResult agreement_; + std::vector agreedSources_; + std::optional target_; + std::vector files_; + std::set> badFiles_; // (name, sha256) refused by the importer + size_t importFailures_ = 0; + uint64_t lastSendAt_ = 0; + size_t rr_ = 0; // first source of the next fill + + // fetch_rate_kbps on requested bytes: a request goes out when nothing is owed, then owes its size + double owed_ = 0; + uint64_t owedAt_ = 0; + + uint64_t dropped_ = 0; + uint64_t lastLogAt_ = 0; + uint64_t lastWaitLogAt_ = 0; + bool fallbackLogged_ = false; + uint64_t lastProgressAt_ = 0; + uint64_t lastProgressBytes_ = 0; + int lastProgressDecile_ = -1; + uint64_t fetchStartedAt_ = 0; +}; + +} // namespace cs + +#endif // SNAPSHOT_FETCHER_HPP diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index b1ee9c4e7..55609a935 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -2247,7 +2248,9 @@ void Node::getSnapshotManifestReply(const uint8_t* data, const size_t size, cons if (!cs::snapshot_p2p::decode(data, size, reply)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed manifest reply", sender); } - // nothing asked for it yet + if (snapshotFetcher_) { + return snapshotFetcher_->onManifestReply(sender, reply); + } logSnapshotDrop(snapshotMessagesDropped_, "an unexpected manifest reply", sender); } @@ -2271,6 +2274,9 @@ void Node::getSnapshotPieceReply(const uint8_t* data, const size_t size, const c if (!cs::snapshot_p2p::decode(data, size, reply)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed piece reply", sender); } + if (snapshotFetcher_) { + return snapshotFetcher_->onPieceReply(sender, std::move(reply)); + } logSnapshotDrop(snapshotMessagesDropped_, "an unexpected piece reply", sender); } diff --git a/csnode/src/snapshot_agreement.cpp b/csnode/src/snapshot_agreement.cpp new file mode 100644 index 000000000..4b4ae05e8 --- /dev/null +++ b/csnode/src/snapshot_agreement.cpp @@ -0,0 +1,157 @@ +#include + +#include +#include +#include +#include +#include + +#include + +#include + +namespace cs { + +namespace { +std::string hostPart(const std::string& address) { + if (!address.empty() && address.front() == '[') { + const auto close = address.find(']'); + return close == std::string::npos ? std::string() : address.substr(1, close - 1); + } + // a.b.c.d:port; an IPv6 address has more than one ':' + const auto colon = address.find(':'); + if (colon != std::string::npos && address.find(':', colon + 1) == std::string::npos) { + return address.substr(0, colon); + } + return address; +} + +std::string unitOf(const SourceManifest& s) { + if (s.trusted) { + return "key:" + EncodeBase58(s.key.data(), s.key.data() + s.key.size()); + } + const auto prefix = netPrefix(s.address); + return prefix.empty() ? std::string("unknown address") : "net:" + prefix; +} + +size_t countUnits(const std::vector& indices, const std::vector& units) { + std::set distinct; + for (auto i : indices) { + distinct.insert(units[i]); + } + return distinct.size(); +} + +std::string shortHex(const std::string& hex) { + return hex.size() > 16 ? hex.substr(0, 16) + "..." : hex; +} +} // namespace + +std::string netPrefix(const std::string& address) { + boost::system::error_code ec; + auto ip = boost::asio::ip::make_address(hostPart(address), ec); + if (ec) { + return {}; + } + if (ip.is_v6() && ip.to_v6().is_v4_mapped()) { + ip = boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, ip.to_v6()); + } + if (ip.is_v4()) { + const auto b = ip.to_v4().to_bytes(); + return std::to_string(b[0]) + "." + std::to_string(b[1]) + "." + std::to_string(b[2]); + } + auto b = ip.to_v6().to_bytes(); + std::fill(b.begin() + 6, b.end(), 0); + return boost::asio::ip::address_v6(b).to_string() + "/48"; +} + +AgreementResult agree(const std::vector& sources, size_t minSources) { + minSources = std::max(minSources, 1); + AgreementResult result; + + std::vector units(sources.size()); + std::set seen; + using BlockKey = std::pair; + using DigestKey = std::pair; + std::map>> blocks; + for (size_t i = 0; i < sources.size(); ++i) { + units[i] = unitOf(sources[i]); + if (!seen.insert(sources[i].key).second) { + continue; + } + const auto& m = sources[i].manifest; + blocks[{m.seq, m.headHash}][{m.stateDigest, m.settingsDigest}].push_back(i); + } + + struct Block { + BlockKey key; + size_t independent = 0; + std::vector groups; + }; + std::vector ordered; + for (const auto& [blockKey, digests] : blocks) { + Block b; + b.key = blockKey; + std::vector all; + for (const auto& [digestKey, indices] : digests) { + AgreementGroup g; + g.seq = blockKey.first; + g.headHash = blockKey.second; + g.stateDigest = digestKey.first; + g.settingsDigest = digestKey.second; + g.sources = indices; + for (auto i : indices) { + g.keys.push_back(sources[i].key); + } + g.independent = countUnits(indices, units); + all.insert(all.end(), indices.begin(), indices.end()); + b.groups.push_back(std::move(g)); + } + b.independent = countUnits(all, units); + std::stable_sort(b.groups.begin(), b.groups.end(), + [](const AgreementGroup& x, const AgreementGroup& y) { return x.independent > y.independent; }); + ordered.push_back(std::move(b)); + } + std::stable_sort(ordered.begin(), ordered.end(), [](const Block& x, const Block& y) { + return x.key.first != y.key.first ? x.key.first > y.key.first : x.independent > y.independent; + }); + + for (auto& b : ordered) { + if (b.independent < minSources) { + continue; + } + auto& top = b.groups.front(); + if (top.independent < minSources) { + continue; + } + if (b.groups.size() > 1 && b.groups[1].independent == top.independent) { + continue; // two digests with as many sources for one block: a conflict, not an agreement + } + top.chosen = true; + result.chosen = top; + break; + } + + size_t best = 0; + for (auto& b : ordered) { + for (auto& g : b.groups) { + best = std::max(best, g.independent); + result.groups.push_back(std::move(g)); + } + } + result.shortfall = result.chosen ? 0 : minSources - std::min(best, minSources); + return result; +} + +std::string describe(const AgreementGroup& group) { + std::ostringstream out; + out << "block " << group.seq << " hash " << shortHex(group.headHash) << " digest " << shortHex(group.stateDigest) << " : " + << group.independent << (group.independent == 1 ? " source" : " sources") << " (keys"; + for (const auto& k : group.keys) { + out << ' ' << EncodeBase58(k.data(), k.data() + k.size()).substr(0, 8) << "..."; + } + out << ')'; + return out.str(); +} + +} // namespace cs diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp new file mode 100644 index 000000000..6ee1c28e6 --- /dev/null +++ b/csnode/src/snapshot_fetcher.cpp @@ -0,0 +1,1008 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; + +uint64_t steadyMs() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +bool powerOfTwo(uint64_t v) { + return v != 0 && (v & (v - 1)) == 0; +} + +std::string megabytes(uint64_t bytes) { + return std::to_string((bytes + 500'000) / 1'000'000); +} + +std::string duration(uint64_t seconds) { + if (seconds >= 60 && seconds % 60 == 0) { + return std::to_string(seconds / 60) + " min"; + } + return std::to_string(seconds) + " s"; +} + +bool sameFile(const SnapshotFile& a, const SnapshotFile& b) { + return a.name == b.name && a.size == b.size && a.sha256 == b.sha256 && a.chunks == b.chunks; +} + +std::string hexOf(const uint8_t* data, size_t size) { + static const char* digits = "0123456789abcdef"; + std::string out; + out.reserve(size * 2); + for (size_t i = 0; i < size; ++i) { + out.push_back(digits[data[i] >> 4]); + out.push_back(digits[data[i] & 0x0f]); + } + return out; +} +} // namespace + +SnapshotFetcher::SnapshotFetcher(const SnapshotData& cfg, fs::path tmpDir, Hooks hooks, bool dryRun) +: cfg_(cfg) +, tmpDir_(std::move(tmpDir)) +, hooks_(std::move(hooks)) +, dryRun_(dryRun) { + if (!hooks_.clock) { + hooks_.clock = steadyMs; + } + for (const auto& key : cfg_.sources) { + if (auto* s = addSource(key)) { + s->configured = true; + s->trusted = true; + } + } + loadSourcesFile(); +} + +SnapshotFetcher::~SnapshotFetcher() { + for (auto& f : files_) { + closeFile(f); + } +} + +void SnapshotFetcher::log(const std::string& line, bool warning) { + if (warning) { + cswarning() << line; + } + else { + cslog() << line; + } + if (hooks_.onLog) { + hooks_.onLog(line); + } +} + +uint64_t SnapshotFetcher::now() const { + return hooks_.clock(); +} + +std::string SnapshotFetcher::keyText(const cs::PublicKey& key) const { + return EncodeBase58(key.data(), key.data() + key.size()); +} + +SnapshotFetcher::Source* SnapshotFetcher::findSource(const cs::PublicKey& key) { + auto it = sources_.find(key); + return it == sources_.end() ? nullptr : &it->second; +} + +SnapshotFetcher::Source* SnapshotFetcher::addSource(const cs::PublicKey& key) { + if (auto* s = findSource(key)) { + return s; + } + if (sources_.size() >= kMaxSources) { + // make room by dropping a discovered peer that is not a neighbour and gave no manifest + auto victim = std::find_if(sources_.begin(), sources_.end(), [this](const auto& entry) { + const auto& s = entry.second; + return !s.configured && !s.manifest && s.outstanding == 0 && neighbours_.count(entry.first) == 0; + }); + if (victim == sources_.end()) { + return nullptr; + } + sources_.erase(victim); + } + return &sources_[key]; +} + +void SnapshotFetcher::loadSourcesFile() { + if (cfg_.sourcesFile.empty()) { + return; + } + std::ifstream in(cfg_.sourcesFile); + if (!in) { + return; // a missing file means no listed sources + } + std::string line; + size_t keys = 0; + size_t lines = 0; + while (keys < kMaxSourcesFileKeys && lines < 10 * kMaxSourcesFileKeys && std::getline(in, line)) { + ++lines; + if (const auto hash = line.find('#'); hash != std::string::npos) { + line.erase(hash); + } + std::istringstream words(line); + std::string word; + std::string last; + while (words >> word) { + last = word; + } + if (last.empty()) { + continue; + } + std::vector bytes; + if (!DecodeBase58(last, bytes) || bytes.size() != cs::PublicKey{}.size()) { + log("SNAPSHOT: " + cfg_.sourcesFile + ": not a node key: " + last.substr(0, 64), true); + continue; + } + cs::PublicKey key; + std::copy(bytes.begin(), bytes.end(), key.begin()); + if (auto* s = addSource(key)) { + s->configured = true; + ++keys; + } + } + if (keys != 0) { + log("SNAPSHOT: " + std::to_string(keys) + " source keys read from " + cfg_.sourcesFile); + } +} + +// --------------------------------------------------------------------------------------------- +// discovery and agreement + +void SnapshotFetcher::tick(const Peers& peers) { + if (stopped_ || phase_ == Phase::Import || phase_ == Phase::Done || phase_ == Phase::Fallback) { + return; + } + const uint64_t t = now(); + if (!started_) { + started_ = true; + startedAt_ = t; + lastLogAt_ = t; + lastWaitLogAt_ = t; + } + neighbours_ = peers.neighbours; + discover(peers, t); + + if (phase_ == Phase::Discover || phase_ == Phase::Collect) { + askManifests(peers, t); + if (tryAgree(t)) { + return; + } + logWaiting(t); + checkWait(t); + return; + } + + verifyFiles(t); + if (phase_ != Phase::Fetch) { + return; + } + expire(t); + if (phase_ != Phase::Fetch) { + return; + } + fill(t); + if (phase_ == Phase::Fetch && outstanding() == 0 && t - lastSendAt_ >= kStallMs) { + backToCollect("no piece could be asked for " + duration(kStallMs / 1000)); + return; + } + logProgress(t, false); +} + +void SnapshotFetcher::discover(const Peers& peers, uint64_t t) { + for (const auto& [key, capabilities] : peers.known) { + if ((capabilities & cs::hostdata::kSnapshotProtocolV1) == 0) { + continue; + } + if (auto* s = addSource(key)) { + s->serving = true; + } + } + std::set add; + for (auto& [key, s] : sources_) { + if (s.banned || neighbours_.count(key) != 0) { + continue; + } + if (!s.added || t - s.addedAt >= kNeighbourRetryMs) { + s.added = true; + s.addedAt = t; + add.insert(key); + } + } + if (!add.empty() && hooks_.addNeighbours) { + hooks_.addNeighbours(add); + } +} + +void SnapshotFetcher::askManifests(const Peers&, uint64_t t) { + for (auto& [key, s] : sources_) { + if (s.banned || neighbours_.count(key) == 0) { + continue; + } + // silence (the serving side may not answer at all) or a reply: ask again after 30 s + if (s.everAsked && t - s.askedAt < kManifestIntervalMs) { + continue; + } + s.everAsked = true; + s.awaiting = true; + s.askedAt = t; + if (!s.loggedAsk) { + s.loggedAsk = true; + log("SNAPSHOT: asking " + keyText(key) + " for its manifest"); + } + if (hooks_.sendManifestRequest) { + hooks_.sendManifestRequest(key, snapshot_p2p::ManifestRequest{}); + } + if (phase_ == Phase::Discover) { + phase_ = Phase::Collect; + } + } +} + +void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapshot_p2p::ManifestReply& reply) { + if (stopped_ || phase_ == Phase::Import || phase_ == Phase::Done || phase_ == Phase::Fallback) { + ++dropped_; + return; + } + auto* s = findSource(sender); + if (s == nullptr || !s->awaiting) { + ++dropped_; + return; + } + s->awaiting = false; + const uint64_t t = now(); + if (reply.status != snapshot_p2p::ManifestStatus::Ready) { + // not serving, not synced or still building: whatever it declared before is not served now + s->manifest.reset(); + return; + } + + std::string why; + auto manifest = SnapshotManifest::fromJson(reply.manifestJson, &why); + if (manifest) { + if (!manifest->verify()) { + why = "bad signature"; + } + else if (manifest->nodeKey != keyText(sender)) { + why = "signed by another node"; + } + else if (manifest->chunkSize > snapshot_p2p::kMaxPieceBytes || !powerOfTwo(manifest->chunkSize)) { + why = "chunk size " + std::to_string(manifest->chunkSize); + } + } + if (!manifest || !why.empty()) { + ++dropped_; + s->manifest.reset(); + log("SNAPSHOT: manifest from " + keyText(sender) + " dropped: " + why, true); + return; + } + s->filesHash = manifest->filesHash(); + s->manifest = std::move(manifest); + s->manifestAt = t; + s->address = hooks_.addressOf ? hooks_.addressOf(sender) : std::string(); + // agreement is tried at the next tick, so that the replies to one round of requests count together +} + +bool SnapshotFetcher::tryAgree(uint64_t t) { + std::vector input; + for (const auto& [key, s] : sources_) { + if (s.banned || !s.manifest) { + continue; + } + SourceManifest m; + m.key = key; + m.address = s.address; + m.trusted = s.trusted; + m.manifest = *s.manifest; + input.push_back(std::move(m)); + } + agreement_ = agree(input, cfg_.minSources); + if (!agreement_.chosen) { + return false; + } + if (dryRun_) { + logGroups(false); + phase_ = Phase::Done; + return true; + } + agreedSources_.clear(); + for (auto i : agreement_.chosen->sources) { + agreedSources_.push_back(input[i]); + } + if (!startFetch(t)) { + agreedSources_.clear(); + return phase_ == Phase::Fallback; + } + return true; +} + +void SnapshotFetcher::logGroups(bool chosenOnly) { + for (const auto& g : agreement_.groups) { + if (g.chosen) { + log("SNAPSHOT: " + describe(g)); + } + else if (!chosenOnly) { + log("SNAPSHOT: " + std::string(agreement_.chosen ? "minority: " : "") + describe(g)); + } + } +} + +std::string SnapshotFetcher::seenText() const { + if (agreement_.groups.empty()) { + return "no manifest"; + } + std::string out; + for (const auto& g : agreement_.groups) { + if (!out.empty()) { + out += "; "; + } + out += std::to_string(g.independent) + (g.independent == 1 ? " source" : " sources") + " for block " + std::to_string(g.seq); + } + return out; +} + +void SnapshotFetcher::logWaiting(uint64_t t) { + const uint64_t waitMs = uint64_t(cfg_.fetchWait) * 1000; + if (t - startedAt_ >= waitMs || t - lastLogAt_ < kLogIntervalMs) { + return; + } + lastLogAt_ = t; + size_t serving = 0; + size_t neighbours = 0; + for (const auto& [key, s] : sources_) { + if (s.serving || s.configured) { + ++serving; + if (neighbours_.count(key) != 0) { + ++neighbours; + } + } + } + log("SNAPSHOT: " + std::to_string(serving) + " peers serve snapshots (" + std::to_string(neighbours) + " neighbours), " + + std::to_string(manifests()) + " manifests collected"); + const std::string left = duration((waitMs - (t - startedAt_) + 59'999) / 60'000 * 60) + " left"; + if (agreement_.chosen) { + log("SNAPSHOT: block " + std::to_string(agreement_.chosen->seq) + + " is agreed, but no file group has a source left: waiting, " + left); + return; + } + const AgreementGroup* best = nullptr; + for (const auto& g : agreement_.groups) { + if (best == nullptr || g.independent > best->independent) { + best = &g; + } + } + if (best != nullptr) { + log("SNAPSHOT: " + std::to_string(best->independent) + (best->independent == 1 ? " source declares" : " sources declare") + + " block " + std::to_string(best->seq) + " (need " + std::to_string(cfg_.minSources) + "): waiting, " + left); + } +} + +void SnapshotFetcher::checkWait(uint64_t t) { + if (t - startedAt_ < uint64_t(cfg_.fetchWait) * 1000) { + return; + } + if (dryRun_) { + logGroups(false); + phase_ = Phase::Done; + return; + } + const std::string head = "SNAPSHOT: no agreement of " + std::to_string(cfg_.minSources) + " community sources after " + + duration((t - startedAt_) / 1000) + " (seen: " + seenText() + ")."; + if (cfg_.fetchFallback == "wait") { + if (!fallbackLogged_ || t - lastWaitLogAt_ >= kWaitLogIntervalMs) { + fallbackLogged_ = true; + lastWaitLogAt_ = t; + log(head + " Waiting for more sources (fetch_fallback=wait). To start from the published snapshot instead: stop the node " + "and run ./node --download-snapshot in this folder, then start it again.", + true); + } + return; + } + log(head + " Syncing from block 0 (about 35 h on a fast machine). To start from the published snapshot instead: stop the node " + "and run ./node --download-snapshot in this folder, then start it again.", + true); + clearInFlight(); + phase_ = Phase::Fallback; +} + +// --------------------------------------------------------------------------------------------- +// download + +bool SnapshotFetcher::everUsable(const Source& s) const { + return !s.banned; +} + +bool SnapshotFetcher::usable(const Source& s, uint64_t t) const { + return !s.banned && s.markedUntil <= t && s.busyUntil <= t && s.outstanding < kPiecesPerSource; +} + +bool SnapshotFetcher::startFetch(uint64_t t) { + if (!chooseTarget(t)) { + return false; + } + phase_ = Phase::Fetch; + fetchStartedAt_ = t; + lastSendAt_ = t; + lastProgressAt_ = t; + lastProgressBytes_ = bytesDone(); + lastProgressDecile_ = bytesTotal() == 0 ? 0 : static_cast(bytesDone() * 10 / bytesTotal()); + logGroups(false); + log("SNAPSHOT: fetching block " + std::to_string(target_->seq) + ": " + megabytes(bytesTotal()) + " MB in " + + std::to_string(files_.size()) + " files from " + std::to_string(agreedSources_.size()) + " sources"); + fill(t); + return true; +} + +bool SnapshotFetcher::chooseTarget(uint64_t t) { + // the files must be those of one signed manifest (the importer checks its signature): among the + // agreed sources' file lists, the one shared by most usable sources whose every file still has + // a usable source; each file then comes from every agreed source declaring the same bytes + const SnapshotManifest* best = nullptr; + std::vector> bestSources; + size_t bestOwners = 0; + size_t bestTotal = 0; + std::set tried; + for (const auto& candidate : agreedSources_) { + const auto& rep = candidate.manifest; + const auto hash = rep.filesHash(); + if (!tried.insert(hash).second) { + continue; + } + bool refused = false; + for (const auto& f : rep.files) { + refused = refused || badFiles_.count({f.name, f.sha256}) != 0; + } + if (refused) { + continue; + } + size_t owners = 0; + for (const auto& other : agreedSources_) { + const auto* s = findSource(other.key); + if (s != nullptr && everUsable(*s) && other.manifest.filesHash() == hash) { + ++owners; + } + } + std::vector> perFile; + size_t total = 0; + bool dry = false; + for (const auto& f : rep.files) { + std::vector keys; + for (const auto& other : agreedSources_) { + const auto* s = findSource(other.key); + const auto* of = other.manifest.file(f.name); + if (s != nullptr && everUsable(*s) && of != nullptr && other.manifest.chunkSize == rep.chunkSize && sameFile(*of, f)) { + keys.push_back(other.key); + } + } + dry = dry || keys.empty(); + total += keys.size(); + perFile.push_back(std::move(keys)); + } + if (dry) { + continue; + } + if (best == nullptr || owners > bestOwners || (owners == bestOwners && total > bestTotal)) { + best = &rep; + bestSources = std::move(perFile); + bestOwners = owners; + bestTotal = total; + } + } + if (best == nullptr) { + return false; + } + + clearInFlight(); + std::vector next; + for (size_t i = 0; i < best->files.size(); ++i) { + const auto& f = best->files[i]; + // equal chunk hashes over equal sizes: the same bytes cut the same way + auto old = std::find_if(files_.begin(), files_.end(), [&](const FileState& s) { return sameFile(s.file, f); }); + FileState state; + bool reuse = old != files_.end(); + if (reuse) { + state = std::move(*old); + old->fd = -1; + // what is on disk is checked again before it is handed over + if (state.verified) { + state.verified = false; + state.verifyPending = true; + } + } + else { + state.file = f; + state.have.assign(f.chunks.size(), false); + state.missing = f.chunks.size(); + state.verifyPending = f.chunks.empty(); + } + state.sources = bestSources[i]; + if (!openFile(state, !reuse)) { + log("SNAPSHOT: cannot write " + (tmpDir_ / f.name).string() + ": syncing from block 0", true); + closeFile(state); + for (auto& n : next) { + closeFile(n); + } + phase_ = Phase::Fallback; + return false; + } + next.push_back(std::move(state)); + } + for (auto& f : files_) { + closeFile(f); + } + files_ = std::move(next); + target_ = *best; + (void)t; + return true; +} + +bool SnapshotFetcher::openFile(FileState& f, bool truncate) { + if (f.fd >= 0) { + return true; + } + std::error_code ec; + fs::create_directories(tmpDir_, ec); + const auto path = tmpDir_ / f.file.name; + f.fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC | (truncate ? O_TRUNC : 0), 0644); + if (f.fd < 0) { + return false; + } + if (::ftruncate(f.fd, static_cast(f.file.size)) != 0) { + closeFile(f); + return false; + } + return true; +} + +void SnapshotFetcher::closeFile(FileState& f) { + if (f.fd >= 0) { + ::close(f.fd); + f.fd = -1; + } +} + +uint64_t SnapshotFetcher::pieceLength(const FileState& f, uint32_t chunk) const { + const uint64_t chunkSize = target_ ? target_->chunkSize : 0; + const uint64_t offset = uint64_t(chunk) * chunkSize; + return offset >= f.file.size ? 0 : std::min(chunkSize, f.file.size - offset); +} + +bool SnapshotFetcher::takeTokens(uint64_t cost, uint64_t t) { + if (cfg_.fetchRateKbps == 0) { + return true; + } + const double rate = double(cfg_.fetchRateKbps) * 1000.0 / 1000.0; // bytes per ms + owed_ = std::max(0.0, owed_ - double(t - owedAt_) * rate); + owedAt_ = t; + if (owed_ > 0) { + return false; + } + owed_ += double(cost); + return true; +} + +void SnapshotFetcher::fill(uint64_t t) { + if (stopped_ || phase_ != Phase::Fetch || !target_) { + return; + } + // round-robin over the sources: each usable source gets one piece per pass, from the file it + // can serve that has the fewest sources, so that a source shared by every file does not take + // the pieces that only it and few others could serve + std::vector keys; + for (const auto& f : files_) { + for (const auto& k : f.sources) { + if (std::find(keys.begin(), keys.end(), k) == keys.end()) { + keys.push_back(k); + } + } + } + if (keys.empty()) { + return; + } + const size_t first = rr_++ % keys.size(); + size_t total = outstanding(); + bool sent = true; + while (sent && total < kPiecesTotal) { + sent = false; + for (size_t n = 0; n < keys.size() && total < kPiecesTotal; ++n) { + const auto& key = keys[(first + n) % keys.size()]; + auto* s = findSource(key); + if (s == nullptr || !usable(*s, t) || neighbours_.count(key) == 0) { + continue; + } + FileState* pick = nullptr; + uint32_t pickChunk = 0; + for (auto& f : files_) { + if (f.missing == f.inFlight.size() || (pick != nullptr && f.sources.size() >= pick->sources.size()) || + std::find(f.sources.begin(), f.sources.end(), key) == f.sources.end()) { + continue; + } + uint32_t chunk = 0; + while (chunk < f.have.size() && (f.have[chunk] || f.inFlight.count(chunk) != 0)) { + ++chunk; + } + if (chunk < f.have.size()) { + pick = &f; + pickChunk = chunk; + } + } + if (pick == nullptr) { + continue; + } + if (!takeTokens(pieceLength(*pick, pickChunk), t)) { + return; + } + snapshot_p2p::PieceRequest request; + request.seq = target_->seq; + request.name = pick->file.name; + request.chunk = pickChunk; + pick->inFlight[pickChunk] = InFlight{key, t}; + ++s->outstanding; + ++total; + lastSendAt_ = t; + sent = true; + if (hooks_.sendPieceRequest) { + hooks_.sendPieceRequest(key, request); + } + } + } +} + +void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::PieceReply&& reply) { + if (stopped_ || phase_ != Phase::Fetch || !target_ || reply.seq != target_->seq) { + ++dropped_; + return; + } + auto file = std::find_if(files_.begin(), files_.end(), [&](const FileState& f) { return f.file.name == reply.name; }); + if (file == files_.end()) { + ++dropped_; + return; + } + auto& f = *file; + auto it = f.inFlight.find(reply.chunk); + if (it == f.inFlight.end() || it->second.source != sender) { + ++dropped_; + return; + } + f.inFlight.erase(it); + auto* s = findSource(sender); + if (s != nullptr && s->outstanding != 0) { + --s->outstanding; + } + const uint64_t t = now(); + + switch (reply.status) { + case snapshot_p2p::PieceStatus::Busy: + if (s != nullptr) { + s->busyUntil = t + std::min(reply.retryMs, kMaxBusyMs); + } + break; + case snapshot_p2p::PieceStatus::Unknown: + mark(sender, t, false, "does not have the piece"); + break; + case snapshot_p2p::PieceStatus::NotServing: + mark(sender, t, false, "does not serve"); + break; + case snapshot_p2p::PieceStatus::Ok: { + const uint64_t length = pieceLength(f, reply.chunk); + if (reply.data.size() != length || sha256Hex(reply.data.data(), reply.data.size()) != f.file.chunks[reply.chunk]) { + log("SNAPSHOT: rejected piece " + f.file.name + ":" + std::to_string(reply.chunk) + " from " + keyText(sender), true); + mark(sender, t, true, "sent a piece that does not match the manifest"); + break; + } + if (f.have[reply.chunk]) { + break; + } + const uint64_t offset = uint64_t(reply.chunk) * target_->chunkSize; + size_t done = 0; + while (done < reply.data.size()) { + const auto n = ::pwrite(f.fd, reply.data.data() + done, reply.data.size() - done, static_cast(offset + done)); + if (n <= 0) { + log("SNAPSHOT: cannot write " + (tmpDir_ / f.file.name).string() + ": syncing from block 0", true); + clearInFlight(); + phase_ = Phase::Fallback; + return; + } + done += static_cast(n); + } + f.have[reply.chunk] = true; + --f.missing; + f.doneBytes += length; + if (f.missing == 0) { + f.verifyPending = true; + } + break; + } + } + if (phase_ != Phase::Fetch) { + return; + } + fill(t); + logProgress(t, false); +} + +void SnapshotFetcher::releaseSource(const cs::PublicKey& key) { + for (auto& f : files_) { + for (auto it = f.inFlight.begin(); it != f.inFlight.end();) { + if (it->second.source == key) { + it = f.inFlight.erase(it); + } + else { + ++it; + } + } + } + if (auto* s = findSource(key)) { + s->outstanding = 0; + } +} + +void SnapshotFetcher::clearInFlight() { + for (auto& f : files_) { + f.inFlight.clear(); + } + for (auto& [key, s] : sources_) { + s.outstanding = 0; + } +} + +void SnapshotFetcher::mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why) { + auto* s = findSource(key); + if (s == nullptr) { + return; + } + if (forever) { + s->banned = true; + } + else { + ++s->marks; + s->markedUntil = t + kMarkMs; + s->banned = s->marks >= 2; + } + log("SNAPSHOT: source " + keyText(key) + " " + why + (s->banned ? ": not asked again" : ": not asked for 1 min")); + if (!s->banned || phase_ != Phase::Fetch) { + return; + } + // its other pieces go to other sources; a file left without a source restarts from another group + releaseSource(key); + for (const auto& f : files_) { + if (f.missing == 0) { + continue; + } + const bool dry = std::none_of(f.sources.begin(), f.sources.end(), [this](const cs::PublicKey& k) { + const auto* other = findSource(k); + return other != nullptr && everUsable(*other); + }); + if (dry) { + const auto name = f.file.name; + if (chooseTarget(t)) { + log("SNAPSHOT: no source left for " + name + ": switching to the files of another source group"); + } + else if (phase_ != Phase::Fallback) { + backToCollect("no source left for " + name); + } + return; + } + } +} + +void SnapshotFetcher::expire(uint64_t t) { + std::set late; + for (auto& f : files_) { + for (auto it = f.inFlight.begin(); it != f.inFlight.end();) { + if (t - it->second.sentAt >= kPieceTimeoutMs) { + late.insert(it->second.source); + if (auto* s = findSource(it->second.source); s != nullptr && s->outstanding != 0) { + --s->outstanding; + } + it = f.inFlight.erase(it); + } + else { + ++it; + } + } + } + for (const auto& key : late) { + if (phase_ != Phase::Fetch) { + break; + } + mark(key, t, false, "did not answer within " + duration(kPieceTimeoutMs / 1000)); + } +} + +void SnapshotFetcher::verifyFiles(uint64_t t) { + for (auto& f : files_) { + if (!f.verifyPending) { + continue; + } + f.verifyPending = false; + crypto_hash_sha256_state state; + crypto_hash_sha256_init(&state); + std::vector buffer(1u << 20); + uint64_t offset = 0; + bool ok = f.fd >= 0; + while (ok && offset < f.file.size) { + const auto want = static_cast(std::min(buffer.size(), f.file.size - offset)); + const auto n = ::pread(f.fd, buffer.data(), want, static_cast(offset)); + if (n <= 0) { + ok = false; + break; + } + crypto_hash_sha256_update(&state, buffer.data(), static_cast(n)); + offset += static_cast(n); + } + std::array digest{}; + crypto_hash_sha256_final(&state, digest.data()); + if (ok && hexOf(digest.data(), digest.size()) == f.file.sha256) { + f.verified = true; + continue; + } + // every piece matched its chunk hash: the file changed on disk after it was written + log("SNAPSHOT: " + f.file.name + " does not have the SHA-256 of the manifest after its last piece", true); + f.have.assign(f.file.chunks.size(), false); + f.missing = f.file.chunks.size(); + f.doneBytes = 0; + f.verified = false; + f.verifyPending = f.file.chunks.empty(); + backToCollect("a downloaded file did not verify"); + return; + } + const bool complete = std::all_of(files_.begin(), files_.end(), [](const FileState& f) { return f.verified; }); + if (complete && target_) { + logProgress(t, true); + std::set used; + for (const auto& f : files_) { + used.insert(f.sources.begin(), f.sources.end()); + } + log("SNAPSHOT: fetched block " + std::to_string(target_->seq) + ": " + megabytes(bytesTotal()) + " MB from " + + std::to_string(used.size()) + " sources in " + std::to_string((t - fetchStartedAt_) / 1000) + " s"); + for (auto& f : files_) { + closeFile(f); + } + clearInFlight(); + phase_ = Phase::Import; + } +} + +void SnapshotFetcher::backToCollect(const std::string& why) { + clearInFlight(); + target_.reset(); + agreedSources_.clear(); + agreement_.chosen.reset(); + phase_ = Phase::Collect; + log("SNAPSHOT: " + why + ": looking at the sources again", true); +} + +void SnapshotFetcher::logProgress(uint64_t t, bool force) { + const uint64_t total = bytesTotal(); + if (!target_ || total == 0) { + return; + } + const uint64_t done = bytesDone(); + const int decile = static_cast(done * 10 / total); + if (!force && decile <= lastProgressDecile_ && t - lastProgressAt_ < kLogIntervalMs) { + return; + } + const double seconds = double(t - lastProgressAt_) / 1000.0; + const double rate = seconds > 0 ? double(done - std::min(done, lastProgressBytes_)) / seconds / 1e6 : 0.0; + std::set active; + for (const auto& f : files_) { + for (const auto& k : f.sources) { + const auto* s = findSource(k); + if (s != nullptr && everUsable(*s)) { + active.insert(k); + } + } + } + char rateText[32]; + std::snprintf(rateText, sizeof(rateText), "%.1f", rate); + log("SNAPSHOT: fetching block " + std::to_string(target_->seq) + ": " + std::to_string(done * 100 / total) + " % (" + + megabytes(done) + " of " + megabytes(total) + " MB), " + std::to_string(active.size()) + " sources, " + rateText + " MB/s"); + lastProgressAt_ = t; + lastProgressBytes_ = done; + lastProgressDecile_ = std::max(lastProgressDecile_, decile); +} + +// --------------------------------------------------------------------------------------------- + +std::optional>> SnapshotFetcher::result() const { + if (phase_ != Phase::Import || !target_) { + return std::nullopt; + } + return std::make_pair(*target_, agreedSources_); +} + +void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { + if (stopped_ || phase_ != Phase::Import) { + return; + } + ++importFailures_; + if (target_) { + if (const auto* f = target_->file(fileAtFault)) { + badFiles_.insert({f->name, f->sha256}); + files_.erase(std::remove_if(files_.begin(), files_.end(), [&](FileState& s) { + if (s.file.name != fileAtFault) { + return false; + } + closeFile(s); + return true; + }), + files_.end()); + } + } + log("SNAPSHOT: the import refused " + (fileAtFault.empty() ? std::string("the snapshot") : fileAtFault) + " (attempt " + + std::to_string(importFailures_) + " of " + std::to_string(kMaxAttempts) + ")", + true); + if (importFailures_ >= kMaxAttempts) { + log("SNAPSHOT: " + std::to_string(kMaxAttempts) + " imports failed: syncing from block 0", true); + phase_ = Phase::Fallback; + return; + } + target_.reset(); + agreedSources_.clear(); + agreement_.chosen.reset(); + phase_ = Phase::Collect; +} + +std::vector SnapshotFetcher::groups() const { + return agreement_.groups; +} + +void SnapshotFetcher::stop() { + stopped_ = true; + clearInFlight(); + for (auto& f : files_) { + closeFile(f); + } +} + +size_t SnapshotFetcher::manifests() const { + return static_cast( + std::count_if(sources_.begin(), sources_.end(), [](const auto& entry) { return !entry.second.banned && entry.second.manifest; })); +} + +size_t SnapshotFetcher::outstanding() const { + size_t n = 0; + for (const auto& f : files_) { + n += f.inFlight.size(); + } + return n; +} + +uint64_t SnapshotFetcher::bytesDone() const { + uint64_t n = 0; + for (const auto& f : files_) { + n += f.doneBytes; + } + return n; +} + +uint64_t SnapshotFetcher::bytesTotal() const { + uint64_t n = 0; + for (const auto& f : files_) { + n += f.file.size; + } + return n; +} + +} // namespace cs diff --git a/tests/csnode/snapshot_agreement_tests.cpp b/tests/csnode/snapshot_agreement_tests.cpp new file mode 100644 index 000000000..c9ba0015e --- /dev/null +++ b/tests/csnode/snapshot_agreement_tests.cpp @@ -0,0 +1,176 @@ +#include + +#include +#include + +#include + +namespace { + +cs::PublicKey key(uint8_t n) { + cs::PublicKey k{}; + k.fill(n); + return k; +} + +cs::SourceManifest source(uint8_t n, const std::string& address, cs::Sequence seq, const std::string& digest = "d1", + const std::string& head = "", bool trusted = false) { + cs::SourceManifest s; + s.key = key(n); + s.address = address; + s.trusted = trusted; + s.manifest.seq = seq; + s.manifest.headHash = head.empty() ? "h" + std::to_string(seq) : head; + s.manifest.stateDigest = digest; + s.manifest.settingsDigest = "s1"; + return s; +} + +} // namespace + +TEST(SnapshotAgreement, NetPrefix) { + EXPECT_EQ(cs::netPrefix("10.1.2.3"), "10.1.2"); + EXPECT_EQ(cs::netPrefix("10.1.2.3:6000"), "10.1.2"); + EXPECT_EQ(cs::netPrefix("::ffff:10.1.2.3"), "10.1.2"); + EXPECT_EQ(cs::netPrefix("2001:db8:1:2::1"), cs::netPrefix("2001:db8:1:ffff::7")); + EXPECT_EQ(cs::netPrefix("[2001:db8:1:2::1]:6000"), cs::netPrefix("2001:db8:1::")); + EXPECT_NE(cs::netPrefix("2001:db8:1::1"), cs::netPrefix("2001:db8:2::1")); + EXPECT_EQ(cs::netPrefix("2001:db8:1::1"), "2001:db8:1::/48"); + EXPECT_EQ(cs::netPrefix(""), ""); + EXPECT_EQ(cs::netPrefix("not an address"), ""); +} + +TEST(SnapshotAgreement, Threshold) { + std::vector v{source(1, "10.0.1.1", 1000)}; + auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.shortfall, 1u); + ASSERT_EQ(r.groups.size(), 1u); + EXPECT_EQ(r.groups[0].independent, 1u); + + v.push_back(source(2, "10.0.2.1", 1000)); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_EQ(r.chosen->independent, 2u); + EXPECT_EQ(r.shortfall, 0u); + EXPECT_TRUE(r.groups[0].chosen); +} + +TEST(SnapshotAgreement, MinSourcesOne) { + const std::vector v{source(1, "10.0.1.1", 1000)}; + const auto r = cs::agree(v, 1); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->keys, std::vector{key(1)}); +} + +TEST(SnapshotAgreement, SameSlash24CountsOnce) { + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.1.200", 1000)}; + const auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.shortfall, 1u); + ASSERT_EQ(r.groups.size(), 1u); + EXPECT_EQ(r.groups[0].keys.size(), 2u); + EXPECT_EQ(r.groups[0].independent, 1u); +} + +TEST(SnapshotAgreement, TrustedKeysCountedSeparately) { + // two trusted keys behind one /24 (an operator's own machines) count twice; an untrusted one there adds one + std::vector v{source(1, "10.0.1.1", 1000, "d1", "", true), source(2, "10.0.1.2", 1000, "d1", "", true)}; + auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->independent, 2u); + v.push_back(source(3, "10.0.1.3", 1000)); + v.push_back(source(4, "10.0.1.4", 1000)); + r = cs::agree(v, 4); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.groups[0].independent, 3u); + EXPECT_EQ(r.shortfall, 1u); +} + +TEST(SnapshotAgreement, Ipv6Slash48) { + std::vector v{source(1, "2001:db8:1:1::1", 1000), source(2, "2001:db8:1:2::1", 1000)}; + auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + v.push_back(source(3, "2001:db8:2::1", 1000)); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->independent, 2u); + EXPECT_EQ(r.chosen->keys.size(), 3u); +} + +TEST(SnapshotAgreement, UnknownAddressesCountTogether) { + const std::vector v{source(1, "", 1000), source(2, "", 1000), source(3, "garbage", 1000)}; + const auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.groups[0].independent, 1u); +} + +TEST(SnapshotAgreement, SameKeyCountsOnce) { + const std::vector v{source(1, "10.0.1.1", 1000), source(1, "10.0.2.1", 1000)}; + const auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); +} + +TEST(SnapshotAgreement, HighestBlockFirst) { + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.2.1", 1000), source(3, "10.0.3.1", 1000), + source(4, "10.0.4.1", 2000), source(5, "10.0.5.1", 2000)}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 2000u); + ASSERT_EQ(r.groups.size(), 2u); + EXPECT_EQ(r.groups[0].seq, 2000u); + EXPECT_TRUE(r.groups[0].chosen); + EXPECT_FALSE(r.groups[1].chosen); +} + +TEST(SnapshotAgreement, OnlyTheLowerBlockReachesTheThreshold) { + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.2.1", 1000), source(3, "10.0.3.1", 2000)}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_EQ(r.groups.size(), 2u); +} + +TEST(SnapshotAgreement, DigestMajorityInsideABlock) { + // 3 of 4 declare one digest, the fourth another: the majority is chosen, the minority reported + const std::vector v{source(1, "10.0.1.1", 1000, "good"), source(2, "10.0.2.1", 1000, "good"), + source(3, "10.0.3.1", 1000, "good"), source(4, "10.0.4.1", 1000, "forged")}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->stateDigest, "good"); + EXPECT_EQ(r.chosen->independent, 3u); + ASSERT_EQ(r.groups.size(), 2u); + EXPECT_EQ(r.groups[1].stateDigest, "forged"); + EXPECT_FALSE(r.groups[1].chosen); + EXPECT_NE(cs::describe(r.groups[1]).find("1 source"), std::string::npos); +} + +TEST(SnapshotAgreement, DigestTieIsNotAnAgreement) { + std::vector v{source(1, "10.0.1.1", 2000, "a"), source(2, "10.0.2.1", 2000, "a"), + source(3, "10.0.3.1", 2000, "b"), source(4, "10.0.4.1", 2000, "b")}; + auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.shortfall, 0u); // groups are large enough, they conflict + // a lower block that agrees is taken instead + v.push_back(source(5, "10.0.5.1", 1000)); + v.push_back(source(6, "10.0.6.1", 1000)); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); +} + +TEST(SnapshotAgreement, DifferentHeadsAreDifferentBlocks) { + const std::vector v{source(1, "10.0.1.1", 1000, "d1", "x"), source(2, "10.0.2.1", 1000, "d1", "y")}; + const auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.groups.size(), 2u); + EXPECT_EQ(r.shortfall, 1u); +} + +TEST(SnapshotAgreement, Shortfall) { + const std::vector v{source(1, "10.0.1.1", 1000)}; + EXPECT_EQ(cs::agree(v, 3).shortfall, 2u); + EXPECT_EQ(cs::agree({}, 2).shortfall, 2u); + EXPECT_TRUE(cs::agree({}, 2).groups.empty()); +} diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp new file mode 100644 index 000000000..19414d029 --- /dev/null +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -0,0 +1,846 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; +using cs::SnapshotFetcher; +using cs::snapshot_p2p::ManifestReply; +using cs::snapshot_p2p::ManifestRequest; +using cs::snapshot_p2p::PieceReply; +using cs::snapshot_p2p::PieceRequest; +using cs::snapshot_p2p::PieceStatus; +using Phase = SnapshotFetcher::Phase; + +const std::string kWallets = "qs-walletscache.dat.lz4"; +const std::string kBlocks = "blocks.bin.lz4"; + +std::string fileText(const fs::path& p) { + std::ifstream f(p, std::ios::binary); + return std::string(std::istreambuf_iterator(f), std::istreambuf_iterator()); +} + +void putU16(std::ofstream& f, uint16_t v) { + f.write(reinterpret_cast(&v), sizeof(v)); +} + +// Five snapshots of block 200 built by buildSnapshot from one generated chain, each signed by its +// own key, with 64 KiB chunks: A1 and A2 from the same checkpoint files (identical bytes), B and +// E with one and two trailing bytes in walletscache.dat (same digest, another sha256 for that +// file), D with another state digest. Built once for all tests. +struct Snapshots { + enum { A1, A2, B, D, E, Count }; + + struct Source { + cs::PrivateKey priv; + cs::PublicKey pub{}; + fs::path dir; + cs::SnapshotManifest manifest; + std::string json; + }; + + fs::path root; + std::array src; + + Snapshots() : root(fs::temp_directory_path() / "snapshot_fetcher_sources") { + fs::remove_all(root); + fs::create_directories(root); + EXPECT_TRUE(cscrypto::cryptoInit()); + + std::map blocks; + cs::Bytes head; + csdb::PoolHash prev; + for (cs::Sequence s = 0; s <= 200; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(100 + s % 50, static_cast('a' + s % 26))); + EXPECT_TRUE(p.compose()); + blocks[s] = p.to_binary(); + head = p.hash().to_binary(); + prev = p.hash(); + } + std::mt19937 rng(11); + std::string noise(1'000'000, '\0'); + for (auto& c : noise) { + c = static_cast(rng()); + } + + auto checkpoint = [&](const std::string& name, size_t extra, uint8_t digestFill) { + const auto qs = root / ("qs-" + name) / "200"; + fs::create_directories(qs); + { + std::ofstream f(qs / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t s = 200; + f.write(reinterpret_cast(&magic), sizeof(magic)); + f.write(reinterpret_cast(&s), sizeof(s)); + putU16(f, static_cast(head.size())); + f.write(reinterpret_cast(head.data()), static_cast(head.size())); + putU16(f, 0); + } + std::ofstream(qs / "walletscache.dat", std::ios::binary) << noise << std::string(extra, 'x'); + std::ofstream(qs / "blockchain.dat", std::ios::binary) << "settings"; + cs::CheckpointDigest d; + d.digest.fill(digestFill); + d.settings.fill(9); + d.wallets = 42; + EXPECT_TRUE(d.write(qs / cs::CheckpointDigest::kFileName)); + return qs; + }; + const auto qsA = checkpoint("a", 0, 7); + const auto qsB = checkpoint("b", 1, 7); + const auto qsD = checkpoint("d", 0, 8); + const auto qsE = checkpoint("e", 2, 7); + const std::array qsOf{qsA, qsA, qsB, qsD, qsE}; + + for (size_t i = 0; i < Count; ++i) { + auto& s = src[i]; + s.priv = cscrypto::generateKeyPair(s.pub); + cs::SnapshotBuildInput in; + in.seq = 200; + in.checkpointDir = qsOf[i]; + in.snapshotBlocks = 150; + in.nodeVersion = "test"; + in.chunkSize = 64 << 10; + in.blockBytes = [&blocks](cs::Sequence seq, cs::Bytes& out) { + auto it = blocks.find(seq); + if (it == blocks.end()) return false; + out = it->second; + return true; + }; + const auto r = cs::buildSnapshot(in, root / ("out" + std::to_string(i)), s.priv); + EXPECT_TRUE(r.ok) << r.error; + s.dir = r.dir; + s.json = fileText(s.dir / "snapshot.json"); + s.manifest = *cs::SnapshotManifest::fromJson(s.json); + } + } + + ~Snapshots() { + std::error_code ec; + fs::remove_all(root, ec); + } +}; + +const Snapshots& snapshots() { + static Snapshots s; + return s; +} + +cs::PublicKey otherKey(uint8_t n) { + cs::PublicKey k{}; + k.fill(n); + return k; +} + +std::string base58(const cs::PublicKey& k) { + return EncodeBase58(k.data(), k.data() + k.size()); +} + +// The joining node's network: the fetcher's hooks lead to real SnapshotProvider instances, one per +// source, that share the test's clock. step() delivers what the fetcher sent since the last step, +// advances the clock and ticks. +struct Net { + struct Source { + size_t index = 0; + std::unique_ptr provider; + std::string address; + bool stall = false; // never answers a piece request + bool corruptAll = false; // flips a byte of every piece + bool busyAll = false; // answers every piece request busy + uint32_t busyMs = 2000; + std::optional manifestJson; // replaces the manifest it serves + size_t outstanding = 0; // piece requests not answered yet + }; + + const cs::PublicKey me = otherKey(0xEE); + uint64_t nowMs = 10'000'000; + std::map sources; + SnapshotFetcher::Peers peers; + std::vector> manifestQueue; + std::vector> pieceQueue; + std::vector> addCalls; + std::vector logs; + std::vector> sent; + std::vector> busySent; // busy replies: when, from whom + size_t maxOutstandingTotal = 0; + size_t maxOutstandingPerSource = 0; + size_t outstandingTotal = 0; + size_t manifestRequests = 0; + + SnapshotFetcher::Hooks hooks() { + SnapshotFetcher::Hooks h; + h.sendManifestRequest = [this](const cs::PublicKey& k, const ManifestRequest& r) { + manifestQueue.emplace_back(k, r); + ++manifestRequests; + }; + h.sendPieceRequest = [this](const cs::PublicKey& k, const PieceRequest& r) { + pieceQueue.emplace_back(k, r); + sent.emplace_back(nowMs, k, r); + if (auto it = sources.find(k); it != sources.end()) { + ++it->second.outstanding; + maxOutstandingPerSource = std::max(maxOutstandingPerSource, it->second.outstanding); + } + ++outstandingTotal; + maxOutstandingTotal = std::max(maxOutstandingTotal, outstandingTotal); + }; + h.addNeighbours = [this](const std::set& keys) { addCalls.push_back(keys); }; + h.addressOf = [this](const cs::PublicKey& k) { + auto it = sources.find(k); + return it == sources.end() ? std::string() : it->second.address; + }; + h.clock = [this]() { return nowMs; }; + h.onLog = [this](const std::string& line) { logs.push_back(line); }; + return h; + } + + const cs::PublicKey& key(size_t index) const { + return snapshots().src[index].pub; + } + + Source& add(size_t index, bool neighbour = true) { + const auto& k = key(index); + auto& s = sources[k]; + s.index = index; + s.address = "10.0." + std::to_string(index + 1) + ".1"; + s.provider = std::make_unique( + [this]() { return std::chrono::steady_clock::time_point{} + std::chrono::milliseconds(nowMs); }); + EXPECT_TRUE(s.provider->load(snapshots().src[index].dir)); + peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + if (neighbour) { + peers.neighbours.insert(k); + } + return s; + } + + static cs::SnapshotProvider::Settings serving() { + cs::SnapshotProvider::Settings s; + s.serve = true; + s.rateKBps = 0; + return s; + } + + static cs::SnapshotNodeState synced() { + cs::SnapshotNodeState s; + s.lastSeq = 1000; + s.round = 1001; + s.synced = true; + return s; + } + + void deliver(SnapshotFetcher& fetcher) { + auto manifests = std::move(manifestQueue); + manifestQueue.clear(); + for (auto& [k, request] : manifests) { + auto it = sources.find(k); + if (it == sources.end()) { + continue; + } + auto reply = it->second.provider->onManifestRequest(me, serving(), synced()); + if (!reply) { + continue; + } + if (it->second.manifestJson && reply->status == cs::snapshot_p2p::ManifestStatus::Ready) { + reply->manifestJson = *it->second.manifestJson; + } + fetcher.onManifestReply(k, *reply); + } + auto pieces = std::move(pieceQueue); + pieceQueue.clear(); + for (auto& [k, request] : pieces) { + auto it = sources.find(k); + if (it == sources.end()) { + continue; + } + auto& s = it->second; + if (s.stall) { + continue; + } + --s.outstanding; + --outstandingTotal; + PieceReply reply; + if (s.busyAll) { + reply.status = PieceStatus::Busy; + reply.retryMs = s.busyMs; + reply.seq = request.seq; + reply.name = request.name; + reply.chunk = request.chunk; + busySent.emplace_back(nowMs, k); + } + else { + reply = s.provider->onPieceRequest(me, request, serving(), synced()); + if (reply.status == PieceStatus::Busy) { + busySent.emplace_back(nowMs, k); + } + if (s.corruptAll && reply.status == PieceStatus::Ok && !reply.data.empty()) { + reply.data[0] ^= 0x5a; + } + } + fetcher.onPieceReply(k, std::move(reply)); + } + } + + void step(SnapshotFetcher& fetcher, uint64_t ms = 100) { + deliver(fetcher); + nowMs += ms; + fetcher.tick(peers); + } + + bool runUntil(SnapshotFetcher& fetcher, const std::function& done, size_t maxSteps = 5000) { + for (size_t i = 0; i < maxSteps; ++i) { + if (done()) { + return true; + } + step(fetcher); + } + return done(); + } + + bool logged(const std::string& text) const { + return std::any_of(logs.begin(), logs.end(), [&](const std::string& l) { return l.find(text) != std::string::npos; }); + } + + size_t countLogged(const std::string& text) const { + return static_cast( + std::count_if(logs.begin(), logs.end(), [&](const std::string& l) { return l.find(text) != std::string::npos; })); + } + + size_t sentTo(const cs::PublicKey& k) const { + return static_cast(std::count_if(sent.begin(), sent.end(), [&](const auto& s) { return std::get<1>(s) == k; })); + } +}; + +SnapshotData fetchConfig(size_t minSources = 2) { + SnapshotData c; + c.minSources = minSources; + c.sourcesFile.clear(); + return c; +} + +struct TmpDir { + fs::path path; + explicit TmpDir(const std::string& name) : path(fs::temp_directory_path() / ("snapshot_fetcher_" + name)) { + fs::remove_all(path); + } + ~TmpDir() { + std::error_code ec; + fs::remove_all(path, ec); + } +}; + +// the fetched files are the manifest's, byte for byte +void expectFilesOf(const fs::path& dir, const cs::SnapshotManifest& m) { + for (const auto& f : m.files) { + const auto bytes = fileText(dir / f.name); + EXPECT_EQ(bytes.size(), f.size) << f.name; + EXPECT_EQ(cs::sha256Hex(reinterpret_cast(bytes.data()), bytes.size()), f.sha256) << f.name; + } +} + +bool isFetching(const SnapshotFetcher& f) { + return f.phase() == Phase::Fetch; +} + +} // namespace + +// (1) discovery: peers with bit 24, [snapshot] sources and sources_file keys, nothing else +TEST(SnapshotFetcher, DiscoveryAddsServingPeersAndConfiguredKeys) { + TmpDir tmp("discovery"); + fs::create_directories(tmp.path); + Net net; + const auto trusted = otherKey(1); + const auto listed = otherKey(2); + const auto official = otherKey(3); + const auto otherBit = otherKey(4); + const auto sourcesFile = tmp.path / "snapshot_sources.txt"; + std::ofstream(sourcesFile) << "# serving nodes\n1.2.3.4:6000 " << base58(listed) << "\n\n5.6.7.8:6000 not-a-key\n"; + + auto cfg = fetchConfig(); + cfg.sources = {trusted}; + cfg.sourcesFile = sourcesFile.string(); + net.add(Snapshots::A1, false); + net.add(Snapshots::A2, false); + net.peers.known.emplace_back(official, 0); + net.peers.known.emplace_back(otherBit, 2); + SnapshotFetcher fetcher(cfg, tmp.path / "tmp", net.hooks()); + + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 1u); + const std::set expected{trusted, listed, net.key(Snapshots::A1), net.key(Snapshots::A2)}; + EXPECT_EQ(net.addCalls[0], expected); + EXPECT_TRUE(net.manifestQueue.empty()); // none is a neighbour yet + EXPECT_EQ(fetcher.phase(), Phase::Discover); + EXPECT_TRUE(net.logged("not a node key")); + + // once per key; a key that became a neighbour is asked for its manifest + net.peers.neighbours.insert(net.key(Snapshots::A1)); + net.nowMs += 500; + fetcher.tick(net.peers); + EXPECT_EQ(net.addCalls.size(), 1u); + ASSERT_EQ(net.manifestQueue.size(), 1u); + EXPECT_EQ(net.manifestQueue[0].first, net.key(Snapshots::A1)); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("asking " + base58(net.key(Snapshots::A1)))); + + // the others are brought in again after a minute; the neighbour is not + net.nowMs += SnapshotFetcher::kNeighbourRetryMs; + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 2u); + EXPECT_EQ(net.addCalls[1].count(net.key(Snapshots::A1)), 0u); + EXPECT_EQ(net.addCalls[1].size(), 3u); +} + +// (2) bad signature, another node's manifest, an oversized chunk, an unsolicited reply: dropped +TEST(SnapshotFetcher, BadManifestsAreDropped) { + TmpDir tmp("badmanifests"); + Net net; + const auto& snaps = snapshots(); + auto& a1 = net.add(Snapshots::A1); + auto& a2 = net.add(Snapshots::A2); + auto& b = net.add(Snapshots::B); + + auto forged = snaps.src[Snapshots::A1].manifest; + forged.wallets += 1; // signature no longer matches + a1.manifestJson = forged.toJson(); + a2.manifestJson = snaps.src[Snapshots::A1].json; // relayed: signed by A1 + auto big = snaps.src[Snapshots::B].manifest; + big.chunkSize = 2u << 20; + for (auto& f : big.files) { + f.chunks.assign(f.size == 0 ? 0 : 1, f.sha256); + } + big.sign(snaps.src[Snapshots::B].priv); + ASSERT_TRUE(cs::SnapshotManifest::fromJson(big.toJson())); + b.manifestJson = big.toJson(); + + SnapshotFetcher fetcher(fetchConfig(1), tmp.path, net.hooks()); + fetcher.onManifestReply(net.key(Snapshots::A1), ManifestReply{}); // not asked yet + EXPECT_EQ(fetcher.dropped(), 1u); + fetcher.tick(net.peers); + net.step(fetcher); + EXPECT_EQ(fetcher.manifests(), 0u); + EXPECT_EQ(fetcher.dropped(), 4u); + EXPECT_TRUE(net.logged("dropped: signature does not verify")); + EXPECT_TRUE(net.logged("dropped: signed by another node")); + EXPECT_TRUE(net.logged("dropped: chunk size")); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + + // the same sources with their own manifests are taken at the next request (30 s later) + a1.manifestJson.reset(); + net.nowMs += SnapshotFetcher::kManifestIntervalMs; + fetcher.tick(net.peers); + net.deliver(fetcher); + EXPECT_EQ(fetcher.manifests(), 1u); + fetcher.tick(net.peers); // agreement is tried at the tick + EXPECT_NE(fetcher.phase(), Phase::Collect); // min_sources=1: agreed +} + +// (3) three sources of four agree, the fourth (another digest) is reported as minority +TEST(SnapshotFetcher, AgreementOfThreeOfFour) { + TmpDir tmp("agreement"); + Net net; + net.add(Snapshots::D); + SnapshotFetcher fetcher(fetchConfig(3), tmp.path, net.hooks()); + net.step(fetcher); + net.step(fetcher); + EXPECT_EQ(fetcher.manifests(), 1u); + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() != Phase::Collect; }, 20)); + EXPECT_EQ(fetcher.phase(), Phase::Fetch); + + const auto groups = fetcher.groups(); + ASSERT_EQ(groups.size(), 2u); + const auto chosen = std::find_if(groups.begin(), groups.end(), [](const auto& g) { return g.chosen; }); + ASSERT_NE(chosen, groups.end()); + EXPECT_EQ(chosen->independent, 3u); + const std::set keys(chosen->keys.begin(), chosen->keys.end()); + EXPECT_EQ(keys, (std::set{net.key(Snapshots::A1), net.key(Snapshots::A2), net.key(Snapshots::B)})); + EXPECT_TRUE(net.logged("minority: block 200")); + EXPECT_EQ(net.sentTo(net.key(Snapshots::D)), 0u); +} + +// (4) pieces go round-robin, at most 2 per source and 8 in all; each file only from sources that +// declare its bytes; the result is the files of the largest group +TEST(SnapshotFetcher, PiecesSpreadAndBounded) { + TmpDir tmp("spread"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + + // the first round already uses every source that can serve + std::set first; + for (const auto& s : net.sent) { + first.insert(std::get<1>(s)); + } + EXPECT_EQ(first.size(), 4u); + + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_LE(net.maxOutstandingPerSource, SnapshotFetcher::kPiecesPerSource); + EXPECT_LE(net.maxOutstandingTotal, SnapshotFetcher::kPiecesTotal); + EXPECT_EQ(net.maxOutstandingTotal, SnapshotFetcher::kPiecesTotal); + + const auto result = fetcher.result(); + ASSERT_TRUE(result); + const auto& m = result->first; + EXPECT_EQ(m.filesHash(), snapshots().src[Snapshots::A1].manifest.filesHash()); + EXPECT_EQ(result->second.size(), 4u); + expectFilesOf(tmp.path, m); + for (const auto& s : net.sent) { + const auto& k = std::get<1>(s); + if (std::get<2>(s).name == kWallets) { + EXPECT_TRUE(k == net.key(Snapshots::A1) || k == net.key(Snapshots::A2)); + } + } + // B and E, whose walletscache differs, served the files they share with A + EXPECT_GT(net.sentTo(net.key(Snapshots::B)), 0u); + EXPECT_GT(net.sentTo(net.key(Snapshots::E)), 0u); + EXPECT_TRUE(net.logged("SNAPSHOT: fetching block 200: 100 %")); + EXPECT_TRUE(net.logged("SNAPSHOT: fetched block 200")); +} + +// (5) a corrupt piece marks its source for the bootstrap; the piece comes from another source +TEST(SnapshotFetcher, CorruptPieceMarksTheSource) { + TmpDir tmp("corrupt"); + Net net; + net.add(Snapshots::A1).corruptAll = true; + net.add(Snapshots::A2); + net.add(Snapshots::B); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_TRUE(net.logged("rejected piece")); + EXPECT_TRUE(net.logged(base58(net.key(Snapshots::A1)) + " sent a piece that does not match")); + EXPECT_LE(net.sentTo(net.key(Snapshots::A1)), SnapshotFetcher::kPiecesPerSource); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (6) a stalled source times out after 30 s, its pieces go elsewhere, it is marked +TEST(SnapshotFetcher, StalledSourceTimesOut) { + TmpDir tmp("stall"); + Net net; + net.add(Snapshots::A1).stall = true; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_GE(net.nowMs - start, SnapshotFetcher::kPieceTimeoutMs); + EXPECT_TRUE(net.logged(base58(net.key(Snapshots::A1)) + " did not answer within 30 s")); + EXPECT_GT(net.sentTo(net.key(Snapshots::A1)), 0u); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (7) busy delays only that source, by retry_ms +TEST(SnapshotFetcher, BusyDelaysOnlyThatSource) { + TmpDir tmp("busy"); + Net net; + auto& a1 = net.add(Snapshots::A1); + a1.busyAll = true; + a1.busyMs = 2000; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + const auto a1Key = net.key(Snapshots::A1); + size_t a1Sends = 0; + for (const auto& s : net.sent) { + if (std::get<1>(s) != a1Key) { + continue; + } + ++a1Sends; + const uint64_t at = std::get<0>(s); + for (const auto& [when, who] : net.busySent) { + if (who == a1Key) { + EXPECT_FALSE(when < at && at < when + 2000) << "asked " << at - when << " ms after a busy reply"; + } + } + } + EXPECT_GE(a1Sends, 2u); + EXPECT_GT(net.sentTo(net.key(Snapshots::A2)), a1Sends); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (8) a source that leaves the neighbour set is not asked +TEST(SnapshotFetcher, SourceThatLeftIsNotAsked) { + TmpDir tmp("left"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + net.peers.neighbours.erase(net.key(Snapshots::A1)); + fetcher.tick(net.peers); + const size_t before = net.sentTo(net.key(Snapshots::A1)); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(net.sentTo(net.key(Snapshots::A1)), before); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (9) a file whose group runs dry restarts from the other group +TEST(SnapshotFetcher, DryGroupSwitchesToTheOther) { + TmpDir tmp("dry"); + Net net; + net.add(Snapshots::A1).corruptAll = true; + net.add(Snapshots::A2).corruptAll = true; + net.add(Snapshots::B); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_TRUE(net.logged("switching to the files of another source group")); + const auto result = fetcher.result(); + ASSERT_TRUE(result); + EXPECT_EQ(result->first.nodeKey, snapshots().src[Snapshots::B].manifest.nodeKey); + expectFilesOf(tmp.path, result->first); +} + +// (10) a file that changes on disk between its last piece and the whole-file check: back to Collect +TEST(SnapshotFetcher, WholeFileMismatchGoesBackToCollect) { + TmpDir tmp("wholefile"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + // up to the last piece; the whole-file check of the last file runs at the next tick + for (size_t i = 0; i < 5000; ++i) { + net.deliver(fetcher); + if (fetcher.bytesTotal() != 0 && fetcher.bytesDone() == fetcher.bytesTotal()) { + break; + } + net.nowMs += 100; + fetcher.tick(net.peers); + } + ASSERT_EQ(fetcher.bytesDone(), fetcher.bytesTotal()); + ASSERT_EQ(fetcher.phase(), Phase::Fetch); + for (const auto& entry : fs::directory_iterator(tmp.path)) { + if (fs::file_size(entry.path()) == 0) { + continue; + } + std::fstream f(entry.path(), std::ios::in | std::ios::out | std::ios::binary); + char c = 0; + f.read(&c, 1); + c ^= 0x01; + f.seekp(0); + f.write(&c, 1); + } + net.nowMs += 100; + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("does not have the SHA-256 of the manifest after its last piece")); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (11) fetch_wait over without agreement: sync falls back, wait keeps waiting with a line every 5 min +TEST(SnapshotFetcher, FetchWaitSync) { + TmpDir tmp("waitsync"); + Net net; + net.add(Snapshots::A1); + auto cfg = fetchConfig(); + cfg.fetchWait = 60; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 700)); + EXPECT_GE(net.nowMs - 10'000'000, 60'000u); + EXPECT_TRUE(net.logged("SNAPSHOT: 1 source declares block 200 (need 2): waiting")); + EXPECT_TRUE(net.logged("SNAPSHOT: no agreement of 2 community sources after 1 min (seen: 1 source for block 200). Syncing from block 0")); + EXPECT_TRUE(net.logged("./node --download-snapshot")); + const auto sends = net.manifestRequests + net.sent.size(); + for (int i = 0; i < 1000; ++i) { + net.step(fetcher); + } + EXPECT_EQ(net.manifestRequests + net.sent.size(), sends); +} + +TEST(SnapshotFetcher, FetchWaitWait) { + TmpDir tmp("waitwait"); + Net net; + net.add(Snapshots::A1); + auto cfg = fetchConfig(); + cfg.fetchWait = 60; + cfg.fetchFallback = "wait"; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + for (int i = 0; i < 6 * 60 * 10 + 10; ++i) { + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_EQ(net.countLogged("Waiting for more sources (fetch_fallback=wait)"), 2u); + // a second source turns up + net.add(Snapshots::A2); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// (12) a refused import marks that file's group; the next attempt takes another group; after +// three refusals the node syncs from block 0 +TEST(SnapshotFetcher, ImportFailures) { + TmpDir tmp("import"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + std::set walletHashes; + for (int attempt = 1; attempt <= 3; ++attempt) { + const size_t sentBefore = net.sent.size(); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })) << attempt; + const auto result = fetcher.result(); + ASSERT_TRUE(result); + expectFilesOf(tmp.path, result->first); + const auto* wallets = result->first.file(kWallets); + ASSERT_NE(wallets, nullptr); + EXPECT_TRUE(walletHashes.insert(wallets->sha256).second) << "the refused file came back"; + if (attempt > 1) { + // files already verified are not asked again + for (size_t i = sentBefore; i < net.sent.size(); ++i) { + EXPECT_EQ(std::get<2>(net.sent[i]).name, kWallets); + } + } + fetcher.onImportFailed(kWallets); + EXPECT_EQ(fetcher.phase(), attempt < 3 ? Phase::Collect : Phase::Fallback); + EXPECT_FALSE(fetcher.result()); + } + EXPECT_TRUE(net.logged("3 imports failed")); +} + +// (13) a dry run stops after Collect and reports the groups +TEST(SnapshotFetcher, DryRunReportsGroups) { + TmpDir tmp("dryrun"); + Net net; + net.add(Snapshots::D); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks(), true); + net.step(fetcher); + net.step(fetcher); + net.add(Snapshots::A1); + net.add(Snapshots::A2); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Done; }, 20)); + EXPECT_TRUE(net.sent.empty()); + const auto groups = fetcher.groups(); + EXPECT_EQ(groups.size(), 2u); + const auto chosen = std::count_if(groups.begin(), groups.end(), [](const cs::GroupReport& g) { return g.chosen; }); + EXPECT_EQ(chosen, 1); + EXPECT_FALSE(fetcher.result()); + EXPECT_FALSE(fs::exists(tmp.path)); + + // a dry run without agreement ends at fetch_wait too + Net lone; + lone.add(Snapshots::A1); + auto cfg = fetchConfig(); + cfg.fetchWait = 10; + SnapshotFetcher alone(cfg, tmp.path, lone.hooks(), true); + ASSERT_TRUE(lone.runUntil(alone, [&] { return alone.phase() == Phase::Done; }, 200)); + EXPECT_EQ(alone.groups().size(), 1u); +} + +// (14) stop in every phase: nothing outstanding, nothing sent, replies dropped +TEST(SnapshotFetcher, StopInEveryPhase) { + TmpDir tmp("stop"); + auto quietAfterStop = [](Net& net, SnapshotFetcher& fetcher) { + fetcher.stop(); + EXPECT_EQ(fetcher.outstanding(), 0u); + const auto manifests = net.manifestRequests; + const auto pieces = net.sent.size(); + const auto dropped = fetcher.dropped(); + const auto queued = net.pieceQueue.size() + net.manifestQueue.size(); + net.deliver(fetcher); // replies to what was asked before the stop + EXPECT_EQ(fetcher.dropped(), dropped + queued); + for (int i = 0; i < 1300; ++i) { // past every timer: piece timeout, marks, manifest interval, stall + net.nowMs += 100; + fetcher.tick(net.peers); + } + EXPECT_EQ(net.manifestRequests, manifests); + EXPECT_EQ(net.sent.size(), pieces); + EXPECT_EQ(fetcher.outstanding(), 0u); + }; + { + Net net; + net.add(Snapshots::A1, false); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Discover); + quietAfterStop(net, fetcher); + EXPECT_EQ(fetcher.phase(), Phase::Discover); + } + { + Net net; + net.add(Snapshots::A1); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + quietAfterStop(net, fetcher); + EXPECT_EQ(fetcher.manifests(), 0u); + EXPECT_EQ(fetcher.dropped(), 1u); // the manifest reply + } + { + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher) && fetcher.outstanding() != 0; }, 20)); + quietAfterStop(net, fetcher); + } + { + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + quietAfterStop(net, fetcher); + fetcher.onImportFailed(kWallets); + EXPECT_EQ(fetcher.phase(), Phase::Import); + } +} + +// (15) fetch_rate_kbps caps the bytes requested +TEST(SnapshotFetcher, RateCap) { + TmpDir tmp("rate"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + auto cfg = fetchConfig(); + cfg.fetchRateKbps = 200; // 200,000 bytes/s + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + + const auto& m = snapshots().src[Snapshots::A1].manifest; + auto length = [&](const PieceRequest& r) { + const auto* f = m.file(r.name); + const uint64_t offset = uint64_t(r.chunk) * m.chunkSize; + return std::min(m.chunkSize, f->size - offset); + }; + std::map perSecond; + uint64_t total = 0; + for (const auto& s : net.sent) { + perSecond[std::get<0>(s) / 1000] += length(std::get<2>(s)); + total += length(std::get<2>(s)); + } + ASSERT_GE(perSecond.size(), 4u); + for (const auto& [second, bytes] : perSecond) { + // a request goes out when nothing is owed: at most one piece above the rate in any second + EXPECT_LE(bytes, 200'000u + m.chunkSize) << "second " << second; + } + const uint64_t span = std::get<0>(net.sent.back()) - std::get<0>(net.sent.front()); + EXPECT_LE(total, 200 * span + m.chunkSize); +} From 816c6a48461decb0da3a15fc57c7f63134aa04ff Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 15:10:50 +0100 Subject: [PATCH 2/9] snapshot fetcher: support over block height in agreement, bounded source table Agreement: among the groups that reach min_sources, a group with a configured key first, then the most independent sources, then the highest block (a tie-breaker only), so that min_sources forged sources at a made-up later block lose to a larger honest group; equal groups are a conflict. Every group not chosen carries the reason, logged with the minority. Source table: configured keys (sources=, sources_file) have their own room; discovered peers are limited to 4 per /24 (/48) and 64 in all; when full, the oldest entry of the most represented prefix makes room, never one with pieces outstanding or in the agreed group during a fetch. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_agreement.hpp | 17 ++-- csnode/include/csnode/snapshot_fetcher.hpp | 9 +- csnode/src/snapshot_agreement.cpp | 87 ++++++++++++-------- csnode/src/snapshot_fetcher.cpp | 73 ++++++++++++---- tests/csnode/snapshot_agreement_tests.cpp | 52 ++++++++++-- tests/csnode/snapshot_fetcher_tests.cpp | 77 ++++++++++++++++- 6 files changed, 246 insertions(+), 69 deletions(-) diff --git a/csnode/include/csnode/snapshot_agreement.hpp b/csnode/include/csnode/snapshot_agreement.hpp index 28741a8d7..18a4f9e09 100644 --- a/csnode/include/csnode/snapshot_agreement.hpp +++ b/csnode/include/csnode/snapshot_agreement.hpp @@ -36,23 +36,28 @@ struct AgreementGroup { std::vector sources; // indices into agree()'s input std::vector keys; // their keys, same order size_t independent = 0; // counting sources: one per /24 or /48, trusted keys once each + bool configured = false; // holds a key of [snapshot] sources bool chosen = false; + std::string reason; // why it was not chosen }; using GroupReport = AgreementGroup; struct AgreementResult { std::optional chosen; - std::vector groups; // every group, highest seq first, then most sources + std::vector groups; // every group, strongest first size_t shortfall = 0; // counting sources the best group lacks (0 when chosen) }; -// Groups by (seq, head_hash), inside by (state_digest, settings_digest). Candidate blocks have at -// least minSources counting sources, highest seq first; in a candidate block the digest group with -// most counting sources is chosen if it reaches minSources and no other group of that block has as -// many (a tie is a conflict: that block is skipped). Sources are counted by netPrefix(address), +// Groups by (seq, head_hash, state_digest, settings_digest). Among the groups with at least +// minSources counting sources, the chosen one is, in this order: a group holding a key of +// [snapshot] sources (trusted), then the group with most counting sources, then the highest seq +// (only a tie-breaker: a later block never wins over more support, so that min_sources forged +// sources at a made-up later block do not beat a larger honest group; an honest network's newest +// snapshot gains sources as the nodes rebuild, and the fetcher asks again every 30 s). Two groups +// equal on all three are a conflict: nothing is chosen. Sources are counted by netPrefix(address), // trusted ones by key; sources whose address does not parse count together as one; a key that -// appears twice counts once. +// appears twice counts once. Every group that is not chosen carries the reason. AgreementResult agree(const std::vector& sources, size_t minSources); // "block S hash H digest D : N sources (keys k1 k2 …)" with shortened hashes and keys diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp index ccf6773f6..b98405855 100644 --- a/csnode/include/csnode/snapshot_fetcher.hpp +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -61,7 +61,8 @@ class SnapshotFetcher { static constexpr uint64_t kStallMs = 120'000; static constexpr size_t kPiecesPerSource = 2; static constexpr size_t kPiecesTotal = 8; - static constexpr size_t kMaxSources = 64; + static constexpr size_t kMaxSources = 64; // discovered sources; configured keys come on top + static constexpr size_t kMaxPerPrefix = 4; // discovered sources per /24 (IPv4) or /48 (IPv6) static constexpr size_t kMaxAttempts = 3; static constexpr size_t kMaxSourcesFileKeys = 64; @@ -100,6 +101,8 @@ class SnapshotFetcher { bool configured = false; // [snapshot] sources or sources_file bool trusted = false; // [snapshot] sources bool serving = false; // announced capability bit 24 + std::string prefix; // netPrefix of the address when discovered + uint64_t discoveredAt = 0; uint64_t addedAt = 0; // last addNeighbours bool added = false; uint64_t askedAt = 0; @@ -138,7 +141,9 @@ class SnapshotFetcher { uint64_t now() const; std::string keyText(const cs::PublicKey& key) const; Source* findSource(const cs::PublicKey& key); - Source* addSource(const cs::PublicKey& key); + Source* addConfigured(const cs::PublicKey& key); + Source* addDiscovered(const cs::PublicKey& key, uint64_t t); + bool isProtected(const cs::PublicKey& key, const Source& s) const; void loadSourcesFile(); void discover(const Peers& peers, uint64_t t); diff --git a/csnode/src/snapshot_agreement.cpp b/csnode/src/snapshot_agreement.cpp index 4b4ae05e8..ac5090d31 100644 --- a/csnode/src/snapshot_agreement.cpp +++ b/csnode/src/snapshot_agreement.cpp @@ -83,16 +83,7 @@ AgreementResult agree(const std::vector& sources, size_t minSour blocks[{m.seq, m.headHash}][{m.stateDigest, m.settingsDigest}].push_back(i); } - struct Block { - BlockKey key; - size_t independent = 0; - std::vector groups; - }; - std::vector ordered; for (const auto& [blockKey, digests] : blocks) { - Block b; - b.key = blockKey; - std::vector all; for (const auto& [digestKey, indices] : digests) { AgreementGroup g; g.seq = blockKey.first; @@ -102,44 +93,65 @@ AgreementResult agree(const std::vector& sources, size_t minSour g.sources = indices; for (auto i : indices) { g.keys.push_back(sources[i].key); + g.configured = g.configured || sources[i].trusted; } g.independent = countUnits(indices, units); - all.insert(all.end(), indices.begin(), indices.end()); - b.groups.push_back(std::move(g)); + result.groups.push_back(std::move(g)); } - b.independent = countUnits(all, units); - std::stable_sort(b.groups.begin(), b.groups.end(), - [](const AgreementGroup& x, const AgreementGroup& y) { return x.independent > y.independent; }); - ordered.push_back(std::move(b)); } - std::stable_sort(ordered.begin(), ordered.end(), [](const Block& x, const Block& y) { - return x.key.first != y.key.first ? x.key.first > y.key.first : x.independent > y.independent; - }); - for (auto& b : ordered) { - if (b.independent < minSources) { - continue; + // The rule (see the header): among the groups that reach minSources, a group with a configured + // key first, then the most independent sources, then the highest block. A higher block alone + // never wins over more support: min_sources made-up sources at a forged later block lose to a + // larger honest group at an earlier one. Equal on all three: a conflict, nothing is chosen. + auto stronger = [](const AgreementGroup& x, const AgreementGroup& y) { + if (x.configured != y.configured) return x.configured; + if (x.independent != y.independent) return x.independent > y.independent; + return x.seq > y.seq; + }; + auto equal = [](const AgreementGroup& x, const AgreementGroup& y) { + return x.configured == y.configured && x.independent == y.independent && x.seq == y.seq; + }; + std::stable_sort(result.groups.begin(), result.groups.end(), stronger); + + const AgreementGroup* best = nullptr; + for (const auto& g : result.groups) { + if (g.independent >= minSources) { + best = &g; + break; } - auto& top = b.groups.front(); - if (top.independent < minSources) { - continue; + } + const bool conflict = best != nullptr && std::any_of(result.groups.begin(), result.groups.end(), [&](const AgreementGroup& g) { + return &g != best && g.independent >= minSources && equal(g, *best); + }); + for (auto& g : result.groups) { + if (g.independent < minSources) { + g.reason = "fewer than " + std::to_string(minSources) + " independent sources"; + } + else if (conflict && equal(g, *best)) { + g.reason = "as much support as another group: conflict"; + } + else if (&g == best) { + g.chosen = true; + } + else if (best->configured && !g.configured) { + g.reason = "no configured source, the strongest group has one"; } - if (b.groups.size() > 1 && b.groups[1].independent == top.independent) { - continue; // two digests with as many sources for one block: a conflict, not an agreement + else if (g.independent < best->independent) { + g.reason = "fewer independent sources than the strongest group"; + } + else { + g.reason = "lower block than the strongest group, with as much support"; } - top.chosen = true; - result.chosen = top; - break; } - - size_t best = 0; - for (auto& b : ordered) { - for (auto& g : b.groups) { - best = std::max(best, g.independent); - result.groups.push_back(std::move(g)); + size_t most = 0; + for (const auto& g : result.groups) { + most = std::max(most, g.independent); + if (g.chosen) { + result.chosen = g; } } - result.shortfall = result.chosen ? 0 : minSources - std::min(best, minSources); + result.shortfall = (result.chosen || conflict) ? 0 : minSources - std::min(most, minSources); return result; } @@ -151,6 +163,9 @@ std::string describe(const AgreementGroup& group) { out << ' ' << EncodeBase58(k.data(), k.data() + k.size()).substr(0, 8) << "..."; } out << ')'; + if (!group.reason.empty()) { + out << ": " << group.reason; + } return out.str(); } diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp index 6ee1c28e6..67b16617d 100644 --- a/csnode/src/snapshot_fetcher.cpp +++ b/csnode/src/snapshot_fetcher.cpp @@ -67,10 +67,7 @@ SnapshotFetcher::SnapshotFetcher(const SnapshotData& cfg, fs::path tmpDir, Hooks hooks_.clock = steadyMs; } for (const auto& key : cfg_.sources) { - if (auto* s = addSource(key)) { - s->configured = true; - s->trusted = true; - } + addConfigured(key)->trusted = true; } loadSourcesFile(); } @@ -106,22 +103,66 @@ SnapshotFetcher::Source* SnapshotFetcher::findSource(const cs::PublicKey& key) { return it == sources_.end() ? nullptr : &it->second; } -SnapshotFetcher::Source* SnapshotFetcher::addSource(const cs::PublicKey& key) { +SnapshotFetcher::Source* SnapshotFetcher::addConfigured(const cs::PublicKey& key) { + // configured keys have their own room: discovered peers never push them out + auto& s = sources_[key]; + s.configured = true; + return &s; +} + +bool SnapshotFetcher::isProtected(const cs::PublicKey& key, const Source& s) const { + if (s.configured || s.outstanding != 0) { + return true; + } + return phase_ == Phase::Fetch && std::any_of(agreedSources_.begin(), agreedSources_.end(), + [&](const SourceManifest& m) { return m.key == key; }); +} + +SnapshotFetcher::Source* SnapshotFetcher::addDiscovered(const cs::PublicKey& key, uint64_t t) { if (auto* s = findSource(key)) { return s; } - if (sources_.size() >= kMaxSources) { - // make room by dropping a discovered peer that is not a neighbour and gave no manifest - auto victim = std::find_if(sources_.begin(), sources_.end(), [this](const auto& entry) { - const auto& s = entry.second; - return !s.configured && !s.manifest && s.outstanding == 0 && neighbours_.count(entry.first) == 0; - }); + // peer keys cost nothing: discovered sources are limited per /24 (/48) and in all, so that one + // network cannot fill the table; when it is full the oldest entry of the most represented + // prefix makes room, unless the newcomer's prefix would become as represented + std::string prefix = netPrefix(hooks_.addressOf ? hooks_.addressOf(key) : std::string()); + if (prefix.empty()) { + prefix = "unknown"; + } + std::map perPrefix; + size_t discovered = 0; + for (const auto& [k, s] : sources_) { + if (!s.configured) { + ++perPrefix[s.prefix]; + ++discovered; + } + } + if (perPrefix[prefix] >= kMaxPerPrefix) { + return nullptr; + } + if (discovered >= kMaxSources) { + auto most = std::max_element(perPrefix.begin(), perPrefix.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); + if (most->second <= perPrefix[prefix] + 1) { + return nullptr; + } + auto victim = sources_.end(); + for (auto it = sources_.begin(); it != sources_.end(); ++it) { + if (it->second.configured || it->second.prefix != most->first || isProtected(it->first, it->second)) { + continue; + } + if (victim == sources_.end() || it->second.discoveredAt < victim->second.discoveredAt) { + victim = it; + } + } if (victim == sources_.end()) { return nullptr; } sources_.erase(victim); } - return &sources_[key]; + auto& s = sources_[key]; + s.prefix = prefix; + s.discoveredAt = t; + return &s; } void SnapshotFetcher::loadSourcesFile() { @@ -156,10 +197,8 @@ void SnapshotFetcher::loadSourcesFile() { } cs::PublicKey key; std::copy(bytes.begin(), bytes.end(), key.begin()); - if (auto* s = addSource(key)) { - s->configured = true; - ++keys; - } + addConfigured(key); + ++keys; } if (keys != 0) { log("SNAPSHOT: " + std::to_string(keys) + " source keys read from " + cfg_.sourcesFile); @@ -214,7 +253,7 @@ void SnapshotFetcher::discover(const Peers& peers, uint64_t t) { if ((capabilities & cs::hostdata::kSnapshotProtocolV1) == 0) { continue; } - if (auto* s = addSource(key)) { + if (auto* s = addDiscovered(key, t)) { s->serving = true; } } diff --git a/tests/csnode/snapshot_agreement_tests.cpp b/tests/csnode/snapshot_agreement_tests.cpp index c9ba0015e..bda9e6859 100644 --- a/tests/csnode/snapshot_agreement_tests.cpp +++ b/tests/csnode/snapshot_agreement_tests.cpp @@ -112,16 +112,49 @@ TEST(SnapshotAgreement, SameKeyCountsOnce) { EXPECT_FALSE(r.chosen); } -TEST(SnapshotAgreement, HighestBlockFirst) { +TEST(SnapshotAgreement, MoreSupportWinsOverAHigherBlock) { + // (a) two forged sources at a made-up later block against three honest ones const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.2.1", 1000), source(3, "10.0.3.1", 1000), - source(4, "10.0.4.1", 2000), source(5, "10.0.5.1", 2000)}; + source(4, "10.0.4.1", 2000, "forged"), source(5, "10.0.5.1", 2000, "forged")}; const auto r = cs::agree(v, 2); ASSERT_TRUE(r.chosen); - EXPECT_EQ(r.chosen->seq, 2000u); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_EQ(r.chosen->independent, 3u); ASSERT_EQ(r.groups.size(), 2u); - EXPECT_EQ(r.groups[0].seq, 2000u); EXPECT_TRUE(r.groups[0].chosen); EXPECT_FALSE(r.groups[1].chosen); + EXPECT_EQ(r.groups[1].seq, 2000u); + EXPECT_NE(cs::describe(r.groups[1]).find("fewer independent sources"), std::string::npos); +} + +TEST(SnapshotAgreement, HigherBlockBreaksATie) { + // (b) equal support: the newer snapshot + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.2.1", 1000), source(4, "10.0.4.1", 2000), + source(5, "10.0.5.1", 2000)}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 2000u); + EXPECT_NE(cs::describe(r.groups[1]).find("lower block"), std::string::npos); +} + +TEST(SnapshotAgreement, ConfiguredKeyWins) { + // (c) a group holding a key of [snapshot] sources wins over one without, both above the threshold + std::vector v{source(1, "10.0.1.1", 1000, "a", "", true), source(2, "10.0.2.1", 1000, "a"), + source(3, "10.0.3.1", 2000, "b"), source(4, "10.0.4.1", 2000, "b")}; + auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_TRUE(r.chosen->configured); + EXPECT_NE(cs::describe(r.groups[1]).find("no configured source"), std::string::npos); + // even when the other group has more support + v.push_back(source(5, "10.0.5.1", 2000, "b")); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + // but a configured key alone does not lift a group below min_sources + r = cs::agree({source(1, "10.0.1.1", 1000, "a", "", true), source(3, "10.0.3.1", 2000, "b"), source(4, "10.0.4.1", 2000, "b")}, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 2000u); } TEST(SnapshotAgreement, OnlyTheLowerBlockReachesTheThreshold) { @@ -146,18 +179,23 @@ TEST(SnapshotAgreement, DigestMajorityInsideABlock) { EXPECT_NE(cs::describe(r.groups[1]).find("1 source"), std::string::npos); } -TEST(SnapshotAgreement, DigestTieIsNotAnAgreement) { +TEST(SnapshotAgreement, TieIsNotAnAgreement) { std::vector v{source(1, "10.0.1.1", 2000, "a"), source(2, "10.0.2.1", 2000, "a"), source(3, "10.0.3.1", 2000, "b"), source(4, "10.0.4.1", 2000, "b")}; auto r = cs::agree(v, 2); EXPECT_FALSE(r.chosen); EXPECT_EQ(r.shortfall, 0u); // groups are large enough, they conflict - // a lower block that agrees is taken instead + EXPECT_NE(cs::describe(r.groups[0]).find("conflict"), std::string::npos); + // a weaker group is not taken instead v.push_back(source(5, "10.0.5.1", 1000)); v.push_back(source(6, "10.0.6.1", 1000)); r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + // one more source for one side settles it + v.push_back(source(7, "10.0.7.1", 2000, "a")); + r = cs::agree(v, 2); ASSERT_TRUE(r.chosen); - EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_EQ(r.chosen->stateDigest, "a"); } TEST(SnapshotAgreement, DifferentHeadsAreDifferentBlocks) { diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp index 19414d029..3154b4849 100644 --- a/tests/csnode/snapshot_fetcher_tests.cpp +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -188,6 +188,7 @@ struct Net { size_t maxOutstandingPerSource = 0; size_t outstandingTotal = 0; size_t manifestRequests = 0; + std::map addresses; // of peers that are not sources SnapshotFetcher::Hooks hooks() { SnapshotFetcher::Hooks h; @@ -208,7 +209,11 @@ struct Net { h.addNeighbours = [this](const std::set& keys) { addCalls.push_back(keys); }; h.addressOf = [this](const cs::PublicKey& k) { auto it = sources.find(k); - return it == sources.end() ? std::string() : it->second.address; + if (it != sources.end()) { + return it->second.address; + } + auto other = addresses.find(k); + return other == addresses.end() ? std::string() : other->second; }; h.clock = [this]() { return nowMs; }; h.onLog = [this](const std::string& line) { logs.push_back(line); }; @@ -411,6 +416,76 @@ TEST(SnapshotFetcher, DiscoveryAddsServingPeersAndConfiguredKeys) { EXPECT_EQ(net.addCalls[1].size(), 3u); } +// a flood of discovered keys from one /24 pushes out neither a configured key nor other networks' sources +TEST(SnapshotFetcher, FloodFromOnePrefixDoesNotFillTheTable) { + TmpDir tmp("flood"); + Net net; + const auto configured = otherKey(1); + auto cfg = fetchConfig(); + cfg.sources = {configured}; + std::vector flood; + for (int i = 0; i < 100; ++i) { + cs::PublicKey k{}; + k.fill(0x40); + k[0] = static_cast(i); + flood.push_back(k); + net.addresses[k] = "10.9.9." + std::to_string(i + 1); + net.peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + net.peers.neighbours.insert(k); // all of them neighbours that never answer + } + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 1u); + const auto& added = net.addCalls[0]; + EXPECT_EQ(added.count(configured), 1u); + // the sources of other networks (neighbours already) are asked; of the flood at most kMaxPerPrefix + std::set asked; + size_t floodAsked = 0; + for (const auto& [k, request] : net.manifestQueue) { + asked.insert(k); + floodAsked += std::count(flood.begin(), flood.end(), k); + } + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::B}) { + EXPECT_EQ(asked.count(net.key(i)), 1u); + } + EXPECT_EQ(floodAsked, SnapshotFetcher::kMaxPerPrefix); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// a full table makes room for a new network by dropping the oldest of the most represented one +TEST(SnapshotFetcher, FullTableMakesRoomForANewPrefix) { + TmpDir tmp("fulltable"); + Net net; + for (int p = 0; p < 16; ++p) { + for (int i = 0; i < 4; ++i) { + cs::PublicKey k{}; + k.fill(0x50); + k[0] = static_cast(p); + k[1] = static_cast(i); + net.addresses[k] = "10.8." + std::to_string(p) + "." + std::to_string(i + 1); + net.peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + } + } + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 1u); + EXPECT_EQ(net.addCalls[0].size(), SnapshotFetcher::kMaxSources); + // two serving nodes from other networks turn up later + net.add(Snapshots::A1, false); + net.add(Snapshots::A2, false); + net.nowMs += 500; + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 2u); + EXPECT_EQ(net.addCalls[1].count(net.key(Snapshots::A1)), 1u); + EXPECT_EQ(net.addCalls[1].count(net.key(Snapshots::A2)), 1u); + net.peers.neighbours.insert(net.key(Snapshots::A1)); + net.peers.neighbours.insert(net.key(Snapshots::A2)); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + // (2) bad signature, another node's manifest, an oversized chunk, an unsolicited reply: dropped TEST(SnapshotFetcher, BadManifestsAreDropped) { TmpDir tmp("badmanifests"); From 878d3185ac7ec5ec66c54e7655021d82ed306490 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 15:23:14 +0100 Subject: [PATCH 3/9] snapshot fetcher: no lockout of newcomers, addresses resolved with the manifest Source table: a full table always makes room, whatever the prefixes: the least useful discovered source goes first (not a neighbour, silent for a minute, no manifest after a minute, then the most represented prefix); discovered sources silent for a minute are let go, and an evicted key is not taken again for 10 min, so that returning keys cannot churn the table. Discovered sources are also limited to 16 per /16 (/32 IPv6). Addresses: an address not known yet is not capped at discovery; it is resolved again when the manifest arrives, and the per-prefix limits and the agreement count both use that address. A source whose address is unknown does not count towards agreement. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_agreement.hpp | 4 +- csnode/include/csnode/snapshot_fetcher.hpp | 12 +- csnode/src/snapshot_agreement.cpp | 26 +++- csnode/src/snapshot_fetcher.cpp | 145 +++++++++++++++---- tests/csnode/snapshot_agreement_tests.cpp | 15 +- tests/csnode/snapshot_fetcher_tests.cpp | 72 ++++++++- 6 files changed, 230 insertions(+), 44 deletions(-) diff --git a/csnode/include/csnode/snapshot_agreement.hpp b/csnode/include/csnode/snapshot_agreement.hpp index 18a4f9e09..8b4e9482f 100644 --- a/csnode/include/csnode/snapshot_agreement.hpp +++ b/csnode/include/csnode/snapshot_agreement.hpp @@ -26,6 +26,8 @@ struct SourceManifest { // IPv6 one ("2001:db8:1::/48"); an IPv4-mapped IPv6 address counts as IPv4. A port after the // address (a.b.c.d:port, [v6]:port) is ignored. Empty for an address that does not parse. std::string netPrefix(const std::string& address); +// the wider block around it: /16 for IPv4, /32 for IPv6; empty when the address does not parse +std::string netBlock(const std::string& address); // sources declaring one (seq, head_hash, state_digest, settings_digest) struct AgreementGroup { @@ -56,7 +58,7 @@ struct AgreementResult { // sources at a made-up later block do not beat a larger honest group; an honest network's newest // snapshot gains sources as the nodes rebuild, and the fetcher asks again every 30 s). Two groups // equal on all three are a conflict: nothing is chosen. Sources are counted by netPrefix(address), -// trusted ones by key; sources whose address does not parse count together as one; a key that +// trusted ones by key; a source whose address is unknown or does not parse does not count; a key that // appears twice counts once. Every group that is not chosen carries the reason. AgreementResult agree(const std::vector& sources, size_t minSources); diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp index b98405855..e1b94d0ed 100644 --- a/csnode/include/csnode/snapshot_fetcher.hpp +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -63,6 +63,9 @@ class SnapshotFetcher { static constexpr size_t kPiecesTotal = 8; static constexpr size_t kMaxSources = 64; // discovered sources; configured keys come on top static constexpr size_t kMaxPerPrefix = 4; // discovered sources per /24 (IPv4) or /48 (IPv6) + static constexpr size_t kMaxPerBlock = 16; // discovered sources per /16 (IPv4) or /32 (IPv6) + static constexpr uint64_t kEvictedMs = 600'000; // an evicted or silent key is not taken again before + static constexpr size_t kMaxEvicted = 1024; static constexpr size_t kMaxAttempts = 3; static constexpr size_t kMaxSourcesFileKeys = 64; @@ -101,7 +104,9 @@ class SnapshotFetcher { bool configured = false; // [snapshot] sources or sources_file bool trusted = false; // [snapshot] sources bool serving = false; // announced capability bit 24 - std::string prefix; // netPrefix of the address when discovered + std::string prefix; // netPrefix and netBlock of the address, empty while unknown; + std::string block; // resolved again when a manifest arrives + bool everAnswered = false; uint64_t discoveredAt = 0; uint64_t addedAt = 0; // last addNeighbours bool added = false; @@ -143,6 +148,10 @@ class SnapshotFetcher { Source* findSource(const cs::PublicKey& key); Source* addConfigured(const cs::PublicKey& key); Source* addDiscovered(const cs::PublicKey& key, uint64_t t); + bool roomFor(const cs::PublicKey& key, const std::string& prefix, const std::string& block) const; + bool evictOne(uint64_t t); + bool silent(const Source& s, uint64_t t) const; + void evict(std::map::iterator it, uint64_t t); bool isProtected(const cs::PublicKey& key, const Source& s) const; void loadSourcesFile(); @@ -182,6 +191,7 @@ class SnapshotFetcher { uint64_t startedAt_ = 0; std::map sources_; std::set neighbours_; // as of the last tick + std::map evicted_; // discovered keys let go, and when AgreementResult agreement_; std::vector agreedSources_; diff --git a/csnode/src/snapshot_agreement.cpp b/csnode/src/snapshot_agreement.cpp index ac5090d31..a0f16488b 100644 --- a/csnode/src/snapshot_agreement.cpp +++ b/csnode/src/snapshot_agreement.cpp @@ -31,13 +31,15 @@ std::string unitOf(const SourceManifest& s) { return "key:" + EncodeBase58(s.key.data(), s.key.data() + s.key.size()); } const auto prefix = netPrefix(s.address); - return prefix.empty() ? std::string("unknown address") : "net:" + prefix; + return prefix.empty() ? std::string() : "net:" + prefix; } size_t countUnits(const std::vector& indices, const std::vector& units) { std::set distinct; for (auto i : indices) { - distinct.insert(units[i]); + if (!units[i].empty()) { // an address not known (yet) is no evidence of independence + distinct.insert(units[i]); + } } return distinct.size(); } @@ -47,7 +49,9 @@ std::string shortHex(const std::string& hex) { } } // namespace -std::string netPrefix(const std::string& address) { +namespace { +// the prefix of the given length in bits (24 or 16 for IPv4 addresses, 48 or 32 for IPv6) +std::string prefixOf(const std::string& address, bool wide) { boost::system::error_code ec; auto ip = boost::asio::ip::make_address(hostPart(address), ec); if (ec) { @@ -58,11 +62,21 @@ std::string netPrefix(const std::string& address) { } if (ip.is_v4()) { const auto b = ip.to_v4().to_bytes(); - return std::to_string(b[0]) + "." + std::to_string(b[1]) + "." + std::to_string(b[2]); + return wide ? std::to_string(b[0]) + "." + std::to_string(b[1]) + : std::to_string(b[0]) + "." + std::to_string(b[1]) + "." + std::to_string(b[2]); } auto b = ip.to_v6().to_bytes(); - std::fill(b.begin() + 6, b.end(), 0); - return boost::asio::ip::address_v6(b).to_string() + "/48"; + std::fill(b.begin() + (wide ? 4 : 6), b.end(), 0); + return boost::asio::ip::address_v6(b).to_string() + (wide ? "/32" : "/48"); +} +} // namespace + +std::string netPrefix(const std::string& address) { + return prefixOf(address, false); +} + +std::string netBlock(const std::string& address) { + return prefixOf(address, true); } AgreementResult agree(const std::vector& sources, size_t minSources) { diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp index 67b16617d..3f43b8e57 100644 --- a/csnode/src/snapshot_fetcher.cpp +++ b/csnode/src/snapshot_fetcher.cpp @@ -118,53 +118,113 @@ bool SnapshotFetcher::isProtected(const cs::PublicKey& key, const Source& s) con [&](const SourceManifest& m) { return m.key == key; }); } +bool SnapshotFetcher::roomFor(const cs::PublicKey& key, const std::string& prefix, const std::string& block) const { + // an address not known yet is not capped here: it is resolved and checked again with its manifest + size_t samePrefix = 0; + size_t sameBlock = 0; + for (const auto& [k, s] : sources_) { + if (k == key || s.configured) { + continue; + } + samePrefix += !prefix.empty() && s.prefix == prefix; + sameBlock += !block.empty() && s.block == block; + } + return samePrefix < kMaxPerPrefix && sameBlock < kMaxPerBlock; +} + SnapshotFetcher::Source* SnapshotFetcher::addDiscovered(const cs::PublicKey& key, uint64_t t) { if (auto* s = findSource(key)) { return s; } - // peer keys cost nothing: discovered sources are limited per /24 (/48) and in all, so that one - // network cannot fill the table; when it is full the oldest entry of the most represented - // prefix makes room, unless the newcomer's prefix would become as represented - std::string prefix = netPrefix(hooks_.addressOf ? hooks_.addressOf(key) : std::string()); - if (prefix.empty()) { - prefix = "unknown"; + if (auto e = evicted_.find(key); e != evicted_.end()) { + if (t - e->second < kEvictedMs) { + return nullptr; + } + evicted_.erase(e); + } + // peer keys cost nothing: discovered sources are limited per /24 (/48), per /16 (/32) and in + // all, so that no network can fill the table + const std::string address = hooks_.addressOf ? hooks_.addressOf(key) : std::string(); + const std::string prefix = netPrefix(address); + const std::string block = netBlock(address); + if (!roomFor(key, prefix, block)) { + return nullptr; } - std::map perPrefix; size_t discovered = 0; for (const auto& [k, s] : sources_) { - if (!s.configured) { - ++perPrefix[s.prefix]; - ++discovered; - } + discovered += !s.configured; } - if (perPrefix[prefix] >= kMaxPerPrefix) { + if (discovered >= kMaxSources && !evictOne(t)) { return nullptr; } - if (discovered >= kMaxSources) { - auto most = std::max_element(perPrefix.begin(), perPrefix.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); - if (most->second <= perPrefix[prefix] + 1) { - return nullptr; - } - auto victim = sources_.end(); - for (auto it = sources_.begin(); it != sources_.end(); ++it) { - if (it->second.configured || it->second.prefix != most->first || isProtected(it->first, it->second)) { - continue; - } - if (victim == sources_.end() || it->second.discoveredAt < victim->second.discoveredAt) { - victim = it; - } - } - if (victim == sources_.end()) { - return nullptr; - } - sources_.erase(victim); - } auto& s = sources_[key]; + s.address = address; s.prefix = prefix; + s.block = block; s.discoveredAt = t; return &s; } +bool SnapshotFetcher::silent(const Source& s, uint64_t t) const { + return !s.configured && s.everAsked && !s.everAnswered && t - s.discoveredAt > kMarkMs; +} + +void SnapshotFetcher::evict(std::map::iterator it, uint64_t t) { + // an evicted key is not taken again for a while, so that keys that come back at every tick + // cannot churn the table + if (evicted_.size() >= kMaxEvicted) { + auto oldest = std::min_element(evicted_.begin(), evicted_.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); + evicted_.erase(oldest); + } + evicted_[it->first] = t; + sources_.erase(it); +} + +bool SnapshotFetcher::evictOne(uint64_t t) { + // the least useful discovered source makes room, whatever its prefix: one that is not a + // neighbour now, then one that never answered in a minute, then one without a manifest after + // a minute, then one without a manifest in the most represented prefix, then the oldest of the + // most represented prefix; never a configured source, one with pieces outstanding or one of the + // agreed group during a fetch + std::map perPrefix; + for (const auto& [k, s] : sources_) { + if (!s.configured) { + ++perPrefix[s.prefix]; + } + } + size_t most = 0; + for (const auto& [prefix, n] : perPrefix) { + most = std::max(most, n); + } + auto tier = [&](const cs::PublicKey& key, const Source& s) { + const bool old = t - s.discoveredAt > kMarkMs; + const bool crowded = perPrefix[s.prefix] == most; + if (neighbours_.count(key) == 0) return 0; + if (silent(s, t)) return 1; + if (old && !s.manifest) return 2; + if (crowded && !s.manifest) return 3; + if (crowded) return 4; + return 5; + }; + auto victim = sources_.end(); + int victimTier = 5; + for (auto it = sources_.begin(); it != sources_.end(); ++it) { + if (isProtected(it->first, it->second)) { + continue; + } + const int k = tier(it->first, it->second); + if (k < victimTier || (k == victimTier && victim != sources_.end() && it->second.discoveredAt < victim->second.discoveredAt)) { + victim = it; + victimTier = k; + } + } + if (victim == sources_.end()) { + return false; + } + evict(victim, t); + return true; +} + void SnapshotFetcher::loadSourcesFile() { if (cfg_.sourcesFile.empty()) { return; @@ -249,6 +309,14 @@ void SnapshotFetcher::tick(const Peers& peers) { } void SnapshotFetcher::discover(const Peers& peers, uint64_t t) { + // discovered sources that never answered a manifest request in a minute are let go + for (auto it = sources_.begin(); it != sources_.end();) { + auto next = std::next(it); + if (silent(it->second, t) && !isProtected(it->first, it->second)) { + evict(it, t); + } + it = next; + } for (const auto& [key, capabilities] : peers.known) { if ((capabilities & cs::hostdata::kSnapshotProtocolV1) == 0) { continue; @@ -309,6 +377,7 @@ void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapsho return; } s->awaiting = false; + s->everAnswered = true; const uint64_t t = now(); if (reply.status != snapshot_p2p::ManifestStatus::Ready) { // not serving, not synced or still building: whatever it declared before is not served now @@ -335,10 +404,22 @@ void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapsho log("SNAPSHOT: manifest from " + keyText(sender) + " dropped: " + why, true); return; } + // the address the table limits and the agreement count are taken from, resolved again now + const std::string address = hooks_.addressOf ? hooks_.addressOf(sender) : std::string(); + const std::string prefix = netPrefix(address); + const std::string block = netBlock(address); + if (!s->configured && !roomFor(sender, prefix, block)) { + ++dropped_; + log("SNAPSHOT: manifest from " + keyText(sender) + " dropped: too many sources at " + prefix); + sources_.erase(sender); + return; + } + s->address = address; + s->prefix = prefix; + s->block = block; s->filesHash = manifest->filesHash(); s->manifest = std::move(manifest); s->manifestAt = t; - s->address = hooks_.addressOf ? hooks_.addressOf(sender) : std::string(); // agreement is tried at the next tick, so that the replies to one round of requests count together } diff --git a/tests/csnode/snapshot_agreement_tests.cpp b/tests/csnode/snapshot_agreement_tests.cpp index bda9e6859..f58c51682 100644 --- a/tests/csnode/snapshot_agreement_tests.cpp +++ b/tests/csnode/snapshot_agreement_tests.cpp @@ -38,6 +38,9 @@ TEST(SnapshotAgreement, NetPrefix) { EXPECT_EQ(cs::netPrefix("2001:db8:1::1"), "2001:db8:1::/48"); EXPECT_EQ(cs::netPrefix(""), ""); EXPECT_EQ(cs::netPrefix("not an address"), ""); + EXPECT_EQ(cs::netBlock("10.1.2.3:6000"), "10.1"); + EXPECT_EQ(cs::netBlock("2001:db8:1:2::1"), "2001:db8::/32"); + EXPECT_EQ(cs::netBlock(""), ""); } TEST(SnapshotAgreement, Threshold) { @@ -99,11 +102,17 @@ TEST(SnapshotAgreement, Ipv6Slash48) { EXPECT_EQ(r.chosen->keys.size(), 3u); } -TEST(SnapshotAgreement, UnknownAddressesCountTogether) { - const std::vector v{source(1, "", 1000), source(2, "", 1000), source(3, "garbage", 1000)}; - const auto r = cs::agree(v, 2); +TEST(SnapshotAgreement, UnknownAddressesDoNotCount) { + std::vector v{source(1, "", 1000), source(2, "garbage", 1000), source(3, "10.0.3.1", 1000)}; + auto r = cs::agree(v, 2); EXPECT_FALSE(r.chosen); EXPECT_EQ(r.groups[0].independent, 1u); + EXPECT_EQ(r.groups[0].keys.size(), 3u); + // a trusted key counts by key whatever its address + v[0].trusted = true; + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->independent, 2u); } TEST(SnapshotAgreement, SameKeyCountsOnce) { diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp index 3154b4849..0f6b972e3 100644 --- a/tests/csnode/snapshot_fetcher_tests.cpp +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -465,7 +465,7 @@ TEST(SnapshotFetcher, FullTableMakesRoomForANewPrefix) { k.fill(0x50); k[0] = static_cast(p); k[1] = static_cast(i); - net.addresses[k] = "10.8." + std::to_string(p) + "." + std::to_string(i + 1); + net.addresses[k] = "10." + std::to_string(100 + p) + ".0." + std::to_string(i + 1); net.peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); } } @@ -486,6 +486,76 @@ TEST(SnapshotFetcher, FullTableMakesRoomForANewPrefix) { ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); } +// 64 keys over 64 networks that never answer do not keep a new honest peer out +TEST(SnapshotFetcher, SilentSpreadKeysDoNotLockNewcomersOut) { + TmpDir tmp("spreadkeys"); + Net net; + for (int p = 0; p < 64; ++p) { + cs::PublicKey k{}; + k.fill(0x60); + k[0] = static_cast(p); + net.addresses[k] = std::to_string(20 + p) + "." + std::to_string(p) + ".0.1"; + net.peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + net.peers.neighbours.insert(k); + } + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + net.step(fetcher); + // a full table makes room at once, before anything has had time to answer + net.add(Snapshots::A1, false); + net.step(fetcher); + ASSERT_FALSE(net.addCalls.empty()); + EXPECT_EQ(net.addCalls.back().count(net.key(Snapshots::A1)), 1u); + net.peers.neighbours.insert(net.key(Snapshots::A1)); + for (int i = 0; i < 610; ++i) { // the others asked, never answering, for a minute + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_EQ(fetcher.manifests(), 1u); + net.add(Snapshots::A2); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// an address not known yet: no cap at discovery, no vote until it is known; the cap is applied +// to the address resolved when the manifest arrives +TEST(SnapshotFetcher, UnknownAddresses) { + TmpDir tmp("unknownaddr"); + Net net; + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::B, Snapshots::D, Snapshots::E}) { + net.add(i).address.clear(); + } + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + EXPECT_EQ(net.manifestQueue.size(), 5u); // five unknown addresses, none refused + net.step(fetcher); + EXPECT_EQ(fetcher.manifests(), 5u); + for (int i = 0; i < 10; ++i) { + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); // nothing counts without an address + EXPECT_FALSE(fetcher.groups().empty()); + EXPECT_EQ(fetcher.groups()[0].independent, 0u); + + // all five turn out to be in one /24: four are kept, they count once + int n = 1; + for (auto& [key, source] : net.sources) { + source.address = "10.0.7." + std::to_string(n++); + } + net.nowMs += SnapshotFetcher::kManifestIntervalMs; + fetcher.tick(net.peers); + net.deliver(fetcher); + EXPECT_EQ(fetcher.manifests(), 4u); + EXPECT_TRUE(net.logged("too many sources at 10.0.7")); + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Collect); // one network: one source + + // known in different networks, they agree + n = 1; + for (auto& [key, source] : net.sources) { + source.address = "10.0." + std::to_string(n++) + ".1"; + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + // (2) bad signature, another node's manifest, an oversized chunk, an unsolicited reply: dropped TEST(SnapshotFetcher, BadManifestsAreDropped) { TmpDir tmp("badmanifests"); From a9dbd627f2da7bb76557ffbe6646c4c9400ae6b6 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 15:36:59 +0100 Subject: [PATCH 4/9] snapshot fetcher: file names checked again where they become paths; no symlinks, mode 0600 Co-Authored-By: Claude Opus 5.5 --- csnode/src/snapshot_fetcher.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp index 3f43b8e57..0dec00874 100644 --- a/csnode/src/snapshot_fetcher.cpp +++ b/csnode/src/snapshot_fetcher.cpp @@ -680,10 +680,15 @@ bool SnapshotFetcher::openFile(FileState& f, bool truncate) { if (f.fd >= 0) { return true; } + // the name comes from a manifest fromJson accepted, so it is one of the snapshot's fixed file + // names; checked again here, where it becomes a path + if (!SnapshotManifest::isKnownFileName(f.file.name) || fs::path(f.file.name).filename() != f.file.name) { + return false; + } std::error_code ec; fs::create_directories(tmpDir_, ec); const auto path = tmpDir_ / f.file.name; - f.fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC | (truncate ? O_TRUNC : 0), 0644); + f.fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC | O_NOFOLLOW | (truncate ? O_TRUNC : 0), 0600); if (f.fd < 0) { return false; } From 4ac14dbb37874f1caaf8cb49456ba1f9ba4d4ca5 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 15:50:24 +0100 Subject: [PATCH 5/9] snapshot fetcher: review fixes (silence, busy, bounded fetch, import bans, logs) - Silence: counted from the first unanswered request (two requests over more than 35 s), not from the discovery; asked again after 10 s twice, then every 30 s; no proactive eviction: silence is only an eviction tier when room is needed; the cool-down after an eviction is 60 s. - Busy: delay floored at 500 ms (cap 10 s); busy for a minute without a piece leaves the source alone for a minute, without counting to a ban. - A fetch always ends: no new byte for 5 min returns to Collect, and every return to Collect counts as an attempt with the refused imports (3 -> Fallback). - A refused import bans every agreed source that declared the refused (name, sha256); without a file at fault, the source that signed the manifest. - A manifest reply over the prefix limits no longer erases a source in use. - Agreement recomputed only when its input changed; filesHash once per source when choosing the files. - Whole-file SHA-256 at most 64 MiB per tick. - Logs: the 30-s line names a conflict, the manifests not counted (address unknown) and the replies dropped; the fallback lines name CREDITS_SNAPSHOT=github; fetch_wait=0 documented. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_fetcher.hpp | 25 ++- csnode/src/snapshot_fetcher.cpp | 208 ++++++++++++++++----- tests/csnode/snapshot_fetcher_tests.cpp | 130 ++++++++++++- 3 files changed, 309 insertions(+), 54 deletions(-) diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp index e1b94d0ed..afe6ae5ed 100644 --- a/csnode/include/csnode/snapshot_fetcher.hpp +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -52,10 +53,16 @@ class SnapshotFetcher { }; static constexpr uint64_t kManifestIntervalMs = 30'000; + static constexpr uint64_t kManifestRetryMs = 10'000; // a request without an answer: asked again sooner + static constexpr size_t kManifestRetries = 2; // ... this many times, then every 30 s + static constexpr uint64_t kSilentMs = kManifestIntervalMs + 5'000; // no answer this long: silent static constexpr uint64_t kNeighbourRetryMs = 60'000; static constexpr uint64_t kPieceTimeoutMs = 30'000; static constexpr uint64_t kMarkMs = 60'000; + static constexpr uint64_t kMinBusyMs = 500; static constexpr uint64_t kMaxBusyMs = 10'000; + static constexpr uint64_t kBusyMarkMs = 60'000; // busy this long without a piece: marked + static constexpr uint64_t kNoProgressMs = 300'000; // a fetch without a new byte this long ends static constexpr uint64_t kLogIntervalMs = 30'000; static constexpr uint64_t kWaitLogIntervalMs = 300'000; static constexpr uint64_t kStallMs = 120'000; @@ -64,11 +71,13 @@ class SnapshotFetcher { static constexpr size_t kMaxSources = 64; // discovered sources; configured keys come on top static constexpr size_t kMaxPerPrefix = 4; // discovered sources per /24 (IPv4) or /48 (IPv6) static constexpr size_t kMaxPerBlock = 16; // discovered sources per /16 (IPv4) or /32 (IPv6) - static constexpr uint64_t kEvictedMs = 600'000; // an evicted or silent key is not taken again before + static constexpr uint64_t kEvictedMs = 60'000; // an evicted key is not taken again before static constexpr size_t kMaxEvicted = 1024; - static constexpr size_t kMaxAttempts = 3; + static constexpr size_t kMaxAttempts = 3; // fetches that end without a snapshot, then Fallback + static constexpr uint64_t kHashBytesPerTick = 64ull << 20; // whole-file SHA-256 per tick static constexpr size_t kMaxSourcesFileKeys = 64; + // cfg.fetchWait = 0: no wait, Fallback (or Done) at the first tick without agreement SnapshotFetcher(const SnapshotData& cfg, std::filesystem::path tmpDir, Hooks hooks, bool dryRun = false); ~SnapshotFetcher(); SnapshotFetcher(const SnapshotFetcher&) = delete; @@ -107,6 +116,8 @@ class SnapshotFetcher { std::string prefix; // netPrefix and netBlock of the address, empty while unknown; std::string block; // resolved again when a manifest arrives bool everAnswered = false; + size_t unanswered = 0; // manifest requests since the last answer + uint64_t firstUnansweredAt = 0; uint64_t discoveredAt = 0; uint64_t addedAt = 0; // last addNeighbours bool added = false; @@ -122,6 +133,7 @@ class SnapshotFetcher { uint64_t markedUntil = 0; bool banned = false; // for the rest of the bootstrap uint64_t busyUntil = 0; + uint64_t busySince = 0; // first busy answer since its last piece; 0: none size_t outstanding = 0; }; @@ -130,6 +142,8 @@ class SnapshotFetcher { uint64_t sentAt = 0; }; + struct HashState; + struct FileState { SnapshotFile file; std::vector sources; // agreed sources declaring this file with this SHA-256 @@ -140,6 +154,8 @@ class SnapshotFetcher { int fd = -1; bool verifyPending = false; bool verified = false; + std::shared_ptr hashing; // whole-file SHA-256 in progress, kHashBytesPerTick per tick + uint64_t hashed = 0; }; void log(const std::string& line, bool warning = false); @@ -174,6 +190,7 @@ class SnapshotFetcher { void releaseSource(const cs::PublicKey& key); void mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why); void backToCollect(const std::string& why); + void banDeclaring(const std::string& name, const std::string& sha256, const std::string& why); void closeFile(FileState& f); bool openFile(FileState& f, bool truncate); uint64_t pieceLength(const FileState& f, uint32_t chunk) const; @@ -198,7 +215,9 @@ class SnapshotFetcher { std::optional target_; std::vector files_; std::set> badFiles_; // (name, sha256) refused by the importer - size_t importFailures_ = 0; + size_t failures_ = 0; // fetches that ended without a snapshot, refused imports included + bool dirty_ = true; // the input of the agreement changed since it was computed + uint64_t lastByteAt_ = 0; uint64_t lastSendAt_ = 0; size_t rr_ = 0; // first source of the next fill diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp index 0dec00874..66e539533 100644 --- a/csnode/src/snapshot_fetcher.cpp +++ b/csnode/src/snapshot_fetcher.cpp @@ -58,6 +58,10 @@ std::string hexOf(const uint8_t* data, size_t size) { } } // namespace +struct SnapshotFetcher::HashState { + crypto_hash_sha256_state state; +}; + SnapshotFetcher::SnapshotFetcher(const SnapshotData& cfg, fs::path tmpDir, Hooks hooks, bool dryRun) : cfg_(cfg) , tmpDir_(std::move(tmpDir)) @@ -166,24 +170,27 @@ SnapshotFetcher::Source* SnapshotFetcher::addDiscovered(const cs::PublicKey& key } bool SnapshotFetcher::silent(const Source& s, uint64_t t) const { - return !s.configured && s.everAsked && !s.everAnswered && t - s.discoveredAt > kMarkMs; + // at least two requests without an answer over more than one request interval, counted from + // the first request, not from the discovery: a peer that became a neighbour late is not silent + return !s.configured && s.unanswered >= 2 && t - s.firstUnansweredAt > kSilentMs; } void SnapshotFetcher::evict(std::map::iterator it, uint64_t t) { - // an evicted key is not taken again for a while, so that keys that come back at every tick - // cannot churn the table + // an evicted key is not taken again for a minute, so that keys that come back at every tick + // cannot churn the table; longer would only hurt honest keys, since keys cost an attacker nothing if (evicted_.size() >= kMaxEvicted) { auto oldest = std::min_element(evicted_.begin(), evicted_.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); evicted_.erase(oldest); } evicted_[it->first] = t; sources_.erase(it); + dirty_ = true; } bool SnapshotFetcher::evictOne(uint64_t t) { // the least useful discovered source makes room, whatever its prefix: one that is not a - // neighbour now, then one that never answered in a minute, then one without a manifest after - // a minute, then one without a manifest in the most represented prefix, then the oldest of the + // neighbour now, then a silent one (silence is a reason only here, when room is needed), then + // one without a manifest after a minute, then one without a manifest in the most represented prefix, then the oldest of the // most represented prefix; never a configured source, one with pieces outstanding or one of the // agreed group during a fetch std::map perPrefix; @@ -301,22 +308,20 @@ void SnapshotFetcher::tick(const Peers& peers) { return; } fill(t); + // a fetch always ends: nothing to ask for 2 min, or no new byte for 5 min (sources busy, + // silent or slow), and it counts as a failed attempt if (phase_ == Phase::Fetch && outstanding() == 0 && t - lastSendAt_ >= kStallMs) { backToCollect("no piece could be asked for " + duration(kStallMs / 1000)); return; } + if (phase_ == Phase::Fetch && t - lastByteAt_ >= kNoProgressMs) { + backToCollect("no data for " + duration(kNoProgressMs / 1000)); + return; + } logProgress(t, false); } void SnapshotFetcher::discover(const Peers& peers, uint64_t t) { - // discovered sources that never answered a manifest request in a minute are let go - for (auto it = sources_.begin(); it != sources_.end();) { - auto next = std::next(it); - if (silent(it->second, t) && !isProtected(it->first, it->second)) { - evict(it, t); - } - it = next; - } for (const auto& [key, capabilities] : peers.known) { if ((capabilities & cs::hostdata::kSnapshotProtocolV1) == 0) { continue; @@ -346,10 +351,19 @@ void SnapshotFetcher::askManifests(const Peers&, uint64_t t) { if (s.banned || neighbours_.count(key) == 0) { continue; } - // silence (the serving side may not answer at all) or a reply: ask again after 30 s - if (s.everAsked && t - s.askedAt < kManifestIntervalMs) { + // a reply: ask again after 30 s; silence (a serving node may not answer when its share for + // idle keys is spent, design 5.3a): again after 10 s, twice, then every 30 s + const uint64_t interval = (s.awaiting && s.unanswered <= kManifestRetries) ? kManifestRetryMs : kManifestIntervalMs; + if (s.everAsked && t - s.askedAt < interval) { continue; } + if (s.awaiting) { + ++s.unanswered; + } + else { + s.unanswered = 1; + s.firstUnansweredAt = t; + } s.everAsked = true; s.awaiting = true; s.askedAt = t; @@ -378,9 +392,11 @@ void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapsho } s->awaiting = false; s->everAnswered = true; + s->unanswered = 0; const uint64_t t = now(); if (reply.status != snapshot_p2p::ManifestStatus::Ready) { // not serving, not synced or still building: whatever it declared before is not served now + dirty_ = dirty_ || s->manifest.has_value(); s->manifest.reset(); return; } @@ -400,6 +416,7 @@ void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapsho } if (!manifest || !why.empty()) { ++dropped_; + dirty_ = dirty_ || s->manifest.has_value(); s->manifest.reset(); log("SNAPSHOT: manifest from " + keyText(sender) + " dropped: " + why, true); return; @@ -411,7 +428,11 @@ void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapsho if (!s->configured && !roomFor(sender, prefix, block)) { ++dropped_; log("SNAPSHOT: manifest from " + keyText(sender) + " dropped: too many sources at " + prefix); - sources_.erase(sender); + if (!isProtected(sender, *s)) { + sources_.erase(sender); + dirty_ = true; + } + // a source in use keeps its entry, its address and the manifest it had return; } s->address = address; @@ -420,10 +441,15 @@ void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapsho s->filesHash = manifest->filesHash(); s->manifest = std::move(manifest); s->manifestAt = t; + dirty_ = true; // agreement is tried at the next tick, so that the replies to one round of requests count together } bool SnapshotFetcher::tryAgree(uint64_t t) { + if (!dirty_) { + return false; // nothing changed since the last agreement that did not lead to a fetch + } + dirty_ = false; std::vector input; for (const auto& [key, s] : sources_) { if (s.banned || !s.manifest) { @@ -497,19 +523,26 @@ void SnapshotFetcher::logWaiting(uint64_t t) { } } } + size_t unknown = 0; + for (const auto& [key, s] : sources_) { + unknown += !s.banned && s.manifest && !s.trusted && netPrefix(s.address).empty(); + } log("SNAPSHOT: " + std::to_string(serving) + " peers serve snapshots (" + std::to_string(neighbours) + " neighbours), " + - std::to_string(manifests()) + " manifests collected"); + std::to_string(manifests()) + " manifests collected, " + std::to_string(unknown) + " not counted: address unknown, " + + std::to_string(dropped_) + " replies dropped"); const std::string left = duration((waitMs - (t - startedAt_) + 59'999) / 60'000 * 60) + " left"; if (agreement_.chosen) { log("SNAPSHOT: block " + std::to_string(agreement_.chosen->seq) + " is agreed, but no file group has a source left: waiting, " + left); return; } - const AgreementGroup* best = nullptr; - for (const auto& g : agreement_.groups) { - if (best == nullptr || g.independent > best->independent) { - best = &g; - } + // groups come strongest first + const AgreementGroup* best = agreement_.groups.empty() ? nullptr : &agreement_.groups.front(); + if (best != nullptr && best->independent >= cfg_.minSources) { + // as strong as another group: a conflict, not a shortage + log("SNAPSHOT: conflict: two groups with " + std::to_string(best->independent) + " sources at block " + + std::to_string(best->seq) + ": waiting, " + left); + return; } if (best != nullptr) { log("SNAPSHOT: " + std::to_string(best->independent) + (best->independent == 1 ? " source declares" : " sources declare") + @@ -533,13 +566,13 @@ void SnapshotFetcher::checkWait(uint64_t t) { fallbackLogged_ = true; lastWaitLogAt_ = t; log(head + " Waiting for more sources (fetch_fallback=wait). To start from the published snapshot instead: stop the node " - "and run ./node --download-snapshot in this folder, then start it again.", + "and run ./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github), then start it again.", true); } return; } log(head + " Syncing from block 0 (about 35 h on a fast machine). To start from the published snapshot instead: stop the node " - "and run ./node --download-snapshot in this folder, then start it again.", + "and run ./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github), then start it again.", true); clearInFlight(); phase_ = Phase::Fallback; @@ -563,6 +596,7 @@ bool SnapshotFetcher::startFetch(uint64_t t) { phase_ = Phase::Fetch; fetchStartedAt_ = t; lastSendAt_ = t; + lastByteAt_ = t; lastProgressAt_ = t; lastProgressBytes_ = bytesDone(); lastProgressDecile_ = bytesTotal() == 0 ? 0 : static_cast(bytesDone() * 10 / bytesTotal()); @@ -581,10 +615,14 @@ bool SnapshotFetcher::chooseTarget(uint64_t t) { std::vector> bestSources; size_t bestOwners = 0; size_t bestTotal = 0; + std::vector hashes; // filesHash of each agreed source, computed once + for (const auto& a : agreedSources_) { + hashes.push_back(a.manifest.filesHash()); + } std::set tried; - for (const auto& candidate : agreedSources_) { - const auto& rep = candidate.manifest; - const auto hash = rep.filesHash(); + for (size_t c = 0; c < agreedSources_.size(); ++c) { + const auto& rep = agreedSources_[c].manifest; + const auto& hash = hashes[c]; if (!tried.insert(hash).second) { continue; } @@ -596,9 +634,9 @@ bool SnapshotFetcher::chooseTarget(uint64_t t) { continue; } size_t owners = 0; - for (const auto& other : agreedSources_) { - const auto* s = findSource(other.key); - if (s != nullptr && everUsable(*s) && other.manifest.filesHash() == hash) { + for (size_t o = 0; o < agreedSources_.size(); ++o) { + const auto* s = findSource(agreedSources_[o].key); + if (s != nullptr && everUsable(*s) && hashes[o] == hash) { ++owners; } } @@ -643,6 +681,7 @@ bool SnapshotFetcher::chooseTarget(uint64_t t) { if (reuse) { state = std::move(*old); old->fd = -1; + state.hashing.reset(); // a check in progress starts over // what is on disk is checked again before it is handed over if (state.verified) { state.verified = false; @@ -819,7 +858,19 @@ void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::Pi switch (reply.status) { case snapshot_p2p::PieceStatus::Busy: if (s != nullptr) { - s->busyUntil = t + std::min(reply.retryMs, kMaxBusyMs); + // at least 500 ms, so that a source answering busy with no delay is not asked at every round trip + s->busyUntil = t + std::clamp(reply.retryMs, kMinBusyMs, kMaxBusyMs); + if (s->busySince == 0) { + s->busySince = t; + } + else if (t - s->busySince >= kBusyMarkMs) { + // busy for a minute without a piece: left alone for a minute, without counting + // towards the ban (a loaded honest node is not a faulty one) + s->busySince = 0; + s->markedUntil = t + kMarkMs; + log("SNAPSHOT: source " + keyText(sender) + " busy for " + duration(kBusyMarkMs / 1000) + + " without a piece: not asked for 1 min"); + } } break; case snapshot_p2p::PieceStatus::Unknown: @@ -832,6 +883,7 @@ void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::Pi const uint64_t length = pieceLength(f, reply.chunk); if (reply.data.size() != length || sha256Hex(reply.data.data(), reply.data.size()) != f.file.chunks[reply.chunk]) { log("SNAPSHOT: rejected piece " + f.file.name + ":" + std::to_string(reply.chunk) + " from " + keyText(sender), true); + // mark() may switch to another file group and replace files_: f is not used after it mark(sender, t, true, "sent a piece that does not match the manifest"); break; } @@ -853,6 +905,10 @@ void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::Pi f.have[reply.chunk] = true; --f.missing; f.doneBytes += length; + lastByteAt_ = t; + if (s != nullptr) { + s->busySince = 0; + } if (f.missing == 0) { f.verifyPending = true; } @@ -891,11 +947,15 @@ void SnapshotFetcher::clearInFlight() { } } +// Keys are not authenticated by the p2p layer (design section 2): a host that spoofs a source's key +// can get that key marked or banned here. The cost is bounded: a source lost for this bootstrap, +// never a wrong snapshot, since every piece is checked against the signed manifest. void SnapshotFetcher::mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why) { auto* s = findSource(key); if (s == nullptr) { return; } + const bool wasBanned = s->banned; if (forever) { s->banned = true; } @@ -904,6 +964,7 @@ void SnapshotFetcher::mark(const cs::PublicKey& key, uint64_t t, bool forever, c s->markedUntil = t + kMarkMs; s->banned = s->marks >= 2; } + dirty_ = dirty_ || (s->banned && !wasBanned); log("SNAPSHOT: source " + keyText(key) + " " + why + (s->banned ? ": not asked again" : ": not asked for 1 min")); if (!s->banned || phase_ != Phase::Fetch) { return; @@ -956,28 +1017,42 @@ void SnapshotFetcher::expire(uint64_t t) { } void SnapshotFetcher::verifyFiles(uint64_t t) { + // the whole-file SHA-256, at most kHashBytesPerTick per tick so that a large file does not hold + // the processor thread; a file that does not match is downloaded again + uint64_t budget = kHashBytesPerTick; + std::vector buffer; + bool bad = false; for (auto& f : files_) { if (!f.verifyPending) { continue; } - f.verifyPending = false; - crypto_hash_sha256_state state; - crypto_hash_sha256_init(&state); - std::vector buffer(1u << 20); - uint64_t offset = 0; + if (!f.hashing) { + f.hashing = std::make_shared(); + crypto_hash_sha256_init(&f.hashing->state); + f.hashed = 0; + } bool ok = f.fd >= 0; - while (ok && offset < f.file.size) { - const auto want = static_cast(std::min(buffer.size(), f.file.size - offset)); - const auto n = ::pread(f.fd, buffer.data(), want, static_cast(offset)); + while (ok && f.hashed < f.file.size && budget != 0) { + if (buffer.empty()) { + buffer.resize(1u << 20); + } + const auto want = static_cast(std::min({buffer.size(), f.file.size - f.hashed, budget})); + const auto n = ::pread(f.fd, buffer.data(), want, static_cast(f.hashed)); if (n <= 0) { ok = false; break; } - crypto_hash_sha256_update(&state, buffer.data(), static_cast(n)); - offset += static_cast(n); + crypto_hash_sha256_update(&f.hashing->state, buffer.data(), static_cast(n)); + f.hashed += static_cast(n); + budget -= static_cast(n); + } + if (ok && f.hashed < f.file.size) { + return; // the rest at the next tick } std::array digest{}; - crypto_hash_sha256_final(&state, digest.data()); + crypto_hash_sha256_final(&f.hashing->state, digest.data()); + f.hashing.reset(); + f.verifyPending = false; if (ok && hexOf(digest.data(), digest.size()) == f.file.sha256) { f.verified = true; continue; @@ -989,6 +1064,9 @@ void SnapshotFetcher::verifyFiles(uint64_t t) { f.doneBytes = 0; f.verified = false; f.verifyPending = f.file.chunks.empty(); + bad = true; + } + if (bad) { backToCollect("a downloaded file did not verify"); return; } @@ -1014,8 +1092,31 @@ void SnapshotFetcher::backToCollect(const std::string& why) { target_.reset(); agreedSources_.clear(); agreement_.chosen.reset(); + dirty_ = true; + ++failures_; + if (failures_ >= kMaxAttempts) { + log("SNAPSHOT: " + why + " (attempt " + std::to_string(failures_) + " of " + std::to_string(kMaxAttempts) + + "): syncing from block 0", + true); + phase_ = Phase::Fallback; + return; + } phase_ = Phase::Collect; - log("SNAPSHOT: " + why + ": looking at the sources again", true); + log("SNAPSHOT: " + why + " (attempt " + std::to_string(failures_) + " of " + std::to_string(kMaxAttempts) + + "): looking at the sources again", + true); +} + +void SnapshotFetcher::banDeclaring(const std::string& name, const std::string& sha256, const std::string& why) { + for (const auto& a : agreedSources_) { + const auto* f = a.manifest.file(name); + auto* s = findSource(a.key); + if (f != nullptr && f->sha256 == sha256 && s != nullptr && !s->banned) { + s->banned = true; + dirty_ = true; + log("SNAPSHOT: source " + keyText(a.key) + " " + why + ": not asked again"); + } + } } void SnapshotFetcher::logProgress(uint64_t t, bool force) { @@ -1061,10 +1162,24 @@ void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { if (stopped_ || phase_ != Phase::Import) { return; } - ++importFailures_; + ++failures_; + if (target_ && fileAtFault.empty()) { + // not a file's fault: the source whose signed manifest this was + std::vector bytes; + if (DecodeBase58(target_->nodeKey, bytes) && bytes.size() == cs::PublicKey{}.size()) { + cs::PublicKey key; + std::copy(bytes.begin(), bytes.end(), key.begin()); + if (auto* s = findSource(key); s != nullptr && !s->banned) { + s->banned = true; + log("SNAPSHOT: source " + keyText(key) + " signed a snapshot the import refused: not asked again"); + } + } + } if (target_) { if (const auto* f = target_->file(fileAtFault)) { badFiles_.insert({f->name, f->sha256}); + // every agreed source that declared these bytes (design 6.7) + banDeclaring(f->name, f->sha256, "declared " + f->name + " that the import refused"); files_.erase(std::remove_if(files_.begin(), files_.end(), [&](FileState& s) { if (s.file.name != fileAtFault) { return false; @@ -1076,9 +1191,9 @@ void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { } } log("SNAPSHOT: the import refused " + (fileAtFault.empty() ? std::string("the snapshot") : fileAtFault) + " (attempt " + - std::to_string(importFailures_) + " of " + std::to_string(kMaxAttempts) + ")", + std::to_string(failures_) + " of " + std::to_string(kMaxAttempts) + ")", true); - if (importFailures_ >= kMaxAttempts) { + if (failures_ >= kMaxAttempts) { log("SNAPSHOT: " + std::to_string(kMaxAttempts) + " imports failed: syncing from block 0", true); phase_ = Phase::Fallback; return; @@ -1086,6 +1201,7 @@ void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { target_.reset(); agreedSources_.clear(); agreement_.chosen.reset(); + dirty_ = true; phase_ = Phase::Collect; } diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp index 0f6b972e3..9780a7bdc 100644 --- a/tests/csnode/snapshot_fetcher_tests.cpp +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -171,6 +171,8 @@ struct Net { bool busyAll = false; // answers every piece request busy uint32_t busyMs = 2000; std::optional manifestJson; // replaces the manifest it serves + size_t silentManifests = 0; // manifest requests left without an answer, from the first + size_t manifestAsks = 0; size_t outstanding = 0; // piece requests not answered yet }; @@ -262,6 +264,11 @@ struct Net { if (it == sources.end()) { continue; } + ++it->second.manifestAsks; + if (it->second.silentManifests != 0) { + --it->second.silentManifests; + continue; + } auto reply = it->second.provider->onManifestRequest(me, serving(), synced()); if (!reply) { continue; @@ -811,7 +818,7 @@ TEST(SnapshotFetcher, FetchWaitSync) { EXPECT_GE(net.nowMs - 10'000'000, 60'000u); EXPECT_TRUE(net.logged("SNAPSHOT: 1 source declares block 200 (need 2): waiting")); EXPECT_TRUE(net.logged("SNAPSHOT: no agreement of 2 community sources after 1 min (seen: 1 source for block 200). Syncing from block 0")); - EXPECT_TRUE(net.logged("./node --download-snapshot")); + EXPECT_TRUE(net.logged("./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github)")); const auto sends = net.manifestRequests + net.sent.size(); for (int i = 0; i < 1000; ++i) { net.step(fetcher); @@ -846,8 +853,10 @@ TEST(SnapshotFetcher, ImportFailures) { net.add(Snapshots::A2); net.add(Snapshots::B); net.add(Snapshots::E); - SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + // the sources that declared a refused file are banned: min_sources=1 keeps one for the third attempt + SnapshotFetcher fetcher(fetchConfig(1), tmp.path, net.hooks()); std::set walletHashes; + std::set banned; for (int attempt = 1; attempt <= 3; ++attempt) { const size_t sentBefore = net.sent.size(); ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })) << attempt; @@ -857,17 +866,128 @@ TEST(SnapshotFetcher, ImportFailures) { const auto* wallets = result->first.file(kWallets); ASSERT_NE(wallets, nullptr); EXPECT_TRUE(walletHashes.insert(wallets->sha256).second) << "the refused file came back"; - if (attempt > 1) { - // files already verified are not asked again - for (size_t i = sentBefore; i < net.sent.size(); ++i) { + for (size_t i = sentBefore; i < net.sent.size(); ++i) { + if (attempt > 1) { + // files already verified are not asked again EXPECT_EQ(std::get<2>(net.sent[i]).name, kWallets); } + EXPECT_EQ(banned.count(std::get<1>(net.sent[i])), 0u) << "a banned source was asked"; + } + for (const auto& [key, source] : net.sources) { + const auto* f = snapshots().src[source.index].manifest.file(kWallets); + if (f->sha256 == wallets->sha256) { + banned.insert(key); + } } fetcher.onImportFailed(kWallets); EXPECT_EQ(fetcher.phase(), attempt < 3 ? Phase::Collect : Phase::Fallback); EXPECT_FALSE(fetcher.result()); } EXPECT_TRUE(net.logged("3 imports failed")); + EXPECT_EQ(net.countLogged("that the import refused: not asked again"), 4u); +} + +// a refusal that is no file's fault bans the source that signed the manifest +TEST(SnapshotFetcher, ImportFailureWithoutAFileBansTheSigner) { + TmpDir tmp("importsigner"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + const auto signer = fetcher.result()->first.nodeKey; + fetcher.onImportFailed(""); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("source " + signer + " signed a snapshot the import refused: not asked again")); + const size_t before = net.sent.size(); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + for (size_t i = before; i < net.sent.size(); ++i) { + EXPECT_NE(base58(std::get<1>(net.sent[i])), signer); + } +} + +// a peer that becomes a neighbour late, and one silent for its first requests, are kept (design 5.3a) +TEST(SnapshotFetcher, LateAndSilentSourcesAreKept) { + TmpDir tmp("late"); + Net net; + net.add(Snapshots::A1, false).silentManifests = 1; + net.add(Snapshots::A2).silentManifests = 2; + net.add(Snapshots::B, false); + SnapshotFetcher fetcher(fetchConfig(3), tmp.path, net.hooks()); + for (int i = 0; i < 250; ++i) { + net.step(fetcher); + } + // A2 was silent twice and answered the third request, 10 and 20 s after the first + EXPECT_EQ(net.sources[net.key(Snapshots::A2)].manifestAsks, 3u); + EXPECT_EQ(fetcher.manifests(), 1u); + for (int i = 0; i < 360; ++i) { // past a minute: A1 and B become neighbours late + net.step(fetcher); + } + EXPECT_EQ(fetcher.manifests(), 1u); + net.peers.neighbours.insert(net.key(Snapshots::A1)); + net.peers.neighbours.insert(net.key(Snapshots::B)); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(fetcher.result()->second.size(), 3u); +} + +// busy with no delay: not asked again before 500 ms +TEST(SnapshotFetcher, BusyWithoutDelayIsFloored) { + TmpDir tmp("busyfloor"); + Net net; + auto& a1 = net.add(Snapshots::A1); + a1.busyAll = true; + a1.busyMs = 0; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + const auto a1Key = net.key(Snapshots::A1); + for (const auto& s : net.sent) { + if (std::get<1>(s) != a1Key) { + continue; + } + for (const auto& [when, who] : net.busySent) { + if (who == a1Key) { + EXPECT_FALSE(when < std::get<0>(s) && std::get<0>(s) < when + SnapshotFetcher::kMinBusyMs); + } + } + } +} + +// sources that are busy forever: each fetch ends after 5 min without data, the third ends the bootstrap +TEST(SnapshotFetcher, BusyForeverEndsTheFetch) { + TmpDir tmp("busyforever"); + Net net; + net.add(Snapshots::A1).busyAll = true; + net.add(Snapshots::A2).busyAll = true; + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; + cfg.fetchFallback = "wait"; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() != Phase::Fetch; }, 4000)); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_GE(net.nowMs - start, SnapshotFetcher::kNoProgressMs); + EXPECT_TRUE(net.logged("no data for 5 min (attempt 1 of 3)")); + EXPECT_TRUE(net.logged("busy for 1 min without a piece")); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 10000)); + EXPECT_TRUE(net.logged("(attempt 3 of 3): syncing from block 0")); +} + +// two groups as strong as each other: the 30-s line says conflict +TEST(SnapshotFetcher, ConflictLine) { + TmpDir tmp("conflict"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::D); + SnapshotFetcher fetcher(fetchConfig(1), tmp.path, net.hooks()); + for (int i = 0; i < 320; ++i) { + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("SNAPSHOT: conflict: two groups with 1 sources at block 200: waiting")); + EXPECT_TRUE(net.logged("2 manifests collected, 0 not counted: address unknown, 0 replies dropped")); } // (13) a dry run stops after Collect and reports the groups From b46bb7f3488cf141787d546eb003adb4cc9a5603 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 16:24:29 +0100 Subject: [PATCH 6/9] snapshot fetcher: adaptive piece timeout; remote failures do not force a sync from 0 Piece timeout: 30 s plus the bytes outstanding to the source at 64 KB/s. A source that timed out but delivered a piece in the last 60 s is not marked: its piece is asked elsewhere and it gets one piece at a time until it delivers two in time; its late reply is still taken (hash-checked) while the piece is missing. A source that never delivered keeps the mark. Attempts: refused imports and failed fetches are counted apart. A fetch that ends with sources banned or marked in it goes back to Collect without counting; other failed fetches count, 3 -> Fallback with fetch_fallback=sync, never with wait. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_fetcher.hpp | 19 +++- csnode/src/snapshot_fetcher.cpp | 107 ++++++++++++++---- tests/csnode/snapshot_fetcher_tests.cpp | 122 +++++++++++++++++++-- 3 files changed, 215 insertions(+), 33 deletions(-) diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp index afe6ae5ed..63cd69490 100644 --- a/csnode/include/csnode/snapshot_fetcher.hpp +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -57,7 +57,10 @@ class SnapshotFetcher { static constexpr size_t kManifestRetries = 2; // ... this many times, then every 30 s static constexpr uint64_t kSilentMs = kManifestIntervalMs + 5'000; // no answer this long: silent static constexpr uint64_t kNeighbourRetryMs = 60'000; - static constexpr uint64_t kPieceTimeoutMs = 30'000; + static constexpr uint64_t kPieceTimeoutMs = 30'000; // plus the bytes outstanding to the source at 64 KB/s + static constexpr uint64_t kTimeoutBytesPerSecond = 64 << 10; + static constexpr uint64_t kRecentPieceMs = 60'000; // a source that delivered this recently is slow, not stalled + static constexpr size_t kMaxLate = 64; // timed-out pieces whose late reply is still taken static constexpr uint64_t kMarkMs = 60'000; static constexpr uint64_t kMinBusyMs = 500; static constexpr uint64_t kMaxBusyMs = 10'000; @@ -73,7 +76,8 @@ class SnapshotFetcher { static constexpr size_t kMaxPerBlock = 16; // discovered sources per /16 (IPv4) or /32 (IPv6) static constexpr uint64_t kEvictedMs = 60'000; // an evicted key is not taken again before static constexpr size_t kMaxEvicted = 1024; - static constexpr size_t kMaxAttempts = 3; // fetches that end without a snapshot, then Fallback + static constexpr size_t kMaxAttempts = 3; // refused imports, and with fetch_fallback=sync failed + // fetches not explained by misbehaving sources: then Fallback static constexpr uint64_t kHashBytesPerTick = 64ull << 20; // whole-file SHA-256 per tick static constexpr size_t kMaxSourcesFileKeys = 64; @@ -134,6 +138,10 @@ class SnapshotFetcher { bool banned = false; // for the rest of the bootstrap uint64_t busyUntil = 0; uint64_t busySince = 0; // first busy answer since its last piece; 0: none + size_t budget = kPiecesPerSource; // pieces it may have outstanding: 1 after a timeout of a source that delivers + size_t streak = 0; // pieces delivered in time in a row + bool delivered = false; + uint64_t lastPieceAt = 0; size_t outstanding = 0; }; @@ -190,6 +198,7 @@ class SnapshotFetcher { void releaseSource(const cs::PublicKey& key); void mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why); void backToCollect(const std::string& why); + void forgetLate(const cs::PublicKey& key); void banDeclaring(const std::string& name, const std::string& sha256, const std::string& why); void closeFile(FileState& f); bool openFile(FileState& f, bool truncate); @@ -215,7 +224,11 @@ class SnapshotFetcher { std::optional target_; std::vector files_; std::set> badFiles_; // (name, sha256) refused by the importer - size_t failures_ = 0; // fetches that ended without a snapshot, refused imports included + size_t importFailures_ = 0; // imports refused (onImportFailed) + size_t fetchFailures_ = 0; // fetches that ended without a snapshot and without a misbehaving source + bool misbehaviour_ = false; // a source was marked or banned during the current fetch + // pieces that timed out, (file, chunk) -> the source asked: its late reply is still taken + std::map, cs::PublicKey> late_; bool dirty_ = true; // the input of the agreement changed since it was computed uint64_t lastByteAt_ = 0; uint64_t lastSendAt_ = 0; diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp index 66e539533..ad034fde9 100644 --- a/csnode/src/snapshot_fetcher.cpp +++ b/csnode/src/snapshot_fetcher.cpp @@ -586,7 +586,7 @@ bool SnapshotFetcher::everUsable(const Source& s) const { } bool SnapshotFetcher::usable(const Source& s, uint64_t t) const { - return !s.banned && s.markedUntil <= t && s.busyUntil <= t && s.outstanding < kPiecesPerSource; + return !s.banned && s.markedUntil <= t && s.busyUntil <= t && s.outstanding < s.budget; } bool SnapshotFetcher::startFetch(uint64_t t) { @@ -594,6 +594,7 @@ bool SnapshotFetcher::startFetch(uint64_t t) { return false; } phase_ = Phase::Fetch; + misbehaviour_ = false; fetchStartedAt_ = t; lastSendAt_ = t; lastByteAt_ = t; @@ -843,15 +844,26 @@ void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::Pi return; } auto& f = *file; + auto* s = findSource(sender); auto it = f.inFlight.find(reply.chunk); - if (it == f.inFlight.end() || it->second.source != sender) { - ++dropped_; - return; + bool lateReply = false; + if (it != f.inFlight.end() && it->second.source == sender) { + f.inFlight.erase(it); + if (s != nullptr && s->outstanding != 0) { + --s->outstanding; + } } - f.inFlight.erase(it); - auto* s = findSource(sender); - if (s != nullptr && s->outstanding != 0) { - --s->outstanding; + else { + // the reply to a piece that timed out on a slow link: still taken while the piece is missing, + // so that the bytes already on their way are not asked for twice + auto l = late_.find({reply.name, reply.chunk}); + if (l == late_.end() || l->second != sender || reply.chunk >= f.have.size() || f.have[reply.chunk] || + reply.status != snapshot_p2p::PieceStatus::Ok) { + ++dropped_; + return; + } + late_.erase(l); + lateReply = true; } const uint64_t t = now(); @@ -908,6 +920,13 @@ void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::Pi lastByteAt_ = t; if (s != nullptr) { s->busySince = 0; + s->delivered = true; + s->lastPieceAt = t; + // two pieces in a row within the timeout: back to two pieces at a time + s->streak = lateReply ? 0 : s->streak + 1; + if (s->budget < kPiecesPerSource && s->streak >= 2) { + s->budget = kPiecesPerSource; + } } if (f.missing == 0) { f.verifyPending = true; @@ -936,9 +955,18 @@ void SnapshotFetcher::releaseSource(const cs::PublicKey& key) { if (auto* s = findSource(key)) { s->outstanding = 0; } + forgetLate(key); +} + +void SnapshotFetcher::forgetLate(const cs::PublicKey& key) { + for (auto it = late_.begin(); it != late_.end();) { + it = it->second == key ? late_.erase(it) : std::next(it); + } } void SnapshotFetcher::clearInFlight() { + // late replies too: after a change of files a chunk index may name other bytes + late_.clear(); for (auto& f : files_) { f.inFlight.clear(); } @@ -956,6 +984,7 @@ void SnapshotFetcher::mark(const cs::PublicKey& key, uint64_t t, bool forever, c return; } const bool wasBanned = s->banned; + misbehaviour_ = misbehaviour_ || phase_ == Phase::Fetch; if (forever) { s->banned = true; } @@ -993,14 +1022,27 @@ void SnapshotFetcher::mark(const cs::PublicKey& key, uint64_t t, bool forever, c } void SnapshotFetcher::expire(uint64_t t) { - std::set late; + // the timeout grows with what the source still has to send us: 30 s plus its outstanding bytes + // at 64 KB/s, so that two large pieces on a slow link are not taken for a stall + std::map bytesTo; + for (const auto& f : files_) { + for (const auto& [chunk, piece] : f.inFlight) { + bytesTo[piece.source] += pieceLength(f, chunk); + } + } + std::set lateSources; for (auto& f : files_) { for (auto it = f.inFlight.begin(); it != f.inFlight.end();) { - if (t - it->second.sentAt >= kPieceTimeoutMs) { - late.insert(it->second.source); + const uint64_t timeout = kPieceTimeoutMs + bytesTo[it->second.source] * 1000 / kTimeoutBytesPerSecond; + if (t - it->second.sentAt >= timeout) { + lateSources.insert(it->second.source); if (auto* s = findSource(it->second.source); s != nullptr && s->outstanding != 0) { --s->outstanding; } + if (late_.size() >= kMaxLate) { + late_.erase(late_.begin()); + } + late_[{f.file.name, it->first}] = it->second.source; it = f.inFlight.erase(it); } else { @@ -1008,11 +1050,21 @@ void SnapshotFetcher::expire(uint64_t t) { } } } - for (const auto& key : late) { + for (const auto& key : lateSources) { if (phase_ != Phase::Fetch) { break; } - mark(key, t, false, "did not answer within " + duration(kPieceTimeoutMs / 1000)); + auto* s = findSource(key); + if (s != nullptr && s->delivered && t - s->lastPieceAt < kRecentPieceMs) { + // it delivers, slowly: one piece at a time until it delivers two in time + s->streak = 0; + if (s->budget != 1) { + s->budget = 1; + log("SNAPSHOT: source " + keyText(key) + " is slow: one piece at a time"); + } + continue; + } + mark(key, t, false, "did not answer within " + duration((kPieceTimeoutMs + bytesTo[key] * 1000 / kTimeoutBytesPerSecond) / 1000)); } } @@ -1093,16 +1145,29 @@ void SnapshotFetcher::backToCollect(const std::string& why) { agreedSources_.clear(); agreement_.chosen.reset(); dirty_ = true; - ++failures_; - if (failures_ >= kMaxAttempts) { - log("SNAPSHOT: " + why + " (attempt " + std::to_string(failures_) + " of " + std::to_string(kMaxAttempts) + + phase_ = Phase::Collect; + if (misbehaviour_) { + // sources were banned or marked: remote peers must not be able to push the node to a sync + // from block 0, so this is not counted + misbehaviour_ = false; + log("SNAPSHOT: " + why + " (sources were banned or marked, not counted): looking at the sources again", true); + return; + } + ++fetchFailures_; + if (cfg_.fetchFallback == "wait") { + log("SNAPSHOT: " + why + " (failed fetch " + std::to_string(fetchFailures_) + + ", fetch_fallback=wait): looking at the sources again", + true); + return; + } + if (fetchFailures_ >= kMaxAttempts) { + log("SNAPSHOT: " + why + " (failed fetch " + std::to_string(fetchFailures_) + " of " + std::to_string(kMaxAttempts) + "): syncing from block 0", true); phase_ = Phase::Fallback; return; } - phase_ = Phase::Collect; - log("SNAPSHOT: " + why + " (attempt " + std::to_string(failures_) + " of " + std::to_string(kMaxAttempts) + + log("SNAPSHOT: " + why + " (failed fetch " + std::to_string(fetchFailures_) + " of " + std::to_string(kMaxAttempts) + "): looking at the sources again", true); } @@ -1162,7 +1227,7 @@ void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { if (stopped_ || phase_ != Phase::Import) { return; } - ++failures_; + ++importFailures_; if (target_ && fileAtFault.empty()) { // not a file's fault: the source whose signed manifest this was std::vector bytes; @@ -1191,9 +1256,9 @@ void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { } } log("SNAPSHOT: the import refused " + (fileAtFault.empty() ? std::string("the snapshot") : fileAtFault) + " (attempt " + - std::to_string(failures_) + " of " + std::to_string(kMaxAttempts) + ")", + std::to_string(importFailures_) + " of " + std::to_string(kMaxAttempts) + ")", true); - if (failures_ >= kMaxAttempts) { + if (importFailures_ >= kMaxAttempts) { log("SNAPSHOT: " + std::to_string(kMaxAttempts) + " imports failed: syncing from block 0", true); phase_ = Phase::Fallback; return; diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp index 9780a7bdc..1bd62fe57 100644 --- a/tests/csnode/snapshot_fetcher_tests.cpp +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -46,12 +46,12 @@ void putU16(std::ofstream& f, uint16_t v) { f.write(reinterpret_cast(&v), sizeof(v)); } -// Five snapshots of block 200 built by buildSnapshot from one generated chain, each signed by its -// own key, with 64 KiB chunks: A1 and A2 from the same checkpoint files (identical bytes), B and +// Snapshots of block 200 built by buildSnapshot from one generated chain, each signed by its +// own key, with 64 KiB chunks: A1, A2 and A3-A6 from the same checkpoint files (identical bytes), B and // E with one and two trailing bytes in walletscache.dat (same digest, another sha256 for that // file), D with another state digest. Built once for all tests. struct Snapshots { - enum { A1, A2, B, D, E, Count }; + enum { A1, A2, B, D, E, A3, A4, A5, A6, Count }; struct Source { cs::PrivateKey priv; @@ -112,7 +112,7 @@ struct Snapshots { const auto qsB = checkpoint("b", 1, 7); const auto qsD = checkpoint("d", 0, 8); const auto qsE = checkpoint("e", 2, 7); - const std::array qsOf{qsA, qsA, qsB, qsD, qsE}; + const std::array qsOf{qsA, qsA, qsB, qsD, qsE, qsA, qsA, qsA, qsA}; for (size_t i = 0; i < Count; ++i) { auto& s = src[i]; @@ -173,6 +173,8 @@ struct Net { std::optional manifestJson; // replaces the manifest it serves size_t silentManifests = 0; // manifest requests left without an answer, from the first size_t manifestAsks = 0; + uint64_t linkBytesPerSecond = 0; // its uplink, when slow + uint64_t linkFreeAt = 0; size_t outstanding = 0; // piece requests not answered yet }; @@ -190,6 +192,9 @@ struct Net { size_t maxOutstandingPerSource = 0; size_t outstandingTotal = 0; size_t manifestRequests = 0; + uint64_t linkBytesPerSecond = 0; + uint64_t linkFreeAt = 0; + std::multimap> link; std::map addresses; // of peers that are not sources SnapshotFetcher::Hooks hooks() { @@ -278,6 +283,12 @@ struct Net { } fetcher.onManifestReply(k, *reply); } + // slow links: the joining node's downlink, or a source's uplink; replies arrive in order at the rate + while (!link.empty() && link.begin()->first <= nowMs) { + auto entry = std::move(link.begin()->second); + link.erase(link.begin()); + fetcher.onPieceReply(entry.first, std::move(entry.second)); + } auto pieces = std::move(pieceQueue); pieceQueue.clear(); for (auto& [k, request] : pieces) { @@ -309,6 +320,13 @@ struct Net { reply.data[0] ^= 0x5a; } } + if (s.linkBytesPerSecond != 0 || linkBytesPerSecond != 0) { + auto& freeAt = s.linkBytesPerSecond != 0 ? s.linkFreeAt : linkFreeAt; + const uint64_t rate = s.linkBytesPerSecond != 0 ? s.linkBytesPerSecond : linkBytesPerSecond; + freeAt = std::max(freeAt, nowMs) + reply.data.size() * 1000 / rate; + link.emplace(freeAt, std::make_pair(k, std::move(reply))); + continue; + } fetcher.onPieceReply(k, std::move(reply)); } } @@ -702,7 +720,8 @@ TEST(SnapshotFetcher, StalledSourceTimesOut) { const uint64_t start = net.nowMs; ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); EXPECT_GE(net.nowMs - start, SnapshotFetcher::kPieceTimeoutMs); - EXPECT_TRUE(net.logged(base58(net.key(Snapshots::A1)) + " did not answer within 30 s")); + // 30 s plus its two outstanding 64-KiB pieces at 64 KB/s + EXPECT_TRUE(net.logged(base58(net.key(Snapshots::A1)) + " did not answer within 32 s")); EXPECT_GT(net.sentTo(net.key(Snapshots::A1)), 0u); expectFilesOf(tmp.path, fetcher.result()->first); } @@ -961,18 +980,103 @@ TEST(SnapshotFetcher, BusyForeverEndsTheFetch) { net.add(Snapshots::A1).busyAll = true; net.add(Snapshots::A2).busyAll = true; auto cfg = fetchConfig(); - cfg.fetchWait = 86'400; - cfg.fetchFallback = "wait"; + cfg.fetchWait = 86'400; // fetch_fallback=sync SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); const uint64_t start = net.nowMs; ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() != Phase::Fetch; }, 4000)); EXPECT_EQ(fetcher.phase(), Phase::Collect); EXPECT_GE(net.nowMs - start, SnapshotFetcher::kNoProgressMs); - EXPECT_TRUE(net.logged("no data for 5 min (attempt 1 of 3)")); + EXPECT_TRUE(net.logged("no data for 5 min (failed fetch 1 of 3)")); EXPECT_TRUE(net.logged("busy for 1 min without a piece")); ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 10000)); - EXPECT_TRUE(net.logged("(attempt 3 of 3): syncing from block 0")); + EXPECT_TRUE(net.logged("(failed fetch 3 of 3): syncing from block 0")); +} + +// with fetch_fallback=wait failed fetches never end the bootstrap +TEST(SnapshotFetcher, BusyForeverWithWaitKeepsTrying) { + TmpDir tmp("busywait"); + Net net; + net.add(Snapshots::A1).busyAll = true; + net.add(Snapshots::A2).busyAll = true; + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; + cfg.fetchFallback = "wait"; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("(failed fetch 4, fetch_fallback=wait)") != 0; }, 20000)); + EXPECT_NE(fetcher.phase(), Phase::Fallback); + // the sources deliver again: the fetch completes + net.sources[net.key(Snapshots::A1)].busyAll = false; + net.sources[net.key(Snapshots::A2)].busyAll = false; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 10000)); +} + +// sources that serve bad pieces are banned wave after wave without pushing the node to a sync from 0 +TEST(SnapshotFetcher, MaliciousWavesDoNotForceFallback) { + TmpDir tmp("waves"); + Net net; + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::A3, Snapshots::A4, Snapshots::A5, Snapshots::A6}) { + net.add(i, i == Snapshots::A1 || i == Snapshots::A2).corruptAll = true; + } + net.add(Snapshots::B, false); + net.add(Snapshots::E, false); + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; // fetch_fallback=sync + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const std::vector> waves{{Snapshots::A3, Snapshots::A4}, {Snapshots::A5, Snapshots::A6}, {Snapshots::B, Snapshots::E}}; + for (size_t w = 0; w < waves.size(); ++w) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("not counted") == w + 1; }, 2000)) << w; + EXPECT_EQ(fetcher.phase(), Phase::Collect); + net.peers.neighbours.insert(net.key(waves[w].first)); + net.peers.neighbours.insert(net.key(waves[w].second)); + if (w + 1 == waves.size()) { + break; + } + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(net.countLogged("failed fetch"), 0u); + EXPECT_EQ(fetcher.result()->first.filesHash() == snapshots().src[Snapshots::B].manifest.filesHash() || + fetcher.result()->first.filesHash() == snapshots().src[Snapshots::E].manifest.filesHash(), + true); +} + +// a slow downlink: about 100 KB/s for eight 512-KiB pieces in flight, scaled here to 64-KiB pieces; +// no source is marked and the fetch completes +TEST(SnapshotFetcher, SlowLinkIsNotMarked) { + TmpDir tmp("slowlink"); + Net net; + net.linkBytesPerSecond = 12'500; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 20000)); + EXPECT_FALSE(net.logged("did not answer")); + EXPECT_FALSE(net.logged("looking at the sources again")); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// sources whose uplink is so slow that their second piece outlives the timeout: they deliver, so +// they go to one piece at a time instead of being marked, and their late pieces are taken +TEST(SnapshotFetcher, SlowSourcesGoToOnePieceAtATime) { + TmpDir tmp("slowsources"); + Net net; + net.add(Snapshots::A1).linkBytesPerSecond = 3'000; + net.add(Snapshots::A2).linkBytesPerSecond = 3'000; + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 20000)); + EXPECT_TRUE(net.logged("is slow: one piece at a time")); + EXPECT_FALSE(net.logged("did not answer")); + EXPECT_FALSE(net.logged("looking at the sources again")); + // late pieces were taken, not asked for again: no more requests than pieces plus the requeued ones + const auto& m = snapshots().src[Snapshots::A1].manifest; + size_t pieces = 0; + for (const auto& f : m.files) { + pieces += f.chunks.size(); + } + EXPECT_LT(net.sent.size(), pieces + pieces / 2); + expectFilesOf(tmp.path, fetcher.result()->first); } // two groups as strong as each other: the 30-s line says conflict From d399c4bdb597815896124352ada65b0d3a1333f8 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 16:35:40 +0100 Subject: [PATCH 7/9] snapshot fetcher: only proven bad data excuses a failed fetch; bounded bootstrap A failed fetch is not counted only when a source was banned in it for bytes that do not match the signed manifest; timeouts, busy and not-serving answers no longer excuse it. Excused aborts have their own cap (10, then Fallback with fetch_fallback=sync). With sync the whole bootstrap ends in Fallback at fetch_wait + 30 min whatever the counters say; with wait it keeps going. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_fetcher.hpp | 7 +- csnode/src/snapshot_fetcher.cpp | 28 ++++++-- tests/csnode/snapshot_fetcher_tests.cpp | 78 ++++++++++++++++++++-- 3 files changed, 101 insertions(+), 12 deletions(-) diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp index 63cd69490..ecd24d1ec 100644 --- a/csnode/include/csnode/snapshot_fetcher.hpp +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -77,7 +77,9 @@ class SnapshotFetcher { static constexpr uint64_t kEvictedMs = 60'000; // an evicted key is not taken again before static constexpr size_t kMaxEvicted = 1024; static constexpr size_t kMaxAttempts = 3; // refused imports, and with fetch_fallback=sync failed - // fetches not explained by misbehaving sources: then Fallback + // fetches without a proven bad source: then Fallback + static constexpr size_t kMaxExcusedAborts = 10; // fetches ended by a proven bad source (fetch_fallback=sync) + static constexpr uint64_t kBootstrapCapMs = 30 * 60'000; // fetch_fallback=sync: Fallback after fetch_wait + this static constexpr uint64_t kHashBytesPerTick = 64ull << 20; // whole-file SHA-256 per tick static constexpr size_t kMaxSourcesFileKeys = 64; @@ -226,7 +228,8 @@ class SnapshotFetcher { std::set> badFiles_; // (name, sha256) refused by the importer size_t importFailures_ = 0; // imports refused (onImportFailed) size_t fetchFailures_ = 0; // fetches that ended without a snapshot and without a misbehaving source - bool misbehaviour_ = false; // a source was marked or banned during the current fetch + size_t excusedAborts_ = 0; // fetches that ended with a source banned for a proven fault + bool misbehaviour_ = false; // a source was banned for a proven fault (a hash against the signed manifest) in this fetch // pieces that timed out, (file, chunk) -> the source asked: its late reply is still taken std::map, cs::PublicKey> late_; bool dirty_ = true; // the input of the agreement changed since it was computed diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp index ad034fde9..e9da8298d 100644 --- a/csnode/src/snapshot_fetcher.cpp +++ b/csnode/src/snapshot_fetcher.cpp @@ -287,6 +287,16 @@ void SnapshotFetcher::tick(const Peers& peers) { lastWaitLogAt_ = t; } neighbours_ = peers.neighbours; + // fetch_fallback=sync: the whole bootstrap is bounded whatever the counters say + if (!dryRun_ && cfg_.fetchFallback != "wait" && t - startedAt_ >= uint64_t(cfg_.fetchWait) * 1000 + kBootstrapCapMs) { + log("SNAPSHOT: no snapshot after " + duration((t - startedAt_) / 1000 / 60 * 60) + + " (fetch_wait + 30 min): syncing from block 0. To start from the published snapshot instead: stop the node and run" + " ./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github), then start it again.", + true); + clearInFlight(); + phase_ = Phase::Fallback; + return; + } discover(peers, t); if (phase_ == Phase::Discover || phase_ == Phase::Collect) { @@ -984,7 +994,9 @@ void SnapshotFetcher::mark(const cs::PublicKey& key, uint64_t t, bool forever, c return; } const bool wasBanned = s->banned; - misbehaviour_ = misbehaviour_ || phase_ == Phase::Fetch; + // only a proven fault (bytes that do not match the signed manifest) excuses the end of a fetch; + // timeouts, busy and "not serving" answers do not, or a peer could keep the node from its fallback + misbehaviour_ = misbehaviour_ || (forever && phase_ == Phase::Fetch); if (forever) { s->banned = true; } @@ -1147,10 +1159,18 @@ void SnapshotFetcher::backToCollect(const std::string& why) { dirty_ = true; phase_ = Phase::Collect; if (misbehaviour_) { - // sources were banned or marked: remote peers must not be able to push the node to a sync - // from block 0, so this is not counted + // a source was banned for bytes that do not match the signed manifest: remote peers must + // not be able to push the node to a sync from block 0, so this is counted apart, with its own cap misbehaviour_ = false; - log("SNAPSHOT: " + why + " (sources were banned or marked, not counted): looking at the sources again", true); + ++excusedAborts_; + if (cfg_.fetchFallback != "wait" && excusedAborts_ >= kMaxExcusedAborts) { + log("SNAPSHOT: " + why + " (a source sent bad data, " + std::to_string(excusedAborts_) + " of " + + std::to_string(kMaxExcusedAborts) + "): syncing from block 0", + true); + phase_ = Phase::Fallback; + return; + } + log("SNAPSHOT: " + why + " (a source sent bad data, not counted as a failed fetch): looking at the sources again", true); return; } ++fetchFailures_; diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp index 1bd62fe57..9ea1c48ec 100644 --- a/tests/csnode/snapshot_fetcher_tests.cpp +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -47,11 +47,11 @@ void putU16(std::ofstream& f, uint16_t v) { } // Snapshots of block 200 built by buildSnapshot from one generated chain, each signed by its -// own key, with 64 KiB chunks: A1, A2 and A3-A6 from the same checkpoint files (identical bytes), B and +// own key, with 64 KiB chunks: A1, A2 and A3-A10 from the same checkpoint files (identical bytes), B and // E with one and two trailing bytes in walletscache.dat (same digest, another sha256 for that // file), D with another state digest. Built once for all tests. struct Snapshots { - enum { A1, A2, B, D, E, A3, A4, A5, A6, Count }; + enum { A1, A2, B, D, E, A3, A4, A5, A6, A7, A8, A9, A10, Count }; struct Source { cs::PrivateKey priv; @@ -112,7 +112,7 @@ struct Snapshots { const auto qsB = checkpoint("b", 1, 7); const auto qsD = checkpoint("d", 0, 8); const auto qsE = checkpoint("e", 2, 7); - const std::array qsOf{qsA, qsA, qsB, qsD, qsE, qsA, qsA, qsA, qsA}; + const std::array qsOf{qsA, qsA, qsB, qsD, qsE, qsA, qsA, qsA, qsA, qsA, qsA, qsA, qsA}; for (size_t i = 0; i < Count; ++i) { auto& s = src[i]; @@ -173,6 +173,8 @@ struct Net { std::optional manifestJson; // replaces the manifest it serves size_t silentManifests = 0; // manifest requests left without an answer, from the first size_t manifestAsks = 0; + uint64_t pieceIntervalMs = 0; // serves one piece per interval, busy otherwise + uint64_t lastServedAt = 0; uint64_t linkBytesPerSecond = 0; // its uplink, when slow uint64_t linkFreeAt = 0; size_t outstanding = 0; // piece requests not answered yet @@ -303,7 +305,8 @@ struct Net { --s.outstanding; --outstandingTotal; PieceReply reply; - if (s.busyAll) { + const bool throttled = s.pieceIntervalMs != 0 && s.lastServedAt != 0 && nowMs - s.lastServedAt < s.pieceIntervalMs; + if (s.busyAll || throttled) { reply.status = PieceStatus::Busy; reply.retryMs = s.busyMs; reply.seq = request.seq; @@ -313,6 +316,9 @@ struct Net { } else { reply = s.provider->onPieceRequest(me, request, serving(), synced()); + if (reply.status == PieceStatus::Ok) { + s.lastServedAt = nowMs; + } if (reply.status == PieceStatus::Busy) { busySent.emplace_back(nowMs, k); } @@ -1025,7 +1031,7 @@ TEST(SnapshotFetcher, MaliciousWavesDoNotForceFallback) { SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); const std::vector> waves{{Snapshots::A3, Snapshots::A4}, {Snapshots::A5, Snapshots::A6}, {Snapshots::B, Snapshots::E}}; for (size_t w = 0; w < waves.size(); ++w) { - ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("not counted") == w + 1; }, 2000)) << w; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("not counted as a failed fetch") == w + 1; }, 2000)) << w; EXPECT_EQ(fetcher.phase(), Phase::Collect); net.peers.neighbours.insert(net.key(waves[w].first)); net.peers.neighbours.insert(net.key(waves[w].second)); @@ -1034,7 +1040,7 @@ TEST(SnapshotFetcher, MaliciousWavesDoNotForceFallback) { } } ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); - EXPECT_EQ(net.countLogged("failed fetch"), 0u); + EXPECT_EQ(net.countLogged("(failed fetch"), 0u); EXPECT_EQ(fetcher.result()->first.filesHash() == snapshots().src[Snapshots::B].manifest.filesHash() || fetcher.result()->first.filesHash() == snapshots().src[Snapshots::E].manifest.filesHash(), true); @@ -1079,6 +1085,66 @@ TEST(SnapshotFetcher, SlowSourcesGoToOnePieceAtATime) { expectFilesOf(tmp.path, fetcher.result()->first); } +// sources that only stop answering cannot keep a node with fetch_fallback=sync from its fallback +TEST(SnapshotFetcher, TimeoutsDoNotExcuseFailedFetches) { + TmpDir tmp("timeoutwaves"); + Net net; + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::A3, Snapshots::A4, Snapshots::A5, Snapshots::A6}) { + net.add(i, i == Snapshots::A1 || i == Snapshots::A2).stall = true; + } + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const std::vector> waves{{Snapshots::A3, Snapshots::A4}, {Snapshots::A5, Snapshots::A6}}; + for (size_t w = 0; w < waves.size(); ++w) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("(failed fetch " + std::to_string(w + 1) + " of 3)") != 0; }, + 10000)) + << w; + net.peers.neighbours.insert(net.key(waves[w].first)); + net.peers.neighbours.insert(net.key(waves[w].second)); + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 10000)); + EXPECT_TRUE(net.logged("(failed fetch 3 of 3): syncing from block 0")); + EXPECT_FALSE(net.logged("not counted as a failed fetch")); +} + +// sources proven bad are not counted as failed fetches, but ten such fetches end the bootstrap (sync) +TEST(SnapshotFetcher, TenBadSourceAbortsEndWithSync) { + TmpDir tmp("badwaves"); + Net net; + const std::vector bad{Snapshots::A1, Snapshots::A2, Snapshots::A3, Snapshots::A4, Snapshots::A5, + Snapshots::A6, Snapshots::A7, Snapshots::A8, Snapshots::A9, Snapshots::A10}; + for (auto i : bad) { + net.add(i, i == Snapshots::A1).corruptAll = true; + } + auto cfg = fetchConfig(1); + cfg.fetchWait = 86'400; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + for (size_t w = 1; w < bad.size(); ++w) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("not counted as a failed fetch") == w; }, 2000)) << w; + net.peers.neighbours.insert(net.key(bad[w])); + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 2000)); + EXPECT_TRUE(net.logged("(a source sent bad data, 10 of 10): syncing from block 0")); + EXPECT_FALSE(net.logged("(failed fetch")); +} + +// a source that keeps a fetch alive with one valid piece every 4 min: fetch_fallback=sync ends the +// bootstrap at fetch_wait + 30 min whatever the counters say +TEST(SnapshotFetcher, BootstrapCapWithSync) { + TmpDir tmp("bootcap"); + Net net; + net.add(Snapshots::A1).pieceIntervalMs = 4 * 60'000; + auto cfg = fetchConfig(1); + cfg.fetchWait = 60; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 20000)); + EXPECT_GE(net.nowMs - start, 31 * 60'000u); + EXPECT_TRUE(net.logged("(fetch_wait + 30 min): syncing from block 0")); + EXPECT_FALSE(net.logged("(failed fetch")); +} + // two groups as strong as each other: the 30-s line says conflict TEST(SnapshotFetcher, ConflictLine) { TmpDir tmp("conflict"); From 0cee43ba80e003d3f881af0251d70065b94e6ba9 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 16:55:10 +0100 Subject: [PATCH 8/9] snapshot fetcher: bootstrap cap makes room for the download; final review fixes - With fetch_fallback=sync the bootstrap cap grows by the agreed snapshot's download time: at fetch_rate_kbps when set, else at 64 KB/s for at most 2 h; a warning at the start of the fetch when the expected time exceeds what is left. - With fetch_fallback=wait refused imports go back to Collect instead of Fallback (the declaring sources are banned). - Late pieces: stored with their send time; for one more timeout they count towards their source's outstanding pieces; the source is not given the same chunk again while another is free; the oldest entry is evicted. - An in-flight piece filled meanwhile by another source's late reply is dropped at expiry without marking anyone. - Tests: the configured rate beyond 30 min, refused imports with wait, the re-assignment race, a symbolic link in the download folder. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/snapshot_fetcher.hpp | 21 ++++- csnode/src/snapshot_fetcher.cpp | 83 +++++++++++++++---- tests/csnode/snapshot_fetcher_tests.cpp | 92 ++++++++++++++++++++-- 3 files changed, 171 insertions(+), 25 deletions(-) diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp index ecd24d1ec..169476734 100644 --- a/csnode/include/csnode/snapshot_fetcher.hpp +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -79,7 +79,10 @@ class SnapshotFetcher { static constexpr size_t kMaxAttempts = 3; // refused imports, and with fetch_fallback=sync failed // fetches without a proven bad source: then Fallback static constexpr size_t kMaxExcusedAborts = 10; // fetches ended by a proven bad source (fetch_fallback=sync) - static constexpr uint64_t kBootstrapCapMs = 30 * 60'000; // fetch_fallback=sync: Fallback after fetch_wait + this + static constexpr uint64_t kBootstrapCapMs = 30 * 60'000; // fetch_fallback=sync: Fallback after fetch_wait + this, + // plus the time the agreed snapshot takes to download: + // at fetch_rate_kbps when set, else at 64 KB/s for at most + static constexpr uint64_t kMaxCapExtensionMs = 2 * 3'600'000; // this (a manifest may declare gigabytes) static constexpr uint64_t kHashBytesPerTick = 64ull << 20; // whole-file SHA-256 per tick static constexpr size_t kMaxSourcesFileKeys = 64; @@ -189,7 +192,9 @@ class SnapshotFetcher { std::string seenText() const; void checkWait(uint64_t t); - bool usable(const Source& s, uint64_t t) const; + bool usable(const cs::PublicKey& key, const Source& s, uint64_t t) const; + size_t lateOnTheWay(const cs::PublicKey& key, uint64_t t) const; + uint64_t capMs() const; bool everUsable(const Source& s) const; bool startFetch(uint64_t t); bool chooseTarget(uint64_t t); @@ -230,8 +235,16 @@ class SnapshotFetcher { size_t fetchFailures_ = 0; // fetches that ended without a snapshot and without a misbehaving source size_t excusedAborts_ = 0; // fetches that ended with a source banned for a proven fault bool misbehaviour_ = false; // a source was banned for a proven fault (a hash against the signed manifest) in this fetch - // pieces that timed out, (file, chunk) -> the source asked: its late reply is still taken - std::map, cs::PublicKey> late_; + // pieces that timed out, (file, chunk) -> the source asked: its late reply is still taken, and + // for one more timeout the piece still counts as on its way from that source + struct Late { + cs::PublicKey source{}; + uint64_t sentAt = 0; + uint64_t expiredAt = 0; + uint64_t timeout = 0; + }; + std::map, Late> late_; + uint64_t capExtensionMs_ = 0; // the bootstrap cap grows by the download time of the agreed snapshot bool dirty_ = true; // the input of the agreement changed since it was computed uint64_t lastByteAt_ = 0; uint64_t lastSendAt_ = 0; diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp index e9da8298d..56955ce45 100644 --- a/csnode/src/snapshot_fetcher.cpp +++ b/csnode/src/snapshot_fetcher.cpp @@ -288,9 +288,9 @@ void SnapshotFetcher::tick(const Peers& peers) { } neighbours_ = peers.neighbours; // fetch_fallback=sync: the whole bootstrap is bounded whatever the counters say - if (!dryRun_ && cfg_.fetchFallback != "wait" && t - startedAt_ >= uint64_t(cfg_.fetchWait) * 1000 + kBootstrapCapMs) { + if (!dryRun_ && cfg_.fetchFallback != "wait" && t - startedAt_ >= capMs()) { log("SNAPSHOT: no snapshot after " + duration((t - startedAt_) / 1000 / 60 * 60) + - " (fetch_wait + 30 min): syncing from block 0. To start from the published snapshot instead: stop the node and run" + " (fetch_wait + 30 min + the download time): syncing from block 0. To start from the published snapshot instead: stop the node and run" " ./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github), then start it again.", true); clearInFlight(); @@ -595,8 +595,21 @@ bool SnapshotFetcher::everUsable(const Source& s) const { return !s.banned; } -bool SnapshotFetcher::usable(const Source& s, uint64_t t) const { - return !s.banned && s.markedUntil <= t && s.busyUntil <= t && s.outstanding < s.budget; +uint64_t SnapshotFetcher::capMs() const { + return uint64_t(cfg_.fetchWait) * 1000 + kBootstrapCapMs + capExtensionMs_; +} + +size_t SnapshotFetcher::lateOnTheWay(const cs::PublicKey& key, uint64_t t) const { + size_t n = 0; + for (const auto& [piece, late] : late_) { + n += late.source == key && t - late.expiredAt < late.timeout; + } + return n; +} + +bool SnapshotFetcher::usable(const cs::PublicKey& key, const Source& s, uint64_t t) const { + // pieces that timed out but are probably still on their way take the source's link too + return !s.banned && s.markedUntil <= t && s.busyUntil <= t && s.outstanding + lateOnTheWay(key, t) < s.budget; } bool SnapshotFetcher::startFetch(uint64_t t) { @@ -614,6 +627,17 @@ bool SnapshotFetcher::startFetch(uint64_t t) { logGroups(false); log("SNAPSHOT: fetching block " + std::to_string(target_->seq) + ": " + megabytes(bytesTotal()) + " MB in " + std::to_string(files_.size()) + " files from " + std::to_string(agreedSources_.size()) + " sources"); + // the bootstrap cap (fetch_fallback=sync) makes room for the download: at the rate the operator + // chose, or at 64 KB/s for at most 2 h, since the size comes from a manifest + const uint64_t remaining = bytesTotal() - std::min(bytesTotal(), bytesDone()); + const uint64_t expected = cfg_.fetchRateKbps != 0 ? remaining / cfg_.fetchRateKbps : remaining * 1000 / kTimeoutBytesPerSecond; + capExtensionMs_ = std::max(capExtensionMs_, cfg_.fetchRateKbps != 0 ? expected : std::min(expected, kMaxCapExtensionMs)); + if (cfg_.fetchFallback != "wait" && t - startedAt_ + expected > capMs()) { + log("SNAPSHOT: fetching block " + std::to_string(target_->seq) + " may take about " + duration(expected / 60'000 * 60) + + ", more than the " + duration((capMs() - std::min(capMs(), t - startedAt_)) / 60'000 * 60) + + " left before the node syncs from block 0", + true); + } fill(t); return true; } @@ -802,7 +826,7 @@ void SnapshotFetcher::fill(uint64_t t) { for (size_t n = 0; n < keys.size() && total < kPiecesTotal; ++n) { const auto& key = keys[(first + n) % keys.size()]; auto* s = findSource(key); - if (s == nullptr || !usable(*s, t) || neighbours_.count(key) == 0) { + if (s == nullptr || !usable(key, *s, t) || neighbours_.count(key) == 0) { continue; } FileState* pick = nullptr; @@ -812,9 +836,22 @@ void SnapshotFetcher::fill(uint64_t t) { std::find(f.sources.begin(), f.sources.end(), key) == f.sources.end()) { continue; } + // a free chunk, not one this source is probably still sending late unless nothing else is free uint32_t chunk = 0; - while (chunk < f.have.size() && (f.have[chunk] || f.inFlight.count(chunk) != 0)) { - ++chunk; + uint32_t fallback = static_cast(f.have.size()); + for (; chunk < f.have.size(); ++chunk) { + if (f.have[chunk] || f.inFlight.count(chunk) != 0) { + continue; + } + auto l = late_.find({f.file.name, chunk}); + if (l != late_.end() && l->second.source == key) { + fallback = std::min(fallback, chunk); + continue; + } + break; + } + if (chunk >= f.have.size()) { + chunk = fallback; } if (chunk < f.have.size()) { pick = &f; @@ -867,7 +904,7 @@ void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::Pi // the reply to a piece that timed out on a slow link: still taken while the piece is missing, // so that the bytes already on their way are not asked for twice auto l = late_.find({reply.name, reply.chunk}); - if (l == late_.end() || l->second != sender || reply.chunk >= f.have.size() || f.have[reply.chunk] || + if (l == late_.end() || l->second.source != sender || reply.chunk >= f.have.size() || f.have[reply.chunk] || reply.status != snapshot_p2p::PieceStatus::Ok) { ++dropped_; return; @@ -970,7 +1007,7 @@ void SnapshotFetcher::releaseSource(const cs::PublicKey& key) { void SnapshotFetcher::forgetLate(const cs::PublicKey& key) { for (auto it = late_.begin(); it != late_.end();) { - it = it->second == key ? late_.erase(it) : std::next(it); + it = it->second.source == key ? late_.erase(it) : std::next(it); } } @@ -1046,15 +1083,24 @@ void SnapshotFetcher::expire(uint64_t t) { for (auto& f : files_) { for (auto it = f.inFlight.begin(); it != f.inFlight.end();) { const uint64_t timeout = kPieceTimeoutMs + bytesTo[it->second.source] * 1000 / kTimeoutBytesPerSecond; - if (t - it->second.sentAt >= timeout) { + if (f.have[it->first]) { + // filled meanwhile by another source's late reply: nothing to wait for, nobody to blame + if (auto* s = findSource(it->second.source); s != nullptr && s->outstanding != 0) { + --s->outstanding; + } + it = f.inFlight.erase(it); + } + else if (t - it->second.sentAt >= timeout) { lateSources.insert(it->second.source); if (auto* s = findSource(it->second.source); s != nullptr && s->outstanding != 0) { --s->outstanding; } if (late_.size() >= kMaxLate) { - late_.erase(late_.begin()); + auto oldest = std::min_element(late_.begin(), late_.end(), + [](const auto& a, const auto& b) { return a.second.sentAt < b.second.sentAt; }); + late_.erase(oldest); } - late_[{f.file.name, it->first}] = it->second.source; + late_[{f.file.name, it->first}] = Late{it->second.source, it->second.sentAt, t, timeout}; it = f.inFlight.erase(it); } else { @@ -1279,9 +1325,16 @@ void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { std::to_string(importFailures_) + " of " + std::to_string(kMaxAttempts) + ")", true); if (importFailures_ >= kMaxAttempts) { - log("SNAPSHOT: " + std::to_string(kMaxAttempts) + " imports failed: syncing from block 0", true); - phase_ = Phase::Fallback; - return; + if (cfg_.fetchFallback == "wait") { + // the sources that declared the refused bytes are banned already: look for others + log("SNAPSHOT: looking at the sources again (refused import " + std::to_string(importFailures_) + ", fetch_fallback=wait)", + true); + } + else { + log("SNAPSHOT: " + std::to_string(kMaxAttempts) + " imports failed: syncing from block 0", true); + phase_ = Phase::Fallback; + return; + } } target_.reset(); agreedSources_.clear(); diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp index 9ea1c48ec..f4d9952d4 100644 --- a/tests/csnode/snapshot_fetcher_tests.cpp +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -49,9 +49,9 @@ void putU16(std::ofstream& f, uint16_t v) { // Snapshots of block 200 built by buildSnapshot from one generated chain, each signed by its // own key, with 64 KiB chunks: A1, A2 and A3-A10 from the same checkpoint files (identical bytes), B and // E with one and two trailing bytes in walletscache.dat (same digest, another sha256 for that -// file), D with another state digest. Built once for all tests. +// file), D with another state digest, F with 1.6 MB more in walletscache.dat. Built once for all tests. struct Snapshots { - enum { A1, A2, B, D, E, A3, A4, A5, A6, A7, A8, A9, A10, Count }; + enum { A1, A2, B, D, E, A3, A4, A5, A6, A7, A8, A9, A10, F, Count }; struct Source { cs::PrivateKey priv; @@ -86,7 +86,7 @@ struct Snapshots { c = static_cast(rng()); } - auto checkpoint = [&](const std::string& name, size_t extra, uint8_t digestFill) { + auto checkpoint = [&](const std::string& name, size_t extra, uint8_t digestFill, size_t moreNoise = 0) { const auto qs = root / ("qs-" + name) / "200"; fs::create_directories(qs); { @@ -99,7 +99,11 @@ struct Snapshots { f.write(reinterpret_cast(head.data()), static_cast(head.size())); putU16(f, 0); } - std::ofstream(qs / "walletscache.dat", std::ios::binary) << noise << std::string(extra, 'x'); + std::string more(moreNoise, '\0'); + for (auto& c : more) { + c = static_cast(rng()); + } + std::ofstream(qs / "walletscache.dat", std::ios::binary) << noise << more << std::string(extra, 'x'); std::ofstream(qs / "blockchain.dat", std::ios::binary) << "settings"; cs::CheckpointDigest d; d.digest.fill(digestFill); @@ -112,7 +116,8 @@ struct Snapshots { const auto qsB = checkpoint("b", 1, 7); const auto qsD = checkpoint("d", 0, 8); const auto qsE = checkpoint("e", 2, 7); - const std::array qsOf{qsA, qsA, qsB, qsD, qsE, qsA, qsA, qsA, qsA, qsA, qsA, qsA, qsA}; + const auto qsF = checkpoint("f", 0, 7, 1'600'000); // a larger snapshot, about 2.6 MB + const std::array qsOf{qsA, qsA, qsB, qsD, qsE, qsA, qsA, qsA, qsA, qsA, qsA, qsA, qsA, qsF}; for (size_t i = 0; i < Count; ++i) { auto& s = src[i]; @@ -1141,10 +1146,85 @@ TEST(SnapshotFetcher, BootstrapCapWithSync) { const uint64_t start = net.nowMs; ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 20000)); EXPECT_GE(net.nowMs - start, 31 * 60'000u); - EXPECT_TRUE(net.logged("(fetch_wait + 30 min): syncing from block 0")); + EXPECT_TRUE(net.logged("(fetch_wait + 30 min + the download time): syncing from block 0")); EXPECT_FALSE(net.logged("(failed fetch")); } +// an honest join limited by fetch_rate_kbps: the cap makes room for the download at that rate +TEST(SnapshotFetcher, BootstrapCapAllowsTheConfiguredRate) { + TmpDir tmp("bootcaprate"); + Net net; + net.add(Snapshots::F); + auto cfg = fetchConfig(1); + cfg.fetchWait = 60; + cfg.fetchRateKbps = 1; // 1000 bytes/s: about 2.6 MB takes about 44 min, more than 30 + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import || fetcher.phase() == Phase::Fallback; }, 40000)); + EXPECT_EQ(fetcher.phase(), Phase::Import); + EXPECT_GT(net.nowMs - start, 31 * 60'000u); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// with fetch_fallback=wait refused imports do not end the bootstrap: other sources are looked for +TEST(SnapshotFetcher, RefusedImportsWithWaitKeepTrying) { + TmpDir tmp("importwait"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + auto cfg = fetchConfig(1); + cfg.fetchFallback = "wait"; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + for (int attempt = 1; attempt <= 3; ++attempt) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })) << attempt; + fetcher.onImportFailed(kWallets); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + } + EXPECT_TRUE(net.logged("(refused import 3, fetch_fallback=wait)")); + // every source that declared a refused file is banned; another one turns up + net.add(Snapshots::F); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(fetcher.result()->first.nodeKey, snapshots().src[Snapshots::F].manifest.nodeKey); +} + +// a slow source and a fast one: pieces taken from the slow one by timeout are fetched once, its +// late replies do not get the fast one blamed, everything is on disk at the end +TEST(SnapshotFetcher, ReassignmentRace) { + TmpDir tmp("race"); + Net net; + net.add(Snapshots::A1).linkBytesPerSecond = 3'000; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 20000)); + EXPECT_EQ(fetcher.bytesDone(), fetcher.bytesTotal()); + EXPECT_FALSE(net.logged(base58(net.key(Snapshots::A2)) + " did not answer")); + const auto& m = snapshots().src[Snapshots::A1].manifest; + size_t pieces = 0; + for (const auto& f : m.files) { + pieces += f.chunks.size(); + } + EXPECT_LT(net.sent.size(), pieces + pieces / 2); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// a symbolic link where a downloaded file goes: the file is not opened, the node syncs from block 0 +TEST(SnapshotFetcher, SymlinkInTmpDirIsRefused) { + TmpDir tmp("symlink"); + fs::create_directories(tmp.path); + const auto target = tmp.path / "elsewhere"; + std::ofstream(target) << "do not touch"; + fs::create_symlink(target, tmp.path / kWallets); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 50)); + EXPECT_TRUE(net.logged("cannot write")); + EXPECT_EQ(fileText(target), "do not touch"); +} + // two groups as strong as each other: the 30-s line says conflict TEST(SnapshotFetcher, ConflictLine) { TmpDir tmp("conflict"); From 50e96b3af3885bf07e24bbfd1eba847c418c43da Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 25 Oct 2026 17:02:12 +0100 Subject: [PATCH 9/9] docs: CHANGES for #104 Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index bd38f7018..c2596ee84 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -38,6 +38,7 @@ lines under the new release. | #100 | net: four p2p messages for snapshot manifests and pieces (types 40-43) with bounded decoders; this build answers requests as not serving and sends none | | #101 | net: `host_data` announces snapshot serving (bit 24, `[snapshot] serve=on` at start) and the community version (bits 32-47, `MINOR_NODE_VERSION`, now 13); `known_peers.json` at a clean stop lists the known peers with what they announce | | #102 | snapshot: a node with `[snapshot] serve=on` builds the snapshot of each periodic checkpoint and serves it to its p2p neighbours, within `serve_rate_kbps` (KB/s), `serve_max_peers` and per-peer limits, at half budget while trusted; snapshots use 512 KiB chunks and no contracts file | +| #104 | snapshot: the p2p fetcher (not started yet): finds the neighbours that serve snapshots, agrees on one (a configured `sources=` key first, then the most independent sources by /24 or /48, the highest block only as a tie-break), downloads its pieces from several sources with every piece and file checked against the signed manifest, an adaptive piece timeout for slow links, limits per network prefix against made-up keys, and the fallback to syncing from block 0 after `fetch_wait` (or a bounded total time) | ## community.12 — 2026-10-24