diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index 0d23c1f8f..a3bde1dc0 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -75,6 +75,8 @@ add_library(csnode include/csnode/snapshot_messages.hpp include/csnode/snapshot_provider.hpp include/csnode/snapshot_service.hpp + include/csnode/snapshot_agreement.hpp + include/csnode/snapshot_fetcher.hpp include/csnode/blockverifier.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp @@ -122,6 +124,8 @@ add_library(csnode src/snapshot_messages.cpp src/snapshot_provider.cpp src/snapshot_service.cpp + src/snapshot_agreement.cpp + src/snapshot_fetcher.cpp src/blockverifier.cpp src/sendcachedata.cpp src/eventreport.cpp diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index b2dfffaf4..cc5b1b6e3 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -28,6 +28,7 @@ class Transport; namespace cs { +class SnapshotFetcher; class SnapshotProvider; class SnapshotService; struct SnapshotNodeState; @@ -510,6 +511,8 @@ public slots: // snapshots served to p2p neighbours ([snapshot] serve) std::unique_ptr snapshotProvider_; std::unique_ptr snapshotService_; + // snapshot fetched from p2p neighbours at a start on an empty database (constructed in bootstrap mode only) + std::unique_ptr snapshotFetcher_; std::chrono::steady_clock::time_point snapshotSummaryAt_{}; uint32_t startPacketRequestPoint_ = 0; diff --git a/csnode/include/csnode/snapshot_agreement.hpp b/csnode/include/csnode/snapshot_agreement.hpp new file mode 100644 index 000000000..8b4e9482f --- /dev/null +++ b/csnode/include/csnode/snapshot_agreement.hpp @@ -0,0 +1,70 @@ +#ifndef SNAPSHOT_AGREEMENT_HPP +#define SNAPSHOT_AGREEMENT_HPP + +#include +#include +#include +#include + +#include +#include + +// Agreement of independent sources on a snapshot (docs/specs/2026-10-25-snapshot-p2p-design.md, +// section 6.3): pure functions over the manifests collected from the p2p neighbours. +namespace cs { + +// a manifest as received from one source: its p2p key, its address in the routing table and +// whether the operator listed the key in [snapshot] sources +struct SourceManifest { + cs::PublicKey key{}; + std::string address; + bool trusted = false; + SnapshotManifest manifest; +}; + +// The independence unit of an address: the /24 of an IPv4 address ("a.b.c"), the /48 of an +// IPv6 one ("2001:db8:1::/48"); an IPv4-mapped IPv6 address counts as IPv4. A port after the +// address (a.b.c.d:port, [v6]:port) is ignored. Empty for an address that does not parse. +std::string netPrefix(const std::string& address); +// the wider block around it: /16 for IPv4, /32 for IPv6; empty when the address does not parse +std::string netBlock(const std::string& address); + +// sources declaring one (seq, head_hash, state_digest, settings_digest) +struct AgreementGroup { + cs::Sequence seq = 0; + std::string headHash; + std::string stateDigest; + std::string settingsDigest; + std::vector sources; // indices into agree()'s input + std::vector keys; // their keys, same order + size_t independent = 0; // counting sources: one per /24 or /48, trusted keys once each + bool configured = false; // holds a key of [snapshot] sources + bool chosen = false; + std::string reason; // why it was not chosen +}; + +using GroupReport = AgreementGroup; + +struct AgreementResult { + std::optional chosen; + std::vector groups; // every group, strongest first + size_t shortfall = 0; // counting sources the best group lacks (0 when chosen) +}; + +// Groups by (seq, head_hash, state_digest, settings_digest). Among the groups with at least +// minSources counting sources, the chosen one is, in this order: a group holding a key of +// [snapshot] sources (trusted), then the group with most counting sources, then the highest seq +// (only a tie-breaker: a later block never wins over more support, so that min_sources forged +// sources at a made-up later block do not beat a larger honest group; an honest network's newest +// snapshot gains sources as the nodes rebuild, and the fetcher asks again every 30 s). Two groups +// equal on all three are a conflict: nothing is chosen. Sources are counted by netPrefix(address), +// trusted ones by key; a source whose address is unknown or does not parse does not count; a key that +// appears twice counts once. Every group that is not chosen carries the reason. +AgreementResult agree(const std::vector& sources, size_t minSources); + +// "block S hash H digest D : N sources (keys k1 k2 …)" with shortened hashes and keys +std::string describe(const AgreementGroup& group); + +} // namespace cs + +#endif // SNAPSHOT_AGREEMENT_HPP diff --git a/csnode/include/csnode/snapshot_fetcher.hpp b/csnode/include/csnode/snapshot_fetcher.hpp new file mode 100644 index 000000000..169476734 --- /dev/null +++ b/csnode/include/csnode/snapshot_fetcher.hpp @@ -0,0 +1,269 @@ +#ifndef SNAPSHOT_FETCHER_HPP +#define SNAPSHOT_FETCHER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace cs { + +// Gets a snapshot from the p2p neighbours of a node that starts on an empty database +// (docs/specs/2026-10-25-snapshot-p2p-design.md, sections 6.2-6.5): brings the peers that serve +// snapshots into the neighbour table, collects their signed manifests, waits for the agreement of +// [snapshot] min_sources independent sources, then downloads the files of the agreed snapshot +// piece by piece into tmpDir, each piece checked against the manifest's chunk hash and each file +// against its SHA-256. Single-threaded: every call comes from the node's processor thread. The +// network, the routing table and the clock are reached only through Hooks. +class SnapshotFetcher { +public: + struct Peers { + std::vector> known; // key, capabilities (net/hostdata.hpp) + std::set neighbours; + }; + + struct Hooks { + std::function sendManifestRequest; + std::function sendPieceRequest; + std::function&)> addNeighbours; + std::function addressOf; // "" when unknown + std::function clock; // ms, steady; default: steady_clock + std::function onLog; // optional: every SNAPSHOT line, for tests + }; + + enum class Phase { + Discover, // no manifest asked yet + Collect, // manifests asked, waiting for agreement + Fetch, // downloading the agreed snapshot + Import, // every file downloaded and verified: result() is set + Done, // dry run: agreement reached or fetch_wait over, groups() has what was seen + Fallback // no snapshot: the node syncs from block 0 + }; + + static constexpr uint64_t kManifestIntervalMs = 30'000; + static constexpr uint64_t kManifestRetryMs = 10'000; // a request without an answer: asked again sooner + static constexpr size_t kManifestRetries = 2; // ... this many times, then every 30 s + static constexpr uint64_t kSilentMs = kManifestIntervalMs + 5'000; // no answer this long: silent + static constexpr uint64_t kNeighbourRetryMs = 60'000; + static constexpr uint64_t kPieceTimeoutMs = 30'000; // plus the bytes outstanding to the source at 64 KB/s + static constexpr uint64_t kTimeoutBytesPerSecond = 64 << 10; + static constexpr uint64_t kRecentPieceMs = 60'000; // a source that delivered this recently is slow, not stalled + static constexpr size_t kMaxLate = 64; // timed-out pieces whose late reply is still taken + static constexpr uint64_t kMarkMs = 60'000; + static constexpr uint64_t kMinBusyMs = 500; + static constexpr uint64_t kMaxBusyMs = 10'000; + static constexpr uint64_t kBusyMarkMs = 60'000; // busy this long without a piece: marked + static constexpr uint64_t kNoProgressMs = 300'000; // a fetch without a new byte this long ends + static constexpr uint64_t kLogIntervalMs = 30'000; + static constexpr uint64_t kWaitLogIntervalMs = 300'000; + static constexpr uint64_t kStallMs = 120'000; + static constexpr size_t kPiecesPerSource = 2; + static constexpr size_t kPiecesTotal = 8; + static constexpr size_t kMaxSources = 64; // discovered sources; configured keys come on top + static constexpr size_t kMaxPerPrefix = 4; // discovered sources per /24 (IPv4) or /48 (IPv6) + static constexpr size_t kMaxPerBlock = 16; // discovered sources per /16 (IPv4) or /32 (IPv6) + static constexpr uint64_t kEvictedMs = 60'000; // an evicted key is not taken again before + static constexpr size_t kMaxEvicted = 1024; + static constexpr size_t kMaxAttempts = 3; // refused imports, and with fetch_fallback=sync failed + // fetches without a proven bad source: then Fallback + static constexpr size_t kMaxExcusedAborts = 10; // fetches ended by a proven bad source (fetch_fallback=sync) + static constexpr uint64_t kBootstrapCapMs = 30 * 60'000; // fetch_fallback=sync: Fallback after fetch_wait + this, + // plus the time the agreed snapshot takes to download: + // at fetch_rate_kbps when set, else at 64 KB/s for at most + static constexpr uint64_t kMaxCapExtensionMs = 2 * 3'600'000; // this (a manifest may declare gigabytes) + static constexpr uint64_t kHashBytesPerTick = 64ull << 20; // whole-file SHA-256 per tick + static constexpr size_t kMaxSourcesFileKeys = 64; + + // cfg.fetchWait = 0: no wait, Fallback (or Done) at the first tick without agreement + SnapshotFetcher(const SnapshotData& cfg, std::filesystem::path tmpDir, Hooks hooks, bool dryRun = false); + ~SnapshotFetcher(); + SnapshotFetcher(const SnapshotFetcher&) = delete; + SnapshotFetcher& operator=(const SnapshotFetcher&) = delete; + + void tick(const Peers& peers); // every 500 ms + void onManifestReply(const cs::PublicKey& sender, const snapshot_p2p::ManifestReply& reply); + void onPieceReply(const cs::PublicKey& sender, snapshot_p2p::PieceReply&& reply); + + Phase phase() const { + return phase_; + } + // the manifest of the downloaded files (in tmpDir) and the agreed sources; set in Import + std::optional>> result() const; + // the importer refused fileAtFault: that file's SHA-256 is not fetched again; back to Collect, + // or Fallback after kMaxAttempts failed imports + void onImportFailed(const std::string& fileAtFault); + std::vector groups() const; + // no request is sent and no reply is taken after this + void stop(); + + // for the logs and the tests + size_t manifests() const; + size_t outstanding() const; + uint64_t dropped() const { + return dropped_; + } + uint64_t bytesDone() const; + uint64_t bytesTotal() const; + +private: + struct Source { + bool configured = false; // [snapshot] sources or sources_file + bool trusted = false; // [snapshot] sources + bool serving = false; // announced capability bit 24 + std::string prefix; // netPrefix and netBlock of the address, empty while unknown; + std::string block; // resolved again when a manifest arrives + bool everAnswered = false; + size_t unanswered = 0; // manifest requests since the last answer + uint64_t firstUnansweredAt = 0; + uint64_t discoveredAt = 0; + uint64_t addedAt = 0; // last addNeighbours + bool added = false; + uint64_t askedAt = 0; + bool everAsked = false; + bool awaiting = false; // a manifest request without a reply yet + bool loggedAsk = false; + std::optional manifest; + uint64_t manifestAt = 0; + std::string filesHash; + std::string address; + size_t marks = 0; + uint64_t markedUntil = 0; + bool banned = false; // for the rest of the bootstrap + uint64_t busyUntil = 0; + uint64_t busySince = 0; // first busy answer since its last piece; 0: none + size_t budget = kPiecesPerSource; // pieces it may have outstanding: 1 after a timeout of a source that delivers + size_t streak = 0; // pieces delivered in time in a row + bool delivered = false; + uint64_t lastPieceAt = 0; + size_t outstanding = 0; + }; + + struct InFlight { + cs::PublicKey source{}; + uint64_t sentAt = 0; + }; + + struct HashState; + + struct FileState { + SnapshotFile file; + std::vector sources; // agreed sources declaring this file with this SHA-256 + std::vector have; + size_t missing = 0; // chunks not on disk yet + uint64_t doneBytes = 0; + std::map inFlight; + int fd = -1; + bool verifyPending = false; + bool verified = false; + std::shared_ptr hashing; // whole-file SHA-256 in progress, kHashBytesPerTick per tick + uint64_t hashed = 0; + }; + + void log(const std::string& line, bool warning = false); + uint64_t now() const; + std::string keyText(const cs::PublicKey& key) const; + Source* findSource(const cs::PublicKey& key); + Source* addConfigured(const cs::PublicKey& key); + Source* addDiscovered(const cs::PublicKey& key, uint64_t t); + bool roomFor(const cs::PublicKey& key, const std::string& prefix, const std::string& block) const; + bool evictOne(uint64_t t); + bool silent(const Source& s, uint64_t t) const; + void evict(std::map::iterator it, uint64_t t); + bool isProtected(const cs::PublicKey& key, const Source& s) const; + void loadSourcesFile(); + + void discover(const Peers& peers, uint64_t t); + void askManifests(const Peers& peers, uint64_t t); + bool tryAgree(uint64_t t); + void logWaiting(uint64_t t); + void logGroups(bool chosenOnly); + std::string seenText() const; + void checkWait(uint64_t t); + + bool usable(const cs::PublicKey& key, const Source& s, uint64_t t) const; + size_t lateOnTheWay(const cs::PublicKey& key, uint64_t t) const; + uint64_t capMs() const; + bool everUsable(const Source& s) const; + bool startFetch(uint64_t t); + bool chooseTarget(uint64_t t); + void fill(uint64_t t); + void expire(uint64_t t); + void verifyFiles(uint64_t t); + void clearInFlight(); + void releaseSource(const cs::PublicKey& key); + void mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why); + void backToCollect(const std::string& why); + void forgetLate(const cs::PublicKey& key); + void banDeclaring(const std::string& name, const std::string& sha256, const std::string& why); + void closeFile(FileState& f); + bool openFile(FileState& f, bool truncate); + uint64_t pieceLength(const FileState& f, uint32_t chunk) const; + void logProgress(uint64_t t, bool force); + bool takeTokens(uint64_t cost, uint64_t t); + + SnapshotData cfg_; + std::filesystem::path tmpDir_; + Hooks hooks_; + bool dryRun_; + + Phase phase_ = Phase::Discover; + bool stopped_ = false; + bool started_ = false; + uint64_t startedAt_ = 0; + std::map sources_; + std::set neighbours_; // as of the last tick + std::map evicted_; // discovered keys let go, and when + + AgreementResult agreement_; + std::vector agreedSources_; + std::optional target_; + std::vector files_; + std::set> badFiles_; // (name, sha256) refused by the importer + size_t importFailures_ = 0; // imports refused (onImportFailed) + size_t fetchFailures_ = 0; // fetches that ended without a snapshot and without a misbehaving source + size_t excusedAborts_ = 0; // fetches that ended with a source banned for a proven fault + bool misbehaviour_ = false; // a source was banned for a proven fault (a hash against the signed manifest) in this fetch + // pieces that timed out, (file, chunk) -> the source asked: its late reply is still taken, and + // for one more timeout the piece still counts as on its way from that source + struct Late { + cs::PublicKey source{}; + uint64_t sentAt = 0; + uint64_t expiredAt = 0; + uint64_t timeout = 0; + }; + std::map, Late> late_; + uint64_t capExtensionMs_ = 0; // the bootstrap cap grows by the download time of the agreed snapshot + bool dirty_ = true; // the input of the agreement changed since it was computed + uint64_t lastByteAt_ = 0; + uint64_t lastSendAt_ = 0; + size_t rr_ = 0; // first source of the next fill + + // fetch_rate_kbps on requested bytes: a request goes out when nothing is owed, then owes its size + double owed_ = 0; + uint64_t owedAt_ = 0; + + uint64_t dropped_ = 0; + uint64_t lastLogAt_ = 0; + uint64_t lastWaitLogAt_ = 0; + bool fallbackLogged_ = false; + uint64_t lastProgressAt_ = 0; + uint64_t lastProgressBytes_ = 0; + int lastProgressDecile_ = -1; + uint64_t fetchStartedAt_ = 0; +}; + +} // namespace cs + +#endif // SNAPSHOT_FETCHER_HPP diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index b1ee9c4e7..55609a935 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -2247,7 +2248,9 @@ void Node::getSnapshotManifestReply(const uint8_t* data, const size_t size, cons if (!cs::snapshot_p2p::decode(data, size, reply)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed manifest reply", sender); } - // nothing asked for it yet + if (snapshotFetcher_) { + return snapshotFetcher_->onManifestReply(sender, reply); + } logSnapshotDrop(snapshotMessagesDropped_, "an unexpected manifest reply", sender); } @@ -2271,6 +2274,9 @@ void Node::getSnapshotPieceReply(const uint8_t* data, const size_t size, const c if (!cs::snapshot_p2p::decode(data, size, reply)) { return logSnapshotDrop(snapshotMessagesDropped_, "a malformed piece reply", sender); } + if (snapshotFetcher_) { + return snapshotFetcher_->onPieceReply(sender, std::move(reply)); + } logSnapshotDrop(snapshotMessagesDropped_, "an unexpected piece reply", sender); } diff --git a/csnode/src/snapshot_agreement.cpp b/csnode/src/snapshot_agreement.cpp new file mode 100644 index 000000000..a0f16488b --- /dev/null +++ b/csnode/src/snapshot_agreement.cpp @@ -0,0 +1,186 @@ +#include + +#include +#include +#include +#include +#include + +#include + +#include + +namespace cs { + +namespace { +std::string hostPart(const std::string& address) { + if (!address.empty() && address.front() == '[') { + const auto close = address.find(']'); + return close == std::string::npos ? std::string() : address.substr(1, close - 1); + } + // a.b.c.d:port; an IPv6 address has more than one ':' + const auto colon = address.find(':'); + if (colon != std::string::npos && address.find(':', colon + 1) == std::string::npos) { + return address.substr(0, colon); + } + return address; +} + +std::string unitOf(const SourceManifest& s) { + if (s.trusted) { + return "key:" + EncodeBase58(s.key.data(), s.key.data() + s.key.size()); + } + const auto prefix = netPrefix(s.address); + return prefix.empty() ? std::string() : "net:" + prefix; +} + +size_t countUnits(const std::vector& indices, const std::vector& units) { + std::set distinct; + for (auto i : indices) { + if (!units[i].empty()) { // an address not known (yet) is no evidence of independence + distinct.insert(units[i]); + } + } + return distinct.size(); +} + +std::string shortHex(const std::string& hex) { + return hex.size() > 16 ? hex.substr(0, 16) + "..." : hex; +} +} // namespace + +namespace { +// the prefix of the given length in bits (24 or 16 for IPv4 addresses, 48 or 32 for IPv6) +std::string prefixOf(const std::string& address, bool wide) { + boost::system::error_code ec; + auto ip = boost::asio::ip::make_address(hostPart(address), ec); + if (ec) { + return {}; + } + if (ip.is_v6() && ip.to_v6().is_v4_mapped()) { + ip = boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, ip.to_v6()); + } + if (ip.is_v4()) { + const auto b = ip.to_v4().to_bytes(); + return wide ? std::to_string(b[0]) + "." + std::to_string(b[1]) + : std::to_string(b[0]) + "." + std::to_string(b[1]) + "." + std::to_string(b[2]); + } + auto b = ip.to_v6().to_bytes(); + std::fill(b.begin() + (wide ? 4 : 6), b.end(), 0); + return boost::asio::ip::address_v6(b).to_string() + (wide ? "/32" : "/48"); +} +} // namespace + +std::string netPrefix(const std::string& address) { + return prefixOf(address, false); +} + +std::string netBlock(const std::string& address) { + return prefixOf(address, true); +} + +AgreementResult agree(const std::vector& sources, size_t minSources) { + minSources = std::max(minSources, 1); + AgreementResult result; + + std::vector units(sources.size()); + std::set seen; + using BlockKey = std::pair; + using DigestKey = std::pair; + std::map>> blocks; + for (size_t i = 0; i < sources.size(); ++i) { + units[i] = unitOf(sources[i]); + if (!seen.insert(sources[i].key).second) { + continue; + } + const auto& m = sources[i].manifest; + blocks[{m.seq, m.headHash}][{m.stateDigest, m.settingsDigest}].push_back(i); + } + + for (const auto& [blockKey, digests] : blocks) { + for (const auto& [digestKey, indices] : digests) { + AgreementGroup g; + g.seq = blockKey.first; + g.headHash = blockKey.second; + g.stateDigest = digestKey.first; + g.settingsDigest = digestKey.second; + g.sources = indices; + for (auto i : indices) { + g.keys.push_back(sources[i].key); + g.configured = g.configured || sources[i].trusted; + } + g.independent = countUnits(indices, units); + result.groups.push_back(std::move(g)); + } + } + + // The rule (see the header): among the groups that reach minSources, a group with a configured + // key first, then the most independent sources, then the highest block. A higher block alone + // never wins over more support: min_sources made-up sources at a forged later block lose to a + // larger honest group at an earlier one. Equal on all three: a conflict, nothing is chosen. + auto stronger = [](const AgreementGroup& x, const AgreementGroup& y) { + if (x.configured != y.configured) return x.configured; + if (x.independent != y.independent) return x.independent > y.independent; + return x.seq > y.seq; + }; + auto equal = [](const AgreementGroup& x, const AgreementGroup& y) { + return x.configured == y.configured && x.independent == y.independent && x.seq == y.seq; + }; + std::stable_sort(result.groups.begin(), result.groups.end(), stronger); + + const AgreementGroup* best = nullptr; + for (const auto& g : result.groups) { + if (g.independent >= minSources) { + best = &g; + break; + } + } + const bool conflict = best != nullptr && std::any_of(result.groups.begin(), result.groups.end(), [&](const AgreementGroup& g) { + return &g != best && g.independent >= minSources && equal(g, *best); + }); + for (auto& g : result.groups) { + if (g.independent < minSources) { + g.reason = "fewer than " + std::to_string(minSources) + " independent sources"; + } + else if (conflict && equal(g, *best)) { + g.reason = "as much support as another group: conflict"; + } + else if (&g == best) { + g.chosen = true; + } + else if (best->configured && !g.configured) { + g.reason = "no configured source, the strongest group has one"; + } + else if (g.independent < best->independent) { + g.reason = "fewer independent sources than the strongest group"; + } + else { + g.reason = "lower block than the strongest group, with as much support"; + } + } + size_t most = 0; + for (const auto& g : result.groups) { + most = std::max(most, g.independent); + if (g.chosen) { + result.chosen = g; + } + } + result.shortfall = (result.chosen || conflict) ? 0 : minSources - std::min(most, minSources); + return result; +} + +std::string describe(const AgreementGroup& group) { + std::ostringstream out; + out << "block " << group.seq << " hash " << shortHex(group.headHash) << " digest " << shortHex(group.stateDigest) << " : " + << group.independent << (group.independent == 1 ? " source" : " sources") << " (keys"; + for (const auto& k : group.keys) { + out << ' ' << EncodeBase58(k.data(), k.data() + k.size()).substr(0, 8) << "..."; + } + out << ')'; + if (!group.reason.empty()) { + out << ": " << group.reason; + } + return out.str(); +} + +} // namespace cs diff --git a/csnode/src/snapshot_fetcher.cpp b/csnode/src/snapshot_fetcher.cpp new file mode 100644 index 000000000..56955ce45 --- /dev/null +++ b/csnode/src/snapshot_fetcher.cpp @@ -0,0 +1,1387 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include + +namespace cs { + +namespace { +namespace fs = std::filesystem; + +uint64_t steadyMs() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +bool powerOfTwo(uint64_t v) { + return v != 0 && (v & (v - 1)) == 0; +} + +std::string megabytes(uint64_t bytes) { + return std::to_string((bytes + 500'000) / 1'000'000); +} + +std::string duration(uint64_t seconds) { + if (seconds >= 60 && seconds % 60 == 0) { + return std::to_string(seconds / 60) + " min"; + } + return std::to_string(seconds) + " s"; +} + +bool sameFile(const SnapshotFile& a, const SnapshotFile& b) { + return a.name == b.name && a.size == b.size && a.sha256 == b.sha256 && a.chunks == b.chunks; +} + +std::string hexOf(const uint8_t* data, size_t size) { + static const char* digits = "0123456789abcdef"; + std::string out; + out.reserve(size * 2); + for (size_t i = 0; i < size; ++i) { + out.push_back(digits[data[i] >> 4]); + out.push_back(digits[data[i] & 0x0f]); + } + return out; +} +} // namespace + +struct SnapshotFetcher::HashState { + crypto_hash_sha256_state state; +}; + +SnapshotFetcher::SnapshotFetcher(const SnapshotData& cfg, fs::path tmpDir, Hooks hooks, bool dryRun) +: cfg_(cfg) +, tmpDir_(std::move(tmpDir)) +, hooks_(std::move(hooks)) +, dryRun_(dryRun) { + if (!hooks_.clock) { + hooks_.clock = steadyMs; + } + for (const auto& key : cfg_.sources) { + addConfigured(key)->trusted = true; + } + loadSourcesFile(); +} + +SnapshotFetcher::~SnapshotFetcher() { + for (auto& f : files_) { + closeFile(f); + } +} + +void SnapshotFetcher::log(const std::string& line, bool warning) { + if (warning) { + cswarning() << line; + } + else { + cslog() << line; + } + if (hooks_.onLog) { + hooks_.onLog(line); + } +} + +uint64_t SnapshotFetcher::now() const { + return hooks_.clock(); +} + +std::string SnapshotFetcher::keyText(const cs::PublicKey& key) const { + return EncodeBase58(key.data(), key.data() + key.size()); +} + +SnapshotFetcher::Source* SnapshotFetcher::findSource(const cs::PublicKey& key) { + auto it = sources_.find(key); + return it == sources_.end() ? nullptr : &it->second; +} + +SnapshotFetcher::Source* SnapshotFetcher::addConfigured(const cs::PublicKey& key) { + // configured keys have their own room: discovered peers never push them out + auto& s = sources_[key]; + s.configured = true; + return &s; +} + +bool SnapshotFetcher::isProtected(const cs::PublicKey& key, const Source& s) const { + if (s.configured || s.outstanding != 0) { + return true; + } + return phase_ == Phase::Fetch && std::any_of(agreedSources_.begin(), agreedSources_.end(), + [&](const SourceManifest& m) { return m.key == key; }); +} + +bool SnapshotFetcher::roomFor(const cs::PublicKey& key, const std::string& prefix, const std::string& block) const { + // an address not known yet is not capped here: it is resolved and checked again with its manifest + size_t samePrefix = 0; + size_t sameBlock = 0; + for (const auto& [k, s] : sources_) { + if (k == key || s.configured) { + continue; + } + samePrefix += !prefix.empty() && s.prefix == prefix; + sameBlock += !block.empty() && s.block == block; + } + return samePrefix < kMaxPerPrefix && sameBlock < kMaxPerBlock; +} + +SnapshotFetcher::Source* SnapshotFetcher::addDiscovered(const cs::PublicKey& key, uint64_t t) { + if (auto* s = findSource(key)) { + return s; + } + if (auto e = evicted_.find(key); e != evicted_.end()) { + if (t - e->second < kEvictedMs) { + return nullptr; + } + evicted_.erase(e); + } + // peer keys cost nothing: discovered sources are limited per /24 (/48), per /16 (/32) and in + // all, so that no network can fill the table + const std::string address = hooks_.addressOf ? hooks_.addressOf(key) : std::string(); + const std::string prefix = netPrefix(address); + const std::string block = netBlock(address); + if (!roomFor(key, prefix, block)) { + return nullptr; + } + size_t discovered = 0; + for (const auto& [k, s] : sources_) { + discovered += !s.configured; + } + if (discovered >= kMaxSources && !evictOne(t)) { + return nullptr; + } + auto& s = sources_[key]; + s.address = address; + s.prefix = prefix; + s.block = block; + s.discoveredAt = t; + return &s; +} + +bool SnapshotFetcher::silent(const Source& s, uint64_t t) const { + // at least two requests without an answer over more than one request interval, counted from + // the first request, not from the discovery: a peer that became a neighbour late is not silent + return !s.configured && s.unanswered >= 2 && t - s.firstUnansweredAt > kSilentMs; +} + +void SnapshotFetcher::evict(std::map::iterator it, uint64_t t) { + // an evicted key is not taken again for a minute, so that keys that come back at every tick + // cannot churn the table; longer would only hurt honest keys, since keys cost an attacker nothing + if (evicted_.size() >= kMaxEvicted) { + auto oldest = std::min_element(evicted_.begin(), evicted_.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); + evicted_.erase(oldest); + } + evicted_[it->first] = t; + sources_.erase(it); + dirty_ = true; +} + +bool SnapshotFetcher::evictOne(uint64_t t) { + // the least useful discovered source makes room, whatever its prefix: one that is not a + // neighbour now, then a silent one (silence is a reason only here, when room is needed), then + // one without a manifest after a minute, then one without a manifest in the most represented prefix, then the oldest of the + // most represented prefix; never a configured source, one with pieces outstanding or one of the + // agreed group during a fetch + std::map perPrefix; + for (const auto& [k, s] : sources_) { + if (!s.configured) { + ++perPrefix[s.prefix]; + } + } + size_t most = 0; + for (const auto& [prefix, n] : perPrefix) { + most = std::max(most, n); + } + auto tier = [&](const cs::PublicKey& key, const Source& s) { + const bool old = t - s.discoveredAt > kMarkMs; + const bool crowded = perPrefix[s.prefix] == most; + if (neighbours_.count(key) == 0) return 0; + if (silent(s, t)) return 1; + if (old && !s.manifest) return 2; + if (crowded && !s.manifest) return 3; + if (crowded) return 4; + return 5; + }; + auto victim = sources_.end(); + int victimTier = 5; + for (auto it = sources_.begin(); it != sources_.end(); ++it) { + if (isProtected(it->first, it->second)) { + continue; + } + const int k = tier(it->first, it->second); + if (k < victimTier || (k == victimTier && victim != sources_.end() && it->second.discoveredAt < victim->second.discoveredAt)) { + victim = it; + victimTier = k; + } + } + if (victim == sources_.end()) { + return false; + } + evict(victim, t); + return true; +} + +void SnapshotFetcher::loadSourcesFile() { + if (cfg_.sourcesFile.empty()) { + return; + } + std::ifstream in(cfg_.sourcesFile); + if (!in) { + return; // a missing file means no listed sources + } + std::string line; + size_t keys = 0; + size_t lines = 0; + while (keys < kMaxSourcesFileKeys && lines < 10 * kMaxSourcesFileKeys && std::getline(in, line)) { + ++lines; + if (const auto hash = line.find('#'); hash != std::string::npos) { + line.erase(hash); + } + std::istringstream words(line); + std::string word; + std::string last; + while (words >> word) { + last = word; + } + if (last.empty()) { + continue; + } + std::vector bytes; + if (!DecodeBase58(last, bytes) || bytes.size() != cs::PublicKey{}.size()) { + log("SNAPSHOT: " + cfg_.sourcesFile + ": not a node key: " + last.substr(0, 64), true); + continue; + } + cs::PublicKey key; + std::copy(bytes.begin(), bytes.end(), key.begin()); + addConfigured(key); + ++keys; + } + if (keys != 0) { + log("SNAPSHOT: " + std::to_string(keys) + " source keys read from " + cfg_.sourcesFile); + } +} + +// --------------------------------------------------------------------------------------------- +// discovery and agreement + +void SnapshotFetcher::tick(const Peers& peers) { + if (stopped_ || phase_ == Phase::Import || phase_ == Phase::Done || phase_ == Phase::Fallback) { + return; + } + const uint64_t t = now(); + if (!started_) { + started_ = true; + startedAt_ = t; + lastLogAt_ = t; + lastWaitLogAt_ = t; + } + neighbours_ = peers.neighbours; + // fetch_fallback=sync: the whole bootstrap is bounded whatever the counters say + if (!dryRun_ && cfg_.fetchFallback != "wait" && t - startedAt_ >= capMs()) { + log("SNAPSHOT: no snapshot after " + duration((t - startedAt_) / 1000 / 60 * 60) + + " (fetch_wait + 30 min + the download time): syncing from block 0. To start from the published snapshot instead: stop the node and run" + " ./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github), then start it again.", + true); + clearInFlight(); + phase_ = Phase::Fallback; + return; + } + discover(peers, t); + + if (phase_ == Phase::Discover || phase_ == Phase::Collect) { + askManifests(peers, t); + if (tryAgree(t)) { + return; + } + logWaiting(t); + checkWait(t); + return; + } + + verifyFiles(t); + if (phase_ != Phase::Fetch) { + return; + } + expire(t); + if (phase_ != Phase::Fetch) { + return; + } + fill(t); + // a fetch always ends: nothing to ask for 2 min, or no new byte for 5 min (sources busy, + // silent or slow), and it counts as a failed attempt + if (phase_ == Phase::Fetch && outstanding() == 0 && t - lastSendAt_ >= kStallMs) { + backToCollect("no piece could be asked for " + duration(kStallMs / 1000)); + return; + } + if (phase_ == Phase::Fetch && t - lastByteAt_ >= kNoProgressMs) { + backToCollect("no data for " + duration(kNoProgressMs / 1000)); + return; + } + logProgress(t, false); +} + +void SnapshotFetcher::discover(const Peers& peers, uint64_t t) { + for (const auto& [key, capabilities] : peers.known) { + if ((capabilities & cs::hostdata::kSnapshotProtocolV1) == 0) { + continue; + } + if (auto* s = addDiscovered(key, t)) { + s->serving = true; + } + } + std::set add; + for (auto& [key, s] : sources_) { + if (s.banned || neighbours_.count(key) != 0) { + continue; + } + if (!s.added || t - s.addedAt >= kNeighbourRetryMs) { + s.added = true; + s.addedAt = t; + add.insert(key); + } + } + if (!add.empty() && hooks_.addNeighbours) { + hooks_.addNeighbours(add); + } +} + +void SnapshotFetcher::askManifests(const Peers&, uint64_t t) { + for (auto& [key, s] : sources_) { + if (s.banned || neighbours_.count(key) == 0) { + continue; + } + // a reply: ask again after 30 s; silence (a serving node may not answer when its share for + // idle keys is spent, design 5.3a): again after 10 s, twice, then every 30 s + const uint64_t interval = (s.awaiting && s.unanswered <= kManifestRetries) ? kManifestRetryMs : kManifestIntervalMs; + if (s.everAsked && t - s.askedAt < interval) { + continue; + } + if (s.awaiting) { + ++s.unanswered; + } + else { + s.unanswered = 1; + s.firstUnansweredAt = t; + } + s.everAsked = true; + s.awaiting = true; + s.askedAt = t; + if (!s.loggedAsk) { + s.loggedAsk = true; + log("SNAPSHOT: asking " + keyText(key) + " for its manifest"); + } + if (hooks_.sendManifestRequest) { + hooks_.sendManifestRequest(key, snapshot_p2p::ManifestRequest{}); + } + if (phase_ == Phase::Discover) { + phase_ = Phase::Collect; + } + } +} + +void SnapshotFetcher::onManifestReply(const cs::PublicKey& sender, const snapshot_p2p::ManifestReply& reply) { + if (stopped_ || phase_ == Phase::Import || phase_ == Phase::Done || phase_ == Phase::Fallback) { + ++dropped_; + return; + } + auto* s = findSource(sender); + if (s == nullptr || !s->awaiting) { + ++dropped_; + return; + } + s->awaiting = false; + s->everAnswered = true; + s->unanswered = 0; + const uint64_t t = now(); + if (reply.status != snapshot_p2p::ManifestStatus::Ready) { + // not serving, not synced or still building: whatever it declared before is not served now + dirty_ = dirty_ || s->manifest.has_value(); + s->manifest.reset(); + return; + } + + std::string why; + auto manifest = SnapshotManifest::fromJson(reply.manifestJson, &why); + if (manifest) { + if (!manifest->verify()) { + why = "bad signature"; + } + else if (manifest->nodeKey != keyText(sender)) { + why = "signed by another node"; + } + else if (manifest->chunkSize > snapshot_p2p::kMaxPieceBytes || !powerOfTwo(manifest->chunkSize)) { + why = "chunk size " + std::to_string(manifest->chunkSize); + } + } + if (!manifest || !why.empty()) { + ++dropped_; + dirty_ = dirty_ || s->manifest.has_value(); + s->manifest.reset(); + log("SNAPSHOT: manifest from " + keyText(sender) + " dropped: " + why, true); + return; + } + // the address the table limits and the agreement count are taken from, resolved again now + const std::string address = hooks_.addressOf ? hooks_.addressOf(sender) : std::string(); + const std::string prefix = netPrefix(address); + const std::string block = netBlock(address); + if (!s->configured && !roomFor(sender, prefix, block)) { + ++dropped_; + log("SNAPSHOT: manifest from " + keyText(sender) + " dropped: too many sources at " + prefix); + if (!isProtected(sender, *s)) { + sources_.erase(sender); + dirty_ = true; + } + // a source in use keeps its entry, its address and the manifest it had + return; + } + s->address = address; + s->prefix = prefix; + s->block = block; + s->filesHash = manifest->filesHash(); + s->manifest = std::move(manifest); + s->manifestAt = t; + dirty_ = true; + // agreement is tried at the next tick, so that the replies to one round of requests count together +} + +bool SnapshotFetcher::tryAgree(uint64_t t) { + if (!dirty_) { + return false; // nothing changed since the last agreement that did not lead to a fetch + } + dirty_ = false; + std::vector input; + for (const auto& [key, s] : sources_) { + if (s.banned || !s.manifest) { + continue; + } + SourceManifest m; + m.key = key; + m.address = s.address; + m.trusted = s.trusted; + m.manifest = *s.manifest; + input.push_back(std::move(m)); + } + agreement_ = agree(input, cfg_.minSources); + if (!agreement_.chosen) { + return false; + } + if (dryRun_) { + logGroups(false); + phase_ = Phase::Done; + return true; + } + agreedSources_.clear(); + for (auto i : agreement_.chosen->sources) { + agreedSources_.push_back(input[i]); + } + if (!startFetch(t)) { + agreedSources_.clear(); + return phase_ == Phase::Fallback; + } + return true; +} + +void SnapshotFetcher::logGroups(bool chosenOnly) { + for (const auto& g : agreement_.groups) { + if (g.chosen) { + log("SNAPSHOT: " + describe(g)); + } + else if (!chosenOnly) { + log("SNAPSHOT: " + std::string(agreement_.chosen ? "minority: " : "") + describe(g)); + } + } +} + +std::string SnapshotFetcher::seenText() const { + if (agreement_.groups.empty()) { + return "no manifest"; + } + std::string out; + for (const auto& g : agreement_.groups) { + if (!out.empty()) { + out += "; "; + } + out += std::to_string(g.independent) + (g.independent == 1 ? " source" : " sources") + " for block " + std::to_string(g.seq); + } + return out; +} + +void SnapshotFetcher::logWaiting(uint64_t t) { + const uint64_t waitMs = uint64_t(cfg_.fetchWait) * 1000; + if (t - startedAt_ >= waitMs || t - lastLogAt_ < kLogIntervalMs) { + return; + } + lastLogAt_ = t; + size_t serving = 0; + size_t neighbours = 0; + for (const auto& [key, s] : sources_) { + if (s.serving || s.configured) { + ++serving; + if (neighbours_.count(key) != 0) { + ++neighbours; + } + } + } + size_t unknown = 0; + for (const auto& [key, s] : sources_) { + unknown += !s.banned && s.manifest && !s.trusted && netPrefix(s.address).empty(); + } + log("SNAPSHOT: " + std::to_string(serving) + " peers serve snapshots (" + std::to_string(neighbours) + " neighbours), " + + std::to_string(manifests()) + " manifests collected, " + std::to_string(unknown) + " not counted: address unknown, " + + std::to_string(dropped_) + " replies dropped"); + const std::string left = duration((waitMs - (t - startedAt_) + 59'999) / 60'000 * 60) + " left"; + if (agreement_.chosen) { + log("SNAPSHOT: block " + std::to_string(agreement_.chosen->seq) + + " is agreed, but no file group has a source left: waiting, " + left); + return; + } + // groups come strongest first + const AgreementGroup* best = agreement_.groups.empty() ? nullptr : &agreement_.groups.front(); + if (best != nullptr && best->independent >= cfg_.minSources) { + // as strong as another group: a conflict, not a shortage + log("SNAPSHOT: conflict: two groups with " + std::to_string(best->independent) + " sources at block " + + std::to_string(best->seq) + ": waiting, " + left); + return; + } + if (best != nullptr) { + log("SNAPSHOT: " + std::to_string(best->independent) + (best->independent == 1 ? " source declares" : " sources declare") + + " block " + std::to_string(best->seq) + " (need " + std::to_string(cfg_.minSources) + "): waiting, " + left); + } +} + +void SnapshotFetcher::checkWait(uint64_t t) { + if (t - startedAt_ < uint64_t(cfg_.fetchWait) * 1000) { + return; + } + if (dryRun_) { + logGroups(false); + phase_ = Phase::Done; + return; + } + const std::string head = "SNAPSHOT: no agreement of " + std::to_string(cfg_.minSources) + " community sources after " + + duration((t - startedAt_) / 1000) + " (seen: " + seenText() + ")."; + if (cfg_.fetchFallback == "wait") { + if (!fallbackLogged_ || t - lastWaitLogAt_ >= kWaitLogIntervalMs) { + fallbackLogged_ = true; + lastWaitLogAt_ = t; + log(head + " Waiting for more sources (fetch_fallback=wait). To start from the published snapshot instead: stop the node " + "and run ./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github), then start it again.", + true); + } + return; + } + log(head + " Syncing from block 0 (about 35 h on a fast machine). To start from the published snapshot instead: stop the node " + "and run ./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github), then start it again.", + true); + clearInFlight(); + phase_ = Phase::Fallback; +} + +// --------------------------------------------------------------------------------------------- +// download + +bool SnapshotFetcher::everUsable(const Source& s) const { + return !s.banned; +} + +uint64_t SnapshotFetcher::capMs() const { + return uint64_t(cfg_.fetchWait) * 1000 + kBootstrapCapMs + capExtensionMs_; +} + +size_t SnapshotFetcher::lateOnTheWay(const cs::PublicKey& key, uint64_t t) const { + size_t n = 0; + for (const auto& [piece, late] : late_) { + n += late.source == key && t - late.expiredAt < late.timeout; + } + return n; +} + +bool SnapshotFetcher::usable(const cs::PublicKey& key, const Source& s, uint64_t t) const { + // pieces that timed out but are probably still on their way take the source's link too + return !s.banned && s.markedUntil <= t && s.busyUntil <= t && s.outstanding + lateOnTheWay(key, t) < s.budget; +} + +bool SnapshotFetcher::startFetch(uint64_t t) { + if (!chooseTarget(t)) { + return false; + } + phase_ = Phase::Fetch; + misbehaviour_ = false; + fetchStartedAt_ = t; + lastSendAt_ = t; + lastByteAt_ = t; + lastProgressAt_ = t; + lastProgressBytes_ = bytesDone(); + lastProgressDecile_ = bytesTotal() == 0 ? 0 : static_cast(bytesDone() * 10 / bytesTotal()); + logGroups(false); + log("SNAPSHOT: fetching block " + std::to_string(target_->seq) + ": " + megabytes(bytesTotal()) + " MB in " + + std::to_string(files_.size()) + " files from " + std::to_string(agreedSources_.size()) + " sources"); + // the bootstrap cap (fetch_fallback=sync) makes room for the download: at the rate the operator + // chose, or at 64 KB/s for at most 2 h, since the size comes from a manifest + const uint64_t remaining = bytesTotal() - std::min(bytesTotal(), bytesDone()); + const uint64_t expected = cfg_.fetchRateKbps != 0 ? remaining / cfg_.fetchRateKbps : remaining * 1000 / kTimeoutBytesPerSecond; + capExtensionMs_ = std::max(capExtensionMs_, cfg_.fetchRateKbps != 0 ? expected : std::min(expected, kMaxCapExtensionMs)); + if (cfg_.fetchFallback != "wait" && t - startedAt_ + expected > capMs()) { + log("SNAPSHOT: fetching block " + std::to_string(target_->seq) + " may take about " + duration(expected / 60'000 * 60) + + ", more than the " + duration((capMs() - std::min(capMs(), t - startedAt_)) / 60'000 * 60) + + " left before the node syncs from block 0", + true); + } + fill(t); + return true; +} + +bool SnapshotFetcher::chooseTarget(uint64_t t) { + // the files must be those of one signed manifest (the importer checks its signature): among the + // agreed sources' file lists, the one shared by most usable sources whose every file still has + // a usable source; each file then comes from every agreed source declaring the same bytes + const SnapshotManifest* best = nullptr; + std::vector> bestSources; + size_t bestOwners = 0; + size_t bestTotal = 0; + std::vector hashes; // filesHash of each agreed source, computed once + for (const auto& a : agreedSources_) { + hashes.push_back(a.manifest.filesHash()); + } + std::set tried; + for (size_t c = 0; c < agreedSources_.size(); ++c) { + const auto& rep = agreedSources_[c].manifest; + const auto& hash = hashes[c]; + if (!tried.insert(hash).second) { + continue; + } + bool refused = false; + for (const auto& f : rep.files) { + refused = refused || badFiles_.count({f.name, f.sha256}) != 0; + } + if (refused) { + continue; + } + size_t owners = 0; + for (size_t o = 0; o < agreedSources_.size(); ++o) { + const auto* s = findSource(agreedSources_[o].key); + if (s != nullptr && everUsable(*s) && hashes[o] == hash) { + ++owners; + } + } + std::vector> perFile; + size_t total = 0; + bool dry = false; + for (const auto& f : rep.files) { + std::vector keys; + for (const auto& other : agreedSources_) { + const auto* s = findSource(other.key); + const auto* of = other.manifest.file(f.name); + if (s != nullptr && everUsable(*s) && of != nullptr && other.manifest.chunkSize == rep.chunkSize && sameFile(*of, f)) { + keys.push_back(other.key); + } + } + dry = dry || keys.empty(); + total += keys.size(); + perFile.push_back(std::move(keys)); + } + if (dry) { + continue; + } + if (best == nullptr || owners > bestOwners || (owners == bestOwners && total > bestTotal)) { + best = &rep; + bestSources = std::move(perFile); + bestOwners = owners; + bestTotal = total; + } + } + if (best == nullptr) { + return false; + } + + clearInFlight(); + std::vector next; + for (size_t i = 0; i < best->files.size(); ++i) { + const auto& f = best->files[i]; + // equal chunk hashes over equal sizes: the same bytes cut the same way + auto old = std::find_if(files_.begin(), files_.end(), [&](const FileState& s) { return sameFile(s.file, f); }); + FileState state; + bool reuse = old != files_.end(); + if (reuse) { + state = std::move(*old); + old->fd = -1; + state.hashing.reset(); // a check in progress starts over + // what is on disk is checked again before it is handed over + if (state.verified) { + state.verified = false; + state.verifyPending = true; + } + } + else { + state.file = f; + state.have.assign(f.chunks.size(), false); + state.missing = f.chunks.size(); + state.verifyPending = f.chunks.empty(); + } + state.sources = bestSources[i]; + if (!openFile(state, !reuse)) { + log("SNAPSHOT: cannot write " + (tmpDir_ / f.name).string() + ": syncing from block 0", true); + closeFile(state); + for (auto& n : next) { + closeFile(n); + } + phase_ = Phase::Fallback; + return false; + } + next.push_back(std::move(state)); + } + for (auto& f : files_) { + closeFile(f); + } + files_ = std::move(next); + target_ = *best; + (void)t; + return true; +} + +bool SnapshotFetcher::openFile(FileState& f, bool truncate) { + if (f.fd >= 0) { + return true; + } + // the name comes from a manifest fromJson accepted, so it is one of the snapshot's fixed file + // names; checked again here, where it becomes a path + if (!SnapshotManifest::isKnownFileName(f.file.name) || fs::path(f.file.name).filename() != f.file.name) { + return false; + } + std::error_code ec; + fs::create_directories(tmpDir_, ec); + const auto path = tmpDir_ / f.file.name; + f.fd = ::open(path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC | O_NOFOLLOW | (truncate ? O_TRUNC : 0), 0600); + if (f.fd < 0) { + return false; + } + if (::ftruncate(f.fd, static_cast(f.file.size)) != 0) { + closeFile(f); + return false; + } + return true; +} + +void SnapshotFetcher::closeFile(FileState& f) { + if (f.fd >= 0) { + ::close(f.fd); + f.fd = -1; + } +} + +uint64_t SnapshotFetcher::pieceLength(const FileState& f, uint32_t chunk) const { + const uint64_t chunkSize = target_ ? target_->chunkSize : 0; + const uint64_t offset = uint64_t(chunk) * chunkSize; + return offset >= f.file.size ? 0 : std::min(chunkSize, f.file.size - offset); +} + +bool SnapshotFetcher::takeTokens(uint64_t cost, uint64_t t) { + if (cfg_.fetchRateKbps == 0) { + return true; + } + const double rate = double(cfg_.fetchRateKbps) * 1000.0 / 1000.0; // bytes per ms + owed_ = std::max(0.0, owed_ - double(t - owedAt_) * rate); + owedAt_ = t; + if (owed_ > 0) { + return false; + } + owed_ += double(cost); + return true; +} + +void SnapshotFetcher::fill(uint64_t t) { + if (stopped_ || phase_ != Phase::Fetch || !target_) { + return; + } + // round-robin over the sources: each usable source gets one piece per pass, from the file it + // can serve that has the fewest sources, so that a source shared by every file does not take + // the pieces that only it and few others could serve + std::vector keys; + for (const auto& f : files_) { + for (const auto& k : f.sources) { + if (std::find(keys.begin(), keys.end(), k) == keys.end()) { + keys.push_back(k); + } + } + } + if (keys.empty()) { + return; + } + const size_t first = rr_++ % keys.size(); + size_t total = outstanding(); + bool sent = true; + while (sent && total < kPiecesTotal) { + sent = false; + for (size_t n = 0; n < keys.size() && total < kPiecesTotal; ++n) { + const auto& key = keys[(first + n) % keys.size()]; + auto* s = findSource(key); + if (s == nullptr || !usable(key, *s, t) || neighbours_.count(key) == 0) { + continue; + } + FileState* pick = nullptr; + uint32_t pickChunk = 0; + for (auto& f : files_) { + if (f.missing == f.inFlight.size() || (pick != nullptr && f.sources.size() >= pick->sources.size()) || + std::find(f.sources.begin(), f.sources.end(), key) == f.sources.end()) { + continue; + } + // a free chunk, not one this source is probably still sending late unless nothing else is free + uint32_t chunk = 0; + uint32_t fallback = static_cast(f.have.size()); + for (; chunk < f.have.size(); ++chunk) { + if (f.have[chunk] || f.inFlight.count(chunk) != 0) { + continue; + } + auto l = late_.find({f.file.name, chunk}); + if (l != late_.end() && l->second.source == key) { + fallback = std::min(fallback, chunk); + continue; + } + break; + } + if (chunk >= f.have.size()) { + chunk = fallback; + } + if (chunk < f.have.size()) { + pick = &f; + pickChunk = chunk; + } + } + if (pick == nullptr) { + continue; + } + if (!takeTokens(pieceLength(*pick, pickChunk), t)) { + return; + } + snapshot_p2p::PieceRequest request; + request.seq = target_->seq; + request.name = pick->file.name; + request.chunk = pickChunk; + pick->inFlight[pickChunk] = InFlight{key, t}; + ++s->outstanding; + ++total; + lastSendAt_ = t; + sent = true; + if (hooks_.sendPieceRequest) { + hooks_.sendPieceRequest(key, request); + } + } + } +} + +void SnapshotFetcher::onPieceReply(const cs::PublicKey& sender, snapshot_p2p::PieceReply&& reply) { + if (stopped_ || phase_ != Phase::Fetch || !target_ || reply.seq != target_->seq) { + ++dropped_; + return; + } + auto file = std::find_if(files_.begin(), files_.end(), [&](const FileState& f) { return f.file.name == reply.name; }); + if (file == files_.end()) { + ++dropped_; + return; + } + auto& f = *file; + auto* s = findSource(sender); + auto it = f.inFlight.find(reply.chunk); + bool lateReply = false; + if (it != f.inFlight.end() && it->second.source == sender) { + f.inFlight.erase(it); + if (s != nullptr && s->outstanding != 0) { + --s->outstanding; + } + } + else { + // the reply to a piece that timed out on a slow link: still taken while the piece is missing, + // so that the bytes already on their way are not asked for twice + auto l = late_.find({reply.name, reply.chunk}); + if (l == late_.end() || l->second.source != sender || reply.chunk >= f.have.size() || f.have[reply.chunk] || + reply.status != snapshot_p2p::PieceStatus::Ok) { + ++dropped_; + return; + } + late_.erase(l); + lateReply = true; + } + const uint64_t t = now(); + + switch (reply.status) { + case snapshot_p2p::PieceStatus::Busy: + if (s != nullptr) { + // at least 500 ms, so that a source answering busy with no delay is not asked at every round trip + s->busyUntil = t + std::clamp(reply.retryMs, kMinBusyMs, kMaxBusyMs); + if (s->busySince == 0) { + s->busySince = t; + } + else if (t - s->busySince >= kBusyMarkMs) { + // busy for a minute without a piece: left alone for a minute, without counting + // towards the ban (a loaded honest node is not a faulty one) + s->busySince = 0; + s->markedUntil = t + kMarkMs; + log("SNAPSHOT: source " + keyText(sender) + " busy for " + duration(kBusyMarkMs / 1000) + + " without a piece: not asked for 1 min"); + } + } + break; + case snapshot_p2p::PieceStatus::Unknown: + mark(sender, t, false, "does not have the piece"); + break; + case snapshot_p2p::PieceStatus::NotServing: + mark(sender, t, false, "does not serve"); + break; + case snapshot_p2p::PieceStatus::Ok: { + const uint64_t length = pieceLength(f, reply.chunk); + if (reply.data.size() != length || sha256Hex(reply.data.data(), reply.data.size()) != f.file.chunks[reply.chunk]) { + log("SNAPSHOT: rejected piece " + f.file.name + ":" + std::to_string(reply.chunk) + " from " + keyText(sender), true); + // mark() may switch to another file group and replace files_: f is not used after it + mark(sender, t, true, "sent a piece that does not match the manifest"); + break; + } + if (f.have[reply.chunk]) { + break; + } + const uint64_t offset = uint64_t(reply.chunk) * target_->chunkSize; + size_t done = 0; + while (done < reply.data.size()) { + const auto n = ::pwrite(f.fd, reply.data.data() + done, reply.data.size() - done, static_cast(offset + done)); + if (n <= 0) { + log("SNAPSHOT: cannot write " + (tmpDir_ / f.file.name).string() + ": syncing from block 0", true); + clearInFlight(); + phase_ = Phase::Fallback; + return; + } + done += static_cast(n); + } + f.have[reply.chunk] = true; + --f.missing; + f.doneBytes += length; + lastByteAt_ = t; + if (s != nullptr) { + s->busySince = 0; + s->delivered = true; + s->lastPieceAt = t; + // two pieces in a row within the timeout: back to two pieces at a time + s->streak = lateReply ? 0 : s->streak + 1; + if (s->budget < kPiecesPerSource && s->streak >= 2) { + s->budget = kPiecesPerSource; + } + } + if (f.missing == 0) { + f.verifyPending = true; + } + break; + } + } + if (phase_ != Phase::Fetch) { + return; + } + fill(t); + logProgress(t, false); +} + +void SnapshotFetcher::releaseSource(const cs::PublicKey& key) { + for (auto& f : files_) { + for (auto it = f.inFlight.begin(); it != f.inFlight.end();) { + if (it->second.source == key) { + it = f.inFlight.erase(it); + } + else { + ++it; + } + } + } + if (auto* s = findSource(key)) { + s->outstanding = 0; + } + forgetLate(key); +} + +void SnapshotFetcher::forgetLate(const cs::PublicKey& key) { + for (auto it = late_.begin(); it != late_.end();) { + it = it->second.source == key ? late_.erase(it) : std::next(it); + } +} + +void SnapshotFetcher::clearInFlight() { + // late replies too: after a change of files a chunk index may name other bytes + late_.clear(); + for (auto& f : files_) { + f.inFlight.clear(); + } + for (auto& [key, s] : sources_) { + s.outstanding = 0; + } +} + +// Keys are not authenticated by the p2p layer (design section 2): a host that spoofs a source's key +// can get that key marked or banned here. The cost is bounded: a source lost for this bootstrap, +// never a wrong snapshot, since every piece is checked against the signed manifest. +void SnapshotFetcher::mark(const cs::PublicKey& key, uint64_t t, bool forever, const std::string& why) { + auto* s = findSource(key); + if (s == nullptr) { + return; + } + const bool wasBanned = s->banned; + // only a proven fault (bytes that do not match the signed manifest) excuses the end of a fetch; + // timeouts, busy and "not serving" answers do not, or a peer could keep the node from its fallback + misbehaviour_ = misbehaviour_ || (forever && phase_ == Phase::Fetch); + if (forever) { + s->banned = true; + } + else { + ++s->marks; + s->markedUntil = t + kMarkMs; + s->banned = s->marks >= 2; + } + dirty_ = dirty_ || (s->banned && !wasBanned); + log("SNAPSHOT: source " + keyText(key) + " " + why + (s->banned ? ": not asked again" : ": not asked for 1 min")); + if (!s->banned || phase_ != Phase::Fetch) { + return; + } + // its other pieces go to other sources; a file left without a source restarts from another group + releaseSource(key); + for (const auto& f : files_) { + if (f.missing == 0) { + continue; + } + const bool dry = std::none_of(f.sources.begin(), f.sources.end(), [this](const cs::PublicKey& k) { + const auto* other = findSource(k); + return other != nullptr && everUsable(*other); + }); + if (dry) { + const auto name = f.file.name; + if (chooseTarget(t)) { + log("SNAPSHOT: no source left for " + name + ": switching to the files of another source group"); + } + else if (phase_ != Phase::Fallback) { + backToCollect("no source left for " + name); + } + return; + } + } +} + +void SnapshotFetcher::expire(uint64_t t) { + // the timeout grows with what the source still has to send us: 30 s plus its outstanding bytes + // at 64 KB/s, so that two large pieces on a slow link are not taken for a stall + std::map bytesTo; + for (const auto& f : files_) { + for (const auto& [chunk, piece] : f.inFlight) { + bytesTo[piece.source] += pieceLength(f, chunk); + } + } + std::set lateSources; + for (auto& f : files_) { + for (auto it = f.inFlight.begin(); it != f.inFlight.end();) { + const uint64_t timeout = kPieceTimeoutMs + bytesTo[it->second.source] * 1000 / kTimeoutBytesPerSecond; + if (f.have[it->first]) { + // filled meanwhile by another source's late reply: nothing to wait for, nobody to blame + if (auto* s = findSource(it->second.source); s != nullptr && s->outstanding != 0) { + --s->outstanding; + } + it = f.inFlight.erase(it); + } + else if (t - it->second.sentAt >= timeout) { + lateSources.insert(it->second.source); + if (auto* s = findSource(it->second.source); s != nullptr && s->outstanding != 0) { + --s->outstanding; + } + if (late_.size() >= kMaxLate) { + auto oldest = std::min_element(late_.begin(), late_.end(), + [](const auto& a, const auto& b) { return a.second.sentAt < b.second.sentAt; }); + late_.erase(oldest); + } + late_[{f.file.name, it->first}] = Late{it->second.source, it->second.sentAt, t, timeout}; + it = f.inFlight.erase(it); + } + else { + ++it; + } + } + } + for (const auto& key : lateSources) { + if (phase_ != Phase::Fetch) { + break; + } + auto* s = findSource(key); + if (s != nullptr && s->delivered && t - s->lastPieceAt < kRecentPieceMs) { + // it delivers, slowly: one piece at a time until it delivers two in time + s->streak = 0; + if (s->budget != 1) { + s->budget = 1; + log("SNAPSHOT: source " + keyText(key) + " is slow: one piece at a time"); + } + continue; + } + mark(key, t, false, "did not answer within " + duration((kPieceTimeoutMs + bytesTo[key] * 1000 / kTimeoutBytesPerSecond) / 1000)); + } +} + +void SnapshotFetcher::verifyFiles(uint64_t t) { + // the whole-file SHA-256, at most kHashBytesPerTick per tick so that a large file does not hold + // the processor thread; a file that does not match is downloaded again + uint64_t budget = kHashBytesPerTick; + std::vector buffer; + bool bad = false; + for (auto& f : files_) { + if (!f.verifyPending) { + continue; + } + if (!f.hashing) { + f.hashing = std::make_shared(); + crypto_hash_sha256_init(&f.hashing->state); + f.hashed = 0; + } + bool ok = f.fd >= 0; + while (ok && f.hashed < f.file.size && budget != 0) { + if (buffer.empty()) { + buffer.resize(1u << 20); + } + const auto want = static_cast(std::min({buffer.size(), f.file.size - f.hashed, budget})); + const auto n = ::pread(f.fd, buffer.data(), want, static_cast(f.hashed)); + if (n <= 0) { + ok = false; + break; + } + crypto_hash_sha256_update(&f.hashing->state, buffer.data(), static_cast(n)); + f.hashed += static_cast(n); + budget -= static_cast(n); + } + if (ok && f.hashed < f.file.size) { + return; // the rest at the next tick + } + std::array digest{}; + crypto_hash_sha256_final(&f.hashing->state, digest.data()); + f.hashing.reset(); + f.verifyPending = false; + if (ok && hexOf(digest.data(), digest.size()) == f.file.sha256) { + f.verified = true; + continue; + } + // every piece matched its chunk hash: the file changed on disk after it was written + log("SNAPSHOT: " + f.file.name + " does not have the SHA-256 of the manifest after its last piece", true); + f.have.assign(f.file.chunks.size(), false); + f.missing = f.file.chunks.size(); + f.doneBytes = 0; + f.verified = false; + f.verifyPending = f.file.chunks.empty(); + bad = true; + } + if (bad) { + backToCollect("a downloaded file did not verify"); + return; + } + const bool complete = std::all_of(files_.begin(), files_.end(), [](const FileState& f) { return f.verified; }); + if (complete && target_) { + logProgress(t, true); + std::set used; + for (const auto& f : files_) { + used.insert(f.sources.begin(), f.sources.end()); + } + log("SNAPSHOT: fetched block " + std::to_string(target_->seq) + ": " + megabytes(bytesTotal()) + " MB from " + + std::to_string(used.size()) + " sources in " + std::to_string((t - fetchStartedAt_) / 1000) + " s"); + for (auto& f : files_) { + closeFile(f); + } + clearInFlight(); + phase_ = Phase::Import; + } +} + +void SnapshotFetcher::backToCollect(const std::string& why) { + clearInFlight(); + target_.reset(); + agreedSources_.clear(); + agreement_.chosen.reset(); + dirty_ = true; + phase_ = Phase::Collect; + if (misbehaviour_) { + // a source was banned for bytes that do not match the signed manifest: remote peers must + // not be able to push the node to a sync from block 0, so this is counted apart, with its own cap + misbehaviour_ = false; + ++excusedAborts_; + if (cfg_.fetchFallback != "wait" && excusedAborts_ >= kMaxExcusedAborts) { + log("SNAPSHOT: " + why + " (a source sent bad data, " + std::to_string(excusedAborts_) + " of " + + std::to_string(kMaxExcusedAborts) + "): syncing from block 0", + true); + phase_ = Phase::Fallback; + return; + } + log("SNAPSHOT: " + why + " (a source sent bad data, not counted as a failed fetch): looking at the sources again", true); + return; + } + ++fetchFailures_; + if (cfg_.fetchFallback == "wait") { + log("SNAPSHOT: " + why + " (failed fetch " + std::to_string(fetchFailures_) + + ", fetch_fallback=wait): looking at the sources again", + true); + return; + } + if (fetchFailures_ >= kMaxAttempts) { + log("SNAPSHOT: " + why + " (failed fetch " + std::to_string(fetchFailures_) + " of " + std::to_string(kMaxAttempts) + + "): syncing from block 0", + true); + phase_ = Phase::Fallback; + return; + } + log("SNAPSHOT: " + why + " (failed fetch " + std::to_string(fetchFailures_) + " of " + std::to_string(kMaxAttempts) + + "): looking at the sources again", + true); +} + +void SnapshotFetcher::banDeclaring(const std::string& name, const std::string& sha256, const std::string& why) { + for (const auto& a : agreedSources_) { + const auto* f = a.manifest.file(name); + auto* s = findSource(a.key); + if (f != nullptr && f->sha256 == sha256 && s != nullptr && !s->banned) { + s->banned = true; + dirty_ = true; + log("SNAPSHOT: source " + keyText(a.key) + " " + why + ": not asked again"); + } + } +} + +void SnapshotFetcher::logProgress(uint64_t t, bool force) { + const uint64_t total = bytesTotal(); + if (!target_ || total == 0) { + return; + } + const uint64_t done = bytesDone(); + const int decile = static_cast(done * 10 / total); + if (!force && decile <= lastProgressDecile_ && t - lastProgressAt_ < kLogIntervalMs) { + return; + } + const double seconds = double(t - lastProgressAt_) / 1000.0; + const double rate = seconds > 0 ? double(done - std::min(done, lastProgressBytes_)) / seconds / 1e6 : 0.0; + std::set active; + for (const auto& f : files_) { + for (const auto& k : f.sources) { + const auto* s = findSource(k); + if (s != nullptr && everUsable(*s)) { + active.insert(k); + } + } + } + char rateText[32]; + std::snprintf(rateText, sizeof(rateText), "%.1f", rate); + log("SNAPSHOT: fetching block " + std::to_string(target_->seq) + ": " + std::to_string(done * 100 / total) + " % (" + + megabytes(done) + " of " + megabytes(total) + " MB), " + std::to_string(active.size()) + " sources, " + rateText + " MB/s"); + lastProgressAt_ = t; + lastProgressBytes_ = done; + lastProgressDecile_ = std::max(lastProgressDecile_, decile); +} + +// --------------------------------------------------------------------------------------------- + +std::optional>> SnapshotFetcher::result() const { + if (phase_ != Phase::Import || !target_) { + return std::nullopt; + } + return std::make_pair(*target_, agreedSources_); +} + +void SnapshotFetcher::onImportFailed(const std::string& fileAtFault) { + if (stopped_ || phase_ != Phase::Import) { + return; + } + ++importFailures_; + if (target_ && fileAtFault.empty()) { + // not a file's fault: the source whose signed manifest this was + std::vector bytes; + if (DecodeBase58(target_->nodeKey, bytes) && bytes.size() == cs::PublicKey{}.size()) { + cs::PublicKey key; + std::copy(bytes.begin(), bytes.end(), key.begin()); + if (auto* s = findSource(key); s != nullptr && !s->banned) { + s->banned = true; + log("SNAPSHOT: source " + keyText(key) + " signed a snapshot the import refused: not asked again"); + } + } + } + if (target_) { + if (const auto* f = target_->file(fileAtFault)) { + badFiles_.insert({f->name, f->sha256}); + // every agreed source that declared these bytes (design 6.7) + banDeclaring(f->name, f->sha256, "declared " + f->name + " that the import refused"); + files_.erase(std::remove_if(files_.begin(), files_.end(), [&](FileState& s) { + if (s.file.name != fileAtFault) { + return false; + } + closeFile(s); + return true; + }), + files_.end()); + } + } + log("SNAPSHOT: the import refused " + (fileAtFault.empty() ? std::string("the snapshot") : fileAtFault) + " (attempt " + + std::to_string(importFailures_) + " of " + std::to_string(kMaxAttempts) + ")", + true); + if (importFailures_ >= kMaxAttempts) { + if (cfg_.fetchFallback == "wait") { + // the sources that declared the refused bytes are banned already: look for others + log("SNAPSHOT: looking at the sources again (refused import " + std::to_string(importFailures_) + ", fetch_fallback=wait)", + true); + } + else { + log("SNAPSHOT: " + std::to_string(kMaxAttempts) + " imports failed: syncing from block 0", true); + phase_ = Phase::Fallback; + return; + } + } + target_.reset(); + agreedSources_.clear(); + agreement_.chosen.reset(); + dirty_ = true; + phase_ = Phase::Collect; +} + +std::vector SnapshotFetcher::groups() const { + return agreement_.groups; +} + +void SnapshotFetcher::stop() { + stopped_ = true; + clearInFlight(); + for (auto& f : files_) { + closeFile(f); + } +} + +size_t SnapshotFetcher::manifests() const { + return static_cast( + std::count_if(sources_.begin(), sources_.end(), [](const auto& entry) { return !entry.second.banned && entry.second.manifest; })); +} + +size_t SnapshotFetcher::outstanding() const { + size_t n = 0; + for (const auto& f : files_) { + n += f.inFlight.size(); + } + return n; +} + +uint64_t SnapshotFetcher::bytesDone() const { + uint64_t n = 0; + for (const auto& f : files_) { + n += f.doneBytes; + } + return n; +} + +uint64_t SnapshotFetcher::bytesTotal() const { + uint64_t n = 0; + for (const auto& f : files_) { + n += f.file.size; + } + return n; +} + +} // namespace cs diff --git a/docs/CHANGES.md b/docs/CHANGES.md index bd38f7018..c2596ee84 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -38,6 +38,7 @@ lines under the new release. | #100 | net: four p2p messages for snapshot manifests and pieces (types 40-43) with bounded decoders; this build answers requests as not serving and sends none | | #101 | net: `host_data` announces snapshot serving (bit 24, `[snapshot] serve=on` at start) and the community version (bits 32-47, `MINOR_NODE_VERSION`, now 13); `known_peers.json` at a clean stop lists the known peers with what they announce | | #102 | snapshot: a node with `[snapshot] serve=on` builds the snapshot of each periodic checkpoint and serves it to its p2p neighbours, within `serve_rate_kbps` (KB/s), `serve_max_peers` and per-peer limits, at half budget while trusted; snapshots use 512 KiB chunks and no contracts file | +| #104 | snapshot: the p2p fetcher (not started yet): finds the neighbours that serve snapshots, agrees on one (a configured `sources=` key first, then the most independent sources by /24 or /48, the highest block only as a tie-break), downloads its pieces from several sources with every piece and file checked against the signed manifest, an adaptive piece timeout for slow links, limits per network prefix against made-up keys, and the fallback to syncing from block 0 after `fetch_wait` (or a bounded total time) | ## community.12 — 2026-10-24 diff --git a/tests/csnode/snapshot_agreement_tests.cpp b/tests/csnode/snapshot_agreement_tests.cpp new file mode 100644 index 000000000..f58c51682 --- /dev/null +++ b/tests/csnode/snapshot_agreement_tests.cpp @@ -0,0 +1,223 @@ +#include + +#include +#include + +#include + +namespace { + +cs::PublicKey key(uint8_t n) { + cs::PublicKey k{}; + k.fill(n); + return k; +} + +cs::SourceManifest source(uint8_t n, const std::string& address, cs::Sequence seq, const std::string& digest = "d1", + const std::string& head = "", bool trusted = false) { + cs::SourceManifest s; + s.key = key(n); + s.address = address; + s.trusted = trusted; + s.manifest.seq = seq; + s.manifest.headHash = head.empty() ? "h" + std::to_string(seq) : head; + s.manifest.stateDigest = digest; + s.manifest.settingsDigest = "s1"; + return s; +} + +} // namespace + +TEST(SnapshotAgreement, NetPrefix) { + EXPECT_EQ(cs::netPrefix("10.1.2.3"), "10.1.2"); + EXPECT_EQ(cs::netPrefix("10.1.2.3:6000"), "10.1.2"); + EXPECT_EQ(cs::netPrefix("::ffff:10.1.2.3"), "10.1.2"); + EXPECT_EQ(cs::netPrefix("2001:db8:1:2::1"), cs::netPrefix("2001:db8:1:ffff::7")); + EXPECT_EQ(cs::netPrefix("[2001:db8:1:2::1]:6000"), cs::netPrefix("2001:db8:1::")); + EXPECT_NE(cs::netPrefix("2001:db8:1::1"), cs::netPrefix("2001:db8:2::1")); + EXPECT_EQ(cs::netPrefix("2001:db8:1::1"), "2001:db8:1::/48"); + EXPECT_EQ(cs::netPrefix(""), ""); + EXPECT_EQ(cs::netPrefix("not an address"), ""); + EXPECT_EQ(cs::netBlock("10.1.2.3:6000"), "10.1"); + EXPECT_EQ(cs::netBlock("2001:db8:1:2::1"), "2001:db8::/32"); + EXPECT_EQ(cs::netBlock(""), ""); +} + +TEST(SnapshotAgreement, Threshold) { + std::vector v{source(1, "10.0.1.1", 1000)}; + auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.shortfall, 1u); + ASSERT_EQ(r.groups.size(), 1u); + EXPECT_EQ(r.groups[0].independent, 1u); + + v.push_back(source(2, "10.0.2.1", 1000)); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_EQ(r.chosen->independent, 2u); + EXPECT_EQ(r.shortfall, 0u); + EXPECT_TRUE(r.groups[0].chosen); +} + +TEST(SnapshotAgreement, MinSourcesOne) { + const std::vector v{source(1, "10.0.1.1", 1000)}; + const auto r = cs::agree(v, 1); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->keys, std::vector{key(1)}); +} + +TEST(SnapshotAgreement, SameSlash24CountsOnce) { + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.1.200", 1000)}; + const auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.shortfall, 1u); + ASSERT_EQ(r.groups.size(), 1u); + EXPECT_EQ(r.groups[0].keys.size(), 2u); + EXPECT_EQ(r.groups[0].independent, 1u); +} + +TEST(SnapshotAgreement, TrustedKeysCountedSeparately) { + // two trusted keys behind one /24 (an operator's own machines) count twice; an untrusted one there adds one + std::vector v{source(1, "10.0.1.1", 1000, "d1", "", true), source(2, "10.0.1.2", 1000, "d1", "", true)}; + auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->independent, 2u); + v.push_back(source(3, "10.0.1.3", 1000)); + v.push_back(source(4, "10.0.1.4", 1000)); + r = cs::agree(v, 4); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.groups[0].independent, 3u); + EXPECT_EQ(r.shortfall, 1u); +} + +TEST(SnapshotAgreement, Ipv6Slash48) { + std::vector v{source(1, "2001:db8:1:1::1", 1000), source(2, "2001:db8:1:2::1", 1000)}; + auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + v.push_back(source(3, "2001:db8:2::1", 1000)); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->independent, 2u); + EXPECT_EQ(r.chosen->keys.size(), 3u); +} + +TEST(SnapshotAgreement, UnknownAddressesDoNotCount) { + std::vector v{source(1, "", 1000), source(2, "garbage", 1000), source(3, "10.0.3.1", 1000)}; + auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.groups[0].independent, 1u); + EXPECT_EQ(r.groups[0].keys.size(), 3u); + // a trusted key counts by key whatever its address + v[0].trusted = true; + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->independent, 2u); +} + +TEST(SnapshotAgreement, SameKeyCountsOnce) { + const std::vector v{source(1, "10.0.1.1", 1000), source(1, "10.0.2.1", 1000)}; + const auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); +} + +TEST(SnapshotAgreement, MoreSupportWinsOverAHigherBlock) { + // (a) two forged sources at a made-up later block against three honest ones + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.2.1", 1000), source(3, "10.0.3.1", 1000), + source(4, "10.0.4.1", 2000, "forged"), source(5, "10.0.5.1", 2000, "forged")}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_EQ(r.chosen->independent, 3u); + ASSERT_EQ(r.groups.size(), 2u); + EXPECT_TRUE(r.groups[0].chosen); + EXPECT_FALSE(r.groups[1].chosen); + EXPECT_EQ(r.groups[1].seq, 2000u); + EXPECT_NE(cs::describe(r.groups[1]).find("fewer independent sources"), std::string::npos); +} + +TEST(SnapshotAgreement, HigherBlockBreaksATie) { + // (b) equal support: the newer snapshot + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.2.1", 1000), source(4, "10.0.4.1", 2000), + source(5, "10.0.5.1", 2000)}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 2000u); + EXPECT_NE(cs::describe(r.groups[1]).find("lower block"), std::string::npos); +} + +TEST(SnapshotAgreement, ConfiguredKeyWins) { + // (c) a group holding a key of [snapshot] sources wins over one without, both above the threshold + std::vector v{source(1, "10.0.1.1", 1000, "a", "", true), source(2, "10.0.2.1", 1000, "a"), + source(3, "10.0.3.1", 2000, "b"), source(4, "10.0.4.1", 2000, "b")}; + auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_TRUE(r.chosen->configured); + EXPECT_NE(cs::describe(r.groups[1]).find("no configured source"), std::string::npos); + // even when the other group has more support + v.push_back(source(5, "10.0.5.1", 2000, "b")); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + // but a configured key alone does not lift a group below min_sources + r = cs::agree({source(1, "10.0.1.1", 1000, "a", "", true), source(3, "10.0.3.1", 2000, "b"), source(4, "10.0.4.1", 2000, "b")}, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 2000u); +} + +TEST(SnapshotAgreement, OnlyTheLowerBlockReachesTheThreshold) { + const std::vector v{source(1, "10.0.1.1", 1000), source(2, "10.0.2.1", 1000), source(3, "10.0.3.1", 2000)}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->seq, 1000u); + EXPECT_EQ(r.groups.size(), 2u); +} + +TEST(SnapshotAgreement, DigestMajorityInsideABlock) { + // 3 of 4 declare one digest, the fourth another: the majority is chosen, the minority reported + const std::vector v{source(1, "10.0.1.1", 1000, "good"), source(2, "10.0.2.1", 1000, "good"), + source(3, "10.0.3.1", 1000, "good"), source(4, "10.0.4.1", 1000, "forged")}; + const auto r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->stateDigest, "good"); + EXPECT_EQ(r.chosen->independent, 3u); + ASSERT_EQ(r.groups.size(), 2u); + EXPECT_EQ(r.groups[1].stateDigest, "forged"); + EXPECT_FALSE(r.groups[1].chosen); + EXPECT_NE(cs::describe(r.groups[1]).find("1 source"), std::string::npos); +} + +TEST(SnapshotAgreement, TieIsNotAnAgreement) { + std::vector v{source(1, "10.0.1.1", 2000, "a"), source(2, "10.0.2.1", 2000, "a"), + source(3, "10.0.3.1", 2000, "b"), source(4, "10.0.4.1", 2000, "b")}; + auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.shortfall, 0u); // groups are large enough, they conflict + EXPECT_NE(cs::describe(r.groups[0]).find("conflict"), std::string::npos); + // a weaker group is not taken instead + v.push_back(source(5, "10.0.5.1", 1000)); + v.push_back(source(6, "10.0.6.1", 1000)); + r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + // one more source for one side settles it + v.push_back(source(7, "10.0.7.1", 2000, "a")); + r = cs::agree(v, 2); + ASSERT_TRUE(r.chosen); + EXPECT_EQ(r.chosen->stateDigest, "a"); +} + +TEST(SnapshotAgreement, DifferentHeadsAreDifferentBlocks) { + const std::vector v{source(1, "10.0.1.1", 1000, "d1", "x"), source(2, "10.0.2.1", 1000, "d1", "y")}; + const auto r = cs::agree(v, 2); + EXPECT_FALSE(r.chosen); + EXPECT_EQ(r.groups.size(), 2u); + EXPECT_EQ(r.shortfall, 1u); +} + +TEST(SnapshotAgreement, Shortfall) { + const std::vector v{source(1, "10.0.1.1", 1000)}; + EXPECT_EQ(cs::agree(v, 3).shortfall, 2u); + EXPECT_EQ(cs::agree({}, 2).shortfall, 2u); + EXPECT_TRUE(cs::agree({}, 2).groups.empty()); +} diff --git a/tests/csnode/snapshot_fetcher_tests.cpp b/tests/csnode/snapshot_fetcher_tests.cpp new file mode 100644 index 000000000..f4d9952d4 --- /dev/null +++ b/tests/csnode/snapshot_fetcher_tests.cpp @@ -0,0 +1,1361 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace fs = std::filesystem; +using cs::SnapshotFetcher; +using cs::snapshot_p2p::ManifestReply; +using cs::snapshot_p2p::ManifestRequest; +using cs::snapshot_p2p::PieceReply; +using cs::snapshot_p2p::PieceRequest; +using cs::snapshot_p2p::PieceStatus; +using Phase = SnapshotFetcher::Phase; + +const std::string kWallets = "qs-walletscache.dat.lz4"; +const std::string kBlocks = "blocks.bin.lz4"; + +std::string fileText(const fs::path& p) { + std::ifstream f(p, std::ios::binary); + return std::string(std::istreambuf_iterator(f), std::istreambuf_iterator()); +} + +void putU16(std::ofstream& f, uint16_t v) { + f.write(reinterpret_cast(&v), sizeof(v)); +} + +// Snapshots of block 200 built by buildSnapshot from one generated chain, each signed by its +// own key, with 64 KiB chunks: A1, A2 and A3-A10 from the same checkpoint files (identical bytes), B and +// E with one and two trailing bytes in walletscache.dat (same digest, another sha256 for that +// file), D with another state digest, F with 1.6 MB more in walletscache.dat. Built once for all tests. +struct Snapshots { + enum { A1, A2, B, D, E, A3, A4, A5, A6, A7, A8, A9, A10, F, Count }; + + struct Source { + cs::PrivateKey priv; + cs::PublicKey pub{}; + fs::path dir; + cs::SnapshotManifest manifest; + std::string json; + }; + + fs::path root; + std::array src; + + Snapshots() : root(fs::temp_directory_path() / "snapshot_fetcher_sources") { + fs::remove_all(root); + fs::create_directories(root); + EXPECT_TRUE(cscrypto::cryptoInit()); + + std::map blocks; + cs::Bytes head; + csdb::PoolHash prev; + for (cs::Sequence s = 0; s <= 200; ++s) { + csdb::Pool p(prev, s); + p.add_user_field(1, std::string(100 + s % 50, static_cast('a' + s % 26))); + EXPECT_TRUE(p.compose()); + blocks[s] = p.to_binary(); + head = p.hash().to_binary(); + prev = p.hash(); + } + std::mt19937 rng(11); + std::string noise(1'000'000, '\0'); + for (auto& c : noise) { + c = static_cast(rng()); + } + + auto checkpoint = [&](const std::string& name, size_t extra, uint8_t digestFill, size_t moreNoise = 0) { + const auto qs = root / ("qs-" + name) / "200"; + fs::create_directories(qs); + { + std::ofstream f(qs / "head.bin", std::ios::binary); + const uint32_t magic = 0x44414548u; + const uint64_t s = 200; + f.write(reinterpret_cast(&magic), sizeof(magic)); + f.write(reinterpret_cast(&s), sizeof(s)); + putU16(f, static_cast(head.size())); + f.write(reinterpret_cast(head.data()), static_cast(head.size())); + putU16(f, 0); + } + std::string more(moreNoise, '\0'); + for (auto& c : more) { + c = static_cast(rng()); + } + std::ofstream(qs / "walletscache.dat", std::ios::binary) << noise << more << std::string(extra, 'x'); + std::ofstream(qs / "blockchain.dat", std::ios::binary) << "settings"; + cs::CheckpointDigest d; + d.digest.fill(digestFill); + d.settings.fill(9); + d.wallets = 42; + EXPECT_TRUE(d.write(qs / cs::CheckpointDigest::kFileName)); + return qs; + }; + const auto qsA = checkpoint("a", 0, 7); + const auto qsB = checkpoint("b", 1, 7); + const auto qsD = checkpoint("d", 0, 8); + const auto qsE = checkpoint("e", 2, 7); + const auto qsF = checkpoint("f", 0, 7, 1'600'000); // a larger snapshot, about 2.6 MB + const std::array qsOf{qsA, qsA, qsB, qsD, qsE, qsA, qsA, qsA, qsA, qsA, qsA, qsA, qsA, qsF}; + + for (size_t i = 0; i < Count; ++i) { + auto& s = src[i]; + s.priv = cscrypto::generateKeyPair(s.pub); + cs::SnapshotBuildInput in; + in.seq = 200; + in.checkpointDir = qsOf[i]; + in.snapshotBlocks = 150; + in.nodeVersion = "test"; + in.chunkSize = 64 << 10; + in.blockBytes = [&blocks](cs::Sequence seq, cs::Bytes& out) { + auto it = blocks.find(seq); + if (it == blocks.end()) return false; + out = it->second; + return true; + }; + const auto r = cs::buildSnapshot(in, root / ("out" + std::to_string(i)), s.priv); + EXPECT_TRUE(r.ok) << r.error; + s.dir = r.dir; + s.json = fileText(s.dir / "snapshot.json"); + s.manifest = *cs::SnapshotManifest::fromJson(s.json); + } + } + + ~Snapshots() { + std::error_code ec; + fs::remove_all(root, ec); + } +}; + +const Snapshots& snapshots() { + static Snapshots s; + return s; +} + +cs::PublicKey otherKey(uint8_t n) { + cs::PublicKey k{}; + k.fill(n); + return k; +} + +std::string base58(const cs::PublicKey& k) { + return EncodeBase58(k.data(), k.data() + k.size()); +} + +// The joining node's network: the fetcher's hooks lead to real SnapshotProvider instances, one per +// source, that share the test's clock. step() delivers what the fetcher sent since the last step, +// advances the clock and ticks. +struct Net { + struct Source { + size_t index = 0; + std::unique_ptr provider; + std::string address; + bool stall = false; // never answers a piece request + bool corruptAll = false; // flips a byte of every piece + bool busyAll = false; // answers every piece request busy + uint32_t busyMs = 2000; + std::optional manifestJson; // replaces the manifest it serves + size_t silentManifests = 0; // manifest requests left without an answer, from the first + size_t manifestAsks = 0; + uint64_t pieceIntervalMs = 0; // serves one piece per interval, busy otherwise + uint64_t lastServedAt = 0; + uint64_t linkBytesPerSecond = 0; // its uplink, when slow + uint64_t linkFreeAt = 0; + size_t outstanding = 0; // piece requests not answered yet + }; + + const cs::PublicKey me = otherKey(0xEE); + uint64_t nowMs = 10'000'000; + std::map sources; + SnapshotFetcher::Peers peers; + std::vector> manifestQueue; + std::vector> pieceQueue; + std::vector> addCalls; + std::vector logs; + std::vector> sent; + std::vector> busySent; // busy replies: when, from whom + size_t maxOutstandingTotal = 0; + size_t maxOutstandingPerSource = 0; + size_t outstandingTotal = 0; + size_t manifestRequests = 0; + uint64_t linkBytesPerSecond = 0; + uint64_t linkFreeAt = 0; + std::multimap> link; + std::map addresses; // of peers that are not sources + + SnapshotFetcher::Hooks hooks() { + SnapshotFetcher::Hooks h; + h.sendManifestRequest = [this](const cs::PublicKey& k, const ManifestRequest& r) { + manifestQueue.emplace_back(k, r); + ++manifestRequests; + }; + h.sendPieceRequest = [this](const cs::PublicKey& k, const PieceRequest& r) { + pieceQueue.emplace_back(k, r); + sent.emplace_back(nowMs, k, r); + if (auto it = sources.find(k); it != sources.end()) { + ++it->second.outstanding; + maxOutstandingPerSource = std::max(maxOutstandingPerSource, it->second.outstanding); + } + ++outstandingTotal; + maxOutstandingTotal = std::max(maxOutstandingTotal, outstandingTotal); + }; + h.addNeighbours = [this](const std::set& keys) { addCalls.push_back(keys); }; + h.addressOf = [this](const cs::PublicKey& k) { + auto it = sources.find(k); + if (it != sources.end()) { + return it->second.address; + } + auto other = addresses.find(k); + return other == addresses.end() ? std::string() : other->second; + }; + h.clock = [this]() { return nowMs; }; + h.onLog = [this](const std::string& line) { logs.push_back(line); }; + return h; + } + + const cs::PublicKey& key(size_t index) const { + return snapshots().src[index].pub; + } + + Source& add(size_t index, bool neighbour = true) { + const auto& k = key(index); + auto& s = sources[k]; + s.index = index; + s.address = "10.0." + std::to_string(index + 1) + ".1"; + s.provider = std::make_unique( + [this]() { return std::chrono::steady_clock::time_point{} + std::chrono::milliseconds(nowMs); }); + EXPECT_TRUE(s.provider->load(snapshots().src[index].dir)); + peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + if (neighbour) { + peers.neighbours.insert(k); + } + return s; + } + + static cs::SnapshotProvider::Settings serving() { + cs::SnapshotProvider::Settings s; + s.serve = true; + s.rateKBps = 0; + return s; + } + + static cs::SnapshotNodeState synced() { + cs::SnapshotNodeState s; + s.lastSeq = 1000; + s.round = 1001; + s.synced = true; + return s; + } + + void deliver(SnapshotFetcher& fetcher) { + auto manifests = std::move(manifestQueue); + manifestQueue.clear(); + for (auto& [k, request] : manifests) { + auto it = sources.find(k); + if (it == sources.end()) { + continue; + } + ++it->second.manifestAsks; + if (it->second.silentManifests != 0) { + --it->second.silentManifests; + continue; + } + auto reply = it->second.provider->onManifestRequest(me, serving(), synced()); + if (!reply) { + continue; + } + if (it->second.manifestJson && reply->status == cs::snapshot_p2p::ManifestStatus::Ready) { + reply->manifestJson = *it->second.manifestJson; + } + fetcher.onManifestReply(k, *reply); + } + // slow links: the joining node's downlink, or a source's uplink; replies arrive in order at the rate + while (!link.empty() && link.begin()->first <= nowMs) { + auto entry = std::move(link.begin()->second); + link.erase(link.begin()); + fetcher.onPieceReply(entry.first, std::move(entry.second)); + } + auto pieces = std::move(pieceQueue); + pieceQueue.clear(); + for (auto& [k, request] : pieces) { + auto it = sources.find(k); + if (it == sources.end()) { + continue; + } + auto& s = it->second; + if (s.stall) { + continue; + } + --s.outstanding; + --outstandingTotal; + PieceReply reply; + const bool throttled = s.pieceIntervalMs != 0 && s.lastServedAt != 0 && nowMs - s.lastServedAt < s.pieceIntervalMs; + if (s.busyAll || throttled) { + reply.status = PieceStatus::Busy; + reply.retryMs = s.busyMs; + reply.seq = request.seq; + reply.name = request.name; + reply.chunk = request.chunk; + busySent.emplace_back(nowMs, k); + } + else { + reply = s.provider->onPieceRequest(me, request, serving(), synced()); + if (reply.status == PieceStatus::Ok) { + s.lastServedAt = nowMs; + } + if (reply.status == PieceStatus::Busy) { + busySent.emplace_back(nowMs, k); + } + if (s.corruptAll && reply.status == PieceStatus::Ok && !reply.data.empty()) { + reply.data[0] ^= 0x5a; + } + } + if (s.linkBytesPerSecond != 0 || linkBytesPerSecond != 0) { + auto& freeAt = s.linkBytesPerSecond != 0 ? s.linkFreeAt : linkFreeAt; + const uint64_t rate = s.linkBytesPerSecond != 0 ? s.linkBytesPerSecond : linkBytesPerSecond; + freeAt = std::max(freeAt, nowMs) + reply.data.size() * 1000 / rate; + link.emplace(freeAt, std::make_pair(k, std::move(reply))); + continue; + } + fetcher.onPieceReply(k, std::move(reply)); + } + } + + void step(SnapshotFetcher& fetcher, uint64_t ms = 100) { + deliver(fetcher); + nowMs += ms; + fetcher.tick(peers); + } + + bool runUntil(SnapshotFetcher& fetcher, const std::function& done, size_t maxSteps = 5000) { + for (size_t i = 0; i < maxSteps; ++i) { + if (done()) { + return true; + } + step(fetcher); + } + return done(); + } + + bool logged(const std::string& text) const { + return std::any_of(logs.begin(), logs.end(), [&](const std::string& l) { return l.find(text) != std::string::npos; }); + } + + size_t countLogged(const std::string& text) const { + return static_cast( + std::count_if(logs.begin(), logs.end(), [&](const std::string& l) { return l.find(text) != std::string::npos; })); + } + + size_t sentTo(const cs::PublicKey& k) const { + return static_cast(std::count_if(sent.begin(), sent.end(), [&](const auto& s) { return std::get<1>(s) == k; })); + } +}; + +SnapshotData fetchConfig(size_t minSources = 2) { + SnapshotData c; + c.minSources = minSources; + c.sourcesFile.clear(); + return c; +} + +struct TmpDir { + fs::path path; + explicit TmpDir(const std::string& name) : path(fs::temp_directory_path() / ("snapshot_fetcher_" + name)) { + fs::remove_all(path); + } + ~TmpDir() { + std::error_code ec; + fs::remove_all(path, ec); + } +}; + +// the fetched files are the manifest's, byte for byte +void expectFilesOf(const fs::path& dir, const cs::SnapshotManifest& m) { + for (const auto& f : m.files) { + const auto bytes = fileText(dir / f.name); + EXPECT_EQ(bytes.size(), f.size) << f.name; + EXPECT_EQ(cs::sha256Hex(reinterpret_cast(bytes.data()), bytes.size()), f.sha256) << f.name; + } +} + +bool isFetching(const SnapshotFetcher& f) { + return f.phase() == Phase::Fetch; +} + +} // namespace + +// (1) discovery: peers with bit 24, [snapshot] sources and sources_file keys, nothing else +TEST(SnapshotFetcher, DiscoveryAddsServingPeersAndConfiguredKeys) { + TmpDir tmp("discovery"); + fs::create_directories(tmp.path); + Net net; + const auto trusted = otherKey(1); + const auto listed = otherKey(2); + const auto official = otherKey(3); + const auto otherBit = otherKey(4); + const auto sourcesFile = tmp.path / "snapshot_sources.txt"; + std::ofstream(sourcesFile) << "# serving nodes\n1.2.3.4:6000 " << base58(listed) << "\n\n5.6.7.8:6000 not-a-key\n"; + + auto cfg = fetchConfig(); + cfg.sources = {trusted}; + cfg.sourcesFile = sourcesFile.string(); + net.add(Snapshots::A1, false); + net.add(Snapshots::A2, false); + net.peers.known.emplace_back(official, 0); + net.peers.known.emplace_back(otherBit, 2); + SnapshotFetcher fetcher(cfg, tmp.path / "tmp", net.hooks()); + + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 1u); + const std::set expected{trusted, listed, net.key(Snapshots::A1), net.key(Snapshots::A2)}; + EXPECT_EQ(net.addCalls[0], expected); + EXPECT_TRUE(net.manifestQueue.empty()); // none is a neighbour yet + EXPECT_EQ(fetcher.phase(), Phase::Discover); + EXPECT_TRUE(net.logged("not a node key")); + + // once per key; a key that became a neighbour is asked for its manifest + net.peers.neighbours.insert(net.key(Snapshots::A1)); + net.nowMs += 500; + fetcher.tick(net.peers); + EXPECT_EQ(net.addCalls.size(), 1u); + ASSERT_EQ(net.manifestQueue.size(), 1u); + EXPECT_EQ(net.manifestQueue[0].first, net.key(Snapshots::A1)); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("asking " + base58(net.key(Snapshots::A1)))); + + // the others are brought in again after a minute; the neighbour is not + net.nowMs += SnapshotFetcher::kNeighbourRetryMs; + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 2u); + EXPECT_EQ(net.addCalls[1].count(net.key(Snapshots::A1)), 0u); + EXPECT_EQ(net.addCalls[1].size(), 3u); +} + +// a flood of discovered keys from one /24 pushes out neither a configured key nor other networks' sources +TEST(SnapshotFetcher, FloodFromOnePrefixDoesNotFillTheTable) { + TmpDir tmp("flood"); + Net net; + const auto configured = otherKey(1); + auto cfg = fetchConfig(); + cfg.sources = {configured}; + std::vector flood; + for (int i = 0; i < 100; ++i) { + cs::PublicKey k{}; + k.fill(0x40); + k[0] = static_cast(i); + flood.push_back(k); + net.addresses[k] = "10.9.9." + std::to_string(i + 1); + net.peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + net.peers.neighbours.insert(k); // all of them neighbours that never answer + } + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 1u); + const auto& added = net.addCalls[0]; + EXPECT_EQ(added.count(configured), 1u); + // the sources of other networks (neighbours already) are asked; of the flood at most kMaxPerPrefix + std::set asked; + size_t floodAsked = 0; + for (const auto& [k, request] : net.manifestQueue) { + asked.insert(k); + floodAsked += std::count(flood.begin(), flood.end(), k); + } + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::B}) { + EXPECT_EQ(asked.count(net.key(i)), 1u); + } + EXPECT_EQ(floodAsked, SnapshotFetcher::kMaxPerPrefix); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// a full table makes room for a new network by dropping the oldest of the most represented one +TEST(SnapshotFetcher, FullTableMakesRoomForANewPrefix) { + TmpDir tmp("fulltable"); + Net net; + for (int p = 0; p < 16; ++p) { + for (int i = 0; i < 4; ++i) { + cs::PublicKey k{}; + k.fill(0x50); + k[0] = static_cast(p); + k[1] = static_cast(i); + net.addresses[k] = "10." + std::to_string(100 + p) + ".0." + std::to_string(i + 1); + net.peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + } + } + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 1u); + EXPECT_EQ(net.addCalls[0].size(), SnapshotFetcher::kMaxSources); + // two serving nodes from other networks turn up later + net.add(Snapshots::A1, false); + net.add(Snapshots::A2, false); + net.nowMs += 500; + fetcher.tick(net.peers); + ASSERT_EQ(net.addCalls.size(), 2u); + EXPECT_EQ(net.addCalls[1].count(net.key(Snapshots::A1)), 1u); + EXPECT_EQ(net.addCalls[1].count(net.key(Snapshots::A2)), 1u); + net.peers.neighbours.insert(net.key(Snapshots::A1)); + net.peers.neighbours.insert(net.key(Snapshots::A2)); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// 64 keys over 64 networks that never answer do not keep a new honest peer out +TEST(SnapshotFetcher, SilentSpreadKeysDoNotLockNewcomersOut) { + TmpDir tmp("spreadkeys"); + Net net; + for (int p = 0; p < 64; ++p) { + cs::PublicKey k{}; + k.fill(0x60); + k[0] = static_cast(p); + net.addresses[k] = std::to_string(20 + p) + "." + std::to_string(p) + ".0.1"; + net.peers.known.emplace_back(k, cs::hostdata::kSnapshotProtocolV1); + net.peers.neighbours.insert(k); + } + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + net.step(fetcher); + // a full table makes room at once, before anything has had time to answer + net.add(Snapshots::A1, false); + net.step(fetcher); + ASSERT_FALSE(net.addCalls.empty()); + EXPECT_EQ(net.addCalls.back().count(net.key(Snapshots::A1)), 1u); + net.peers.neighbours.insert(net.key(Snapshots::A1)); + for (int i = 0; i < 610; ++i) { // the others asked, never answering, for a minute + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_EQ(fetcher.manifests(), 1u); + net.add(Snapshots::A2); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// an address not known yet: no cap at discovery, no vote until it is known; the cap is applied +// to the address resolved when the manifest arrives +TEST(SnapshotFetcher, UnknownAddresses) { + TmpDir tmp("unknownaddr"); + Net net; + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::B, Snapshots::D, Snapshots::E}) { + net.add(i).address.clear(); + } + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + EXPECT_EQ(net.manifestQueue.size(), 5u); // five unknown addresses, none refused + net.step(fetcher); + EXPECT_EQ(fetcher.manifests(), 5u); + for (int i = 0; i < 10; ++i) { + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); // nothing counts without an address + EXPECT_FALSE(fetcher.groups().empty()); + EXPECT_EQ(fetcher.groups()[0].independent, 0u); + + // all five turn out to be in one /24: four are kept, they count once + int n = 1; + for (auto& [key, source] : net.sources) { + source.address = "10.0.7." + std::to_string(n++); + } + net.nowMs += SnapshotFetcher::kManifestIntervalMs; + fetcher.tick(net.peers); + net.deliver(fetcher); + EXPECT_EQ(fetcher.manifests(), 4u); + EXPECT_TRUE(net.logged("too many sources at 10.0.7")); + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Collect); // one network: one source + + // known in different networks, they agree + n = 1; + for (auto& [key, source] : net.sources) { + source.address = "10.0." + std::to_string(n++) + ".1"; + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// (2) bad signature, another node's manifest, an oversized chunk, an unsolicited reply: dropped +TEST(SnapshotFetcher, BadManifestsAreDropped) { + TmpDir tmp("badmanifests"); + Net net; + const auto& snaps = snapshots(); + auto& a1 = net.add(Snapshots::A1); + auto& a2 = net.add(Snapshots::A2); + auto& b = net.add(Snapshots::B); + + auto forged = snaps.src[Snapshots::A1].manifest; + forged.wallets += 1; // signature no longer matches + a1.manifestJson = forged.toJson(); + a2.manifestJson = snaps.src[Snapshots::A1].json; // relayed: signed by A1 + auto big = snaps.src[Snapshots::B].manifest; + big.chunkSize = 2u << 20; + for (auto& f : big.files) { + f.chunks.assign(f.size == 0 ? 0 : 1, f.sha256); + } + big.sign(snaps.src[Snapshots::B].priv); + ASSERT_TRUE(cs::SnapshotManifest::fromJson(big.toJson())); + b.manifestJson = big.toJson(); + + SnapshotFetcher fetcher(fetchConfig(1), tmp.path, net.hooks()); + fetcher.onManifestReply(net.key(Snapshots::A1), ManifestReply{}); // not asked yet + EXPECT_EQ(fetcher.dropped(), 1u); + fetcher.tick(net.peers); + net.step(fetcher); + EXPECT_EQ(fetcher.manifests(), 0u); + EXPECT_EQ(fetcher.dropped(), 4u); + EXPECT_TRUE(net.logged("dropped: signature does not verify")); + EXPECT_TRUE(net.logged("dropped: signed by another node")); + EXPECT_TRUE(net.logged("dropped: chunk size")); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + + // the same sources with their own manifests are taken at the next request (30 s later) + a1.manifestJson.reset(); + net.nowMs += SnapshotFetcher::kManifestIntervalMs; + fetcher.tick(net.peers); + net.deliver(fetcher); + EXPECT_EQ(fetcher.manifests(), 1u); + fetcher.tick(net.peers); // agreement is tried at the tick + EXPECT_NE(fetcher.phase(), Phase::Collect); // min_sources=1: agreed +} + +// (3) three sources of four agree, the fourth (another digest) is reported as minority +TEST(SnapshotFetcher, AgreementOfThreeOfFour) { + TmpDir tmp("agreement"); + Net net; + net.add(Snapshots::D); + SnapshotFetcher fetcher(fetchConfig(3), tmp.path, net.hooks()); + net.step(fetcher); + net.step(fetcher); + EXPECT_EQ(fetcher.manifests(), 1u); + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() != Phase::Collect; }, 20)); + EXPECT_EQ(fetcher.phase(), Phase::Fetch); + + const auto groups = fetcher.groups(); + ASSERT_EQ(groups.size(), 2u); + const auto chosen = std::find_if(groups.begin(), groups.end(), [](const auto& g) { return g.chosen; }); + ASSERT_NE(chosen, groups.end()); + EXPECT_EQ(chosen->independent, 3u); + const std::set keys(chosen->keys.begin(), chosen->keys.end()); + EXPECT_EQ(keys, (std::set{net.key(Snapshots::A1), net.key(Snapshots::A2), net.key(Snapshots::B)})); + EXPECT_TRUE(net.logged("minority: block 200")); + EXPECT_EQ(net.sentTo(net.key(Snapshots::D)), 0u); +} + +// (4) pieces go round-robin, at most 2 per source and 8 in all; each file only from sources that +// declare its bytes; the result is the files of the largest group +TEST(SnapshotFetcher, PiecesSpreadAndBounded) { + TmpDir tmp("spread"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + + // the first round already uses every source that can serve + std::set first; + for (const auto& s : net.sent) { + first.insert(std::get<1>(s)); + } + EXPECT_EQ(first.size(), 4u); + + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_LE(net.maxOutstandingPerSource, SnapshotFetcher::kPiecesPerSource); + EXPECT_LE(net.maxOutstandingTotal, SnapshotFetcher::kPiecesTotal); + EXPECT_EQ(net.maxOutstandingTotal, SnapshotFetcher::kPiecesTotal); + + const auto result = fetcher.result(); + ASSERT_TRUE(result); + const auto& m = result->first; + EXPECT_EQ(m.filesHash(), snapshots().src[Snapshots::A1].manifest.filesHash()); + EXPECT_EQ(result->second.size(), 4u); + expectFilesOf(tmp.path, m); + for (const auto& s : net.sent) { + const auto& k = std::get<1>(s); + if (std::get<2>(s).name == kWallets) { + EXPECT_TRUE(k == net.key(Snapshots::A1) || k == net.key(Snapshots::A2)); + } + } + // B and E, whose walletscache differs, served the files they share with A + EXPECT_GT(net.sentTo(net.key(Snapshots::B)), 0u); + EXPECT_GT(net.sentTo(net.key(Snapshots::E)), 0u); + EXPECT_TRUE(net.logged("SNAPSHOT: fetching block 200: 100 %")); + EXPECT_TRUE(net.logged("SNAPSHOT: fetched block 200")); +} + +// (5) a corrupt piece marks its source for the bootstrap; the piece comes from another source +TEST(SnapshotFetcher, CorruptPieceMarksTheSource) { + TmpDir tmp("corrupt"); + Net net; + net.add(Snapshots::A1).corruptAll = true; + net.add(Snapshots::A2); + net.add(Snapshots::B); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_TRUE(net.logged("rejected piece")); + EXPECT_TRUE(net.logged(base58(net.key(Snapshots::A1)) + " sent a piece that does not match")); + EXPECT_LE(net.sentTo(net.key(Snapshots::A1)), SnapshotFetcher::kPiecesPerSource); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (6) a stalled source times out after 30 s, its pieces go elsewhere, it is marked +TEST(SnapshotFetcher, StalledSourceTimesOut) { + TmpDir tmp("stall"); + Net net; + net.add(Snapshots::A1).stall = true; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_GE(net.nowMs - start, SnapshotFetcher::kPieceTimeoutMs); + // 30 s plus its two outstanding 64-KiB pieces at 64 KB/s + EXPECT_TRUE(net.logged(base58(net.key(Snapshots::A1)) + " did not answer within 32 s")); + EXPECT_GT(net.sentTo(net.key(Snapshots::A1)), 0u); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (7) busy delays only that source, by retry_ms +TEST(SnapshotFetcher, BusyDelaysOnlyThatSource) { + TmpDir tmp("busy"); + Net net; + auto& a1 = net.add(Snapshots::A1); + a1.busyAll = true; + a1.busyMs = 2000; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + const auto a1Key = net.key(Snapshots::A1); + size_t a1Sends = 0; + for (const auto& s : net.sent) { + if (std::get<1>(s) != a1Key) { + continue; + } + ++a1Sends; + const uint64_t at = std::get<0>(s); + for (const auto& [when, who] : net.busySent) { + if (who == a1Key) { + EXPECT_FALSE(when < at && at < when + 2000) << "asked " << at - when << " ms after a busy reply"; + } + } + } + EXPECT_GE(a1Sends, 2u); + EXPECT_GT(net.sentTo(net.key(Snapshots::A2)), a1Sends); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (8) a source that leaves the neighbour set is not asked +TEST(SnapshotFetcher, SourceThatLeftIsNotAsked) { + TmpDir tmp("left"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + net.peers.neighbours.erase(net.key(Snapshots::A1)); + fetcher.tick(net.peers); + const size_t before = net.sentTo(net.key(Snapshots::A1)); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(net.sentTo(net.key(Snapshots::A1)), before); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (9) a file whose group runs dry restarts from the other group +TEST(SnapshotFetcher, DryGroupSwitchesToTheOther) { + TmpDir tmp("dry"); + Net net; + net.add(Snapshots::A1).corruptAll = true; + net.add(Snapshots::A2).corruptAll = true; + net.add(Snapshots::B); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_TRUE(net.logged("switching to the files of another source group")); + const auto result = fetcher.result(); + ASSERT_TRUE(result); + EXPECT_EQ(result->first.nodeKey, snapshots().src[Snapshots::B].manifest.nodeKey); + expectFilesOf(tmp.path, result->first); +} + +// (10) a file that changes on disk between its last piece and the whole-file check: back to Collect +TEST(SnapshotFetcher, WholeFileMismatchGoesBackToCollect) { + TmpDir tmp("wholefile"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + // up to the last piece; the whole-file check of the last file runs at the next tick + for (size_t i = 0; i < 5000; ++i) { + net.deliver(fetcher); + if (fetcher.bytesTotal() != 0 && fetcher.bytesDone() == fetcher.bytesTotal()) { + break; + } + net.nowMs += 100; + fetcher.tick(net.peers); + } + ASSERT_EQ(fetcher.bytesDone(), fetcher.bytesTotal()); + ASSERT_EQ(fetcher.phase(), Phase::Fetch); + for (const auto& entry : fs::directory_iterator(tmp.path)) { + if (fs::file_size(entry.path()) == 0) { + continue; + } + std::fstream f(entry.path(), std::ios::in | std::ios::out | std::ios::binary); + char c = 0; + f.read(&c, 1); + c ^= 0x01; + f.seekp(0); + f.write(&c, 1); + } + net.nowMs += 100; + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("does not have the SHA-256 of the manifest after its last piece")); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// (11) fetch_wait over without agreement: sync falls back, wait keeps waiting with a line every 5 min +TEST(SnapshotFetcher, FetchWaitSync) { + TmpDir tmp("waitsync"); + Net net; + net.add(Snapshots::A1); + auto cfg = fetchConfig(); + cfg.fetchWait = 60; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 700)); + EXPECT_GE(net.nowMs - 10'000'000, 60'000u); + EXPECT_TRUE(net.logged("SNAPSHOT: 1 source declares block 200 (need 2): waiting")); + EXPECT_TRUE(net.logged("SNAPSHOT: no agreement of 2 community sources after 1 min (seen: 1 source for block 200). Syncing from block 0")); + EXPECT_TRUE(net.logged("./node --download-snapshot in this folder (Docker: CREDITS_SNAPSHOT=github)")); + const auto sends = net.manifestRequests + net.sent.size(); + for (int i = 0; i < 1000; ++i) { + net.step(fetcher); + } + EXPECT_EQ(net.manifestRequests + net.sent.size(), sends); +} + +TEST(SnapshotFetcher, FetchWaitWait) { + TmpDir tmp("waitwait"); + Net net; + net.add(Snapshots::A1); + auto cfg = fetchConfig(); + cfg.fetchWait = 60; + cfg.fetchFallback = "wait"; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + for (int i = 0; i < 6 * 60 * 10 + 10; ++i) { + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_EQ(net.countLogged("Waiting for more sources (fetch_fallback=wait)"), 2u); + // a second source turns up + net.add(Snapshots::A2); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); +} + +// (12) a refused import marks that file's group; the next attempt takes another group; after +// three refusals the node syncs from block 0 +TEST(SnapshotFetcher, ImportFailures) { + TmpDir tmp("import"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + // the sources that declared a refused file are banned: min_sources=1 keeps one for the third attempt + SnapshotFetcher fetcher(fetchConfig(1), tmp.path, net.hooks()); + std::set walletHashes; + std::set banned; + for (int attempt = 1; attempt <= 3; ++attempt) { + const size_t sentBefore = net.sent.size(); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })) << attempt; + const auto result = fetcher.result(); + ASSERT_TRUE(result); + expectFilesOf(tmp.path, result->first); + const auto* wallets = result->first.file(kWallets); + ASSERT_NE(wallets, nullptr); + EXPECT_TRUE(walletHashes.insert(wallets->sha256).second) << "the refused file came back"; + for (size_t i = sentBefore; i < net.sent.size(); ++i) { + if (attempt > 1) { + // files already verified are not asked again + EXPECT_EQ(std::get<2>(net.sent[i]).name, kWallets); + } + EXPECT_EQ(banned.count(std::get<1>(net.sent[i])), 0u) << "a banned source was asked"; + } + for (const auto& [key, source] : net.sources) { + const auto* f = snapshots().src[source.index].manifest.file(kWallets); + if (f->sha256 == wallets->sha256) { + banned.insert(key); + } + } + fetcher.onImportFailed(kWallets); + EXPECT_EQ(fetcher.phase(), attempt < 3 ? Phase::Collect : Phase::Fallback); + EXPECT_FALSE(fetcher.result()); + } + EXPECT_TRUE(net.logged("3 imports failed")); + EXPECT_EQ(net.countLogged("that the import refused: not asked again"), 4u); +} + +// a refusal that is no file's fault bans the source that signed the manifest +TEST(SnapshotFetcher, ImportFailureWithoutAFileBansTheSigner) { + TmpDir tmp("importsigner"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + const auto signer = fetcher.result()->first.nodeKey; + fetcher.onImportFailed(""); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("source " + signer + " signed a snapshot the import refused: not asked again")); + const size_t before = net.sent.size(); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + for (size_t i = before; i < net.sent.size(); ++i) { + EXPECT_NE(base58(std::get<1>(net.sent[i])), signer); + } +} + +// a peer that becomes a neighbour late, and one silent for its first requests, are kept (design 5.3a) +TEST(SnapshotFetcher, LateAndSilentSourcesAreKept) { + TmpDir tmp("late"); + Net net; + net.add(Snapshots::A1, false).silentManifests = 1; + net.add(Snapshots::A2).silentManifests = 2; + net.add(Snapshots::B, false); + SnapshotFetcher fetcher(fetchConfig(3), tmp.path, net.hooks()); + for (int i = 0; i < 250; ++i) { + net.step(fetcher); + } + // A2 was silent twice and answered the third request, 10 and 20 s after the first + EXPECT_EQ(net.sources[net.key(Snapshots::A2)].manifestAsks, 3u); + EXPECT_EQ(fetcher.manifests(), 1u); + for (int i = 0; i < 360; ++i) { // past a minute: A1 and B become neighbours late + net.step(fetcher); + } + EXPECT_EQ(fetcher.manifests(), 1u); + net.peers.neighbours.insert(net.key(Snapshots::A1)); + net.peers.neighbours.insert(net.key(Snapshots::B)); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(fetcher.result()->second.size(), 3u); +} + +// busy with no delay: not asked again before 500 ms +TEST(SnapshotFetcher, BusyWithoutDelayIsFloored) { + TmpDir tmp("busyfloor"); + Net net; + auto& a1 = net.add(Snapshots::A1); + a1.busyAll = true; + a1.busyMs = 0; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + const auto a1Key = net.key(Snapshots::A1); + for (const auto& s : net.sent) { + if (std::get<1>(s) != a1Key) { + continue; + } + for (const auto& [when, who] : net.busySent) { + if (who == a1Key) { + EXPECT_FALSE(when < std::get<0>(s) && std::get<0>(s) < when + SnapshotFetcher::kMinBusyMs); + } + } + } +} + +// sources that are busy forever: each fetch ends after 5 min without data, the third ends the bootstrap +TEST(SnapshotFetcher, BusyForeverEndsTheFetch) { + TmpDir tmp("busyforever"); + Net net; + net.add(Snapshots::A1).busyAll = true; + net.add(Snapshots::A2).busyAll = true; + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; // fetch_fallback=sync + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher); }, 20)); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() != Phase::Fetch; }, 4000)); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_GE(net.nowMs - start, SnapshotFetcher::kNoProgressMs); + EXPECT_TRUE(net.logged("no data for 5 min (failed fetch 1 of 3)")); + EXPECT_TRUE(net.logged("busy for 1 min without a piece")); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 10000)); + EXPECT_TRUE(net.logged("(failed fetch 3 of 3): syncing from block 0")); +} + +// with fetch_fallback=wait failed fetches never end the bootstrap +TEST(SnapshotFetcher, BusyForeverWithWaitKeepsTrying) { + TmpDir tmp("busywait"); + Net net; + net.add(Snapshots::A1).busyAll = true; + net.add(Snapshots::A2).busyAll = true; + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; + cfg.fetchFallback = "wait"; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("(failed fetch 4, fetch_fallback=wait)") != 0; }, 20000)); + EXPECT_NE(fetcher.phase(), Phase::Fallback); + // the sources deliver again: the fetch completes + net.sources[net.key(Snapshots::A1)].busyAll = false; + net.sources[net.key(Snapshots::A2)].busyAll = false; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 10000)); +} + +// sources that serve bad pieces are banned wave after wave without pushing the node to a sync from 0 +TEST(SnapshotFetcher, MaliciousWavesDoNotForceFallback) { + TmpDir tmp("waves"); + Net net; + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::A3, Snapshots::A4, Snapshots::A5, Snapshots::A6}) { + net.add(i, i == Snapshots::A1 || i == Snapshots::A2).corruptAll = true; + } + net.add(Snapshots::B, false); + net.add(Snapshots::E, false); + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; // fetch_fallback=sync + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const std::vector> waves{{Snapshots::A3, Snapshots::A4}, {Snapshots::A5, Snapshots::A6}, {Snapshots::B, Snapshots::E}}; + for (size_t w = 0; w < waves.size(); ++w) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("not counted as a failed fetch") == w + 1; }, 2000)) << w; + EXPECT_EQ(fetcher.phase(), Phase::Collect); + net.peers.neighbours.insert(net.key(waves[w].first)); + net.peers.neighbours.insert(net.key(waves[w].second)); + if (w + 1 == waves.size()) { + break; + } + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(net.countLogged("(failed fetch"), 0u); + EXPECT_EQ(fetcher.result()->first.filesHash() == snapshots().src[Snapshots::B].manifest.filesHash() || + fetcher.result()->first.filesHash() == snapshots().src[Snapshots::E].manifest.filesHash(), + true); +} + +// a slow downlink: about 100 KB/s for eight 512-KiB pieces in flight, scaled here to 64-KiB pieces; +// no source is marked and the fetch completes +TEST(SnapshotFetcher, SlowLinkIsNotMarked) { + TmpDir tmp("slowlink"); + Net net; + net.linkBytesPerSecond = 12'500; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 20000)); + EXPECT_FALSE(net.logged("did not answer")); + EXPECT_FALSE(net.logged("looking at the sources again")); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// sources whose uplink is so slow that their second piece outlives the timeout: they deliver, so +// they go to one piece at a time instead of being marked, and their late pieces are taken +TEST(SnapshotFetcher, SlowSourcesGoToOnePieceAtATime) { + TmpDir tmp("slowsources"); + Net net; + net.add(Snapshots::A1).linkBytesPerSecond = 3'000; + net.add(Snapshots::A2).linkBytesPerSecond = 3'000; + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 20000)); + EXPECT_TRUE(net.logged("is slow: one piece at a time")); + EXPECT_FALSE(net.logged("did not answer")); + EXPECT_FALSE(net.logged("looking at the sources again")); + // late pieces were taken, not asked for again: no more requests than pieces plus the requeued ones + const auto& m = snapshots().src[Snapshots::A1].manifest; + size_t pieces = 0; + for (const auto& f : m.files) { + pieces += f.chunks.size(); + } + EXPECT_LT(net.sent.size(), pieces + pieces / 2); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// sources that only stop answering cannot keep a node with fetch_fallback=sync from its fallback +TEST(SnapshotFetcher, TimeoutsDoNotExcuseFailedFetches) { + TmpDir tmp("timeoutwaves"); + Net net; + for (auto i : {Snapshots::A1, Snapshots::A2, Snapshots::A3, Snapshots::A4, Snapshots::A5, Snapshots::A6}) { + net.add(i, i == Snapshots::A1 || i == Snapshots::A2).stall = true; + } + auto cfg = fetchConfig(); + cfg.fetchWait = 86'400; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const std::vector> waves{{Snapshots::A3, Snapshots::A4}, {Snapshots::A5, Snapshots::A6}}; + for (size_t w = 0; w < waves.size(); ++w) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("(failed fetch " + std::to_string(w + 1) + " of 3)") != 0; }, + 10000)) + << w; + net.peers.neighbours.insert(net.key(waves[w].first)); + net.peers.neighbours.insert(net.key(waves[w].second)); + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 10000)); + EXPECT_TRUE(net.logged("(failed fetch 3 of 3): syncing from block 0")); + EXPECT_FALSE(net.logged("not counted as a failed fetch")); +} + +// sources proven bad are not counted as failed fetches, but ten such fetches end the bootstrap (sync) +TEST(SnapshotFetcher, TenBadSourceAbortsEndWithSync) { + TmpDir tmp("badwaves"); + Net net; + const std::vector bad{Snapshots::A1, Snapshots::A2, Snapshots::A3, Snapshots::A4, Snapshots::A5, + Snapshots::A6, Snapshots::A7, Snapshots::A8, Snapshots::A9, Snapshots::A10}; + for (auto i : bad) { + net.add(i, i == Snapshots::A1).corruptAll = true; + } + auto cfg = fetchConfig(1); + cfg.fetchWait = 86'400; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + for (size_t w = 1; w < bad.size(); ++w) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return net.countLogged("not counted as a failed fetch") == w; }, 2000)) << w; + net.peers.neighbours.insert(net.key(bad[w])); + } + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 2000)); + EXPECT_TRUE(net.logged("(a source sent bad data, 10 of 10): syncing from block 0")); + EXPECT_FALSE(net.logged("(failed fetch")); +} + +// a source that keeps a fetch alive with one valid piece every 4 min: fetch_fallback=sync ends the +// bootstrap at fetch_wait + 30 min whatever the counters say +TEST(SnapshotFetcher, BootstrapCapWithSync) { + TmpDir tmp("bootcap"); + Net net; + net.add(Snapshots::A1).pieceIntervalMs = 4 * 60'000; + auto cfg = fetchConfig(1); + cfg.fetchWait = 60; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 20000)); + EXPECT_GE(net.nowMs - start, 31 * 60'000u); + EXPECT_TRUE(net.logged("(fetch_wait + 30 min + the download time): syncing from block 0")); + EXPECT_FALSE(net.logged("(failed fetch")); +} + +// an honest join limited by fetch_rate_kbps: the cap makes room for the download at that rate +TEST(SnapshotFetcher, BootstrapCapAllowsTheConfiguredRate) { + TmpDir tmp("bootcaprate"); + Net net; + net.add(Snapshots::F); + auto cfg = fetchConfig(1); + cfg.fetchWait = 60; + cfg.fetchRateKbps = 1; // 1000 bytes/s: about 2.6 MB takes about 44 min, more than 30 + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + const uint64_t start = net.nowMs; + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import || fetcher.phase() == Phase::Fallback; }, 40000)); + EXPECT_EQ(fetcher.phase(), Phase::Import); + EXPECT_GT(net.nowMs - start, 31 * 60'000u); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// with fetch_fallback=wait refused imports do not end the bootstrap: other sources are looked for +TEST(SnapshotFetcher, RefusedImportsWithWaitKeepTrying) { + TmpDir tmp("importwait"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + net.add(Snapshots::B); + net.add(Snapshots::E); + auto cfg = fetchConfig(1); + cfg.fetchFallback = "wait"; + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + for (int attempt = 1; attempt <= 3; ++attempt) { + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })) << attempt; + fetcher.onImportFailed(kWallets); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + } + EXPECT_TRUE(net.logged("(refused import 3, fetch_fallback=wait)")); + // every source that declared a refused file is banned; another one turns up + net.add(Snapshots::F); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + EXPECT_EQ(fetcher.result()->first.nodeKey, snapshots().src[Snapshots::F].manifest.nodeKey); +} + +// a slow source and a fast one: pieces taken from the slow one by timeout are fetched once, its +// late replies do not get the fast one blamed, everything is on disk at the end +TEST(SnapshotFetcher, ReassignmentRace) { + TmpDir tmp("race"); + Net net; + net.add(Snapshots::A1).linkBytesPerSecond = 3'000; + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; }, 20000)); + EXPECT_EQ(fetcher.bytesDone(), fetcher.bytesTotal()); + EXPECT_FALSE(net.logged(base58(net.key(Snapshots::A2)) + " did not answer")); + const auto& m = snapshots().src[Snapshots::A1].manifest; + size_t pieces = 0; + for (const auto& f : m.files) { + pieces += f.chunks.size(); + } + EXPECT_LT(net.sent.size(), pieces + pieces / 2); + expectFilesOf(tmp.path, fetcher.result()->first); +} + +// a symbolic link where a downloaded file goes: the file is not opened, the node syncs from block 0 +TEST(SnapshotFetcher, SymlinkInTmpDirIsRefused) { + TmpDir tmp("symlink"); + fs::create_directories(tmp.path); + const auto target = tmp.path / "elsewhere"; + std::ofstream(target) << "do not touch"; + fs::create_symlink(target, tmp.path / kWallets); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Fallback; }, 50)); + EXPECT_TRUE(net.logged("cannot write")); + EXPECT_EQ(fileText(target), "do not touch"); +} + +// two groups as strong as each other: the 30-s line says conflict +TEST(SnapshotFetcher, ConflictLine) { + TmpDir tmp("conflict"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::D); + SnapshotFetcher fetcher(fetchConfig(1), tmp.path, net.hooks()); + for (int i = 0; i < 320; ++i) { + net.step(fetcher); + } + EXPECT_EQ(fetcher.phase(), Phase::Collect); + EXPECT_TRUE(net.logged("SNAPSHOT: conflict: two groups with 1 sources at block 200: waiting")); + EXPECT_TRUE(net.logged("2 manifests collected, 0 not counted: address unknown, 0 replies dropped")); +} + +// (13) a dry run stops after Collect and reports the groups +TEST(SnapshotFetcher, DryRunReportsGroups) { + TmpDir tmp("dryrun"); + Net net; + net.add(Snapshots::D); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks(), true); + net.step(fetcher); + net.step(fetcher); + net.add(Snapshots::A1); + net.add(Snapshots::A2); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Done; }, 20)); + EXPECT_TRUE(net.sent.empty()); + const auto groups = fetcher.groups(); + EXPECT_EQ(groups.size(), 2u); + const auto chosen = std::count_if(groups.begin(), groups.end(), [](const cs::GroupReport& g) { return g.chosen; }); + EXPECT_EQ(chosen, 1); + EXPECT_FALSE(fetcher.result()); + EXPECT_FALSE(fs::exists(tmp.path)); + + // a dry run without agreement ends at fetch_wait too + Net lone; + lone.add(Snapshots::A1); + auto cfg = fetchConfig(); + cfg.fetchWait = 10; + SnapshotFetcher alone(cfg, tmp.path, lone.hooks(), true); + ASSERT_TRUE(lone.runUntil(alone, [&] { return alone.phase() == Phase::Done; }, 200)); + EXPECT_EQ(alone.groups().size(), 1u); +} + +// (14) stop in every phase: nothing outstanding, nothing sent, replies dropped +TEST(SnapshotFetcher, StopInEveryPhase) { + TmpDir tmp("stop"); + auto quietAfterStop = [](Net& net, SnapshotFetcher& fetcher) { + fetcher.stop(); + EXPECT_EQ(fetcher.outstanding(), 0u); + const auto manifests = net.manifestRequests; + const auto pieces = net.sent.size(); + const auto dropped = fetcher.dropped(); + const auto queued = net.pieceQueue.size() + net.manifestQueue.size(); + net.deliver(fetcher); // replies to what was asked before the stop + EXPECT_EQ(fetcher.dropped(), dropped + queued); + for (int i = 0; i < 1300; ++i) { // past every timer: piece timeout, marks, manifest interval, stall + net.nowMs += 100; + fetcher.tick(net.peers); + } + EXPECT_EQ(net.manifestRequests, manifests); + EXPECT_EQ(net.sent.size(), pieces); + EXPECT_EQ(fetcher.outstanding(), 0u); + }; + { + Net net; + net.add(Snapshots::A1, false); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Discover); + quietAfterStop(net, fetcher); + EXPECT_EQ(fetcher.phase(), Phase::Discover); + } + { + Net net; + net.add(Snapshots::A1); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + fetcher.tick(net.peers); + EXPECT_EQ(fetcher.phase(), Phase::Collect); + quietAfterStop(net, fetcher); + EXPECT_EQ(fetcher.manifests(), 0u); + EXPECT_EQ(fetcher.dropped(), 1u); // the manifest reply + } + { + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return isFetching(fetcher) && fetcher.outstanding() != 0; }, 20)); + quietAfterStop(net, fetcher); + } + { + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + SnapshotFetcher fetcher(fetchConfig(), tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + quietAfterStop(net, fetcher); + fetcher.onImportFailed(kWallets); + EXPECT_EQ(fetcher.phase(), Phase::Import); + } +} + +// (15) fetch_rate_kbps caps the bytes requested +TEST(SnapshotFetcher, RateCap) { + TmpDir tmp("rate"); + Net net; + net.add(Snapshots::A1); + net.add(Snapshots::A2); + auto cfg = fetchConfig(); + cfg.fetchRateKbps = 200; // 200,000 bytes/s + SnapshotFetcher fetcher(cfg, tmp.path, net.hooks()); + ASSERT_TRUE(net.runUntil(fetcher, [&] { return fetcher.phase() == Phase::Import; })); + + const auto& m = snapshots().src[Snapshots::A1].manifest; + auto length = [&](const PieceRequest& r) { + const auto* f = m.file(r.name); + const uint64_t offset = uint64_t(r.chunk) * m.chunkSize; + return std::min(m.chunkSize, f->size - offset); + }; + std::map perSecond; + uint64_t total = 0; + for (const auto& s : net.sent) { + perSecond[std::get<0>(s) / 1000] += length(std::get<2>(s)); + total += length(std::get<2>(s)); + } + ASSERT_GE(perSecond.size(), 4u); + for (const auto& [second, bytes] : perSecond) { + // a request goes out when nothing is owed: at most one piece above the rate in any second + EXPECT_LE(bytes, 200'000u + m.chunkSize) << "second " << second; + } + const uint64_t span = std::get<0>(net.sent.back()) - std::get<0>(net.sent.front()); + EXPECT_LE(total, 200 * span + m.chunkSize); +}