From e2908bdf66b9679e03396d3f03abb86d9e2360e2 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 30 Oct 2026 15:32:53 +0100 Subject: [PATCH 1/4] csnode: forward sync paces its requests to responders with an anti-duplicate log (order 35) A responder with special order 35 active refuses a request whose last sequence is not above the last one it logged for the same requester, while that entry lives (5 s after its reply), and for ever after it accepted a request (AwaitAnswer) and had nothing to send. A node syncing from block 0 has not applied order 35 yet: Node::getSyncroMessage dropped the responders' SyncroMsg, and the synchronizer kept no request log, so it went on asking a full peer for blocks 1..100 right after blocks 301..400, refused every time (pruned-peers-join: a joiner that never got blocks 1..100 in 75 minutes, seen once). - Node::getSyncroMessage passes every SyncroMsg to the synchronizer, whatever this node knows of order 35 (a responder sends them only when its order 35 is active). - cs::SyncroPacing (new): a peer that sent a SyncroMsg is active; a request to it whose last sequence is at or below its live entry, or below an AwaitAnswer-then-NoSuchBlocks entry, is not sent: the range goes to the next neighbour. A peer that never sent one is asked as before. - PoolSynchronizer keeps its request log for active peers even before order 35 applies here (synchroLogFor); logs once per peer when it becomes active. Nothing changes where no responder sends SyncroMsg (order 35 not issued). Tests: syncropacing_tests.cpp. robustness.yml: pruned-peers-join-official (valid with node_alt_ref=official: official full responders serve the low blocks). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 5 ++ csnode/CMakeLists.txt | 2 + csnode/include/csnode/poolsynchronizer.hpp | 5 ++ csnode/include/csnode/syncropacing.hpp | 48 +++++++++++ csnode/src/node.cpp | 11 ++- csnode/src/poolsynchronizer.cpp | 45 +++++++---- csnode/src/syncropacing.cpp | 84 +++++++++++++++++++ tests/csnode/syncropacing_tests.cpp | 93 ++++++++++++++++++++++ 8 files changed, 275 insertions(+), 18 deletions(-) create mode 100644 csnode/include/csnode/syncropacing.hpp create mode 100644 csnode/src/syncropacing.cpp create mode 100644 tests/csnode/syncropacing_tests.cpp diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 2ae54d67b..40d31cd84 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -209,6 +209,11 @@ jobs: - name: pruned-peers-join official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=lmdb,lmdb,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000 + # the same with node_alt_ref=official: the even nodes, which answer the joiner (n9, this build) + # with the low blocks, are official responders with their anti-duplicate log (special order 35) + - name: pruned-peers-join-official + env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,lmdb:pruned,berkeleydb,lmdb:pruned,berkeleydb,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000 + fixed: true fixed: true # kills that land while the node prunes, or right after a checkpoint it prunes from - name: rocksdb-pruned-crash diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index 82fd20bd9..81ca8d47f 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -54,6 +54,7 @@ add_library(csnode include/csnode/walletsids_serializer.hpp include/csnode/blockhashes.hpp include/csnode/poolsynchronizer.hpp + include/csnode/syncropacing.hpp include/csnode/fee.hpp include/csnode/transactionsvalidator.hpp include/csnode/walletsstate.hpp @@ -103,6 +104,7 @@ add_library(csnode src/walletsids_serializer.cpp src/blockhashes.cpp src/poolsynchronizer.cpp + src/syncropacing.cpp src/fee.cpp src/transactionsvalidator.cpp src/transactionsindex.cpp diff --git a/csnode/include/csnode/poolsynchronizer.hpp b/csnode/include/csnode/poolsynchronizer.hpp index 8af32d1b6..7f5675c7b 100644 --- a/csnode/include/csnode/poolsynchronizer.hpp +++ b/csnode/include/csnode/poolsynchronizer.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -60,6 +61,9 @@ private slots: void updateSynchroLog(); bool removeSynchroLog(const cs::PublicKey& sender); bool checkSynchroLog(const cs::PublicKey& sender); + // the request log is kept for this peer: special order 35 applied here, or the peer sent a + // SyncroMsg (its own anti-duplicate log is active) + bool synchroLogFor(const cs::PublicKey& sender) const; bool showSyncronizationProgress(Sequence lastWrittenSequence) const; void manageSyncBlocks(cs::PoolsBlock&& poolsBlock); @@ -81,6 +85,7 @@ private slots: std::unordered_map neighbours_; std::map> synchroLog_; + SyncroPacing pacing_; Timer timer_; }; } // namespace cs diff --git a/csnode/include/csnode/syncropacing.hpp b/csnode/include/csnode/syncropacing.hpp new file mode 100644 index 000000000..d29ca3fe8 --- /dev/null +++ b/csnode/include/csnode/syncropacing.hpp @@ -0,0 +1,48 @@ +#ifndef SYNCROPACING_HPP +#define SYNCROPACING_HPP + +#include +#include +#include + +#include +#include + +namespace cs { + +// Forward sync against responders whose anti-duplicate log is active (special order 35): such a +// responder refuses a request whose last sequence is not above the last one it logged for the same +// requester, while that entry lives (5 s after its reply), and for ever when it accepted a request +// (AwaitAnswer) and then had nothing to send (NoSuchBlocks). A requester that has not applied +// order 35 itself (a node syncing from block 0 before that block) knows it only from the SyncroMsg +// the responder sends: each peer that sent one is active, and the rules below apply to it alone. +class SyncroPacing { +public: + static constexpr uint64_t kSentLifeMs = 6'000; // the responder keeps a Sent entry 5 s + + void onSyncroMessage(const PublicKey& peer, SyncroMessage msg, uint64_t nowMs); + void onRequest(const PublicKey& peer, Sequence highest, uint64_t nowMs); + void onReply(const PublicKey& peer, uint64_t nowMs); + void forget(const PublicKey& peer); + + // the peer sent a SyncroMsg: its anti-duplicate log is active + bool active(const PublicKey& peer) const; + bool anyActive() const; + // a request to that peer ending at `highest` would not be refused as a duplicate + bool allows(const PublicKey& peer, Sequence highest, uint64_t nowMs) const; + +private: + struct State { + Sequence lastHighest = 0; // the last sequence of our latest request to it + bool requested = false; + uint64_t loggedUntil = 0; // its entry for that request lives until then + bool awaiting = false; // AwaitAnswer for the request in flight + Sequence refusedUpTo = 0; // never again up to this (NoSuchBlocks after AwaitAnswer) + bool refusedForever = false; + }; + std::map peers_; +}; + +} // namespace cs + +#endif // SYNCROPACING_HPP diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 7adc6e11e..227ee568a 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -2156,12 +2156,15 @@ void Node::sendSyncroMessage(cs::Byte msg, const cs::PublicKey& target) { void Node::getSyncroMessage(const uint8_t* data, const size_t size, const cs::PublicKey& sender) { //csdebug() << __func__; - if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound) { - return; - } - cs::Byte msg; + cs::Byte msg = cs::SyncroMessage::NoAnswer; cs::IDataStream stream(data, size); stream >> msg; + if (!stream.isValid() || msg > cs::SyncroMessage::Sent) { + return; + } + // a responder sends these only once special order 35 is active on its side, which a node that + // syncs from block 0 does not know yet: the synchronizer reads them whatever this node knows of + // that round, and paces its requests to that responder poolSynchronizer_->getSyncroMessage(sender, static_cast(msg)); } diff --git a/csnode/src/poolsynchronizer.cpp b/csnode/src/poolsynchronizer.cpp index b048c03bd..297de2fe9 100644 --- a/csnode/src/poolsynchronizer.cpp +++ b/csnode/src/poolsynchronizer.cpp @@ -165,6 +165,7 @@ void PoolSynchronizer::getBlockReply(cs::PoolsBlock&& poolsBlock, const cs::Publ << ", seqs: [" << poolsBlock.front().sequence() << ", " << poolsBlock.back().sequence() << "]"; removeSynchroLog(sender); + pacing_.onReply(sender, cs::Utils::currentTimestamp()); //std::thread sThread(&PoolSynchronizer::manageSyncBlocks, this, std::move(poolsBlock));; //sThread.detach(); manageSyncBlocks(std::move(poolsBlock)); @@ -356,15 +357,22 @@ void PoolSynchronizer::sendBlockRequest() { csdetails() << "SYNC: All sequences already requested"; break; } + const auto now = cs::Utils::currentTimestamp(); + if (!pacing_.allows(neighbour->first, neededSequences.back(), now)) { + // its anti-duplicate log would refuse a request ending there: the range goes to the next + csdebug() << "SYNC: " << cs::Utils::byteStreamToHex(neighbour->first) << " would refuse blocks up to " + << neededSequences.back() << " as a duplicate, asking another neighbour"; + } else { maxRequestedSequence_ = neededSequences.back(); - } - if (checkSynchroLog(neighbour->first)) { - cslog() << "SYNC: requesting for " << neededSequences.size() - << " blocks [" << neededSequences.front() << ", " << neededSequences.back() - << "] from " << cs::Utils::byteStreamToHex(neighbour->first); - emit sendRequest(neighbour->first, neededSequences); - addSynchroLog(neighbour->first, neededSequences, SyncroMessage::NoAnswer); + if (checkSynchroLog(neighbour->first)) { + cslog() << "SYNC: requesting for " << neededSequences.size() + << " blocks [" << neededSequences.front() << ", " << neededSequences.back() + << "] from " << cs::Utils::byteStreamToHex(neighbour->first); + emit sendRequest(neighbour->first, neededSequences); + pacing_.onRequest(neighbour->first, neededSequences.back(), now); + addSynchroLog(neighbour->first, neededSequences, SyncroMessage::NoAnswer); + } } ++neighbour; @@ -444,10 +452,15 @@ void PoolSynchronizer::stop() { cslog() << "SYNC: shutdown — pool synchronizer stopped"; } +// A responder sends SyncroMsg only once special order 35 is active on its side: read whatever this +// node knows of that round, so that a node syncing from block 0 (before the order) paces its requests +// to that responder as the responder expects void PoolSynchronizer::getSyncroMessage(const cs::PublicKey& sender, SyncroMessage msg) { - if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound) { - return; + if (!pacing_.active(sender)) { + cslog() << "SYNC: " << cs::Utils::byteStreamToHex(sender) + << " keeps an anti-duplicate log of requests (special order 35): requests to it are paced"; } + pacing_.onSyncroMessage(sender, msg, cs::Utils::currentTimestamp()); csdebug() << __func__; if (msg == SyncroMessage::AwaitAnswer) { changeSynchroLog(sender, msg); @@ -458,7 +471,7 @@ void PoolSynchronizer::getSyncroMessage(const cs::PublicKey& sender, SyncroMessa } void PoolSynchronizer::addSynchroLog(const cs::PublicKey& sender, cs::PoolsRequestedSequences& sequences, SyncroMessage msg) { - if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound) { + if (!synchroLogFor(sender)) { return; } //csdebug() << __func__; @@ -467,7 +480,7 @@ void PoolSynchronizer::addSynchroLog(const cs::PublicKey& sender, cs::PoolsReque } bool PoolSynchronizer::changeSynchroLog(const cs::PublicKey& sender, SyncroMessage msg) { - if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound) { + if (!synchroLogFor(sender)) { return true; } //csdebug() << __func__; @@ -480,7 +493,7 @@ bool PoolSynchronizer::changeSynchroLog(const cs::PublicKey& sender, SyncroMessa } void PoolSynchronizer::updateSynchroLog() { - if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound) { + if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound && !pacing_.anyActive()) { return; } auto timeNow = cs::Utils::currentTimestamp(); @@ -498,7 +511,7 @@ void PoolSynchronizer::updateSynchroLog() { } bool PoolSynchronizer::removeSynchroLog(const cs::PublicKey& sender) { - if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound) { + if (!synchroLogFor(sender)) { return true; } //csdebug() << __func__; @@ -511,8 +524,12 @@ bool PoolSynchronizer::removeSynchroLog(const cs::PublicKey& sender) { return true; } +bool PoolSynchronizer::synchroLogFor(const cs::PublicKey& sender) const { + return cs::Conveyer::instance().currentRoundNumber() >= Consensus::syncroChangeRound || pacing_.active(sender); +} + bool PoolSynchronizer::checkSynchroLog(const cs::PublicKey& sender) { //is true when we can request blocks's set - if (cs::Conveyer::instance().currentRoundNumber() < Consensus::syncroChangeRound) { + if (!synchroLogFor(sender)) { return true; } auto it = synchroLog_.find(sender); diff --git a/csnode/src/syncropacing.cpp b/csnode/src/syncropacing.cpp new file mode 100644 index 000000000..bf3cc39ec --- /dev/null +++ b/csnode/src/syncropacing.cpp @@ -0,0 +1,84 @@ +#include + +#include + +namespace cs { + +void SyncroPacing::onSyncroMessage(const PublicKey& peer, SyncroMessage msg, uint64_t nowMs) { + auto& s = peers_[peer]; + switch (msg) { + case SyncroMessage::AwaitAnswer: + s.awaiting = true; + break; + case SyncroMessage::NoSuchBlocks: + if (s.awaiting && s.requested) { + // the entry stays AwaitAnswer in its log: every request up to it is refused + s.refusedForever = true; + s.refusedUpTo = std::max(s.refusedUpTo, s.lastHighest); + } + s.awaiting = false; + break; + case SyncroMessage::DuplicatedRequest: + // its entry from an earlier request still lives + s.loggedUntil = std::max(s.loggedUntil, nowMs + kSentLifeMs); + s.awaiting = false; + break; + default: + s.awaiting = false; + break; + } +} + +void SyncroPacing::onRequest(const PublicKey& peer, Sequence highest, uint64_t nowMs) { + const auto it = peers_.find(peer); + if (it == peers_.end()) { + return; + } + auto& s = it->second; + // a request it accepts replaces the last logged sequence, a lower one keeps the old entry + if (!s.requested || highest > s.lastHighest || nowMs >= s.loggedUntil) { + s.lastHighest = highest; + } + s.requested = true; + s.awaiting = false; + // until the reply comes the entry lives anyway; the reply sets the 5 s + s.loggedUntil = std::max(s.loggedUntil, nowMs + kSentLifeMs); +} + +void SyncroPacing::onReply(const PublicKey& peer, uint64_t nowMs) { + const auto it = peers_.find(peer); + if (it == peers_.end()) { + return; + } + it->second.awaiting = false; + it->second.loggedUntil = nowMs + kSentLifeMs; +} + +void SyncroPacing::forget(const PublicKey& peer) { + peers_.erase(peer); +} + +bool SyncroPacing::active(const PublicKey& peer) const { + return peers_.count(peer) != 0; +} + +bool SyncroPacing::anyActive() const { + return !peers_.empty(); +} + +bool SyncroPacing::allows(const PublicKey& peer, Sequence highest, uint64_t nowMs) const { + const auto it = peers_.find(peer); + if (it == peers_.end()) { + return true; + } + const auto& s = it->second; + if (s.refusedForever && highest <= s.refusedUpTo) { + return false; + } + if (s.requested && nowMs < s.loggedUntil && highest <= s.lastHighest) { + return false; + } + return true; +} + +} // namespace cs diff --git a/tests/csnode/syncropacing_tests.cpp b/tests/csnode/syncropacing_tests.cpp new file mode 100644 index 000000000..c1b1e2b7e --- /dev/null +++ b/tests/csnode/syncropacing_tests.cpp @@ -0,0 +1,93 @@ +#include + +#include + +// Forward sync against responders with an active anti-duplicate log (special order 35), known only +// from the SyncroMsg they send. +namespace { +cs::PublicKey key(uint8_t n) { + cs::PublicKey k{}; + k.fill(n); + return k; +} +} // namespace + +// a peer that never sent a SyncroMsg is asked as before +TEST(SyncroPacing, APeerWithoutSyncroMessagesIsNotPaced) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onRequest(p, 400, 1000); + pacing.onReply(p, 1100); + EXPECT_FALSE(pacing.active(p)); + EXPECT_FALSE(pacing.anyActive()); + EXPECT_TRUE(pacing.allows(p, 100, 1200)); +} + +// after a reply ending at 400, a request ending at or below 400 waits until the responder's entry +// expired; a higher one goes at once +TEST(SyncroPacing, ALowerRequestWaitsForTheResponderEntry) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 900); + pacing.onRequest(p, 400, 1000); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 1010); + pacing.onReply(p, 1100); + EXPECT_TRUE(pacing.active(p)); + EXPECT_TRUE(pacing.allows(p, 500, 1200)); + EXPECT_FALSE(pacing.allows(p, 100, 1200)); + EXPECT_FALSE(pacing.allows(p, 400, 1100 + cs::SyncroPacing::kSentLifeMs - 1)); + EXPECT_TRUE(pacing.allows(p, 100, 1100 + cs::SyncroPacing::kSentLifeMs)); + // the higher request becomes the entry + pacing.onRequest(p, 500, 1200); + pacing.onReply(p, 1300); + EXPECT_FALSE(pacing.allows(p, 450, 2000)); + EXPECT_TRUE(pacing.allows(p, 450, 1300 + cs::SyncroPacing::kSentLifeMs)); +} + +// a lower request sent while the entry lives does not lower what the responder logged +TEST(SyncroPacing, ALowerRequestDoesNotReplaceTheEntry) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 0); + pacing.onRequest(p, 400, 1000); + pacing.onReply(p, 1100); + pacing.onRequest(p, 100, 1200); // sent anyway (e.g. by an earlier build): refused by the responder + pacing.onSyncroMessage(p, cs::SyncroMessage::DuplicatedRequest, 1250); + EXPECT_FALSE(pacing.allows(p, 400, 1300)); + EXPECT_TRUE(pacing.allows(p, 401, 1300)); + EXPECT_TRUE(pacing.allows(p, 100, 1250 + cs::SyncroPacing::kSentLifeMs)); +} + +// NoSuchBlocks after AwaitAnswer: the responder keeps the entry for ever, every request up to it is +// refused; higher ones are not +TEST(SyncroPacing, NoSuchBlocksAfterAwaitAnswerIsForEver) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onSyncroMessage(p, cs::SyncroMessage::NoSuchBlocks, 0); // a pruned peer, no request of ours: nothing + pacing.onRequest(p, 300, 1000); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 1010); + pacing.onSyncroMessage(p, cs::SyncroMessage::NoSuchBlocks, 1020); + const uint64_t later = 10'000'000; + EXPECT_FALSE(pacing.allows(p, 300, later)); + EXPECT_FALSE(pacing.allows(p, 1, later)); + EXPECT_TRUE(pacing.allows(p, 301, later)); + // a plain NoSuchBlocks (refused before it logged anything) leaves no such mark + const auto q = key(2); + pacing.onSyncroMessage(q, cs::SyncroMessage::AwaitAnswer, 0); + pacing.onRequest(q, 300, 1000); + pacing.onSyncroMessage(q, cs::SyncroMessage::NoSuchBlocks, 1020); + EXPECT_TRUE(pacing.allows(q, 100, later)); +} + +// peers are independent +TEST(SyncroPacing, PeersAreIndependent) { + cs::SyncroPacing pacing; + pacing.onSyncroMessage(key(1), cs::SyncroMessage::AwaitAnswer, 0); + pacing.onRequest(key(1), 400, 1000); + pacing.onReply(key(1), 1100); + pacing.onRequest(key(2), 400, 1000); + EXPECT_FALSE(pacing.allows(key(1), 100, 1200)); + EXPECT_TRUE(pacing.allows(key(2), 100, 1200)); + pacing.forget(key(1)); + EXPECT_TRUE(pacing.allows(key(1), 100, 1200)); +} From 498b8d9bae1dd9321394a28cbc01e6f4fbda058b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 30 Oct 2026 15:33:24 +0100 Subject: [PATCH 2/4] docs: CHANGES line for #136 Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 816f8ef07..364fbae74 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -45,6 +45,7 @@ lines under the new release. | #127 | ci: token tests: `ci/contracts/executor_harness.py` compares contract executors on token contracts without a node, and `TOKENS=1` runs `ci/testnet/tokens.py` (v1 and v0 tokens, balances and states on every node, the token API; `TOKENS_VIA` through another node) on the private network | | #128 | api: the token cache (`[api] tokens_cache=true`, any build) follows v1/v2 tokens (BigDecimal amounts, standards 3 and 4) and only successful calls. It reads balances from the contract for the wallets a call involves, refreshes them in the background at start, and is rebuilt from the chain when the checkpoint holds none. `approve()` without arguments no longer reads out of bounds. `TokensListGet` filters before paging, and the token lists sort without exceptions or invalid orderings. `TokenTransfersListGet` reports the signer as the sender. See [tokens.md](tokens.md) | | #131 | packaging: a guided set-up in every package (`setup.sh`, `setup.bat`): pruned or full, database, how the first start gets the chain, snapshot sharing, and the contract executor (community or official: it shows the build in the folder, can download either, checked, and sets `executor_commit_min/max` so that the node starts it; an upgrade by copying only the node binary keeps the old executor) | +| #136 | csnode: forward sync reads the responders' SyncroMsg before it has applied special order 35 itself, and does not ask a responder with an anti-duplicate log for a range that responder would refuse as a duplicate; it asks another neighbour instead (a node syncing from block 0 among mostly pruned peers could otherwise wait for ever for its first blocks) | ## community.14 — 2026-10-27 From 6826a5ce23aa255a74310ba086242a575cbbcbb3 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 30 Oct 2026 15:34:22 +0100 Subject: [PATCH 3/4] ci: pruned-peers-join keeps its fixed: true (the new row's insertion took it) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/robustness.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 40d31cd84..a566b96ec 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -209,12 +209,12 @@ jobs: - name: pruned-peers-join official: 'no' # needs a feature of this fork on an even node env: NODE_STORAGE=lmdb,lmdb,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb:pruned,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000 + fixed: true # the same with node_alt_ref=official: the even nodes, which answer the joiner (n9, this build) # with the low blocks, are official responders with their anti-duplicate log (special order 35) - name: pruned-peers-join-official env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,lmdb:pruned,berkeleydb,lmdb:pruned,berkeleydb,lmdb JOIN_PRUNED_AT=2300 JOIN_FULL=1 SYNCRO_ROUND=100 TARGET_SEQ=3000 fixed: true - fixed: true # kills that land while the node prunes, or right after a checkpoint it prunes from - name: rocksdb-pruned-crash official: 'no' # needs a feature of this fork on an even node From b95abcd108b531c33f342b04803eebbe6f15692d Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 30 Oct 2026 22:31:02 +0100 Subject: [PATCH 4/4] csnode: forward-sync pacing review fixes (#136) - M2: the backfill and the forward sync both read every SyncroMsg now. An answer counts for the forward sync only while a forward request to that peer is in flight (SyncroPacing::inFlight); any other one marks the peer active and changes neither the pacing nor synchroLog_. - M3: every forward request is recorded, also to a peer that has not sent a SyncroMsg yet, so that AwaitAnswer then NoSuchBlocks on the first request counts. A refusal the responder does not log (NoSuchBlocks without AwaitAnswer, DuplicatedRequest, IncorrectRequest) leaves the entry it held. - L1: a neighbour removed is forgotten. - L2: "for ever" (NoSuchBlocks after AwaitAnswer) lasts an hour here. - L4: an AwaitAnswer entry of synchroLog_ (order 35 known) expires after 30 s, as a forward request in flight does, so that a peer whose reply was lost is asked again. Tests: a request sent before the peer's first SyncroMsg; a DuplicatedRequest with no request recorded; answers to the backfill leave the forward state alone; a request without an answer expires; the hour of "for ever". Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/syncropacing.hpp | 26 +++++++++--- csnode/src/poolsynchronizer.cpp | 13 +++++- csnode/src/syncropacing.cpp | 59 ++++++++++++++++++++------ tests/csnode/syncropacing_tests.cpp | 59 ++++++++++++++++++++++++-- 4 files changed, 132 insertions(+), 25 deletions(-) diff --git a/csnode/include/csnode/syncropacing.hpp b/csnode/include/csnode/syncropacing.hpp index d29ca3fe8..7c2ad3eba 100644 --- a/csnode/include/csnode/syncropacing.hpp +++ b/csnode/include/csnode/syncropacing.hpp @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -15,10 +14,16 @@ namespace cs { // requester, while that entry lives (5 s after its reply), and for ever when it accepted a request // (AwaitAnswer) and then had nothing to send (NoSuchBlocks). A requester that has not applied // order 35 itself (a node syncing from block 0 before that block) knows it only from the SyncroMsg -// the responder sends: each peer that sent one is active, and the rules below apply to it alone. +// the responder sends: each peer that sent one is active, and requests to it are paced. +// +// Every forward request is recorded, also to a peer not known as active yet, so that the answer to +// it counts. A SyncroMsg that arrives while no forward request to that peer is in flight answers +// another request (the backfill's): it marks the peer active and changes nothing else. class SyncroPacing { public: - static constexpr uint64_t kSentLifeMs = 6'000; // the responder keeps a Sent entry 5 s + static constexpr uint64_t kSentLifeMs = 6'000; // the responder keeps a Sent entry 5 s + static constexpr uint64_t kInFlightMs = 30'000; // a request without an answer this long is over + static constexpr uint64_t kRefusedForMs = 3'600'000; // "for ever" on the responder: an hour here void onSyncroMessage(const PublicKey& peer, SyncroMessage msg, uint64_t nowMs); void onRequest(const PublicKey& peer, Sequence highest, uint64_t nowMs); @@ -28,18 +33,27 @@ class SyncroPacing { // the peer sent a SyncroMsg: its anti-duplicate log is active bool active(const PublicKey& peer) const; bool anyActive() const; + // a forward request to that peer waits for its answer + bool inFlight(const PublicKey& peer, uint64_t nowMs) const; // a request to that peer ending at `highest` would not be refused as a duplicate bool allows(const PublicKey& peer, Sequence highest, uint64_t nowMs) const; private: struct State { - Sequence lastHighest = 0; // the last sequence of our latest request to it + bool active = false; bool requested = false; + Sequence lastHighest = 0; // the last sequence of our latest request it logged + Sequence lastHighestBefore = 0; // ... and before that request, if it refuses to log it + uint64_t loggedUntilBefore = 0; + uint64_t requestedAt = 0; + bool inFlight = false; uint64_t loggedUntil = 0; // its entry for that request lives until then bool awaiting = false; // AwaitAnswer for the request in flight - Sequence refusedUpTo = 0; // never again up to this (NoSuchBlocks after AwaitAnswer) - bool refusedForever = false; + Sequence refusedUpTo = 0; // not again up to this (NoSuchBlocks after AwaitAnswer) ... + uint64_t refusedUntil = 0; // ... until then }; + bool inFlight(const State& s, uint64_t nowMs) const; + std::map peers_; }; diff --git a/csnode/src/poolsynchronizer.cpp b/csnode/src/poolsynchronizer.cpp index 297de2fe9..87dbc7298 100644 --- a/csnode/src/poolsynchronizer.cpp +++ b/csnode/src/poolsynchronizer.cpp @@ -297,6 +297,7 @@ void PoolSynchronizer::onPingReceived(cs::Sequence sequence, const cs::PublicKey void PoolSynchronizer::onNeighbourRemoved(const cs::PublicKey& publicKey) { neighbours_.erase(publicKey); + pacing_.forget(publicKey); } // @@ -460,8 +461,14 @@ void PoolSynchronizer::getSyncroMessage(const cs::PublicKey& sender, SyncroMessa cslog() << "SYNC: " << cs::Utils::byteStreamToHex(sender) << " keeps an anti-duplicate log of requests (special order 35): requests to it are paced"; } - pacing_.onSyncroMessage(sender, msg, cs::Utils::currentTimestamp()); + const auto now = cs::Utils::currentTimestamp(); + // an answer to a forward request in flight; any other (the backfill's) only marks the peer active + const bool forward = pacing_.inFlight(sender, now); + pacing_.onSyncroMessage(sender, msg, now); csdebug() << __func__; + if (!forward) { + return; + } if (msg == SyncroMessage::AwaitAnswer) { changeSynchroLog(sender, msg); } @@ -501,7 +508,9 @@ void PoolSynchronizer::updateSynchroLog() { while (it != synchroLog_.end()) { auto msg = std::get<1>(it->second); auto timeEvent = std::get<2>(it->second); - if ((msg != SyncroMessage::AwaitAnswer && timeNow > timeEvent + 50000) || ((std::get<1>(it->second) == SyncroMessage::NoAnswer && timeNow > timeEvent + 2000))) { + // an AwaitAnswer whose reply was lost: asked again after 30 s + if ((msg != SyncroMessage::AwaitAnswer && timeNow > timeEvent + 50000) || ((std::get<1>(it->second) == SyncroMessage::NoAnswer && timeNow > timeEvent + 2000)) + || (msg == SyncroMessage::AwaitAnswer && timeNow > timeEvent + SyncroPacing::kInFlightMs)) { it = synchroLog_.erase(it); } else { diff --git a/csnode/src/syncropacing.cpp b/csnode/src/syncropacing.cpp index bf3cc39ec..a4f964915 100644 --- a/csnode/src/syncropacing.cpp +++ b/csnode/src/syncropacing.cpp @@ -4,42 +4,66 @@ namespace cs { +bool SyncroPacing::inFlight(const State& s, uint64_t nowMs) const { + return s.inFlight && nowMs < s.requestedAt + kInFlightMs; +} + void SyncroPacing::onSyncroMessage(const PublicKey& peer, SyncroMessage msg, uint64_t nowMs) { auto& s = peers_[peer]; + s.active = true; + if (!inFlight(s, nowMs)) { + // not an answer to a forward request (the backfill's, or a late one): nothing else changes + s.inFlight = false; + s.awaiting = false; + return; + } switch (msg) { case SyncroMessage::AwaitAnswer: s.awaiting = true; break; case SyncroMessage::NoSuchBlocks: - if (s.awaiting && s.requested) { + if (s.awaiting) { // the entry stays AwaitAnswer in its log: every request up to it is refused - s.refusedForever = true; s.refusedUpTo = std::max(s.refusedUpTo, s.lastHighest); + s.refusedUntil = nowMs + kRefusedForMs; + } + else { + // refused before it logged the request: its log holds what it held before + s.lastHighest = s.lastHighestBefore; + s.loggedUntil = s.loggedUntilBefore; } s.awaiting = false; + s.inFlight = false; break; case SyncroMessage::DuplicatedRequest: - // its entry from an earlier request still lives - s.loggedUntil = std::max(s.loggedUntil, nowMs + kSentLifeMs); + // not logged: its entry from an earlier request still lives + s.lastHighest = std::max(s.lastHighestBefore, s.lastHighest); + s.loggedUntil = std::max(s.loggedUntilBefore, nowMs + kSentLifeMs); s.awaiting = false; + s.inFlight = false; break; - default: + case SyncroMessage::IncorrectRequest: + s.lastHighest = s.lastHighestBefore; + s.loggedUntil = s.loggedUntilBefore; s.awaiting = false; + s.inFlight = false; + break; + default: break; } } void SyncroPacing::onRequest(const PublicKey& peer, Sequence highest, uint64_t nowMs) { - const auto it = peers_.find(peer); - if (it == peers_.end()) { - return; - } - auto& s = it->second; + auto& s = peers_[peer]; + s.lastHighestBefore = s.lastHighest; + s.loggedUntilBefore = s.loggedUntil; // a request it accepts replaces the last logged sequence, a lower one keeps the old entry if (!s.requested || highest > s.lastHighest || nowMs >= s.loggedUntil) { s.lastHighest = highest; } s.requested = true; + s.requestedAt = nowMs; + s.inFlight = true; s.awaiting = false; // until the reply comes the entry lives anyway; the reply sets the 5 s s.loggedUntil = std::max(s.loggedUntil, nowMs + kSentLifeMs); @@ -51,6 +75,7 @@ void SyncroPacing::onReply(const PublicKey& peer, uint64_t nowMs) { return; } it->second.awaiting = false; + it->second.inFlight = false; it->second.loggedUntil = nowMs + kSentLifeMs; } @@ -59,20 +84,26 @@ void SyncroPacing::forget(const PublicKey& peer) { } bool SyncroPacing::active(const PublicKey& peer) const { - return peers_.count(peer) != 0; + const auto it = peers_.find(peer); + return it != peers_.end() && it->second.active; } bool SyncroPacing::anyActive() const { - return !peers_.empty(); + return std::any_of(peers_.begin(), peers_.end(), [](const auto& p) { return p.second.active; }); +} + +bool SyncroPacing::inFlight(const PublicKey& peer, uint64_t nowMs) const { + const auto it = peers_.find(peer); + return it != peers_.end() && inFlight(it->second, nowMs); } bool SyncroPacing::allows(const PublicKey& peer, Sequence highest, uint64_t nowMs) const { const auto it = peers_.find(peer); - if (it == peers_.end()) { + if (it == peers_.end() || !it->second.active) { return true; } const auto& s = it->second; - if (s.refusedForever && highest <= s.refusedUpTo) { + if (nowMs < s.refusedUntil && highest <= s.refusedUpTo) { return false; } if (s.requested && nowMs < s.loggedUntil && highest <= s.lastHighest) { diff --git a/tests/csnode/syncropacing_tests.cpp b/tests/csnode/syncropacing_tests.cpp index c1b1e2b7e..0a4278ca9 100644 --- a/tests/csnode/syncropacing_tests.cpp +++ b/tests/csnode/syncropacing_tests.cpp @@ -59,7 +59,7 @@ TEST(SyncroPacing, ALowerRequestDoesNotReplaceTheEntry) { } // NoSuchBlocks after AwaitAnswer: the responder keeps the entry for ever, every request up to it is -// refused; higher ones are not +// refused (here for an hour); higher ones are not TEST(SyncroPacing, NoSuchBlocksAfterAwaitAnswerIsForEver) { cs::SyncroPacing pacing; const auto p = key(1); @@ -67,16 +67,18 @@ TEST(SyncroPacing, NoSuchBlocksAfterAwaitAnswerIsForEver) { pacing.onRequest(p, 300, 1000); pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 1010); pacing.onSyncroMessage(p, cs::SyncroMessage::NoSuchBlocks, 1020); - const uint64_t later = 10'000'000; + const uint64_t later = 1020 + cs::SyncroPacing::kRefusedForMs - 1; EXPECT_FALSE(pacing.allows(p, 300, later)); EXPECT_FALSE(pacing.allows(p, 1, later)); EXPECT_TRUE(pacing.allows(p, 301, later)); + // an hour later it is asked again (its log may have been cleared by a restart) + EXPECT_TRUE(pacing.allows(p, 1, 1020 + cs::SyncroPacing::kRefusedForMs)); // a plain NoSuchBlocks (refused before it logged anything) leaves no such mark const auto q = key(2); pacing.onSyncroMessage(q, cs::SyncroMessage::AwaitAnswer, 0); pacing.onRequest(q, 300, 1000); pacing.onSyncroMessage(q, cs::SyncroMessage::NoSuchBlocks, 1020); - EXPECT_TRUE(pacing.allows(q, 100, later)); + EXPECT_TRUE(pacing.allows(q, 100, 2000)); } // peers are independent @@ -91,3 +93,54 @@ TEST(SyncroPacing, PeersAreIndependent) { pacing.forget(key(1)); EXPECT_TRUE(pacing.allows(key(1), 100, 1200)); } + +// the first request goes out before the peer ever sent a SyncroMsg: its answer still counts +TEST(SyncroPacing, ARequestSentBeforeThePeersFirstSyncroMessage) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onRequest(p, 300, 1000); + EXPECT_FALSE(pacing.active(p)); + EXPECT_TRUE(pacing.inFlight(p, 1001)); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 1010); + pacing.onSyncroMessage(p, cs::SyncroMessage::NoSuchBlocks, 1020); + EXPECT_TRUE(pacing.active(p)); + EXPECT_FALSE(pacing.inFlight(p, 1030)); + EXPECT_FALSE(pacing.allows(p, 200, 5000)); + EXPECT_TRUE(pacing.allows(p, 301, 5000)); +} + +// a DuplicatedRequest with no request of ours in flight marks the peer active, nothing else +TEST(SyncroPacing, ADuplicatedRequestWithNoRecordedRequest) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onSyncroMessage(p, cs::SyncroMessage::DuplicatedRequest, 0); + EXPECT_TRUE(pacing.active(p)); + EXPECT_TRUE(pacing.allows(p, 100, 1)); +} + +// answers to another request (the backfill's) while no forward request is in flight change nothing +TEST(SyncroPacing, AnswersToTheBackfillLeaveTheForwardStateAlone) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 0); + pacing.onRequest(p, 400, 1000); + pacing.onReply(p, 1100); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 2000); + pacing.onSyncroMessage(p, cs::SyncroMessage::NoSuchBlocks, 2010); + pacing.onSyncroMessage(p, cs::SyncroMessage::DuplicatedRequest, 2020); + EXPECT_TRUE(pacing.allows(p, 100, 1100 + cs::SyncroPacing::kSentLifeMs)); + EXPECT_TRUE(pacing.allows(p, 500, 2100)); +} + +// a forward request without an answer for 30 s is no longer in flight: a later answer is not its +TEST(SyncroPacing, ARequestWithoutAnswerExpires) { + cs::SyncroPacing pacing; + const auto p = key(1); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 0); + pacing.onRequest(p, 300, 1000); + pacing.onSyncroMessage(p, cs::SyncroMessage::AwaitAnswer, 1010); + EXPECT_TRUE(pacing.inFlight(p, 1000 + cs::SyncroPacing::kInFlightMs - 1)); + EXPECT_FALSE(pacing.inFlight(p, 1000 + cs::SyncroPacing::kInFlightMs)); + pacing.onSyncroMessage(p, cs::SyncroMessage::NoSuchBlocks, 1000 + cs::SyncroPacing::kInFlightMs); + EXPECT_TRUE(pacing.allows(p, 100, 1000 + cs::SyncroPacing::kInFlightMs + cs::SyncroPacing::kSentLifeMs)); +}