From d3b8d79a675c85001f60d235ba1663d4c49362c1 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sat, 31 Oct 2026 15:55:46 +0100 Subject: [PATCH] csnode: the backfill turn budget follows the node's role With 50 ms per turn the mainnet test node backfilled at about 250 blocks/s instead of about 550 (each block costs ~1.7 ms of the processor thread). A node outside the round's trusted set has little consensus work: it now takes 90 ms per turn (backfill_idle_turn_budget_ms), and keeps 50 ms (backfill_turn_budget_ms) while it is a confidant or the writer. Co-Authored-By: Claude Opus 5.5 --- client/config/config.cpp | 1 + client/config/config.hpp | 3 ++- csnode/include/csnode/backwardsync.hpp | 10 ++++++++-- csnode/src/backwardsync.cpp | 5 ++++- csnode/src/node.cpp | 2 ++ docs/CHANGES.md | 1 + docs/differences.md | 2 +- docs/storage.md | 3 ++- tests/csnode/backfillconfig_tests.cpp | 5 ++++- tests/csnode/backwardsync_tests.cpp | 22 ++++++++++++++++++++-- 10 files changed, 45 insertions(+), 9 deletions(-) diff --git a/client/config/config.cpp b/client/config/config.cpp index 4031136a4..4452902b6 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -1071,6 +1071,7 @@ void Config::readBackfillSettings(const boost::property_tree::ptree& data, Stora number("backfill_rate", storage.backfillRate, 0, 1'000'000); number("backfill_store_per_turn", storage.backfillStorePerTurn, 1, 1000); number("backfill_turn_budget_ms", storage.backfillTurnBudgetMs, 0, 1000); + number("backfill_idle_turn_budget_ms", storage.backfillIdleTurnBudgetMs, 0, 1000); } namespace { diff --git a/client/config/config.hpp b/client/config/config.hpp index 1a829b0fd..df76a71f3 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -74,7 +74,8 @@ struct StorageData { size_t backfillBatch = 100; // blocks per request, 1..1000 size_t backfillRate = 1000; // blocks requested per second, 0 = unlimited size_t backfillStorePerTurn = 50; // blocks verified and stored per processor-thread turn, 1..1000 - size_t backfillTurnBudgetMs = 50; // and ms per turn, 0 (no limit)..1000 + size_t backfillTurnBudgetMs = 50; // and ms per turn while a confidant, 0 (no limit)..1000 + size_t backfillIdleTurnBudgetMs = 90; // and ms per turn otherwise, 0 (no limit)..1000 }; // [snapshot]: snapshots served by this node to its p2p neighbours and fetched from them at a first diff --git a/csnode/include/csnode/backwardsync.hpp b/csnode/include/csnode/backwardsync.hpp index c32db3563..82a7e6d1c 100644 --- a/csnode/include/csnode/backwardsync.hpp +++ b/csnode/include/csnode/backwardsync.hpp @@ -38,6 +38,7 @@ class BackwardSync { bool bootstrapping = false; // the node fetches a snapshot: its chain is not open RoundNumber round = 0; Sequence lastSeq = 0; + bool trusted = false; // the node is a confidant (or the writer) of this round }; // the first stored block and the hash of the block below it (empty: unknown) @@ -62,9 +63,13 @@ class BackwardSync { // blocks verified and stored per call (a tick or a reply), so that the processor thread, // which also runs consensus, is never held long; the rest waits for the next call size_t storePerTurn = 50; - // and for at most this long (ms) per call, 0: no limit, at most 1000. A turn ends after the block that - // passes it: a mainnet turn of 50 blocks took up to 1.7 s, more than a round + // and for at most this long (ms) per call while the node is a confidant, 0: no limit, at most + // 1000. A turn ends after the block that passes it: a mainnet turn of 50 blocks took up to 1.7 s, + // more than a round uint64_t turnBudgetMs = 50; + // the same while the node is not a confidant, with little consensus work: about 450 blocks/s + // on mainnet instead of about 250 + uint64_t idleTurnBudgetMs = 90; }; struct Stats { @@ -188,6 +193,7 @@ class BackwardSync { std::atomic busyUs_{0}; std::atomic longestTurnUs_{0}; std::atomic slowTurns_{0}; + bool trusted_ = false; // from the last tick: the budget of the turns until the next one std::atomic longestFlushUs_{0}; }; diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index d47e16efb..95b2f37c6 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -33,6 +33,7 @@ BackwardSync::BackwardSync(Hooks hooks, Settings settings) settings_.batch = std::clamp(settings_.batch, 1, 1000); settings_.storePerTurn = std::clamp(settings_.storePerTurn, 1, 1000); settings_.turnBudgetMs = std::min(settings_.turnBudgetMs, 1000); + settings_.idleTurnBudgetMs = std::min(settings_.idleTurnBudgetMs, 1000); } uint64_t BackwardSync::now() const { @@ -114,6 +115,7 @@ void BackwardSync::tick(const Context& ctx) { if (stopped_ || done_) { return; } + trusted_ = ctx.trusted; if (inFlight_.empty() && buffer_.empty() && !refreshAnchor(ctx)) { return; } @@ -503,13 +505,14 @@ void BackwardSync::drain() { }; const auto startedAt = Clock::now(); const uint64_t startedMs = now(); // the budget and the slow-turn decision: the hooks' clock (tests drive it) + const uint64_t budgetMs = trusted_ ? settings_.turnBudgetMs : settings_.idleTurnBudgetMs; size_t storedNow = 0; size_t transactions = 0; uint64_t verifyUs = 0; uint64_t storeUs = 0; uint64_t confirmUs = 0; while (!done_ && storedNow < settings_.storePerTurn) { - if (storedNow > 0 && settings_.turnBudgetMs > 0 && now() >= startedMs + settings_.turnBudgetMs) { + if (storedNow > 0 && budgetMs > 0 && now() >= startedMs + budgetMs) { break; } const auto it = buffer_.find(next_); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 7bfe574f1..90bb3cb4f 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -440,6 +440,7 @@ void Node::startBackfill() { settings.rate = sto.backfillRate; settings.storePerTurn = sto.backfillStorePerTurn; settings.turnBudgetMs = sto.backfillTurnBudgetMs; + settings.idleTurnBudgetMs = sto.backfillIdleTurnBudgetMs; // created under the lock that stop() takes to read the pointer (finishBootstrap runs this on the // processor thread while a stop may come from another one) std::lock_guard lock(backfillTimerMutex_); @@ -482,6 +483,7 @@ void Node::onBackfillTick() { ctx.bootstrapping = bootstrapping_; ctx.round = cs::Conveyer::instance().currentRoundNumber(); ctx.lastSeq = blockChain_.getLastSeq(); + ctx.trusted = myLevel_ != Level::Normal; backwardSync_->tick(ctx); } diff --git a/docs/CHANGES.md b/docs/CHANGES.md index c4721869c..eeb605443 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -65,6 +65,7 @@ lines under the new release. | #143 | node: a `Utility` message (ban or unban a node key) is obeyed only with the blockchain key's signature, an exact length and a round at most 100 rounds old (its signature was not checked since upstream `af4de3f8`, so it was not authenticated) | | #144 | csnode: stream reads bounded by the data: `IDataStream` checks a length without overflow, a vector reserves no more than the bytes left and stops at the first missing element, a smart stage-one message whose fee count exceeds its size or that lacks its hash is refused, the API checks a string field's length before reading it | | #145 | csnode: backward sync asks for no more blocks than its buffer holds (a request could overrun it by one batch); blocks a reply brings when the buffer is full are asked for again at once and no longer count against the peer that sent them (with slow stores, on mainnet after a restart, peers that had answered were left out for an hour and the backfill fell to ~30 blocks/s) | +| #146 | csnode: the backfill turn budget depends on the node's role: 50 ms while it is a confidant (`backfill_turn_budget_ms`), 90 ms otherwise (`backfill_idle_turn_budget_ms`); on mainnet a node outside the trusted set backfills at about 450 blocks/s instead of about 250 | ## community.14 — 2026-10-27 diff --git a/docs/differences.md b/docs/differences.md index bfef87eb7..15eb8ca6c 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -32,7 +32,7 @@ each key and its default. | `db_backend` | Chooses the block database: `berkeleydb` (the default, the official format), `lmdb` or `rocksdb`. RocksDB is about a third of the BerkeleyDB size on mainnet. | | `prune_keep_blocks` | Pruned storage: the node keeps only the latest blocks and removes older ones at each periodic checkpoint. Pruned storage runs on LMDB. | | `async_write_queue_size`, `write_batch_size`, `rocksdb_block_cache_mb`, `rocksdb_memtable_mb` | Tune the database writer and RocksDB. | -| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn`, `backfill_turn_budget_ms` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. | +| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn`, `backfill_turn_budget_ms`, `backfill_idle_turn_budget_ms` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. | ### Converting the block database diff --git a/docs/storage.md b/docs/storage.md index e53a4edc8..c1d70a64a 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -39,7 +39,8 @@ official node. | `backfill_batch` | `100` | Blocks per request, `1` to `1000`. | | `backfill_rate` | `1000` | Blocks requested per second over all peers (`0` = no limit). | | `backfill_store_per_turn` | `50` | Blocks checked and stored per turn of the thread that also runs consensus, `1` to `1000`; the rest waits for the next turn, 100 ms later. A turn over 200 ms is logged (`BACKFILL: slow turn of ...`) with the time spent verifying, storing and checking confirmations. | -| `backfill_turn_budget_ms` | `50` | A turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`. A shorter turn leaves more of the thread to consensus; on mainnet 50 ms gives about 400 blocks/s. | +| `backfill_turn_budget_ms` | `50` | While the node is a confidant, a turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`: consensus keeps most of the thread; on mainnet about 250 blocks/s. | +| `backfill_idle_turn_budget_ms` | `90` | The same while the node is not a confidant, with little consensus work: on mainnet about 450 blocks/s. | ### Choosing a backend diff --git a/tests/csnode/backfillconfig_tests.cpp b/tests/csnode/backfillconfig_tests.cpp index f9cad419f..5e1442326 100644 --- a/tests/csnode/backfillconfig_tests.cpp +++ b/tests/csnode/backfillconfig_tests.cpp @@ -27,12 +27,14 @@ TEST(BackfillConfig, Defaults) { EXPECT_EQ(d.backfillRate, 1000u); EXPECT_EQ(d.backfillStorePerTurn, 50u); EXPECT_EQ(d.backfillTurnBudgetMs, 50u); + EXPECT_EQ(d.backfillIdleTurnBudgetMs, 90u); } TEST(BackfillConfig, EveryKey) { - const auto d = parse("backfill=off\nbackfill_batch=250\nbackfill_rate=0\nbackfill_store_per_turn=20\nbackfill_turn_budget_ms=0\n"); + const auto d = parse("backfill=off\nbackfill_batch=250\nbackfill_rate=0\nbackfill_store_per_turn=20\nbackfill_turn_budget_ms=0\nbackfill_idle_turn_budget_ms=200\n"); EXPECT_FALSE(d.backfill); EXPECT_EQ(d.backfillTurnBudgetMs, 0u); + EXPECT_EQ(d.backfillIdleTurnBudgetMs, 200u); EXPECT_EQ(d.backfillBatch, 250u); EXPECT_EQ(d.backfillRate, 0u); EXPECT_EQ(d.backfillStorePerTurn, 20u); @@ -44,6 +46,7 @@ TEST(BackfillConfig, BadValuesKeepTheDefaultOrAreClamped) { EXPECT_TRUE(parse("backfill=maybe\n").backfill); EXPECT_EQ(parse("backfill_batch=5000\n").backfillBatch, 1000u); EXPECT_EQ(parse("backfill_turn_budget_ms=5000\n").backfillTurnBudgetMs, 1000u); + EXPECT_EQ(parse("backfill_idle_turn_budget_ms=5000\n").backfillIdleTurnBudgetMs, 1000u); EXPECT_EQ(parse("backfill_batch=0\n").backfillBatch, 1u); EXPECT_EQ(parse("backfill_batch=many\n").backfillBatch, 100u); EXPECT_EQ(parse("backfill_rate=-5\n").backfillRate, 1000u); diff --git a/tests/csnode/backwardsync_tests.cpp b/tests/csnode/backwardsync_tests.cpp index 3a9bbfa13..4d1cab6fe 100644 --- a/tests/csnode/backwardsync_tests.cpp +++ b/tests/csnode/backwardsync_tests.cpp @@ -116,6 +116,7 @@ cs::BackwardSync::Settings settingsOf(size_t batch = 100, size_t rate = 1000, si s.rate = rate; s.storePerTurn = perTurn; s.turnBudgetMs = 0; // no time limit unless a test sets one + s.idleTurnBudgetMs = 0; return s; } @@ -170,6 +171,7 @@ class BackfillTest : public ::testing::Test { ctx.lastSeq = kHead; ctx.forwardSyncRunning = forwardSync_; ctx.bootstrapping = bootstrapping_; + ctx.trusted = trusted_; return ctx; } void tick(uint64_t advanceMs = 500) { @@ -215,6 +217,7 @@ class BackfillTest : public ::testing::Test { uint64_t storeDelayMs_ = 0; // a slow disk: simulated time spent in every store bool forwardSync_ = false; bool bootstrapping_ = false; + bool trusted_ = false; }; } // namespace @@ -559,7 +562,7 @@ TEST_F(BackfillTest, TheWorkPerTurnIsBounded) { // so no block is dropped and no peer that answered is taken for one that did not TEST_F(BackfillTest, SlowStoresNeitherDropBlocksNorMarkPeers) { auto settings = settingsOf(20, 0, 50); - settings.turnBudgetMs = 30; + settings.idleTurnBudgetMs = 30; make(settings); storeDelayMs_ = 12; // three blocks a turn for (int i = 0; i < 400 && !sync_->stats().done; ++i) { @@ -573,9 +576,24 @@ TEST_F(BackfillTest, SlowStoresNeitherDropBlocksNorMarkPeers) { EXPECT_EQ(logged("did not answer"), 0u); } +// a confidant keeps the shorter budget, a node outside the round's trusted the longer one +TEST_F(BackfillTest, AConfidantTakesTheShorterBudget) { + auto settings = settingsOf(100, 0, 50); + settings.turnBudgetMs = 20; + settings.idleTurnBudgetMs = 60; + make(settings); + storeDelayMs_ = 10; + tick(); + sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); + EXPECT_EQ(kFirst - first_, 6u); // not trusted: 60 ms, six blocks of 10 ms + trusted_ = true; + tick(); + EXPECT_EQ(kFirst - first_, 8u); // trusted: 20 ms, two more +} + TEST_F(BackfillTest, ASlowTurnStopsAtItsTimeBudget) { auto settings = settingsOf(100, 0, 50); - settings.turnBudgetMs = 30; + settings.idleTurnBudgetMs = 30; make(settings); storeDelayMs_ = 12; tick();