diff --git a/client/config/config.cpp b/client/config/config.cpp index 0c24ba61a..4031136a4 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -1070,6 +1070,7 @@ void Config::readBackfillSettings(const boost::property_tree::ptree& data, Stora number("backfill_batch", storage.backfillBatch, 1, 1000); number("backfill_rate", storage.backfillRate, 0, 1'000'000); number("backfill_store_per_turn", storage.backfillStorePerTurn, 1, 1000); + number("backfill_turn_budget_ms", storage.backfillTurnBudgetMs, 0, 1000); } namespace { diff --git a/client/config/config.hpp b/client/config/config.hpp index ad4aae00e..1a829b0fd 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -74,6 +74,7 @@ struct StorageData { size_t backfillBatch = 100; // blocks per request, 1..1000 size_t backfillRate = 1000; // blocks requested per second, 0 = unlimited size_t backfillStorePerTurn = 50; // blocks verified and stored per processor-thread turn, 1..1000 + size_t backfillTurnBudgetMs = 50; // and ms per turn, 0 (no limit)..1000 }; // [snapshot]: snapshots served by this node to its p2p neighbours and fetched from them at a first diff --git a/csnode/include/csnode/backwardsync.hpp b/csnode/include/csnode/backwardsync.hpp index ae0423036..c32db3563 100644 --- a/csnode/include/csnode/backwardsync.hpp +++ b/csnode/include/csnode/backwardsync.hpp @@ -62,6 +62,9 @@ class BackwardSync { // blocks verified and stored per call (a tick or a reply), so that the processor thread, // which also runs consensus, is never held long; the rest waits for the next call size_t storePerTurn = 50; + // and for at most this long (ms) per call, 0: no limit, at most 1000. A turn ends after the block that + // passes it: a mainnet turn of 50 blocks took up to 1.7 s, more than a round + uint64_t turnBudgetMs = 50; }; struct Stats { @@ -74,6 +77,8 @@ class BackwardSync { size_t buffered = 0; uint64_t busyUs = 0; // processor-thread time spent verifying and storing uint64_t longestTurnUs = 0; // the longest single call + size_t slowTurns = 0; // calls longer than kSlowTurnMs + uint64_t longestFlushUs = 0; // the longest storage flush (every kFlushBlocks or kFlushMs) bool started = false; bool done = false; }; @@ -91,6 +96,7 @@ class BackwardSync { static constexpr size_t kFlushBlocks = 10'000; static constexpr uint64_t kFlushMs = 60'000; static constexpr size_t kProgressBlocks = 100'000; + static constexpr uint64_t kSlowTurnMs = 200; // a longer turn is logged, with where its time went BackwardSync(Hooks hooks, Settings settings); @@ -181,6 +187,8 @@ class BackwardSync { std::atomic bufferedCount_{0}; std::atomic busyUs_{0}; std::atomic longestTurnUs_{0}; + std::atomic slowTurns_{0}; + std::atomic longestFlushUs_{0}; }; } // namespace cs diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index e04a3b565..616c15482 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -32,6 +32,7 @@ BackwardSync::BackwardSync(Hooks hooks, Settings settings) } settings_.batch = std::clamp(settings_.batch, 1, 1000); settings_.storePerTurn = std::clamp(settings_.storePerTurn, 1, 1000); + settings_.turnBudgetMs = std::min(settings_.turnBudgetMs, 1000); } uint64_t BackwardSync::now() const { @@ -73,6 +74,8 @@ BackwardSync::Stats BackwardSync::stats() const { s.buffered = bufferedCount_; s.busyUs = busyUs_; s.longestTurnUs = longestTurnUs_; + s.slowTurns = slowTurns_; + s.longestFlushUs = longestFlushUs_; s.started = started_; s.done = done_; return s; @@ -167,7 +170,9 @@ void BackwardSync::tick(const Context& ctx) { log("BACKFILL: stored down to #" + std::to_string(next_ + 1) + ", " + std::to_string(static_cast(rate)) + " blocks/s over the last " + std::to_string(stored_ - progressStored_) + ", " + std::to_string(inFlight_.size()) + " peers in use, about " + std::to_string(static_cast(hoursLeft + 0.5)) + " h left, " + - std::to_string(busyUs_ / 1000) + " ms of processor thread so far (longest turn " + std::to_string(longestTurnUs_ / 1000) + " ms)"); + std::to_string(busyUs_ / 1000) + " ms of processor thread so far (longest turn " + std::to_string(longestTurnUs_ / 1000) + " ms, " + + std::to_string(slowTurns_.load()) + " over " + std::to_string(kSlowTurnMs) + " ms; longest flush " + + std::to_string(longestFlushUs_ / 1000) + " ms)"); progressAt_ = t; progressStored_ = stored_; } @@ -482,19 +487,37 @@ void BackwardSync::drain() { return; } } - const auto startedAt = std::chrono::steady_clock::now(); + using Clock = std::chrono::steady_clock; + const auto usSince = [](Clock::time_point from) { + return static_cast(std::chrono::duration_cast(Clock::now() - from).count()); + }; + const auto startedAt = Clock::now(); + const uint64_t startedMs = now(); // the budget and the slow-turn decision: the hooks' clock (tests drive it) size_t storedNow = 0; + size_t transactions = 0; + uint64_t verifyUs = 0; + uint64_t storeUs = 0; + uint64_t confirmUs = 0; while (!done_ && storedNow < settings_.storePerTurn) { + if (storedNow > 0 && settings_.turnBudgetMs > 0 && now() >= startedMs + settings_.turnBudgetMs) { + break; + } const auto it = buffer_.find(next_); if (it == buffer_.end()) { break; } + auto stepAt = Clock::now(); const std::string why = verify(it->second.pool); + verifyUs += usSince(stepAt); if (!why.empty()) { - reject(it->second.peer, next_, why); + const PublicKey peer = it->second.peer; // reject() erases the node this iterator points to + reject(peer, next_, why); continue; } - if (!hooks_.store || !hooks_.store(it->second.pool)) { + stepAt = Clock::now(); + const bool stored = hooks_.store && hooks_.store(it->second.pool); + storeUs += usSince(stepAt); + if (!stored) { if (!storeFailing_) { log("BACKFILL: block " + std::to_string(next_) + " could not be stored: trying again in 60 s", true); } @@ -508,6 +531,7 @@ void BackwardSync::drain() { } // the round confirmations of the block stored before are signed by the confidants of this // one: inside the hash, so a mismatch is the chain's, not the peer's. Counted, never refused + stepAt = Clock::now(); if (lastStored_) { std::string bad; if (!BlockVerifier::verifyConfirmations(*lastStored_, it->second.pool.confidants(), bad)) { @@ -519,6 +543,8 @@ void BackwardSync::drain() { } } } + confirmUs += usSince(stepAt); + transactions += it->second.pool.transactions_count(); lastStored_ = std::move(it->second.pool); expectedHash_ = lastStored_->previous_hash(); buffer_.erase(it); @@ -533,12 +559,20 @@ void BackwardSync::drain() { --next_; } bufferedCount_ = buffer_.size(); - const auto us = static_cast( - std::chrono::duration_cast(std::chrono::steady_clock::now() - startedAt).count()); + const uint64_t us = usSince(startedAt); busyUs_ += us; if (us > longestTurnUs_) { longestTurnUs_ = us; } + if (now() >= startedMs + kSlowTurnMs) { + const size_t n = ++slowTurns_; + if (n <= 20 || n % 100 == 0) { + log("BACKFILL: slow turn of " + std::to_string(now() - startedMs) + " ms (" + std::to_string(n) + " so far): " + + std::to_string(storedNow) + " blocks down to #" + std::to_string(first_.load()) + ", " + + std::to_string(transactions) + " transactions; verify " + std::to_string(verifyUs / 1000) + " ms, store " + + std::to_string(storeUs / 1000) + " ms, confirmations " + std::to_string(confirmUs / 1000) + " ms"); + } + } flushIfDue(false); } @@ -550,10 +584,18 @@ void BackwardSync::flushIfDue(bool force) { if (!force && storedSinceFlush_ < kFlushBlocks && t < lastFlushAt_ + kFlushMs) { return; } + // on the processor thread too, outside the turn budget: timed, so that a slow disk shows + const auto flushAt = std::chrono::steady_clock::now(); const bool ok = !hooks_.flush || hooks_.flush(); + const auto us = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - flushAt).count()); + busyUs_ += us; + if (us > longestFlushUs_) { + longestFlushUs_ = us; + } lastFlushAt_ = t; log("BACKFILL: flushed, " + std::to_string(storedSinceFlush_) + " blocks since the last flush, down to #" + - std::to_string(first_.load()) + (ok ? "" : ": the storage flush failed"), + std::to_string(first_.load()) + " in " + std::to_string(us / 1000) + " ms" + (ok ? "" : ": the storage flush failed"), !ok); storedSinceFlush_ = 0; } diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 1dd9dffc0..9b278b5cb 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -438,6 +438,7 @@ void Node::startBackfill() { settings.batch = sto.backfillBatch; settings.rate = sto.backfillRate; settings.storePerTurn = sto.backfillStorePerTurn; + settings.turnBudgetMs = sto.backfillTurnBudgetMs; // created under the lock that stop() takes to read the pointer (finishBootstrap runs this on the // processor thread while a stop may come from another one) std::lock_guard lock(backfillTimerMutex_); diff --git a/docs/CHANGES.md b/docs/CHANGES.md index d330d9ebc..f8c8aa8a4 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -59,6 +59,7 @@ lines under the new release. | #136 | csnode: forward sync reads the responders' SyncroMsg before it has applied special order 35 itself, and does not ask a responder with an anti-duplicate log for a range that responder would refuse as a duplicate; it asks another neighbour instead (a node syncing from block 0 among mostly pruned peers could otherwise wait for ever for its first blocks) | | #138 | ci: macOS jobs run on `macos-15` (GitHub retires the macOS 14 images on 2026-11-02 and fails them during brownouts until then), with binaries built for macOS 14 and later, and time out after their measured duration plus a margin (build 60, platform sync 40, private network 60 minutes instead of 240, 90, 150) | | #139 | ci: the backfill crash steps (kill, power cut...) wait until the node stored a block since its last start, so that every restart resumes lower; in `fund.py` a transfer rejected by consensus right after an earlier transfer of the same sender ended with an unknown outcome (a partition longer than the 60 s wait) counts with the pending answers, not as a fault (net-partition) | +| #140 | csnode: a backfill turn also ends after 50 ms (`backfill_turn_budget_ms`) (on mainnet a turn of 50 blocks took up to 1.7 s, longer than a round, on a node that may be a confidant); a turn over 200 ms is logged with its blocks, transactions and the time spent verifying, storing and checking confirmations, and the progress line counts them; the storage flush every 10,000 blocks is timed in its log line; fix: a rejected block's peer key was read after its buffer entry was freed | ## community.14 — 2026-10-27 diff --git a/docs/differences.md b/docs/differences.md index 5396f1a32..bfef87eb7 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -32,7 +32,7 @@ each key and its default. | `db_backend` | Chooses the block database: `berkeleydb` (the default, the official format), `lmdb` or `rocksdb`. RocksDB is about a third of the BerkeleyDB size on mainnet. | | `prune_keep_blocks` | Pruned storage: the node keeps only the latest blocks and removes older ones at each periodic checkpoint. Pruned storage runs on LMDB. | | `async_write_queue_size`, `write_batch_size`, `rocksdb_block_cache_mb`, `rocksdb_memtable_mb` | Tune the database writer and RocksDB. | -| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. | +| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn`, `backfill_turn_budget_ms` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. | ### Converting the block database diff --git a/docs/storage.md b/docs/storage.md index 987ac50c4..e53a4edc8 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -38,7 +38,8 @@ official node. | `backfill` | `on` | Backward sync: a node whose oldest stored block is above its floor fetches the older blocks from its neighbours (see [Backward sync](#backward-sync-backfill)). `off` never does. | | `backfill_batch` | `100` | Blocks per request, `1` to `1000`. | | `backfill_rate` | `1000` | Blocks requested per second over all peers (`0` = no limit). | -| `backfill_store_per_turn` | `50` | Blocks checked and stored per turn of the thread that also runs consensus, `1` to `1000`; the rest waits for the next turn, 100 ms later. | +| `backfill_store_per_turn` | `50` | Blocks checked and stored per turn of the thread that also runs consensus, `1` to `1000`; the rest waits for the next turn, 100 ms later. A turn over 200 ms is logged (`BACKFILL: slow turn of ...`) with the time spent verifying, storing and checking confirmations. | +| `backfill_turn_budget_ms` | `50` | A turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`. A shorter turn leaves more of the thread to consensus; on mainnet 50 ms gives about 400 blocks/s. | ### Choosing a backend diff --git a/tests/csnode/backfillconfig_tests.cpp b/tests/csnode/backfillconfig_tests.cpp index 913dc2448..f9cad419f 100644 --- a/tests/csnode/backfillconfig_tests.cpp +++ b/tests/csnode/backfillconfig_tests.cpp @@ -26,11 +26,13 @@ TEST(BackfillConfig, Defaults) { EXPECT_EQ(d.backfillBatch, 100u); EXPECT_EQ(d.backfillRate, 1000u); EXPECT_EQ(d.backfillStorePerTurn, 50u); + EXPECT_EQ(d.backfillTurnBudgetMs, 50u); } TEST(BackfillConfig, EveryKey) { - const auto d = parse("backfill=off\nbackfill_batch=250\nbackfill_rate=0\nbackfill_store_per_turn=20\n"); + const auto d = parse("backfill=off\nbackfill_batch=250\nbackfill_rate=0\nbackfill_store_per_turn=20\nbackfill_turn_budget_ms=0\n"); EXPECT_FALSE(d.backfill); + EXPECT_EQ(d.backfillTurnBudgetMs, 0u); EXPECT_EQ(d.backfillBatch, 250u); EXPECT_EQ(d.backfillRate, 0u); EXPECT_EQ(d.backfillStorePerTurn, 20u); @@ -41,6 +43,7 @@ TEST(BackfillConfig, EveryKey) { TEST(BackfillConfig, BadValuesKeepTheDefaultOrAreClamped) { EXPECT_TRUE(parse("backfill=maybe\n").backfill); EXPECT_EQ(parse("backfill_batch=5000\n").backfillBatch, 1000u); + EXPECT_EQ(parse("backfill_turn_budget_ms=5000\n").backfillTurnBudgetMs, 1000u); EXPECT_EQ(parse("backfill_batch=0\n").backfillBatch, 1u); EXPECT_EQ(parse("backfill_batch=many\n").backfillBatch, 100u); EXPECT_EQ(parse("backfill_rate=-5\n").backfillRate, 1000u); diff --git a/tests/csnode/backwardsync_tests.cpp b/tests/csnode/backwardsync_tests.cpp index 08102a5a0..7e41ffb4b 100644 --- a/tests/csnode/backwardsync_tests.cpp +++ b/tests/csnode/backwardsync_tests.cpp @@ -115,6 +115,7 @@ cs::BackwardSync::Settings settingsOf(size_t batch = 100, size_t rate = 1000, si s.batch = batch; s.rate = rate; s.storePerTurn = perTurn; + s.turnBudgetMs = 0; // no time limit unless a test sets one return s; } @@ -143,6 +144,7 @@ class BackfillTest : public ::testing::Test { if (failStore_) { return false; } + now_ += storeDelayMs_; EXPECT_EQ(pool.sequence() + 1, first_) << "not strictly downwards"; EXPECT_EQ(pool.hash(), chain().blocks[pool.sequence()].hash()); first_ = pool.sequence(); @@ -210,6 +212,7 @@ class BackfillTest : public ::testing::Test { size_t storedCount_ = 0; size_t flushes_ = 0; bool failStore_ = false; + uint64_t storeDelayMs_ = 0; // a slow disk: simulated time spent in every store bool forwardSync_ = false; bool bootstrapping_ = false; }; @@ -550,6 +553,30 @@ TEST_F(BackfillTest, TheWorkPerTurnIsBounded) { EXPECT_GT(sync_->stats().longestTurnUs, 0u); } +// a slow store ends the turn once it has used its time, however many blocks it may still store; +// such a turn is logged with where its time went +TEST_F(BackfillTest, ASlowTurnStopsAtItsTimeBudget) { + auto settings = settingsOf(100, 0, 50); + settings.turnBudgetMs = 30; + make(settings); + storeDelayMs_ = 12; + tick(); + sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); + // 12 ms a block: 36 ms after the third block, past the budget of 30 ms + EXPECT_EQ(kFirst - first_, 3u); + storeDelayMs_ = 0; + tick(); + EXPECT_EQ(kFirst - first_, 53u); // a fast turn stores up to storePerTurn + EXPECT_EQ(sync_->stats().slowTurns, 0u); + + // a turn over kSlowTurnMs is counted and logged with where its time went + storeDelayMs_ = 210; + tick(); + EXPECT_EQ(kFirst - first_, 54u); // one block, then the budget is spent + EXPECT_EQ(sync_->stats().slowTurns, 1u); + EXPECT_EQ(logged("BACKFILL: slow turn of 210 ms (1 so far): 1 blocks"), 1u); +} + // L7: pruning raised the floor above blocks waiting in the buffer: the backfill ends, nothing is // stored below the floor TEST_F(BackfillTest, AFloorRaisedByPruningEndsIt) {