From e9093f395b095a5d3295f8fd06674780cd9b92e6 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 1 Nov 2026 09:49:30 +0100 Subject: [PATCH 1/2] csnode: the periodic backfill flush runs on its own thread On the mainnet test node the backfill flush (transaction index, block hashes, storage sync) took up to 5.6 s on the processor thread. It now runs on a thread of BackwardSync: one at a time (the blocks stored meanwhile go with the next one), the final flush at the end of the backfill waits for the one under way and runs on the calling thread, and stop() / the destructor join the thread; Node::destroy stops the backfill before it flushes and closes the storage. The engines' flushes are thread-safe (LMDB syncs under the map lock shared with transactions, RocksDB SyncWAL, BerkeleyDB with DB_THREAD; error state is per thread). Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/backwardsync.hpp | 26 ++++++++- csnode/src/backwardsync.cpp | 72 +++++++++++++++++++++++++ csnode/src/node.cpp | 3 ++ docs/CHANGES.md | 1 + docs/storage.md | 2 + tests/csnode/backwardsync_tests.cpp | 75 ++++++++++++++++++++++++-- 6 files changed, 174 insertions(+), 5 deletions(-) diff --git a/csnode/include/csnode/backwardsync.hpp b/csnode/include/csnode/backwardsync.hpp index 62d97f638..df64742b5 100644 --- a/csnode/include/csnode/backwardsync.hpp +++ b/csnode/include/csnode/backwardsync.hpp @@ -2,12 +2,15 @@ #define BACKWARDSYNC_HPP #include +#include #include #include #include +#include #include #include #include +#include #include #include @@ -21,8 +24,9 @@ namespace cs { // with BlockRequest, newest first, checks each against the hash chain and its signatures, and // stores it strictly downwards, never applying it to the caches. Requests go out only while the // node is synced; replies already asked for are stored in any case. Single-threaded: every call -// comes from the node's processor thread, but stop() and stats(). The chain, the network and the -// clock are reached only through Hooks. +// comes from the node's processor thread, but stop() and stats(); the periodic flush runs on a +// thread of its own (Settings::asyncFlush). The chain, the network and the clock are reached only +// through Hooks. class BackwardSync { public: struct Peer { @@ -70,6 +74,9 @@ class BackwardSync { // the same while the node is not a confidant, with little consensus work: about 450 blocks/s // on mainnet instead of about 250 uint64_t idleTurnBudgetMs = 90; + // the periodic flush (up to several seconds on mainnet) runs on a thread of its own instead of + // the processor thread; the flushes at the end and at a stop stay where they are + bool asyncFlush = true; }; struct Stats { @@ -107,6 +114,9 @@ class BackwardSync { static constexpr uint64_t kSlowTurnMs = 200; // a longer turn is logged, with where its time went BackwardSync(Hooks hooks, Settings settings); + ~BackwardSync(); + BackwardSync(const BackwardSync&) = delete; + BackwardSync& operator=(const BackwardSync&) = delete; // every 500 ms void tick(const Context& ctx); @@ -149,6 +159,8 @@ class BackwardSync { void mark(const PublicKey& peer, uint64_t until, const std::string& why); void expire(); void flushIfDue(bool force); + void flushWorker(); + void stopFlusher(); void request(const Context& ctx); std::optional pickPeer(const Context& ctx, Sequence highest); void finish(); @@ -198,6 +210,16 @@ class BackwardSync { std::atomic slowTurns_{0}; bool trusted_ = false; // from the last tick: the budget of the turns until the next one std::atomic longestFlushUs_{0}; + + // the flush thread: started at the first periodic flush, joined by stop() and the destructor + std::mutex flushMutex_; + std::condition_variable flushCv_; + std::thread flushThread_; + bool flushRequested_ = false; // the fields below: under flushMutex_ + bool flushRunning_ = false; + bool flushStopping_ = false; + size_t flushBlocks_ = 0; // what the requested flush covers, for its log line + Sequence flushDown_ = 0; }; } // namespace cs diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index ae122a7fb..17846925f 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -60,8 +60,59 @@ bool BackwardSync::isActive() const { return started_ && !done_ && !stopped_; } +BackwardSync::~BackwardSync() { + stopFlusher(); +} + void BackwardSync::stop() { stopped_ = true; + stopFlusher(); +} + +// a flush under way ends first: nothing flushes the storage behind the node's back once it stops +void BackwardSync::stopFlusher() { + std::thread worker; + { + std::lock_guard lock(flushMutex_); + flushStopping_ = true; + worker = std::move(flushThread_); + } + flushCv_.notify_all(); + if (worker.joinable() && worker.get_id() != std::this_thread::get_id()) { + worker.join(); + } +} + +void BackwardSync::flushWorker() { + std::unique_lock lock(flushMutex_); + for (;;) { + flushCv_.wait(lock, [this] { return flushRequested_ || flushStopping_; }); + if (flushStopping_) { + flushRequested_ = false; // the node flushes once more before its storage closes + flushCv_.notify_all(); + return; + } + flushRequested_ = false; + flushRunning_ = true; + const size_t blocks = flushBlocks_; + const Sequence down = flushDown_; + lock.unlock(); + + const auto at = std::chrono::steady_clock::now(); + const bool ok = !hooks_.flush || hooks_.flush(); + const auto us = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - at).count()); + if (us > longestFlushUs_) { + longestFlushUs_ = us; + } + log("BACKFILL: flushed, " + std::to_string(blocks) + " blocks since the last flush, down to #" + std::to_string(down) + + " in " + std::to_string(us / 1000) + " ms on the flush thread" + (ok ? "" : ": the storage flush failed"), + !ok); + + lock.lock(); + flushRunning_ = false; + flushCv_.notify_all(); + } } BackwardSync::Stats BackwardSync::stats() const { @@ -590,6 +641,11 @@ void BackwardSync::drain() { } void BackwardSync::flushIfDue(bool force) { + if (force && settings_.asyncFlush) { + // the last flush covers every block: after the one under way, if any + std::unique_lock lock(flushMutex_); + flushCv_.wait(lock, [this] { return (!flushRunning_ && !flushRequested_) || (flushStopping_ && !flushRunning_); }); + } if (storedSinceFlush_ == 0) { return; } @@ -600,6 +656,22 @@ void BackwardSync::flushIfDue(bool force) { if (!force && trusted_ && t < lastFlushAt_ + kTrustedFlushDeferMs) { return; // consensus first: flushed at the first turn outside the trusted set } + if (!force && settings_.asyncFlush) { + std::lock_guard lock(flushMutex_); + if (flushStopping_ || flushRunning_ || flushRequested_) { + return; // one at a time: the blocks stored meanwhile go with the next one + } + if (!flushThread_.joinable()) { + flushThread_ = std::thread([this] { flushWorker(); }); + } + flushRequested_ = true; + flushBlocks_ = storedSinceFlush_; + flushDown_ = first_; + lastFlushAt_ = t; + storedSinceFlush_ = 0; + flushCv_.notify_one(); + return; + } // on the processor thread too, outside the turn budget: timed, so that a slow disk shows const auto flushAt = std::chrono::steady_clock::now(); const bool ok = !hooks_.flush || hooks_.flush(); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 90bb3cb4f..331b6bf4b 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -893,6 +893,9 @@ void Node::destroy() { solver_->finish(); cswarning() << "[SOLVER STOPPED]"; + if (backwardSync_) { + backwardSync_->stop(); // its flush thread ends before the storage closes + } if (backwardSync_ && backwardSync_->stats().stored > 0) { // the index and the block hashes of the blocks backfilled since the last flush, before the storage closes blockChain_.flushBackfill(); diff --git a/docs/CHANGES.md b/docs/CHANGES.md index c869529ab..cda3ae320 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -67,6 +67,7 @@ lines under the new release. | #145 | csnode: backward sync asks for no more blocks than its buffer holds (a request could overrun it by one batch); blocks a reply brings when the buffer is full are asked for again at once and no longer count against the peer that sent them (with slow stores, on mainnet after a restart, peers that had answered were left out for an hour and the backfill fell to ~30 blocks/s) | | #146 | csnode: the backfill turn budget depends on the node's role: 50 ms while it is a confidant (`backfill_turn_budget_ms`), 90 ms otherwise (`backfill_idle_turn_budget_ms`); on mainnet a node outside the trusted set backfills at about 450 blocks/s instead of about 250 | | #147 | csnode: while the node is a confidant, the backfill storage flush (median 358 ms, up to 2.4 s on mainnet, on the processor thread) waits until the node is out of the trusted set, 10 minutes at most; blocks not flushed are fetched again after a crash | +| #148 | csnode: the periodic backfill flush runs on a thread of its own instead of the processor thread (on mainnet it took up to 5.6 s, with the storing of blocks waiting); one at a time, the final flush waits for the one under way, and the node stops the thread before its storage closes | ## community.14 — 2026-10-27 diff --git a/docs/storage.md b/docs/storage.md index c1d70a64a..5f1c835bb 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -42,6 +42,8 @@ official node. | `backfill_turn_budget_ms` | `50` | While the node is a confidant, a turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`: consensus keeps most of the thread; on mainnet about 250 blocks/s. | | `backfill_idle_turn_budget_ms` | `90` | The same while the node is not a confidant, with little consensus work: on mainnet about 450 blocks/s. | +The periodic flush of the backfilled blocks (every 10,000 blocks or a minute; up to several seconds on mainnet) runs on a thread of its own, not on the thread that runs consensus; one flush at a time, and while the node is a confidant it waits until the node is out of the trusted set (10 minutes at most). The log line says `on the flush thread`. + ### Choosing a backend | Backend | Use it for | Notes | diff --git a/tests/csnode/backwardsync_tests.cpp b/tests/csnode/backwardsync_tests.cpp index 6bbcf14b8..b7eceee71 100644 --- a/tests/csnode/backwardsync_tests.cpp +++ b/tests/csnode/backwardsync_tests.cpp @@ -1,8 +1,13 @@ #include #include +#include +#include +#include +#include #include #include +#include #include #include #include @@ -117,6 +122,7 @@ cs::BackwardSync::Settings settingsOf(size_t batch = 100, size_t rate = 1000, si s.storePerTurn = perTurn; s.turnBudgetMs = 0; // no time limit unless a test sets one s.idleTurnBudgetMs = 0; + s.asyncFlush = false; // flushes on the calling thread unless a test says otherwise return s; } @@ -152,13 +158,22 @@ class BackfillTest : public ::testing::Test { ++storedCount_; return true; }; - hooks.flush = [this] { ++flushes_; return true; }; + hooks.flush = [this] { + if (flushGate_) { + flushGate_(); // a slow disk: the test decides when the flush ends + } + ++flushes_; + return true; + }; hooks.floor = [this] { return floor_; }; hooks.sendRequest = [this](const cs::PublicKey& to, const cs::PoolsRequestedSequences& seqs) { requests_.emplace_back(to, seqs); }; hooks.clock = [this] { return now_; }; - hooks.onLog = [this](const std::string& line) { logs_.push_back(line); }; + hooks.onLog = [this](const std::string& line) { + std::lock_guard lock(logsMutex_); // the flush thread logs too + logs_.push_back(line); + }; sync_ = std::make_unique(std::move(hooks), settings); } @@ -194,6 +209,7 @@ class BackfillTest : public ::testing::Test { } } size_t logged(const std::string& text) const { + std::lock_guard lock(logsMutex_); return static_cast(std::count_if(logs_.begin(), logs_.end(), [&](const std::string& l) { return l.find(text) != std::string::npos; })); } // ticks and answers until the backfill is done or `limit` simulated seconds pass @@ -208,11 +224,13 @@ class BackfillTest : public ::testing::Test { std::unique_ptr sync_; std::vector> requests_; std::vector logs_; + mutable std::mutex logsMutex_; + std::function flushGate_; cs::Sequence first_ = kFirst; cs::Sequence floor_ = 0; uint64_t now_ = 1'000'000; size_t storedCount_ = 0; - size_t flushes_ = 0; + std::atomic flushes_{0}; bool failStore_ = false; uint64_t storeDelayMs_ = 0; // a slow disk: simulated time spent in every store bool forwardSync_ = false; @@ -618,6 +636,57 @@ TEST_F(BackfillTest, AConfidantDefersTheFlush) { EXPECT_EQ(flushes_, before + 2); } +// the periodic flush runs on its own thread: storing goes on while it lasts, a second one waits +// for the first, and the last flush (end of the backfill) waits for the one under way +TEST_F(BackfillTest, ThePeriodicFlushRunsOnItsOwnThread) { + std::mutex m; + std::condition_variable cv; + bool release = false; + std::atomic started{0}; + struct Release { // a failed assertion must not leave the flush thread waiting for ever + std::function f; + ~Release() { f(); } + } guard{[&] { + { + std::lock_guard lock(m); + release = true; + } + cv.notify_all(); + }}; + flushGate_ = [&] { + ++started; + std::unique_lock lock(m); + cv.wait(lock, [&] { return release; }); + }; + auto settings = settingsOf(100, 0, 50); + settings.asyncFlush = true; + make(settings); + tick(); + sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); + tick(cs::BackwardSync::kFlushMs + 1); // a flush is due: handed to the flush thread + const cs::Sequence storedWhenDue = first_; + for (int i = 0; i < 200 && started.load() == 0; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + ASSERT_EQ(started.load(), 1) << "the flush thread took the flush"; + + // the flush hangs: blocks are still stored, and no second flush starts + answerAll(); + tick(cs::BackwardSync::kFlushMs + 1); + tick(cs::BackwardSync::kFlushMs + 1); + EXPECT_LT(first_, storedWhenDue) << "storing goes on during the flush"; + EXPECT_EQ(started.load(), 1) << "one flush at a time"; + EXPECT_EQ(flushes_.load(), 0u); + + guard.f(); + // the end of the backfill waits for the flush under way, then flushes the rest itself + runToEnd(); + EXPECT_TRUE(sync_->stats().done); + EXPECT_GE(flushes_.load(), 2u); + EXPECT_GE(logged("on the flush thread"), 1u); + sync_->stop(); // joins the flush thread +} + TEST_F(BackfillTest, ASlowTurnStopsAtItsTimeBudget) { auto settings = settingsOf(100, 0, 50); settings.idleTurnBudgetMs = 30; From ff0ed1c742d0c1ddeec7273bb835b82ea974b6f3 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Sun, 1 Nov 2026 10:19:22 +0100 Subject: [PATCH 2/2] csnode: review fixes for the backfill flush thread - the test's gate is shared with the flush thread by value and waits up to 5 s for the thread to start (a failed assertion no longer leaves it on a dead mutex) - stopFlusher detaches instead of terminating when called on the flush thread itself; a thread that cannot be created falls back to the synchronous flush; an exception from the flush is caught and logged - BlockHashes::failures_ is atomic (the flush thread may report an LMDB failure); comment on the flush skipped after stop(); docs: the one remaining wait (LMDB map growth during a sync) Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/blockhashes.hpp | 3 +- csnode/src/backwardsync.cpp | 50 ++++++++++++++++++++------- docs/storage.md | 2 +- tests/csnode/backwardsync_tests.cpp | 48 ++++++++++++++----------- 4 files changed, 67 insertions(+), 36 deletions(-) diff --git a/csnode/include/csnode/blockhashes.hpp b/csnode/include/csnode/blockhashes.hpp index b2e3f68a8..b251d224c 100644 --- a/csnode/include/csnode/blockhashes.hpp +++ b/csnode/include/csnode/blockhashes.hpp @@ -1,6 +1,7 @@ #ifndef BLOCKHASHES_HPP #define BLOCKHASHES_HPP +#include #include #include #include @@ -68,7 +69,7 @@ private slots: std::string path_; std::unique_ptr seqDb_; std::unique_ptr hashDb_; - size_t failures_ = 0; + std::atomic failures_{0}; // the backfill flush thread reports LMDB failures too }; } // namespace cs diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index 17846925f..c98c90402 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include @@ -78,8 +80,13 @@ void BackwardSync::stopFlusher() { worker = std::move(flushThread_); } flushCv_.notify_all(); - if (worker.joinable() && worker.get_id() != std::this_thread::get_id()) { - worker.join(); + if (worker.joinable()) { + if (worker.get_id() == std::this_thread::get_id()) { + worker.detach(); // stopped from the flush thread itself: it returns on its own + } + else { + worker.join(); + } } } @@ -99,7 +106,13 @@ void BackwardSync::flushWorker() { lock.unlock(); const auto at = std::chrono::steady_clock::now(); - const bool ok = !hooks_.flush || hooks_.flush(); + bool ok = false; + try { + ok = !hooks_.flush || hooks_.flush(); + } + catch (const std::exception& e) { + log(std::string("BACKFILL: the flush thread caught: ") + e.what(), true); + } const auto us = static_cast( std::chrono::duration_cast(std::chrono::steady_clock::now() - at).count()); if (us > longestFlushUs_) { @@ -658,19 +671,30 @@ void BackwardSync::flushIfDue(bool force) { } if (!force && settings_.asyncFlush) { std::lock_guard lock(flushMutex_); - if (flushStopping_ || flushRunning_ || flushRequested_) { + if (flushStopping_) { + return; // stopped: the node flushes once more itself before its storage closes (Node::destroy) + } + if (flushRunning_ || flushRequested_) { return; // one at a time: the blocks stored meanwhile go with the next one } if (!flushThread_.joinable()) { - flushThread_ = std::thread([this] { flushWorker(); }); - } - flushRequested_ = true; - flushBlocks_ = storedSinceFlush_; - flushDown_ = first_; - lastFlushAt_ = t; - storedSinceFlush_ = 0; - flushCv_.notify_one(); - return; + try { + flushThread_ = std::thread([this] { flushWorker(); }); + } + catch (const std::system_error& e) { + log(std::string("BACKFILL: no flush thread (") + e.what() + "): flushing on the processor thread", true); + settings_.asyncFlush = false; + } + } + if (flushThread_.joinable()) { + flushRequested_ = true; + flushBlocks_ = storedSinceFlush_; + flushDown_ = first_; + lastFlushAt_ = t; + storedSinceFlush_ = 0; + flushCv_.notify_one(); + return; + } } // on the processor thread too, outside the turn budget: timed, so that a slow disk shows const auto flushAt = std::chrono::steady_clock::now(); diff --git a/docs/storage.md b/docs/storage.md index 5f1c835bb..a75c499be 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -42,7 +42,7 @@ official node. | `backfill_turn_budget_ms` | `50` | While the node is a confidant, a turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`: consensus keeps most of the thread; on mainnet about 250 blocks/s. | | `backfill_idle_turn_budget_ms` | `90` | The same while the node is not a confidant, with little consensus work: on mainnet about 450 blocks/s. | -The periodic flush of the backfilled blocks (every 10,000 blocks or a minute; up to several seconds on mainnet) runs on a thread of its own, not on the thread that runs consensus; one flush at a time, and while the node is a confidant it waits until the node is out of the trusted set (10 minutes at most). The log line says `on the flush thread`. +The periodic flush of the backfilled blocks (every 10,000 blocks or a minute; up to several seconds on mainnet) runs on a thread of its own, not on the thread that runs consensus; one flush at a time, and while the node is a confidant it waits until the node is out of the trusted set (10 minutes at most). The log line says `on the flush thread`. One wait remains possible: an LMDB index that must grow its map while the flush thread syncs it waits for that sync (rare: the map grows in steps). ### Choosing a backend diff --git a/tests/csnode/backwardsync_tests.cpp b/tests/csnode/backwardsync_tests.cpp index b7eceee71..d77d7f25f 100644 --- a/tests/csnode/backwardsync_tests.cpp +++ b/tests/csnode/backwardsync_tests.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -639,24 +640,29 @@ TEST_F(BackfillTest, AConfidantDefersTheFlush) { // the periodic flush runs on its own thread: storing goes on while it lasts, a second one waits // for the first, and the last flush (end of the backfill) waits for the one under way TEST_F(BackfillTest, ThePeriodicFlushRunsOnItsOwnThread) { - std::mutex m; - std::condition_variable cv; - bool release = false; - std::atomic started{0}; - struct Release { // a failed assertion must not leave the flush thread waiting for ever - std::function f; - ~Release() { f(); } - } guard{[&] { - { - std::lock_guard lock(m); - release = true; + // shared with the flush thread by value: it may outlive this body if an assertion fails + struct Gate { + std::mutex m; + std::condition_variable cv; + bool release = false; + std::atomic started{0}; + void open() { + { + std::lock_guard lock(m); + release = true; + } + cv.notify_all(); } - cv.notify_all(); - }}; - flushGate_ = [&] { - ++started; - std::unique_lock lock(m); - cv.wait(lock, [&] { return release; }); + }; + const auto gate = std::make_shared(); + struct Release { // a failed assertion must not leave the flush thread waiting for ever + std::shared_ptr g; + ~Release() { g->open(); } + } guard{gate}; + flushGate_ = [gate] { + ++gate->started; + std::unique_lock lock(gate->m); + gate->cv.wait(lock, [&] { return gate->release; }); }; auto settings = settingsOf(100, 0, 50); settings.asyncFlush = true; @@ -665,20 +671,20 @@ TEST_F(BackfillTest, ThePeriodicFlushRunsOnItsOwnThread) { sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); tick(cs::BackwardSync::kFlushMs + 1); // a flush is due: handed to the flush thread const cs::Sequence storedWhenDue = first_; - for (int i = 0; i < 200 && started.load() == 0; ++i) { + for (int i = 0; i < 1000 && gate->started.load() == 0; ++i) { // up to 5 s on a busy runner std::this_thread::sleep_for(std::chrono::milliseconds(5)); } - ASSERT_EQ(started.load(), 1) << "the flush thread took the flush"; + ASSERT_EQ(gate->started.load(), 1) << "the flush thread took the flush"; // the flush hangs: blocks are still stored, and no second flush starts answerAll(); tick(cs::BackwardSync::kFlushMs + 1); tick(cs::BackwardSync::kFlushMs + 1); EXPECT_LT(first_, storedWhenDue) << "storing goes on during the flush"; - EXPECT_EQ(started.load(), 1) << "one flush at a time"; + EXPECT_EQ(gate->started.load(), 1) << "one flush at a time"; EXPECT_EQ(flushes_.load(), 0u); - guard.f(); + gate->open(); // the end of the backfill waits for the flush under way, then flushes the rest itself runToEnd(); EXPECT_TRUE(sync_->stats().done);