diff --git a/csnode/include/csnode/backwardsync.hpp b/csnode/include/csnode/backwardsync.hpp index 62d97f638..df64742b5 100644 --- a/csnode/include/csnode/backwardsync.hpp +++ b/csnode/include/csnode/backwardsync.hpp @@ -2,12 +2,15 @@ #define BACKWARDSYNC_HPP #include +#include #include #include #include +#include #include #include #include +#include #include #include @@ -21,8 +24,9 @@ namespace cs { // with BlockRequest, newest first, checks each against the hash chain and its signatures, and // stores it strictly downwards, never applying it to the caches. Requests go out only while the // node is synced; replies already asked for are stored in any case. Single-threaded: every call -// comes from the node's processor thread, but stop() and stats(). The chain, the network and the -// clock are reached only through Hooks. +// comes from the node's processor thread, but stop() and stats(); the periodic flush runs on a +// thread of its own (Settings::asyncFlush). The chain, the network and the clock are reached only +// through Hooks. class BackwardSync { public: struct Peer { @@ -70,6 +74,9 @@ class BackwardSync { // the same while the node is not a confidant, with little consensus work: about 450 blocks/s // on mainnet instead of about 250 uint64_t idleTurnBudgetMs = 90; + // the periodic flush (up to several seconds on mainnet) runs on a thread of its own instead of + // the processor thread; the flushes at the end and at a stop stay where they are + bool asyncFlush = true; }; struct Stats { @@ -107,6 +114,9 @@ class BackwardSync { static constexpr uint64_t kSlowTurnMs = 200; // a longer turn is logged, with where its time went BackwardSync(Hooks hooks, Settings settings); + ~BackwardSync(); + BackwardSync(const BackwardSync&) = delete; + BackwardSync& operator=(const BackwardSync&) = delete; // every 500 ms void tick(const Context& ctx); @@ -149,6 +159,8 @@ class BackwardSync { void mark(const PublicKey& peer, uint64_t until, const std::string& why); void expire(); void flushIfDue(bool force); + void flushWorker(); + void stopFlusher(); void request(const Context& ctx); std::optional pickPeer(const Context& ctx, Sequence highest); void finish(); @@ -198,6 +210,16 @@ class BackwardSync { std::atomic slowTurns_{0}; bool trusted_ = false; // from the last tick: the budget of the turns until the next one std::atomic longestFlushUs_{0}; + + // the flush thread: started at the first periodic flush, joined by stop() and the destructor + std::mutex flushMutex_; + std::condition_variable flushCv_; + std::thread flushThread_; + bool flushRequested_ = false; // the fields below: under flushMutex_ + bool flushRunning_ = false; + bool flushStopping_ = false; + size_t flushBlocks_ = 0; // what the requested flush covers, for its log line + Sequence flushDown_ = 0; }; } // namespace cs diff --git a/csnode/include/csnode/blockhashes.hpp b/csnode/include/csnode/blockhashes.hpp index b2e3f68a8..b251d224c 100644 --- a/csnode/include/csnode/blockhashes.hpp +++ b/csnode/include/csnode/blockhashes.hpp @@ -1,6 +1,7 @@ #ifndef BLOCKHASHES_HPP #define BLOCKHASHES_HPP +#include #include #include #include @@ -68,7 +69,7 @@ private slots: std::string path_; std::unique_ptr seqDb_; std::unique_ptr hashDb_; - size_t failures_ = 0; + std::atomic failures_{0}; // the backfill flush thread reports LMDB failures too }; } // namespace cs diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index ae122a7fb..c98c90402 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include @@ -60,8 +62,70 @@ bool BackwardSync::isActive() const { return started_ && !done_ && !stopped_; } +BackwardSync::~BackwardSync() { + stopFlusher(); +} + void BackwardSync::stop() { stopped_ = true; + stopFlusher(); +} + +// a flush under way ends first: nothing flushes the storage behind the node's back once it stops +void BackwardSync::stopFlusher() { + std::thread worker; + { + std::lock_guard lock(flushMutex_); + flushStopping_ = true; + worker = std::move(flushThread_); + } + flushCv_.notify_all(); + if (worker.joinable()) { + if (worker.get_id() == std::this_thread::get_id()) { + worker.detach(); // stopped from the flush thread itself: it returns on its own + } + else { + worker.join(); + } + } +} + +void BackwardSync::flushWorker() { + std::unique_lock lock(flushMutex_); + for (;;) { + flushCv_.wait(lock, [this] { return flushRequested_ || flushStopping_; }); + if (flushStopping_) { + flushRequested_ = false; // the node flushes once more before its storage closes + flushCv_.notify_all(); + return; + } + flushRequested_ = false; + flushRunning_ = true; + const size_t blocks = flushBlocks_; + const Sequence down = flushDown_; + lock.unlock(); + + const auto at = std::chrono::steady_clock::now(); + bool ok = false; + try { + ok = !hooks_.flush || hooks_.flush(); + } + catch (const std::exception& e) { + log(std::string("BACKFILL: the flush thread caught: ") + e.what(), true); + } + const auto us = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - at).count()); + if (us > longestFlushUs_) { + longestFlushUs_ = us; + } + log("BACKFILL: flushed, " + std::to_string(blocks) + " blocks since the last flush, down to #" + std::to_string(down) + + " in " + std::to_string(us / 1000) + " ms on the flush thread" + (ok ? "" : ": the storage flush failed"), + !ok); + + lock.lock(); + flushRunning_ = false; + flushCv_.notify_all(); + } } BackwardSync::Stats BackwardSync::stats() const { @@ -590,6 +654,11 @@ void BackwardSync::drain() { } void BackwardSync::flushIfDue(bool force) { + if (force && settings_.asyncFlush) { + // the last flush covers every block: after the one under way, if any + std::unique_lock lock(flushMutex_); + flushCv_.wait(lock, [this] { return (!flushRunning_ && !flushRequested_) || (flushStopping_ && !flushRunning_); }); + } if (storedSinceFlush_ == 0) { return; } @@ -600,6 +669,33 @@ void BackwardSync::flushIfDue(bool force) { if (!force && trusted_ && t < lastFlushAt_ + kTrustedFlushDeferMs) { return; // consensus first: flushed at the first turn outside the trusted set } + if (!force && settings_.asyncFlush) { + std::lock_guard lock(flushMutex_); + if (flushStopping_) { + return; // stopped: the node flushes once more itself before its storage closes (Node::destroy) + } + if (flushRunning_ || flushRequested_) { + return; // one at a time: the blocks stored meanwhile go with the next one + } + if (!flushThread_.joinable()) { + try { + flushThread_ = std::thread([this] { flushWorker(); }); + } + catch (const std::system_error& e) { + log(std::string("BACKFILL: no flush thread (") + e.what() + "): flushing on the processor thread", true); + settings_.asyncFlush = false; + } + } + if (flushThread_.joinable()) { + flushRequested_ = true; + flushBlocks_ = storedSinceFlush_; + flushDown_ = first_; + lastFlushAt_ = t; + storedSinceFlush_ = 0; + flushCv_.notify_one(); + return; + } + } // on the processor thread too, outside the turn budget: timed, so that a slow disk shows const auto flushAt = std::chrono::steady_clock::now(); const bool ok = !hooks_.flush || hooks_.flush(); diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 90bb3cb4f..331b6bf4b 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -893,6 +893,9 @@ void Node::destroy() { solver_->finish(); cswarning() << "[SOLVER STOPPED]"; + if (backwardSync_) { + backwardSync_->stop(); // its flush thread ends before the storage closes + } if (backwardSync_ && backwardSync_->stats().stored > 0) { // the index and the block hashes of the blocks backfilled since the last flush, before the storage closes blockChain_.flushBackfill(); diff --git a/docs/CHANGES.md b/docs/CHANGES.md index c869529ab..cda3ae320 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -67,6 +67,7 @@ lines under the new release. | #145 | csnode: backward sync asks for no more blocks than its buffer holds (a request could overrun it by one batch); blocks a reply brings when the buffer is full are asked for again at once and no longer count against the peer that sent them (with slow stores, on mainnet after a restart, peers that had answered were left out for an hour and the backfill fell to ~30 blocks/s) | | #146 | csnode: the backfill turn budget depends on the node's role: 50 ms while it is a confidant (`backfill_turn_budget_ms`), 90 ms otherwise (`backfill_idle_turn_budget_ms`); on mainnet a node outside the trusted set backfills at about 450 blocks/s instead of about 250 | | #147 | csnode: while the node is a confidant, the backfill storage flush (median 358 ms, up to 2.4 s on mainnet, on the processor thread) waits until the node is out of the trusted set, 10 minutes at most; blocks not flushed are fetched again after a crash | +| #148 | csnode: the periodic backfill flush runs on a thread of its own instead of the processor thread (on mainnet it took up to 5.6 s, with the storing of blocks waiting); one at a time, the final flush waits for the one under way, and the node stops the thread before its storage closes | ## community.14 — 2026-10-27 diff --git a/docs/storage.md b/docs/storage.md index c1d70a64a..a75c499be 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -42,6 +42,8 @@ official node. | `backfill_turn_budget_ms` | `50` | While the node is a confidant, a turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`: consensus keeps most of the thread; on mainnet about 250 blocks/s. | | `backfill_idle_turn_budget_ms` | `90` | The same while the node is not a confidant, with little consensus work: on mainnet about 450 blocks/s. | +The periodic flush of the backfilled blocks (every 10,000 blocks or a minute; up to several seconds on mainnet) runs on a thread of its own, not on the thread that runs consensus; one flush at a time, and while the node is a confidant it waits until the node is out of the trusted set (10 minutes at most). The log line says `on the flush thread`. One wait remains possible: an LMDB index that must grow its map while the flush thread syncs it waits for that sync (rare: the map grows in steps). + ### Choosing a backend | Backend | Use it for | Notes | diff --git a/tests/csnode/backwardsync_tests.cpp b/tests/csnode/backwardsync_tests.cpp index 6bbcf14b8..d77d7f25f 100644 --- a/tests/csnode/backwardsync_tests.cpp +++ b/tests/csnode/backwardsync_tests.cpp @@ -1,9 +1,15 @@ #include #include +#include +#include +#include +#include #include #include +#include #include +#include #include #include #include @@ -117,6 +123,7 @@ cs::BackwardSync::Settings settingsOf(size_t batch = 100, size_t rate = 1000, si s.storePerTurn = perTurn; s.turnBudgetMs = 0; // no time limit unless a test sets one s.idleTurnBudgetMs = 0; + s.asyncFlush = false; // flushes on the calling thread unless a test says otherwise return s; } @@ -152,13 +159,22 @@ class BackfillTest : public ::testing::Test { ++storedCount_; return true; }; - hooks.flush = [this] { ++flushes_; return true; }; + hooks.flush = [this] { + if (flushGate_) { + flushGate_(); // a slow disk: the test decides when the flush ends + } + ++flushes_; + return true; + }; hooks.floor = [this] { return floor_; }; hooks.sendRequest = [this](const cs::PublicKey& to, const cs::PoolsRequestedSequences& seqs) { requests_.emplace_back(to, seqs); }; hooks.clock = [this] { return now_; }; - hooks.onLog = [this](const std::string& line) { logs_.push_back(line); }; + hooks.onLog = [this](const std::string& line) { + std::lock_guard lock(logsMutex_); // the flush thread logs too + logs_.push_back(line); + }; sync_ = std::make_unique(std::move(hooks), settings); } @@ -194,6 +210,7 @@ class BackfillTest : public ::testing::Test { } } size_t logged(const std::string& text) const { + std::lock_guard lock(logsMutex_); return static_cast(std::count_if(logs_.begin(), logs_.end(), [&](const std::string& l) { return l.find(text) != std::string::npos; })); } // ticks and answers until the backfill is done or `limit` simulated seconds pass @@ -208,11 +225,13 @@ class BackfillTest : public ::testing::Test { std::unique_ptr sync_; std::vector> requests_; std::vector logs_; + mutable std::mutex logsMutex_; + std::function flushGate_; cs::Sequence first_ = kFirst; cs::Sequence floor_ = 0; uint64_t now_ = 1'000'000; size_t storedCount_ = 0; - size_t flushes_ = 0; + std::atomic flushes_{0}; bool failStore_ = false; uint64_t storeDelayMs_ = 0; // a slow disk: simulated time spent in every store bool forwardSync_ = false; @@ -618,6 +637,62 @@ TEST_F(BackfillTest, AConfidantDefersTheFlush) { EXPECT_EQ(flushes_, before + 2); } +// the periodic flush runs on its own thread: storing goes on while it lasts, a second one waits +// for the first, and the last flush (end of the backfill) waits for the one under way +TEST_F(BackfillTest, ThePeriodicFlushRunsOnItsOwnThread) { + // shared with the flush thread by value: it may outlive this body if an assertion fails + struct Gate { + std::mutex m; + std::condition_variable cv; + bool release = false; + std::atomic started{0}; + void open() { + { + std::lock_guard lock(m); + release = true; + } + cv.notify_all(); + } + }; + const auto gate = std::make_shared(); + struct Release { // a failed assertion must not leave the flush thread waiting for ever + std::shared_ptr g; + ~Release() { g->open(); } + } guard{gate}; + flushGate_ = [gate] { + ++gate->started; + std::unique_lock lock(gate->m); + gate->cv.wait(lock, [&] { return gate->release; }); + }; + auto settings = settingsOf(100, 0, 50); + settings.asyncFlush = true; + make(settings); + tick(); + sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); + tick(cs::BackwardSync::kFlushMs + 1); // a flush is due: handed to the flush thread + const cs::Sequence storedWhenDue = first_; + for (int i = 0; i < 1000 && gate->started.load() == 0; ++i) { // up to 5 s on a busy runner + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + ASSERT_EQ(gate->started.load(), 1) << "the flush thread took the flush"; + + // the flush hangs: blocks are still stored, and no second flush starts + answerAll(); + tick(cs::BackwardSync::kFlushMs + 1); + tick(cs::BackwardSync::kFlushMs + 1); + EXPECT_LT(first_, storedWhenDue) << "storing goes on during the flush"; + EXPECT_EQ(gate->started.load(), 1) << "one flush at a time"; + EXPECT_EQ(flushes_.load(), 0u); + + gate->open(); + // the end of the backfill waits for the flush under way, then flushes the rest itself + runToEnd(); + EXPECT_TRUE(sync_->stats().done); + EXPECT_GE(flushes_.load(), 2u); + EXPECT_GE(logged("on the flush thread"), 1u); + sync_->stop(); // joins the flush thread +} + TEST_F(BackfillTest, ASlowTurnStopsAtItsTimeBudget) { auto settings = settingsOf(100, 0, 50); settings.idleTurnBudgetMs = 30;