Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions csnode/include/csnode/backwardsync.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@
#define BACKWARDSYNC_HPP

#include <atomic>
#include <condition_variable>
#include <cstdint>
#include <functional>
#include <map>
#include <mutex>
#include <optional>
#include <set>
#include <string>
#include <thread>
#include <vector>

#include <csdb/pool.hpp>
Expand All @@ -21,8 +24,9 @@ namespace cs {
// with BlockRequest, newest first, checks each against the hash chain and its signatures, and
// stores it strictly downwards, never applying it to the caches. Requests go out only while the
// node is synced; replies already asked for are stored in any case. Single-threaded: every call
// comes from the node's processor thread, but stop() and stats(). The chain, the network and the
// clock are reached only through Hooks.
// comes from the node's processor thread, but stop() and stats(); the periodic flush runs on a
// thread of its own (Settings::asyncFlush). The chain, the network and the clock are reached only
// through Hooks.
class BackwardSync {
public:
struct Peer {
Expand Down Expand Up @@ -70,6 +74,9 @@ class BackwardSync {
// the same while the node is not a confidant, with little consensus work: about 450 blocks/s
// on mainnet instead of about 250
uint64_t idleTurnBudgetMs = 90;
// the periodic flush (up to several seconds on mainnet) runs on a thread of its own instead of
// the processor thread; the flushes at the end and at a stop stay where they are
bool asyncFlush = true;
};

struct Stats {
Expand Down Expand Up @@ -107,6 +114,9 @@ class BackwardSync {
static constexpr uint64_t kSlowTurnMs = 200; // a longer turn is logged, with where its time went

BackwardSync(Hooks hooks, Settings settings);
~BackwardSync();
BackwardSync(const BackwardSync&) = delete;
BackwardSync& operator=(const BackwardSync&) = delete;

// every 500 ms
void tick(const Context& ctx);
Expand Down Expand Up @@ -149,6 +159,8 @@ class BackwardSync {
void mark(const PublicKey& peer, uint64_t until, const std::string& why);
void expire();
void flushIfDue(bool force);
void flushWorker();
void stopFlusher();
void request(const Context& ctx);
std::optional<PublicKey> pickPeer(const Context& ctx, Sequence highest);
void finish();
Expand Down Expand Up @@ -198,6 +210,16 @@ class BackwardSync {
std::atomic<size_t> slowTurns_{0};
bool trusted_ = false; // from the last tick: the budget of the turns until the next one
std::atomic<uint64_t> longestFlushUs_{0};

// the flush thread: started at the first periodic flush, joined by stop() and the destructor
std::mutex flushMutex_;
std::condition_variable flushCv_;
std::thread flushThread_;
bool flushRequested_ = false; // the fields below: under flushMutex_
bool flushRunning_ = false;
bool flushStopping_ = false;
size_t flushBlocks_ = 0; // what the requested flush covers, for its log line
Sequence flushDown_ = 0;
};

} // namespace cs
Expand Down
3 changes: 2 additions & 1 deletion csnode/include/csnode/blockhashes.hpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#ifndef BLOCKHASHES_HPP
#define BLOCKHASHES_HPP

#include <atomic>
#include <functional>
#include <map>
#include <memory>
Expand Down Expand Up @@ -68,7 +69,7 @@ private slots:
std::string path_;
std::unique_ptr<cs::Lmdb> seqDb_;
std::unique_ptr<cs::Lmdb> hashDb_;
size_t failures_ = 0;
std::atomic<size_t> failures_{0}; // the backfill flush thread reports LMDB failures too
};
} // namespace cs

Expand Down
96 changes: 96 additions & 0 deletions csnode/src/backwardsync.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
#include <algorithm>
#include <chrono>
#include <limits>
#include <stdexcept>
#include <system_error>

#include <base58.h>

Expand Down Expand Up @@ -60,8 +62,70 @@ bool BackwardSync::isActive() const {
return started_ && !done_ && !stopped_;
}

BackwardSync::~BackwardSync() {
stopFlusher();
}

void BackwardSync::stop() {
stopped_ = true;
stopFlusher();
}

// a flush under way ends first: nothing flushes the storage behind the node's back once it stops
void BackwardSync::stopFlusher() {
std::thread worker;
{
std::lock_guard<std::mutex> lock(flushMutex_);
flushStopping_ = true;
worker = std::move(flushThread_);
}
flushCv_.notify_all();
if (worker.joinable()) {
if (worker.get_id() == std::this_thread::get_id()) {
worker.detach(); // stopped from the flush thread itself: it returns on its own
}
else {
worker.join();
}
}
}

void BackwardSync::flushWorker() {
std::unique_lock<std::mutex> lock(flushMutex_);
for (;;) {
flushCv_.wait(lock, [this] { return flushRequested_ || flushStopping_; });
if (flushStopping_) {
flushRequested_ = false; // the node flushes once more before its storage closes
flushCv_.notify_all();
return;
}
flushRequested_ = false;
flushRunning_ = true;
const size_t blocks = flushBlocks_;
const Sequence down = flushDown_;
lock.unlock();

const auto at = std::chrono::steady_clock::now();
bool ok = false;
try {
ok = !hooks_.flush || hooks_.flush();
}
catch (const std::exception& e) {
log(std::string("BACKFILL: the flush thread caught: ") + e.what(), true);
}
const auto us = static_cast<uint64_t>(
std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - at).count());
if (us > longestFlushUs_) {
longestFlushUs_ = us;
}
log("BACKFILL: flushed, " + std::to_string(blocks) + " blocks since the last flush, down to #" + std::to_string(down) +
" in " + std::to_string(us / 1000) + " ms on the flush thread" + (ok ? "" : ": the storage flush failed"),
!ok);

lock.lock();
flushRunning_ = false;
flushCv_.notify_all();
}
}

BackwardSync::Stats BackwardSync::stats() const {
Expand Down Expand Up @@ -590,6 +654,11 @@ void BackwardSync::drain() {
}

void BackwardSync::flushIfDue(bool force) {
if (force && settings_.asyncFlush) {
// the last flush covers every block: after the one under way, if any
std::unique_lock<std::mutex> lock(flushMutex_);
flushCv_.wait(lock, [this] { return (!flushRunning_ && !flushRequested_) || (flushStopping_ && !flushRunning_); });
}
if (storedSinceFlush_ == 0) {
return;
}
Expand All @@ -600,6 +669,33 @@ void BackwardSync::flushIfDue(bool force) {
if (!force && trusted_ && t < lastFlushAt_ + kTrustedFlushDeferMs) {
return; // consensus first: flushed at the first turn outside the trusted set
}
if (!force && settings_.asyncFlush) {
std::lock_guard<std::mutex> lock(flushMutex_);
if (flushStopping_) {
return; // stopped: the node flushes once more itself before its storage closes (Node::destroy)
}
if (flushRunning_ || flushRequested_) {
return; // one at a time: the blocks stored meanwhile go with the next one
}
if (!flushThread_.joinable()) {
try {
flushThread_ = std::thread([this] { flushWorker(); });
}
catch (const std::system_error& e) {
log(std::string("BACKFILL: no flush thread (") + e.what() + "): flushing on the processor thread", true);
settings_.asyncFlush = false;
}
}
if (flushThread_.joinable()) {
flushRequested_ = true;
flushBlocks_ = storedSinceFlush_;
flushDown_ = first_;
lastFlushAt_ = t;
storedSinceFlush_ = 0;
flushCv_.notify_one();
return;
}
}
// on the processor thread too, outside the turn budget: timed, so that a slow disk shows
const auto flushAt = std::chrono::steady_clock::now();
const bool ok = !hooks_.flush || hooks_.flush();
Expand Down
3 changes: 3 additions & 0 deletions csnode/src/node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,9 @@ void Node::destroy() {
solver_->finish();
cswarning() << "[SOLVER STOPPED]";

if (backwardSync_) {
backwardSync_->stop(); // its flush thread ends before the storage closes
}
if (backwardSync_ && backwardSync_->stats().stored > 0) {
// the index and the block hashes of the blocks backfilled since the last flush, before the storage closes
blockChain_.flushBackfill();
Expand Down
1 change: 1 addition & 0 deletions docs/CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ lines under the new release.
| #145 | csnode: backward sync asks for no more blocks than its buffer holds (a request could overrun it by one batch); blocks a reply brings when the buffer is full are asked for again at once and no longer count against the peer that sent them (with slow stores, on mainnet after a restart, peers that had answered were left out for an hour and the backfill fell to ~30 blocks/s) |
| #146 | csnode: the backfill turn budget depends on the node's role: 50 ms while it is a confidant (`backfill_turn_budget_ms`), 90 ms otherwise (`backfill_idle_turn_budget_ms`); on mainnet a node outside the trusted set backfills at about 450 blocks/s instead of about 250 |
| #147 | csnode: while the node is a confidant, the backfill storage flush (median 358 ms, up to 2.4 s on mainnet, on the processor thread) waits until the node is out of the trusted set, 10 minutes at most; blocks not flushed are fetched again after a crash |
| #148 | csnode: the periodic backfill flush runs on a thread of its own instead of the processor thread (on mainnet it took up to 5.6 s, with the storing of blocks waiting); one at a time, the final flush waits for the one under way, and the node stops the thread before its storage closes |

## community.14 — 2026-10-27

Expand Down
2 changes: 2 additions & 0 deletions docs/storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ official node.
| `backfill_turn_budget_ms` | `50` | While the node is a confidant, a turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`: consensus keeps most of the thread; on mainnet about 250 blocks/s. |
| `backfill_idle_turn_budget_ms` | `90` | The same while the node is not a confidant, with little consensus work: on mainnet about 450 blocks/s. |

The periodic flush of the backfilled blocks (every 10,000 blocks or a minute; up to several seconds on mainnet) runs on a thread of its own, not on the thread that runs consensus; one flush at a time, and while the node is a confidant it waits until the node is out of the trusted set (10 minutes at most). The log line says `on the flush thread`. One wait remains possible: an LMDB index that must grow its map while the flush thread syncs it waits for that sync (rare: the map grows in steps).

### Choosing a backend

| Backend | Use it for | Notes |
Expand Down
81 changes: 78 additions & 3 deletions tests/csnode/backwardsync_tests.cpp
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
#include <gtest/gtest.h>

#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <future>
#include <map>
#include <memory>
#include <mutex>
#include <set>
#include <thread>
#include <string>
#include <utility>
#include <vector>
Expand Down Expand Up @@ -117,6 +123,7 @@ cs::BackwardSync::Settings settingsOf(size_t batch = 100, size_t rate = 1000, si
s.storePerTurn = perTurn;
s.turnBudgetMs = 0; // no time limit unless a test sets one
s.idleTurnBudgetMs = 0;
s.asyncFlush = false; // flushes on the calling thread unless a test says otherwise
return s;
}

Expand Down Expand Up @@ -152,13 +159,22 @@ class BackfillTest : public ::testing::Test {
++storedCount_;
return true;
};
hooks.flush = [this] { ++flushes_; return true; };
hooks.flush = [this] {
if (flushGate_) {
flushGate_(); // a slow disk: the test decides when the flush ends
}
++flushes_;
return true;
};
hooks.floor = [this] { return floor_; };
hooks.sendRequest = [this](const cs::PublicKey& to, const cs::PoolsRequestedSequences& seqs) {
requests_.emplace_back(to, seqs);
};
hooks.clock = [this] { return now_; };
hooks.onLog = [this](const std::string& line) { logs_.push_back(line); };
hooks.onLog = [this](const std::string& line) {
std::lock_guard<std::mutex> lock(logsMutex_); // the flush thread logs too
logs_.push_back(line);
};
sync_ = std::make_unique<cs::BackwardSync>(std::move(hooks), settings);
}

Expand Down Expand Up @@ -194,6 +210,7 @@ class BackfillTest : public ::testing::Test {
}
}
size_t logged(const std::string& text) const {
std::lock_guard<std::mutex> lock(logsMutex_);
return static_cast<size_t>(std::count_if(logs_.begin(), logs_.end(), [&](const std::string& l) { return l.find(text) != std::string::npos; }));
}
// ticks and answers until the backfill is done or `limit` simulated seconds pass
Expand All @@ -208,11 +225,13 @@ class BackfillTest : public ::testing::Test {
std::unique_ptr<cs::BackwardSync> sync_;
std::vector<std::pair<cs::PublicKey, cs::PoolsRequestedSequences>> requests_;
std::vector<std::string> logs_;
mutable std::mutex logsMutex_;
std::function<void()> flushGate_;
cs::Sequence first_ = kFirst;
cs::Sequence floor_ = 0;
uint64_t now_ = 1'000'000;
size_t storedCount_ = 0;
size_t flushes_ = 0;
std::atomic<size_t> flushes_{0};
bool failStore_ = false;
uint64_t storeDelayMs_ = 0; // a slow disk: simulated time spent in every store
bool forwardSync_ = false;
Expand Down Expand Up @@ -618,6 +637,62 @@ TEST_F(BackfillTest, AConfidantDefersTheFlush) {
EXPECT_EQ(flushes_, before + 2);
}

// the periodic flush runs on its own thread: storing goes on while it lasts, a second one waits
// for the first, and the last flush (end of the backfill) waits for the one under way
TEST_F(BackfillTest, ThePeriodicFlushRunsOnItsOwnThread) {
// shared with the flush thread by value: it may outlive this body if an assertion fails
struct Gate {
std::mutex m;
std::condition_variable cv;
bool release = false;
std::atomic<int> started{0};
void open() {
{
std::lock_guard<std::mutex> lock(m);
release = true;
}
cv.notify_all();
}
};
const auto gate = std::make_shared<Gate>();
struct Release { // a failed assertion must not leave the flush thread waiting for ever
std::shared_ptr<Gate> g;
~Release() { g->open(); }
} guard{gate};
flushGate_ = [gate] {
++gate->started;
std::unique_lock<std::mutex> lock(gate->m);
gate->cv.wait(lock, [&] { return gate->release; });
};
auto settings = settingsOf(100, 0, 50);
settings.asyncFlush = true;
make(settings);
tick();
sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second));
tick(cs::BackwardSync::kFlushMs + 1); // a flush is due: handed to the flush thread
const cs::Sequence storedWhenDue = first_;
for (int i = 0; i < 1000 && gate->started.load() == 0; ++i) { // up to 5 s on a busy runner
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
ASSERT_EQ(gate->started.load(), 1) << "the flush thread took the flush";

// the flush hangs: blocks are still stored, and no second flush starts
answerAll();
tick(cs::BackwardSync::kFlushMs + 1);
tick(cs::BackwardSync::kFlushMs + 1);
EXPECT_LT(first_, storedWhenDue) << "storing goes on during the flush";
EXPECT_EQ(gate->started.load(), 1) << "one flush at a time";
EXPECT_EQ(flushes_.load(), 0u);

gate->open();
// the end of the backfill waits for the flush under way, then flushes the rest itself
runToEnd();
EXPECT_TRUE(sync_->stats().done);
EXPECT_GE(flushes_.load(), 2u);
EXPECT_GE(logged("on the flush thread"), 1u);
sync_->stop(); // joins the flush thread
}

TEST_F(BackfillTest, ASlowTurnStopsAtItsTimeBudget) {
auto settings = settingsOf(100, 0, 50);
settings.idleTurnBudgetMs = 30;
Expand Down
Loading