Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions client/config/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1071,6 +1071,7 @@ void Config::readBackfillSettings(const boost::property_tree::ptree& data, Stora
number("backfill_rate", storage.backfillRate, 0, 1'000'000);
number("backfill_store_per_turn", storage.backfillStorePerTurn, 1, 1000);
number("backfill_turn_budget_ms", storage.backfillTurnBudgetMs, 0, 1000);
number("backfill_idle_turn_budget_ms", storage.backfillIdleTurnBudgetMs, 0, 1000);
}

namespace {
Expand Down
3 changes: 2 additions & 1 deletion client/config/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ struct StorageData {
size_t backfillBatch = 100; // blocks per request, 1..1000
size_t backfillRate = 1000; // blocks requested per second, 0 = unlimited
size_t backfillStorePerTurn = 50; // blocks verified and stored per processor-thread turn, 1..1000
size_t backfillTurnBudgetMs = 50; // and ms per turn, 0 (no limit)..1000
size_t backfillTurnBudgetMs = 50; // and ms per turn while a confidant, 0 (no limit)..1000
size_t backfillIdleTurnBudgetMs = 90; // and ms per turn otherwise, 0 (no limit)..1000
};

// [snapshot]: snapshots served by this node to its p2p neighbours and fetched from them at a first
Expand Down
10 changes: 8 additions & 2 deletions csnode/include/csnode/backwardsync.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class BackwardSync {
bool bootstrapping = false; // the node fetches a snapshot: its chain is not open
RoundNumber round = 0;
Sequence lastSeq = 0;
bool trusted = false; // the node is a confidant (or the writer) of this round
};

// the first stored block and the hash of the block below it (empty: unknown)
Expand All @@ -62,9 +63,13 @@ class BackwardSync {
// blocks verified and stored per call (a tick or a reply), so that the processor thread,
// which also runs consensus, is never held long; the rest waits for the next call
size_t storePerTurn = 50;
// and for at most this long (ms) per call, 0: no limit, at most 1000. A turn ends after the block that
// passes it: a mainnet turn of 50 blocks took up to 1.7 s, more than a round
// and for at most this long (ms) per call while the node is a confidant, 0: no limit, at most
// 1000. A turn ends after the block that passes it: a mainnet turn of 50 blocks took up to 1.7 s,
// more than a round
uint64_t turnBudgetMs = 50;
// the same while the node is not a confidant, with little consensus work: about 450 blocks/s
// on mainnet instead of about 250
uint64_t idleTurnBudgetMs = 90;
};

struct Stats {
Expand Down Expand Up @@ -188,6 +193,7 @@ class BackwardSync {
std::atomic<uint64_t> busyUs_{0};
std::atomic<uint64_t> longestTurnUs_{0};
std::atomic<size_t> slowTurns_{0};
bool trusted_ = false; // from the last tick: the budget of the turns until the next one
std::atomic<uint64_t> longestFlushUs_{0};
};

Expand Down
5 changes: 4 additions & 1 deletion csnode/src/backwardsync.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ BackwardSync::BackwardSync(Hooks hooks, Settings settings)
settings_.batch = std::clamp<size_t>(settings_.batch, 1, 1000);
settings_.storePerTurn = std::clamp<size_t>(settings_.storePerTurn, 1, 1000);
settings_.turnBudgetMs = std::min<uint64_t>(settings_.turnBudgetMs, 1000);
settings_.idleTurnBudgetMs = std::min<uint64_t>(settings_.idleTurnBudgetMs, 1000);
}

uint64_t BackwardSync::now() const {
Expand Down Expand Up @@ -114,6 +115,7 @@ void BackwardSync::tick(const Context& ctx) {
if (stopped_ || done_) {
return;
}
trusted_ = ctx.trusted;
if (inFlight_.empty() && buffer_.empty() && !refreshAnchor(ctx)) {
return;
}
Expand Down Expand Up @@ -503,13 +505,14 @@ void BackwardSync::drain() {
};
const auto startedAt = Clock::now();
const uint64_t startedMs = now(); // the budget and the slow-turn decision: the hooks' clock (tests drive it)
const uint64_t budgetMs = trusted_ ? settings_.turnBudgetMs : settings_.idleTurnBudgetMs;
size_t storedNow = 0;
size_t transactions = 0;
uint64_t verifyUs = 0;
uint64_t storeUs = 0;
uint64_t confirmUs = 0;
while (!done_ && storedNow < settings_.storePerTurn) {
if (storedNow > 0 && settings_.turnBudgetMs > 0 && now() >= startedMs + settings_.turnBudgetMs) {
if (storedNow > 0 && budgetMs > 0 && now() >= startedMs + budgetMs) {
break;
}
const auto it = buffer_.find(next_);
Expand Down
2 changes: 2 additions & 0 deletions csnode/src/node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,7 @@ void Node::startBackfill() {
settings.rate = sto.backfillRate;
settings.storePerTurn = sto.backfillStorePerTurn;
settings.turnBudgetMs = sto.backfillTurnBudgetMs;
settings.idleTurnBudgetMs = sto.backfillIdleTurnBudgetMs;
// created under the lock that stop() takes to read the pointer (finishBootstrap runs this on the
// processor thread while a stop may come from another one)
std::lock_guard<std::mutex> lock(backfillTimerMutex_);
Expand Down Expand Up @@ -482,6 +483,7 @@ void Node::onBackfillTick() {
ctx.bootstrapping = bootstrapping_;
ctx.round = cs::Conveyer::instance().currentRoundNumber();
ctx.lastSeq = blockChain_.getLastSeq();
ctx.trusted = myLevel_ != Level::Normal;
backwardSync_->tick(ctx);
}

Expand Down
1 change: 1 addition & 0 deletions docs/CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ lines under the new release.
| #143 | node: a `Utility` message (ban or unban a node key) is obeyed only with the blockchain key's signature, an exact length and a round at most 100 rounds old (its signature was not checked since upstream `af4de3f8`, so it was not authenticated) |
| #144 | csnode: stream reads bounded by the data: `IDataStream` checks a length without overflow, a vector reserves no more than the bytes left and stops at the first missing element, a smart stage-one message whose fee count exceeds its size or that lacks its hash is refused, the API checks a string field's length before reading it |
| #145 | csnode: backward sync asks for no more blocks than its buffer holds (a request could overrun it by one batch); blocks a reply brings when the buffer is full are asked for again at once and no longer count against the peer that sent them (with slow stores, on mainnet after a restart, peers that had answered were left out for an hour and the backfill fell to ~30 blocks/s) |
| #146 | csnode: the backfill turn budget depends on the node's role: 50 ms while it is a confidant (`backfill_turn_budget_ms`), 90 ms otherwise (`backfill_idle_turn_budget_ms`); on mainnet a node outside the trusted set backfills at about 450 blocks/s instead of about 250 |

## community.14 — 2026-10-27

Expand Down
2 changes: 1 addition & 1 deletion docs/differences.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ each key and its default.
| `db_backend` | Chooses the block database: `berkeleydb` (the default, the official format), `lmdb` or `rocksdb`. RocksDB is about a third of the BerkeleyDB size on mainnet. |
| `prune_keep_blocks` | Pruned storage: the node keeps only the latest blocks and removes older ones at each periodic checkpoint. Pruned storage runs on LMDB. |
| `async_write_queue_size`, `write_batch_size`, `rocksdb_block_cache_mb`, `rocksdb_memtable_mb` | Tune the database writer and RocksDB. |
| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn`, `backfill_turn_budget_ms` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. |
| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn`, `backfill_turn_budget_ms`, `backfill_idle_turn_budget_ms` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. |

### Converting the block database

Expand Down
3 changes: 2 additions & 1 deletion docs/storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ official node.
| `backfill_batch` | `100` | Blocks per request, `1` to `1000`. |
| `backfill_rate` | `1000` | Blocks requested per second over all peers (`0` = no limit). |
| `backfill_store_per_turn` | `50` | Blocks checked and stored per turn of the thread that also runs consensus, `1` to `1000`; the rest waits for the next turn, 100 ms later. A turn over 200 ms is logged (`BACKFILL: slow turn of ...`) with the time spent verifying, storing and checking confirmations. |
| `backfill_turn_budget_ms` | `50` | A turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`. A shorter turn leaves more of the thread to consensus; on mainnet 50 ms gives about 400 blocks/s. |
| `backfill_turn_budget_ms` | `50` | While the node is a confidant, a turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`: consensus keeps most of the thread; on mainnet about 250 blocks/s. |
| `backfill_idle_turn_budget_ms` | `90` | The same while the node is not a confidant, with little consensus work: on mainnet about 450 blocks/s. |

### Choosing a backend

Expand Down
5 changes: 4 additions & 1 deletion tests/csnode/backfillconfig_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,14 @@ TEST(BackfillConfig, Defaults) {
EXPECT_EQ(d.backfillRate, 1000u);
EXPECT_EQ(d.backfillStorePerTurn, 50u);
EXPECT_EQ(d.backfillTurnBudgetMs, 50u);
EXPECT_EQ(d.backfillIdleTurnBudgetMs, 90u);
}

TEST(BackfillConfig, EveryKey) {
const auto d = parse("backfill=off\nbackfill_batch=250\nbackfill_rate=0\nbackfill_store_per_turn=20\nbackfill_turn_budget_ms=0\n");
const auto d = parse("backfill=off\nbackfill_batch=250\nbackfill_rate=0\nbackfill_store_per_turn=20\nbackfill_turn_budget_ms=0\nbackfill_idle_turn_budget_ms=200\n");
EXPECT_FALSE(d.backfill);
EXPECT_EQ(d.backfillTurnBudgetMs, 0u);
EXPECT_EQ(d.backfillIdleTurnBudgetMs, 200u);
EXPECT_EQ(d.backfillBatch, 250u);
EXPECT_EQ(d.backfillRate, 0u);
EXPECT_EQ(d.backfillStorePerTurn, 20u);
Expand All @@ -44,6 +46,7 @@ TEST(BackfillConfig, BadValuesKeepTheDefaultOrAreClamped) {
EXPECT_TRUE(parse("backfill=maybe\n").backfill);
EXPECT_EQ(parse("backfill_batch=5000\n").backfillBatch, 1000u);
EXPECT_EQ(parse("backfill_turn_budget_ms=5000\n").backfillTurnBudgetMs, 1000u);
EXPECT_EQ(parse("backfill_idle_turn_budget_ms=5000\n").backfillIdleTurnBudgetMs, 1000u);
EXPECT_EQ(parse("backfill_batch=0\n").backfillBatch, 1u);
EXPECT_EQ(parse("backfill_batch=many\n").backfillBatch, 100u);
EXPECT_EQ(parse("backfill_rate=-5\n").backfillRate, 1000u);
Expand Down
22 changes: 20 additions & 2 deletions tests/csnode/backwardsync_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ cs::BackwardSync::Settings settingsOf(size_t batch = 100, size_t rate = 1000, si
s.rate = rate;
s.storePerTurn = perTurn;
s.turnBudgetMs = 0; // no time limit unless a test sets one
s.idleTurnBudgetMs = 0;
return s;
}

Expand Down Expand Up @@ -170,6 +171,7 @@ class BackfillTest : public ::testing::Test {
ctx.lastSeq = kHead;
ctx.forwardSyncRunning = forwardSync_;
ctx.bootstrapping = bootstrapping_;
ctx.trusted = trusted_;
return ctx;
}
void tick(uint64_t advanceMs = 500) {
Expand Down Expand Up @@ -215,6 +217,7 @@ class BackfillTest : public ::testing::Test {
uint64_t storeDelayMs_ = 0; // a slow disk: simulated time spent in every store
bool forwardSync_ = false;
bool bootstrapping_ = false;
bool trusted_ = false;
};

} // namespace
Expand Down Expand Up @@ -559,7 +562,7 @@ TEST_F(BackfillTest, TheWorkPerTurnIsBounded) {
// so no block is dropped and no peer that answered is taken for one that did not
TEST_F(BackfillTest, SlowStoresNeitherDropBlocksNorMarkPeers) {
auto settings = settingsOf(20, 0, 50);
settings.turnBudgetMs = 30;
settings.idleTurnBudgetMs = 30;
make(settings);
storeDelayMs_ = 12; // three blocks a turn
for (int i = 0; i < 400 && !sync_->stats().done; ++i) {
Expand All @@ -573,9 +576,24 @@ TEST_F(BackfillTest, SlowStoresNeitherDropBlocksNorMarkPeers) {
EXPECT_EQ(logged("did not answer"), 0u);
}

// a confidant keeps the shorter budget, a node outside the round's trusted the longer one
TEST_F(BackfillTest, AConfidantTakesTheShorterBudget) {
auto settings = settingsOf(100, 0, 50);
settings.turnBudgetMs = 20;
settings.idleTurnBudgetMs = 60;
make(settings);
storeDelayMs_ = 10;
tick();
sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second));
EXPECT_EQ(kFirst - first_, 6u); // not trusted: 60 ms, six blocks of 10 ms
trusted_ = true;
tick();
EXPECT_EQ(kFirst - first_, 8u); // trusted: 20 ms, two more
}

TEST_F(BackfillTest, ASlowTurnStopsAtItsTimeBudget) {
auto settings = settingsOf(100, 0, 50);
settings.turnBudgetMs = 30;
settings.idleTurnBudgetMs = 30;
make(settings);
storeDelayMs_ = 12;
tick();
Expand Down
Loading