From 92e7cb3d4cc9886f1f8870726d48c42454b8d5ba Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 2 Nov 2026 07:20:22 +0100 Subject: [PATCH 1/4] csnode: backfill signatures verified on worker threads The backfill held the processor thread 97-98 % of the time on mainnet, about 2.5 ms a block: 1 ms for the block's signatures, 1 ms for the round confirmations of the block stored before, 0.5 ms to store it. A pool of verifier threads (Settings::verifyThreads, [storage] backfill_verify_threads, 0..16, default min(4, cores - 1); 0 keeps everything on the processor thread) now checks the signatures of each block as it enters the buffer, highest blocks first, on a copy of the block whose hash was computed before it was shared. drain() keeps the sequence and hash-chain checks, takes the result when it is ready and ends the turn otherwise, never waiting. The result lives in a shared_ptr owned by the buffer entry: a block refused, replaced or dropped leaves its result to nobody, and the verifier skips a check nobody waits for. The round confirmations of the block stored before are checked on the verifier threads after the store, with the same predicate; a mismatch is counted and logged as before, never refused. stop() and the destructor join the threads (a check under way ends first, the queued ones are dropped); after the end of the backfill the threads finish the queued confirmations and return. The per-phase averages of #151 add the verifier threads' time atomically; busyUs and the busy share count only the processor thread. Co-Authored-By: Claude Opus 5.5 --- client/config/config.cpp | 19 ++- client/config/config.hpp | 3 + csnode/include/csnode/backwardsync.hpp | 66 +++++++- csnode/src/backwardsync.cpp | 209 +++++++++++++++++++++---- csnode/src/node.cpp | 3 + docs/differences.md | 2 +- docs/storage.md | 7 + tests/csnode/backfillconfig_tests.cpp | 10 ++ tests/csnode/backwardsync_tests.cpp | 176 ++++++++++++++++++--- 9 files changed, 432 insertions(+), 63 deletions(-) diff --git a/client/config/config.cpp b/client/config/config.cpp index d2c6b627d..76cfaba13 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -1046,9 +1046,10 @@ void Config::readBackfillSettings(const boost::property_tree::ptree& data, Stora cswarning() << "config: [storage] backfill=" << text << " is not on or off, using " << (storage.backfill ? "on" : "off"); } } - auto number = [&data](const char* key, size_t& value, size_t min, size_t max) { + // false: missing, or not a number (a warning, the value as it was, or `fallback` when given) + auto number = [&data](const char* key, size_t& value, size_t min, size_t max, const char* fallback = nullptr) { if (!data.count(key)) { - return; + return false; } const auto text = data.get(key); try { @@ -1060,18 +1061,28 @@ void Config::readBackfillSettings(const boost::property_tree::ptree& data, Stora cswarning() << "config: [storage] " << key << "=" << text << " is not in [" << min << ", " << max << "], using " << clamped; } value = static_cast(clamped); - return; + return true; } } catch (const std::exception&) { } - cswarning() << "config: [storage] " << key << "=" << text << " is not a number, using " << value; + if (fallback) { + cswarning() << "config: [storage] " << key << "=" << text << " is not a number, using " << fallback; + } + else { + cswarning() << "config: [storage] " << key << "=" << text << " is not a number, using " << value; + } + return false; }; number("backfill_batch", storage.backfillBatch, 1, 1000); number("backfill_rate", storage.backfillRate, 0, 1'000'000); number("backfill_store_per_turn", storage.backfillStorePerTurn, 1, 1000); number("backfill_turn_budget_ms", storage.backfillTurnBudgetMs, 0, 1000); number("backfill_idle_turn_budget_ms", storage.backfillIdleTurnBudgetMs, 0, 1000); + size_t threads = 0; + if (number("backfill_verify_threads", threads, 0, 16, "the default (min(4, cores - 1))")) { + storage.backfillVerifyThreads = threads; + } } namespace { diff --git a/client/config/config.hpp b/client/config/config.hpp index df76a71f3..422f38c62 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -2,6 +2,7 @@ #ifndef CONFIG_HPP #define CONFIG_HPP +#include #include #include @@ -76,6 +77,8 @@ struct StorageData { size_t backfillStorePerTurn = 50; // blocks verified and stored per processor-thread turn, 1..1000 size_t backfillTurnBudgetMs = 50; // and ms per turn while a confidant, 0 (no limit)..1000 size_t backfillIdleTurnBudgetMs = 90; // and ms per turn otherwise, 0 (no limit)..1000 + // threads that check the backfilled blocks' signatures, 0 (the processor thread)..16; none: min(4, cores - 1) + std::optional backfillVerifyThreads; }; // [snapshot]: snapshots served by this node to its p2p neighbours and fetched from them at a first diff --git a/csnode/include/csnode/backwardsync.hpp b/csnode/include/csnode/backwardsync.hpp index 840c34f95..83321f67d 100644 --- a/csnode/include/csnode/backwardsync.hpp +++ b/csnode/include/csnode/backwardsync.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -19,14 +20,18 @@ namespace cs { +// min(4, cores - 1), at least 1: the verifier threads of a backward sync unless configured +size_t defaultBackfillVerifyThreads(); + // Backward sync (docs/specs/2026-10-17-backward-sync-design.md): a node whose storage starts above // its floor (block 0 for a full node) fetches the blocks below its lowest one from its neighbours // with BlockRequest, newest first, checks each against the hash chain and its signatures, and // stores it strictly downwards, never applying it to the caches. Requests go out only while the // node is synced; replies already asked for are stored in any case. Single-threaded: every call // comes from the node's processor thread, but stop() and stats(); the periodic flush runs on a -// thread of its own (Settings::asyncFlush). The chain, the network and the clock are reached only -// through Hooks. +// thread of its own (Settings::asyncFlush), and the signatures are checked on verifier threads +// (Settings::verifyThreads), which see only copies of the blocks and hand back their results through +// shared state. The chain, the network and the clock are reached only through Hooks. class BackwardSync { public: struct Peer { @@ -59,6 +64,8 @@ class BackwardSync { std::function sendRequest; std::function clock; // ms, steady; default: steady_clock std::function onLog; // optional: every BACKFILL line, for tests + // optional, for tests: called on a verifier thread before it checks a block's signatures + std::function onVerify; }; struct Settings { @@ -77,6 +84,10 @@ class BackwardSync { // the periodic flush (up to several seconds on mainnet) runs on a thread of its own instead of // the processor thread; the flushes at the end and at a stop stay where they are bool asyncFlush = true; + // threads that check the signatures of the buffered blocks and the round confirmations of the + // stored ones (about 2 ms a block on mainnet), so that the processor thread only links and + // stores them; 0: everything on the processor thread, at most 16 + size_t verifyThreads = defaultBackfillVerifyThreads(); }; struct Stats { @@ -87,7 +98,7 @@ class BackwardSync { size_t confirmationErrors = 0; size_t inFlight = 0; size_t buffered = 0; - uint64_t busyUs = 0; // processor-thread time spent verifying and storing + uint64_t busyUs = 0; // processor-thread time spent verifying and storing (not the verifier threads') uint64_t longestTurnUs = 0; // the longest single call size_t slowTurns = 0; // calls longer than kSlowTurnMs uint64_t longestFlushUs = 0; // the longest storage flush (every kFlushBlocks or kFlushMs) @@ -112,6 +123,7 @@ class BackwardSync { static constexpr uint64_t kTrustedFlushDeferMs = 600'000; static constexpr size_t kProgressBlocks = 100'000; static constexpr uint64_t kSlowTurnMs = 200; // a longer turn is logged, with where its time went + static constexpr size_t kMaxVerifyThreads = 16; BackwardSync(Hooks hooks, Settings settings); ~BackwardSync(); @@ -143,17 +155,44 @@ class BackwardSync { bool paced = false; // AwaitAnswer seen: its anti-duplicate log is active (order 35) bool awaitingAnswer = false; // AwaitAnswer for the batch in flight }; + // a block's signature check on a verifier thread: `why` is written before `ready` is set, and + // read by the processor thread only after it sees `ready` + struct SignatureCheck { + std::atomic ready{false}; + std::string why; // empty: the signatures verify + }; struct Buffered { csdb::Pool pool; PublicKey peer; + // none: checked on the processor thread when the block is stored. Owned by this entry: a + // block replaced or dropped leaves its result to nobody + std::shared_ptr check; + }; + // the work of a verifier thread, on copies of the blocks (csdb::Pool shares its data with an + // atomic count; each thread reads only through its own copy, the hash computed beforehand) + struct VerifyJob { + csdb::Pool block; // its signatures, or its round confirmations + csdb::Pool below; // confirmations: the block whose confidants signed them + std::shared_ptr check; // signatures: where the result goes }; uint64_t now() const; void log(const std::string& text, bool error = false) const; bool refreshAnchor(const Context& ctx); void drain(); - // why the block below the last stored one is refused, empty if it is accepted - std::string verify(const csdb::Pool& block) const; + // why the block below the last stored one does not link to the chain, empty if it does + std::string verifyLink(const csdb::Pool& block) const; + // why its signatures are refused, empty if they verify; any thread + static std::string verifySignatures(const csdb::Pool& block); + // the round confirmations of `above` against the confidants of `block`, the block below it: + // counted and logged when they do not verify, never refused; any thread + void checkConfirmations(const csdb::Pool& above, const csdb::Pool& block); + // hands a job to the verifier threads, started at the first one; false: none runs, the caller + // does the work itself + bool submit(VerifyJob&& job); + void verifyWorker(); + void runJob(VerifyJob& job); + void stopVerifiers(); void reject(const PublicKey& peer, Sequence seq, const std::string& why); void reassign(const PublicKey& peer); void mark(const PublicKey& peer, uint64_t until, const std::string& why); @@ -194,10 +233,11 @@ class BackwardSync { size_t storedSinceFlush_ = 0; uint64_t progressAt_ = 0; size_t progressStored_ = 0; - // the phases of storing since the last progress line, for its averages per block - uint64_t phaseVerifyUs_ = 0; + // the phases of storing since the last progress line, for its averages per block; verify and + // confirmations are added by the verifier threads when they run there + std::atomic phaseVerifyUs_{0}; uint64_t phaseStoreUs_ = 0; - uint64_t phaseConfirmUs_ = 0; + std::atomic phaseConfirmUs_{0}; uint64_t phaseTransactions_ = 0; uint64_t progressBusyUs_ = 0; @@ -226,6 +266,16 @@ class BackwardSync { bool flushStopping_ = false; size_t flushBlocks_ = 0; // what the requested flush covers, for its log line Sequence flushDown_ = 0; + + // the verifier threads: started at the first job, joined by stop() and the destructor; after the + // end of the backfill they finish the jobs queued and return + std::mutex verifyMutex_; + std::condition_variable verifyCv_; + std::vector verifiers_; // the fields below: under verifyMutex_ + // the highest blocks first: the storage waits for the highest buffered one + std::multimap> verifyQueue_; + bool verifyStopping_ = false; // stop(): the jobs queued are dropped + bool verifyClosing_ = false; // finish(): the jobs queued are done, then the threads return }; } // namespace cs diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index 93726dc06..d8b539c1e 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -24,8 +24,19 @@ std::string keyText(const PublicKey& key) { } constexpr uint64_t kForever = std::numeric_limits::max(); + +using Clock = std::chrono::steady_clock; + +uint64_t usSince(Clock::time_point from) { + return static_cast(std::chrono::duration_cast(Clock::now() - from).count()); +} } // namespace +size_t defaultBackfillVerifyThreads() { + const unsigned cores = std::thread::hardware_concurrency(); // 0: unknown + return cores > 2 ? std::min(4, cores - 1) : 1; +} + BackwardSync::BackwardSync(Hooks hooks, Settings settings) : hooks_(std::move(hooks)) , settings_(settings) { @@ -36,6 +47,7 @@ BackwardSync::BackwardSync(Hooks hooks, Settings settings) settings_.storePerTurn = std::clamp(settings_.storePerTurn, 1, 1000); settings_.turnBudgetMs = std::min(settings_.turnBudgetMs, 1000); settings_.idleTurnBudgetMs = std::min(settings_.idleTurnBudgetMs, 1000); + settings_.verifyThreads = std::min(settings_.verifyThreads, kMaxVerifyThreads); } uint64_t BackwardSync::now() const { @@ -64,11 +76,108 @@ bool BackwardSync::isActive() const { BackwardSync::~BackwardSync() { stopFlusher(); + stopVerifiers(); } void BackwardSync::stop() { stopped_ = true; stopFlusher(); + stopVerifiers(); +} + +// a verifier in the middle of a check ends it first (a few ms); the jobs still queued are dropped +void BackwardSync::stopVerifiers() { + std::vector workers; + decltype(verifyQueue_) dropped; + { + std::lock_guard lock(verifyMutex_); + verifyStopping_ = true; + workers = std::move(verifiers_); + dropped.swap(verifyQueue_); + } + verifyCv_.notify_all(); + for (auto& worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } +} + +bool BackwardSync::submit(VerifyJob&& job) { + if (settings_.verifyThreads == 0) { + return false; + } + { + std::lock_guard lock(verifyMutex_); + if (verifyStopping_ || verifyClosing_) { + return false; + } + if (verifiers_.empty()) { + try { + while (verifiers_.size() < settings_.verifyThreads) { + verifiers_.emplace_back([this] { verifyWorker(); }); + } + } + catch (const std::system_error& e) { + log(std::string("BACKFILL: verifier thread ") + std::to_string(verifiers_.size() + 1) + " not started (" + e.what() + ")" + + (verifiers_.empty() ? ": verifying on the processor thread" : ""), + true); + } + if (verifiers_.empty()) { + settings_.verifyThreads = 0; + return false; + } + } + const Sequence seq = job.block.sequence(); + verifyQueue_.emplace(seq, std::move(job)); + } + verifyCv_.notify_one(); + return true; +} + +void BackwardSync::verifyWorker() { + std::unique_lock lock(verifyMutex_); + for (;;) { + verifyCv_.wait(lock, [this] { return verifyStopping_ || verifyClosing_ || !verifyQueue_.empty(); }); + if (verifyStopping_ || verifyQueue_.empty()) { + return; + } + { + auto node = verifyQueue_.extract(verifyQueue_.begin()); + VerifyJob job = std::move(node.mapped()); + lock.unlock(); + runJob(job); + } // the job's copies of the blocks go before the lock is taken again + lock.lock(); + } +} + +void BackwardSync::runJob(VerifyJob& job) { + try { + if (job.check) { + if (job.check.use_count() == 1) { + return; // the block was refused, replaced or dropped meanwhile: nobody reads the result + } + if (hooks_.onVerify) { + hooks_.onVerify(job.block.sequence()); + } + const auto at = Clock::now(); + job.check->why = verifySignatures(job.block); + phaseVerifyUs_ += usSince(at); + job.check->ready.store(true, std::memory_order_release); + } + else { + checkConfirmations(job.block, job.below); + } + } + catch (const std::exception& e) { + log(std::string("BACKFILL: a verifier thread caught: ") + e.what(), true); + if (job.check) { + // not the sender's doing, but the block cannot be stored unchecked: it is asked for again + job.check->why = std::string("signatures: not checked (") + e.what() + ")"; + job.check->ready.store(true, std::memory_order_release); + } + } } // a flush under way ends first: nothing flushes the storage behind the node's back once it stops @@ -170,7 +279,8 @@ bool BackwardSync::refreshAnchor(const Context& ctx) { progressAt_ = lastFlushAt_; log("BACKFILL: " + std::string(floor_ == 0 ? "full node" : "pruned node") + " with blocks " + std::to_string(anchor.first) + ".." + std::to_string(ctx.lastSeq) + ": fetching " + std::to_string(floor_) + ".." + std::to_string(next_) + - " from peers in the background"); + " from peers in the background" + + (settings_.verifyThreads > 0 ? ", signatures checked on " + std::to_string(settings_.verifyThreads) + " verifier threads" : "")); } return true; } @@ -242,12 +352,15 @@ void BackwardSync::tick(const Context& ctx) { // where the time of a block goes, on average: what to move off the processor thread next const uint64_t blocks = std::max(1, stored_ - progressStored_); const uint64_t busy = busyUs_ - progressBusyUs_; - log("BACKFILL: per block over the last " + std::to_string(blocks) + ": verify " + std::to_string(phaseVerifyUs_ / blocks) + - " us, store " + std::to_string(phaseStoreUs_ / blocks) + " us, confirmations " + std::to_string(phaseConfirmUs_ / blocks) + - " us, " + std::to_string(phaseTransactions_ * 100 / blocks / 100) + "." + + const uint64_t verifyUs = phaseVerifyUs_.exchange(0); + const uint64_t confirmUs = phaseConfirmUs_.exchange(0); + log("BACKFILL: per block over the last " + std::to_string(blocks) + ": verify " + std::to_string(verifyUs / blocks) + + " us, store " + std::to_string(phaseStoreUs_ / blocks) + " us, confirmations " + std::to_string(confirmUs / blocks) + + " us" + (settings_.verifyThreads > 0 ? " (verify and confirmations on " + std::to_string(settings_.verifyThreads) + " verifier threads)" : "") + + ", " + std::to_string(phaseTransactions_ * 100 / blocks / 100) + "." + std::to_string(phaseTransactions_ * 100 / blocks % 100 / 10) + " transactions; processor thread busy " + std::to_string(busy / 10 / std::max(1, t - progressAt_)) + " % of the time"); - phaseVerifyUs_ = phaseStoreUs_ = phaseConfirmUs_ = phaseTransactions_ = 0; + phaseStoreUs_ = phaseTransactions_ = 0; progressBusyUs_ = busyUs_; progressAt_ = t; progressStored_ = stored_; @@ -371,13 +484,14 @@ std::optional BackwardSync::pickPeer(const Context& ctx, Sequence hig void BackwardSync::onBlocks(const PublicKey& sender, PoolsBlock&& blocks) { const auto it = inFlight_.find(sender); - if (it == inFlight_.end()) { + if (it == inFlight_.end() || stopped_) { dropped_ += blocks.size(); return; } Batch& batch = it->second; const size_t limit = kBufferBatches * settings_.batch; std::set noRoom; // sent, but not kept: asked for again, not held against the peer + uint64_t handedUs = 0; // hashing the blocks and queuing their checks, on this thread for (auto& block : blocks) { const Sequence seq = block.sequence(); if (!block.is_valid() || batch.missing.count(seq) == 0) { @@ -394,8 +508,19 @@ void BackwardSync::onBlocks(const PublicKey& sender, PoolsBlock&& blocks) { } continue; } - buffer_[seq] = Buffered{std::move(block), sender}; + Buffered& entry = buffer_[seq]; + entry = Buffered{std::move(block), sender, nullptr}; batch.missing.erase(seq); + if (settings_.verifyThreads > 0) { + // the hash is computed and cached here, before the data is shared: a verifier never writes it + const auto at = Clock::now(); + (void)entry.pool.hash(); + auto check = std::make_shared(); + if (submit(VerifyJob{entry.pool, csdb::Pool{}, check})) { + entry.check = std::move(check); + } + handedUs += usSince(at); + } } if (!noRoom.empty()) { for (const Sequence seq : noRoom) { @@ -418,6 +543,7 @@ void BackwardSync::onBlocks(const PublicKey& sender, PoolsBlock&& blocks) { } inFlightCount_ = inFlight_.size(); bufferedCount_ = buffer_.size(); + busyUs_ += handedUs; drain(); } @@ -517,13 +643,17 @@ void BackwardSync::expire() { } } -std::string BackwardSync::verify(const csdb::Pool& block) const { +std::string BackwardSync::verifyLink(const csdb::Pool& block) const { if (!block.is_valid() || block.sequence() != next_) { return "sequence"; } if (expectedHash_.is_empty() || block.hash() != expectedHash_) { return "hash"; } + return {}; +} + +std::string BackwardSync::verifySignatures(const csdb::Pool& block) { std::string why; if (!BlockVerifier::verifySignatures(block, why)) { return "signatures: " + why; @@ -531,6 +661,23 @@ std::string BackwardSync::verify(const csdb::Pool& block) const { return {}; } +// the round confirmations of the block stored before are signed by the confidants of the one below +// it: inside the hash, so a mismatch is the chain's, not the peer's. Counted, never refused +void BackwardSync::checkConfirmations(const csdb::Pool& above, const csdb::Pool& block) { + const auto at = Clock::now(); + std::string bad; + if (!BlockVerifier::verifyConfirmations(above, block.confidants(), bad)) { + const size_t n = ++confirmationErrors_; + if (n <= 10 || n % 10'000 == 0) { + log("BACKFILL: the round confirmations of block " + std::to_string(above.sequence()) + + " do not verify against the confidants of block " + std::to_string(block.sequence()) + " (" + bad + "), " + + std::to_string(n) + " so far", + true); + } + } + phaseConfirmUs_ += usSince(at); +} + // A block that fails its checks: every block that peer sent and that waits here goes, the peer is // left out for an hour, its blocks are asked for again from others. void BackwardSync::reject(const PublicKey& peer, Sequence seq, const std::string& why) { @@ -573,10 +720,6 @@ void BackwardSync::drain() { return; } } - using Clock = std::chrono::steady_clock; - const auto usSince = [](Clock::time_point from) { - return static_cast(std::chrono::duration_cast(Clock::now() - from).count()); - }; const auto startedAt = Clock::now(); const uint64_t startedMs = now(); // the budget and the slow-turn decision: the hooks' clock (tests drive it) const uint64_t budgetMs = trusted_ ? settings_.turnBudgetMs : settings_.idleTurnBudgetMs; @@ -593,8 +736,21 @@ void BackwardSync::drain() { if (it == buffer_.end()) { break; } + // the link to the chain here, the signatures by a verifier thread when one took them: the + // turn ends at a block whose check is not done yet, the next tick or reply goes on auto stepAt = Clock::now(); - const std::string why = verify(it->second.pool); + std::string why = verifyLink(it->second.pool); + if (why.empty()) { + if (it->second.check) { + if (!it->second.check->ready.load(std::memory_order_acquire)) { + break; + } + why = it->second.check->why; + } + else { + why = verifySignatures(it->second.pool); + } + } verifyUs += usSince(stepAt); if (!why.empty()) { const PublicKey peer = it->second.peer; // reject() erases the node this iterator points to @@ -616,19 +772,11 @@ void BackwardSync::drain() { storeFailing_ = false; log("BACKFILL: storing again from block " + std::to_string(next_)); } - // the round confirmations of the block stored before are signed by the confidants of this - // one: inside the hash, so a mismatch is the chain's, not the peer's. Counted, never refused + // the round confirmations of the block stored before, against the confidants of this one: on + // a verifier thread when there is one, as they decide nothing here stepAt = Clock::now(); - if (lastStored_) { - std::string bad; - if (!BlockVerifier::verifyConfirmations(*lastStored_, it->second.pool.confidants(), bad)) { - const size_t n = ++confirmationErrors_; - if (n <= 10 || n % 10'000 == 0) { - log("BACKFILL: the round confirmations of block " + std::to_string(lastStored_->sequence()) + - " do not verify against the confidants of block " + std::to_string(next_) + " (" + bad + "), " + - std::to_string(n) + " so far", true); - } - } + if (lastStored_ && !submit(VerifyJob{*lastStored_, it->second.pool, nullptr})) { + checkConfirmations(*lastStored_, it->second.pool); } confirmUs += usSince(stepAt); transactions += it->second.pool.transactions_count(); @@ -651,9 +799,10 @@ void BackwardSync::drain() { if (us > longestTurnUs_) { longestTurnUs_ = us; } - phaseVerifyUs_ += verifyUs; + if (settings_.verifyThreads == 0) { + phaseVerifyUs_ += verifyUs; // checkConfirmations() added its own time + } phaseStoreUs_ += storeUs; - phaseConfirmUs_ += confirmUs; phaseTransactions_ += transactions; if (now() >= startedMs + kSlowTurnMs) { const size_t n = ++slowTurns_; @@ -728,6 +877,12 @@ void BackwardSync::flushIfDue(bool force) { void BackwardSync::finish() { done_ = true; + { + // the confirmations still queued are checked, then the verifier threads return + std::lock_guard lock(verifyMutex_); + verifyClosing_ = true; + } + verifyCv_.notify_all(); flushIfDue(true); const std::string cost = ", " + std::to_string(stored_.load()) + " blocks stored in " + std::to_string(busyUs_ / 1000) + " ms of processor thread, longest turn " + std::to_string(longestTurnUs_ / 1000) + " ms"; diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 331b6bf4b..8d1c83c91 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -441,6 +441,9 @@ void Node::startBackfill() { settings.storePerTurn = sto.backfillStorePerTurn; settings.turnBudgetMs = sto.backfillTurnBudgetMs; settings.idleTurnBudgetMs = sto.backfillIdleTurnBudgetMs; + if (sto.backfillVerifyThreads) { + settings.verifyThreads = *sto.backfillVerifyThreads; + } // created under the lock that stop() takes to read the pointer (finishBootstrap runs this on the // processor thread while a stop may come from another one) std::lock_guard lock(backfillTimerMutex_); diff --git a/docs/differences.md b/docs/differences.md index 15eb8ca6c..19b5f0fa9 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -32,7 +32,7 @@ each key and its default. | `db_backend` | Chooses the block database: `berkeleydb` (the default, the official format), `lmdb` or `rocksdb`. RocksDB is about a third of the BerkeleyDB size on mainnet. | | `prune_keep_blocks` | Pruned storage: the node keeps only the latest blocks and removes older ones at each periodic checkpoint. Pruned storage runs on LMDB. | | `async_write_queue_size`, `write_batch_size`, `rocksdb_block_cache_mb`, `rocksdb_memtable_mb` | Tune the database writer and RocksDB. | -| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn`, `backfill_turn_budget_ms`, `backfill_idle_turn_budget_ms` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. | +| `backfill`, `backfill_batch`, `backfill_rate`, `backfill_store_per_turn`, `backfill_turn_budget_ms`, `backfill_idle_turn_budget_ms`, `backfill_verify_threads` | Backward sync: a full node whose oldest block is above block 0 (started from a snapshot on lmdb or rocksdb, or switched from pruned to full) fetches the older blocks from its neighbours in the background, with the existing block requests, checks each against the chain's hashes and its signatures and stores it without applying it; the API serves each block and history as it arrives. On by default; `backfill=off` keeps only the snapshot's blocks. | ### Converting the block database diff --git a/docs/storage.md b/docs/storage.md index a0c65ca1a..1584a967c 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -41,6 +41,7 @@ official node. | `backfill_store_per_turn` | `50` | Blocks checked and stored per turn of the thread that also runs consensus, `1` to `1000`; the rest waits for the next turn, 100 ms later. A turn over 200 ms is logged (`BACKFILL: slow turn of ...`) with the time spent verifying, storing and checking confirmations. | | `backfill_turn_budget_ms` | `50` | While the node is a confidant, a turn also ends once it has taken this long (after at least one block), `0` (no limit) to `1000`: consensus keeps most of the thread; on mainnet about 250 blocks/s. | | `backfill_idle_turn_budget_ms` | `90` | The same while the node is not a confidant, with little consensus work: on mainnet about 450 blocks/s. | +| `backfill_verify_threads` | min(4, cores − 1) | Threads that check the signatures and round confirmations of the fetched blocks, `0` to `16`; `0` checks them on the thread that runs consensus, as before. | The periodic flush of the backfilled blocks (every 10,000 blocks or a minute; up to several seconds on mainnet) runs on a thread of its own, not on the thread that runs consensus; one flush at a time, and while the node is a confidant it waits until the node is out of the trusted set (10 minutes at most). The log line says `on the flush thread`. One wait remains possible: an LMDB index that must grow its map while the flush thread syncs it waits for that sync (rare: the map grows in steps). @@ -98,6 +99,12 @@ restart. prune_keep_blocks` below its head at start), which a snapshot usually holds already. - Once block 1 is stored, the node knows the chain's identifier and, as a node synced from block 0, no longer accepts neighbours that announce another chain. +- The signature checks (about 2 ms a block on mainnet, 80 % of the cost of a block) run on + `backfill_verify_threads` threads of their own as soon as a block arrives; the thread that runs + consensus only links each block to the chain and stores it, and never waits for a check: a block + not checked yet waits for the next turn. The round confirmations of each stored block are checked + there too, after it is stored (a mismatch is counted and logged, as before; it never refuses a + block). - The log shows `BACKFILL: full node with blocks N..M: fetching 0..N-1`, a line every 100,000 blocks, every peer left out, and at the end `BACKFILL: full history stored`; the status line gets `, oldest stored = K`. A restart resumes below the oldest stored block. diff --git a/tests/csnode/backfillconfig_tests.cpp b/tests/csnode/backfillconfig_tests.cpp index 5e1442326..e9484ea97 100644 --- a/tests/csnode/backfillconfig_tests.cpp +++ b/tests/csnode/backfillconfig_tests.cpp @@ -51,3 +51,13 @@ TEST(BackfillConfig, BadValuesKeepTheDefaultOrAreClamped) { EXPECT_EQ(parse("backfill_batch=many\n").backfillBatch, 100u); EXPECT_EQ(parse("backfill_rate=-5\n").backfillRate, 1000u); } + +// backfill_verify_threads: 0..16; missing or not a number: BackwardSync's default +TEST(BackfillConfig, VerifyThreads) { + EXPECT_FALSE(parse("db_backend=lmdb\n").backfillVerifyThreads.has_value()); + EXPECT_EQ(parse("backfill_verify_threads=0\n").backfillVerifyThreads.value_or(99), 0u); + EXPECT_EQ(parse("backfill_verify_threads=3\n").backfillVerifyThreads.value_or(99), 3u); + EXPECT_EQ(parse("backfill_verify_threads=64\n").backfillVerifyThreads.value_or(99), 16u); + EXPECT_FALSE(parse("backfill_verify_threads=many\n").backfillVerifyThreads.has_value()); + EXPECT_FALSE(parse("backfill_verify_threads=-1\n").backfillVerifyThreads.has_value()); +} diff --git a/tests/csnode/backwardsync_tests.cpp b/tests/csnode/backwardsync_tests.cpp index d77d7f25f..0f35f9fcb 100644 --- a/tests/csnode/backwardsync_tests.cpp +++ b/tests/csnode/backwardsync_tests.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -124,9 +125,35 @@ cs::BackwardSync::Settings settingsOf(size_t batch = 100, size_t rate = 1000, si s.turnBudgetMs = 0; // no time limit unless a test sets one s.idleTurnBudgetMs = 0; s.asyncFlush = false; // flushes on the calling thread unless a test says otherwise + s.verifyThreads = 0; // and verifies there return s; } +// a gate a test holds a background thread at, shared with that thread by value: it may outlive the +// test body if an assertion fails +struct Gate { + std::mutex m; + std::condition_variable cv; + bool release = false; + std::atomic started{0}; + void open() { + { + std::lock_guard lock(m); + release = true; + } + cv.notify_all(); + } + void pass() { + ++started; + std::unique_lock lock(m); + cv.wait(lock, [this] { return release; }); + } +}; +struct Release { // a failed assertion must not leave the thread waiting for ever + std::shared_ptr g; + ~Release() { g->open(); } +}; + cs::PublicKey peerKey(uint8_t n) { cs::PublicKey k{}; k.fill(n); @@ -142,6 +169,10 @@ class BackfillTest : public ::testing::Test { } make(); } + // the verifier and flush threads end while the fixture they log to is whole + void TearDown() override { + sync_.reset(); + } void make(cs::BackwardSync::Settings settings = settingsOf()) { cs::BackwardSync::Hooks hooks; @@ -175,6 +206,7 @@ class BackfillTest : public ::testing::Test { std::lock_guard lock(logsMutex_); // the flush thread logs too logs_.push_back(line); }; + hooks.onVerify = verifyGate_; // copied: a verifier thread never reads the fixture's sync_ = std::make_unique(std::move(hooks), settings); } @@ -220,6 +252,25 @@ class BackfillTest : public ::testing::Test { answerAll(); } } + // with verifier threads: ticks (1 ms of the clock each, so that no batch times out while the + // verifiers work), answering when `answer`, until `until` holds or 60 s of real time pass + bool settle(const std::function& until, bool answer = false) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (!until()) { + if (std::chrono::steady_clock::now() > deadline) { + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + tick(1); + if (answer) { + answerAll(); + } + } + return true; + } + bool runThreadedToEnd() { + return settle([this] { return sync_->stats().done; }, true); + } std::vector peers_; std::unique_ptr sync_; @@ -227,6 +278,7 @@ class BackfillTest : public ::testing::Test { std::vector logs_; mutable std::mutex logsMutex_; std::function flushGate_; + std::function verifyGate_; // set before make() cs::Sequence first_ = kFirst; cs::Sequence floor_ = 0; uint64_t now_ = 1'000'000; @@ -640,30 +692,9 @@ TEST_F(BackfillTest, AConfidantDefersTheFlush) { // the periodic flush runs on its own thread: storing goes on while it lasts, a second one waits // for the first, and the last flush (end of the backfill) waits for the one under way TEST_F(BackfillTest, ThePeriodicFlushRunsOnItsOwnThread) { - // shared with the flush thread by value: it may outlive this body if an assertion fails - struct Gate { - std::mutex m; - std::condition_variable cv; - bool release = false; - std::atomic started{0}; - void open() { - { - std::lock_guard lock(m); - release = true; - } - cv.notify_all(); - } - }; const auto gate = std::make_shared(); - struct Release { // a failed assertion must not leave the flush thread waiting for ever - std::shared_ptr g; - ~Release() { g->open(); } - } guard{gate}; - flushGate_ = [gate] { - ++gate->started; - std::unique_lock lock(gate->m); - gate->cv.wait(lock, [&] { return gate->release; }); - }; + Release guard{gate}; + flushGate_ = [gate] { gate->pass(); }; auto settings = settingsOf(100, 0, 50); settings.asyncFlush = true; make(settings); @@ -754,3 +785,102 @@ TEST_F(BackfillTest, OtherSyncroMessagesAreIgnored) { sync_->onSyncroMessage(stranger, cs::SyncroMessage::DuplicatedRequest); EXPECT_EQ(logged("BACKFILL: peer"), 0u); } + +// verifier threads: the same blocks stored in the same order down to block 0, replies out of order, +// the confirmation mismatch of block 350 counted once +TEST_F(BackfillTest, VerifierThreadsStoreTheSameBlocks) { + auto settings = settingsOf(); + settings.verifyThreads = 3; + make(settings); + tick(); + ASSERT_EQ(requests_.size(), 4u); + EXPECT_EQ(logged("signatures checked on 3 verifier threads"), 1u); + sync_->onBlocks(requests_[1].first, blocksFor(requests_[1].second)); + sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); + requests_.erase(requests_.begin(), requests_.begin() + 2); + ASSERT_TRUE(settle([this] { return first_ == 201u; })); + EXPECT_EQ(sync_->stats().buffered, 0u); + ASSERT_TRUE(runThreadedToEnd()); + EXPECT_EQ(first_, 0u); + EXPECT_EQ(storedCount_, kFirst); + EXPECT_EQ(sync_->stats().rejected, 0u); + EXPECT_EQ(logged("BACKFILL: full history stored"), 1u); + // the confirmations are checked after the blocks are stored, on the verifier threads + ASSERT_TRUE(settle([this] { return sync_->stats().confirmationErrors >= 1; })); + EXPECT_EQ(logged("the round confirmations of block 350 do not verify against the confidants of block 349"), 1u); + sync_->stop(); + EXPECT_EQ(sync_->stats().confirmationErrors, 1u); +} + +// verifier threads: a bad signature is refused as on the processor thread, the peer marked, the +// blocks asked for again from another peer +TEST_F(BackfillTest, VerifierThreadsRefuseBadSignatures) { + auto settings = settingsOf(); + settings.verifyThreads = 2; + make(settings); + tick(); + const auto [badPeer, seqs] = requests_[0]; + auto blocks = blocksFor(seqs); + blocks.back() = chain().badSignature(seqs.back()); // block 400 + requests_.erase(requests_.begin()); + sync_->onBlocks(badPeer, std::move(blocks)); + ASSERT_TRUE(settle([this] { return sync_->stats().rejected == 1u; })); + EXPECT_EQ(first_, kFirst); + EXPECT_EQ(logged("block 400 from peer"), 1u); + EXPECT_EQ(logged("rejected (signatures"), 1u); + EXPECT_GE(logged("marked for 1 h"), 1u); + ASSERT_TRUE(runThreadedToEnd()); + EXPECT_EQ(first_, 0u); + EXPECT_EQ(sync_->stats().rejected, 1u); + for (const auto& [peer, s] : requests_) { + EXPECT_NE(peer, badPeer); + } +} + +// a turn never waits for a verifier: the blocks whose signatures are not checked yet wait in the +// buffer, and the next calls store them once they are +TEST_F(BackfillTest, ATurnDoesNotWaitForTheVerifiers) { + const auto gate = std::make_shared(); + Release guard{gate}; + verifyGate_ = [gate](cs::Sequence) { gate->pass(); }; + auto settings = settingsOf(); + settings.verifyThreads = 2; + make(settings); + tick(); + sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); + tick(); + tick(); + EXPECT_EQ(first_, kFirst) << "stored before its signatures were checked"; + EXPECT_EQ(sync_->stats().buffered, 100u); + EXPECT_EQ(sync_->stats().rejected, 0u); + gate->open(); + ASSERT_TRUE(settle([this] { return first_ == 301u; })); + EXPECT_EQ(sync_->stats().rejected, 0u); +} + +// stop() while a verifier is in the middle of a check: it returns once that check ends, the checks +// queued are dropped and nothing more is stored +TEST_F(BackfillTest, StopEndsTheVerifiers) { + const auto gate = std::make_shared(); + Release guard{gate}; + verifyGate_ = [gate](cs::Sequence) { gate->pass(); }; + auto settings = settingsOf(); + settings.verifyThreads = 1; + make(settings); + tick(); + sync_->onBlocks(requests_[0].first, blocksFor(requests_[0].second)); + for (int i = 0; i < 1000 && gate->started.load() == 0; ++i) { // up to 5 s on a busy runner + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + ASSERT_EQ(gate->started.load(), 1) << "a verifier took the first check"; + std::thread opener([gate] { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + gate->open(); + }); + sync_->stop(); + opener.join(); + tick(); + answerAll(); + EXPECT_EQ(first_, kFirst); + EXPECT_FALSE(sync_->isActive()); +} From 95ce7207c0fa96c945464f647276196598e43096 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 2 Nov 2026 07:20:58 +0100 Subject: [PATCH 2/4] docs: CHANGES row for #152 Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index f5796d962..b4917da24 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -43,6 +43,7 @@ lines under the new release. | PR | Change | |---|---| | #151 | csnode: every backfill progress line (each 100,000 blocks) is followed by the average cost of a block per phase (verify, store, confirmations), the transactions per block and the share of the processor thread the backfill took | +| #152 | csnode: the backfill checks the signatures of the fetched blocks, and the round confirmations of the stored ones, on verifier threads (`backfill_verify_threads`, default min(4, cores − 1), `0` = as before); the thread that runs consensus only links and stores the blocks, about 0.5 ms a block instead of 2.5 ms on mainnet | ## community.15 — 2026-11-01 From 44b252097f1a160691c88b4bdd9afc232751632b Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 2 Nov 2026 07:51:40 +0100 Subject: [PATCH 3/4] csnode: review fixes for the backfill verifier threads - signature checks are served before the confirmation checks (only the former gate the storage), each highest block first - a failure to grow the thread vector falls back like a thread that cannot start (reserve, catch std::exception) - the slow-turn line no longer reports a confirmations time when the signatures run on the verifier threads - test: wait for the confirmation line itself, not only the counter that goes up before it is logged Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/backwardsync.hpp | 5 ++++- csnode/src/backwardsync.cpp | 11 +++++++---- tests/csnode/backwardsync_tests.cpp | 5 ++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/csnode/include/csnode/backwardsync.hpp b/csnode/include/csnode/backwardsync.hpp index 83321f67d..99f91f5e3 100644 --- a/csnode/include/csnode/backwardsync.hpp +++ b/csnode/include/csnode/backwardsync.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -273,7 +274,9 @@ class BackwardSync { std::condition_variable verifyCv_; std::vector verifiers_; // the fields below: under verifyMutex_ // the highest blocks first: the storage waits for the highest buffered one - std::multimap> verifyQueue_; + // signature checks first (the storage waits for them), the highest block first; the confirmation + // checks, counted only, after them + std::multimap, VerifyJob, std::greater>> verifyQueue_; bool verifyStopping_ = false; // stop(): the jobs queued are dropped bool verifyClosing_ = false; // finish(): the jobs queued are done, then the threads return }; diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index d8b539c1e..acf2a566b 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -114,11 +114,12 @@ bool BackwardSync::submit(VerifyJob&& job) { } if (verifiers_.empty()) { try { + verifiers_.reserve(settings_.verifyThreads); while (verifiers_.size() < settings_.verifyThreads) { verifiers_.emplace_back([this] { verifyWorker(); }); } } - catch (const std::system_error& e) { + catch (const std::exception& e) { log(std::string("BACKFILL: verifier thread ") + std::to_string(verifiers_.size() + 1) + " not started (" + e.what() + ")" + (verifiers_.empty() ? ": verifying on the processor thread" : ""), true); @@ -128,8 +129,8 @@ bool BackwardSync::submit(VerifyJob&& job) { return false; } } - const Sequence seq = job.block.sequence(); - verifyQueue_.emplace(seq, std::move(job)); + const auto key = std::make_pair(job.check != nullptr, job.block.sequence()); + verifyQueue_.emplace(key, std::move(job)); } verifyCv_.notify_one(); return true; @@ -810,7 +811,9 @@ void BackwardSync::drain() { log("BACKFILL: slow turn of " + std::to_string(now() - startedMs) + " ms (" + std::to_string(n) + " so far): " + std::to_string(storedNow) + " blocks down to #" + std::to_string(first_.load()) + ", " + std::to_string(transactions) + " transactions; verify " + std::to_string(verifyUs / 1000) + " ms, store " + - std::to_string(storeUs / 1000) + " ms, confirmations " + std::to_string(confirmUs / 1000) + " ms"); + std::to_string(storeUs / 1000) + " ms" + + (settings_.verifyThreads > 0 ? std::string(" (signatures on the verifier threads)") + : ", confirmations " + std::to_string(confirmUs / 1000) + " ms")); } } flushIfDue(false); diff --git a/tests/csnode/backwardsync_tests.cpp b/tests/csnode/backwardsync_tests.cpp index 0f35f9fcb..202cdd6b5 100644 --- a/tests/csnode/backwardsync_tests.cpp +++ b/tests/csnode/backwardsync_tests.cpp @@ -806,7 +806,10 @@ TEST_F(BackfillTest, VerifierThreadsStoreTheSameBlocks) { EXPECT_EQ(sync_->stats().rejected, 0u); EXPECT_EQ(logged("BACKFILL: full history stored"), 1u); // the confirmations are checked after the blocks are stored, on the verifier threads - ASSERT_TRUE(settle([this] { return sync_->stats().confirmationErrors >= 1; })); + // the counter goes up before the line is logged: wait for the line + ASSERT_TRUE(settle([this] { + return logged("the round confirmations of block 350 do not verify against the confidants of block 349") >= 1; + })); EXPECT_EQ(logged("the round confirmations of block 350 do not verify against the confidants of block 349"), 1u); sync_->stop(); EXPECT_EQ(sync_->stats().confirmationErrors, 1u); From 3e75a8dbca86be313dce754186c4a6f380c0ff6f Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 2 Nov 2026 07:53:15 +0100 Subject: [PATCH 4/4] csnode: at most 2000 confirmation checks wait for the verifier threads Confirmation jobs are served after the signature jobs; on a single slow verifier they could pile up, each holding two blocks. Past 2000 queued, the processor thread checks the confirmations of the next stored blocks itself (the existing fallback when submit() refuses), which also slows the storing down until the verifiers catch up. Co-Authored-By: Claude Opus 5.5 --- csnode/include/csnode/backwardsync.hpp | 4 ++++ csnode/src/backwardsync.cpp | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/csnode/include/csnode/backwardsync.hpp b/csnode/include/csnode/backwardsync.hpp index 99f91f5e3..283402f91 100644 --- a/csnode/include/csnode/backwardsync.hpp +++ b/csnode/include/csnode/backwardsync.hpp @@ -123,6 +123,9 @@ class BackwardSync { // not flushed yet are fetched again after a crash, nothing else is lost static constexpr uint64_t kTrustedFlushDeferMs = 600'000; static constexpr size_t kProgressBlocks = 100'000; + // confirmation jobs wait behind the signature jobs: past this many queued, the processor thread + // checks them itself, so that they cannot pile up (each holds two blocks) on a slow verifier + static constexpr size_t kMaxPendingConfirmations = 2'000; static constexpr uint64_t kSlowTurnMs = 200; // a longer turn is logged, with where its time went static constexpr size_t kMaxVerifyThreads = 16; @@ -278,6 +281,7 @@ class BackwardSync { // checks, counted only, after them std::multimap, VerifyJob, std::greater>> verifyQueue_; bool verifyStopping_ = false; // stop(): the jobs queued are dropped + size_t pendingConfirmations_ = 0; // confirmation jobs queued, at most kMaxPendingConfirmations bool verifyClosing_ = false; // finish(): the jobs queued are done, then the threads return }; diff --git a/csnode/src/backwardsync.cpp b/csnode/src/backwardsync.cpp index acf2a566b..6999bc204 100644 --- a/csnode/src/backwardsync.cpp +++ b/csnode/src/backwardsync.cpp @@ -94,6 +94,7 @@ void BackwardSync::stopVerifiers() { verifyStopping_ = true; workers = std::move(verifiers_); dropped.swap(verifyQueue_); + pendingConfirmations_ = 0; } verifyCv_.notify_all(); for (auto& worker : workers) { @@ -112,6 +113,9 @@ bool BackwardSync::submit(VerifyJob&& job) { if (verifyStopping_ || verifyClosing_) { return false; } + if (!job.check && pendingConfirmations_ >= kMaxPendingConfirmations) { + return false; // the caller checks these confirmations itself + } if (verifiers_.empty()) { try { verifiers_.reserve(settings_.verifyThreads); @@ -130,6 +134,9 @@ bool BackwardSync::submit(VerifyJob&& job) { } } const auto key = std::make_pair(job.check != nullptr, job.block.sequence()); + if (!job.check) { + ++pendingConfirmations_; + } verifyQueue_.emplace(key, std::move(job)); } verifyCv_.notify_one(); @@ -146,6 +153,9 @@ void BackwardSync::verifyWorker() { { auto node = verifyQueue_.extract(verifyQueue_.begin()); VerifyJob job = std::move(node.mapped()); + if (!job.check) { + --pendingConfirmations_; + } lock.unlock(); runJob(job); } // the job's copies of the blocks go before the lock is taken again