diff --git a/csdb/src/database_berkeleydb.cpp b/csdb/src/database_berkeleydb.cpp index a149843b9..6f7689e26 100644 --- a/csdb/src/database_berkeleydb.cpp +++ b/csdb/src/database_berkeleydb.cpp @@ -332,23 +332,31 @@ bool DatabaseBerkeleyDB::remove(const cs::Bytes &key) { return false; } - Dbt_copy db_key(key); - Dbt_copy db_seq_no; - int status = db_seq_no_->get(nullptr, &db_key, &db_seq_no, 0); - if (status != 0) { - set_last_error_from_berkeleydb(status); - return false; - } + // BerkeleyDB throws when its buffer cache is exhausted (BDB3017, seen on mainnet while a pruned + // node removed old blocks during a fast sync): report it, the caller may retry later + try { + Dbt_copy db_key(key); + Dbt_copy db_seq_no; + int status = db_seq_no_->get(nullptr, &db_key, &db_seq_no, 0); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } - status = db_seq_no_->del(nullptr, &db_key, 0); - if (status != 0) { - set_last_error_from_berkeleydb(status); - return false; - } + status = db_seq_no_->del(nullptr, &db_key, 0); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } - status = db_blocks_->del(nullptr, &db_seq_no, 0); - if (status != 0) { - set_last_error_from_berkeleydb(status); + status = db_blocks_->del(nullptr, &db_seq_no, 0); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } + } + catch (const std::exception& e) { + set_last_error(UnknownError, "remove: %s", e.what()); return false; } @@ -362,14 +370,39 @@ bool DatabaseBerkeleyDB::compact() { return false; } - // DB_FREE_SPACE gives emptied pages back to the file system - for (Db* db : { db_blocks_.get(), db_seq_no_.get() }) { - int status = db->compact(nullptr, nullptr, nullptr, nullptr, DB_FREE_SPACE, nullptr); - if (status != 0) { - set_last_error_from_berkeleydb(status); - return false; + // DB_FREE_SPACE gives emptied pages back to the file system. Compacting a large file in one call + // pins more pages than the environment's buffer cache holds (BDB3017, "Db::compact: Cannot + // allocate memory"), so it goes in bounded steps, each resuming where the previous one stopped. + // Compaction only returns space (BerkeleyDB reuses freed pages anyway): a failure is reported + // and never lets an exception terminate the node. + constexpr u_int32_t kPagesPerStep = 64; + constexpr int kMaxSteps = 1'000'000; + try { + for (Db* db : { db_blocks_.get(), db_seq_no_.get() }) { + cs::Bytes resume; + for (int step = 0; step < kMaxSteps; ++step) { + DB_COMPACT progress{}; + progress.compact_pages = kPagesPerStep; + Dbt from(resume.data(), static_cast(resume.size())); + Dbt_safe stopped; + int status = db->compact(nullptr, resume.empty() ? nullptr : &from, nullptr, &progress, DB_FREE_SPACE, &stopped); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } + // pages left over in the budget: the pass reached the end of the database + if (progress.compact_pages > 0 || stopped.get_size() == 0) { + break; + } + auto begin = static_cast(stopped.get_data()); + resume.assign(begin, begin + stopped.get_size()); + } } } + catch (const std::exception& e) { + set_last_error(UnknownError, "compact: %s", e.what()); + return false; + } set_last_error(); return true; diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index c86fa80de..28ba08580 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -1109,7 +1109,8 @@ bool Storage::compact() { return false; } if (!d->db->compact()) { - d->set_last_error(DatabaseError, "%s: %s", funcName(), d->db->last_error_message().c_str()); + // not a fault: compaction only returns space, and freed pages are reused for new blocks anyway + cswarning() << "Storage> compaction skipped: " << d->db->last_error_message(); return false; } return true; diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index e1dfcb3db..16fcd47be 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -270,9 +270,15 @@ void BlockChain::pruneStep() { if (pool.is_valid() && trxIndex_) { trxIndex_->onPruneBlock(pool); } - if (hash.is_empty() || !storage_.pool_prune(hash)) { + if (hash.is_empty()) { csdebug() << kLogPrefix << "pruned storage: block " << from << " not found"; } + else if (!storage_.pool_prune(hash) && pool.is_valid()) { + // the block is there but could not be removed now (e.g. the storage is short of buffer + // cache during a fast sync): keep it and retry from here on the next step + cswarning() << kLogPrefix << "pruned storage: block " << from << " not removed yet, will retry"; + break; + } // getHashBySequence() then answers an empty hash, as for any block not stored blockHashes_->remove(from); }