From f1bb3c98420b5359aa5563c51d269d7851485773 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 12 Oct 2026 07:14:27 +0200 Subject: [PATCH 1/4] csdb: compact pruned storage in bounded steps, never terminate on it On mainnet the first pruning cycle of a node syncing from block 0 (blocks before 99'000 removed at the 200'000 checkpoint) terminated the node: compacting the 600 MB blockchain.db in one Db::compact call needed more of the environment's buffer cache than it holds, and BerkeleyDB threw (BDB3017 unable to allocate space from the buffer cache; terminate called after throwing an instance of 'DbException', what(): Db::compact: Cannot allocate memory). DatabaseBerkeleyDB::compact now works through each database 64 pages per call, resuming from the key the previous call stopped at, and turns any exception into an error result: compaction only returns space to the file system (BerkeleyDB reuses freed pages anyway), so pruneStep logs the failure and the node carries on. Co-Authored-By: Claude Opus 5.5 --- csdb/src/database_berkeleydb.cpp | 37 ++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/csdb/src/database_berkeleydb.cpp b/csdb/src/database_berkeleydb.cpp index a149843b9..c265e7e4a 100644 --- a/csdb/src/database_berkeleydb.cpp +++ b/csdb/src/database_berkeleydb.cpp @@ -362,14 +362,39 @@ bool DatabaseBerkeleyDB::compact() { return false; } - // DB_FREE_SPACE gives emptied pages back to the file system - for (Db* db : { db_blocks_.get(), db_seq_no_.get() }) { - int status = db->compact(nullptr, nullptr, nullptr, nullptr, DB_FREE_SPACE, nullptr); - if (status != 0) { - set_last_error_from_berkeleydb(status); - return false; + // DB_FREE_SPACE gives emptied pages back to the file system. Compacting a large file in one call + // pins more pages than the environment's buffer cache holds (BDB3017, "Db::compact: Cannot + // allocate memory"), so it goes in bounded steps, each resuming where the previous one stopped. + // Compaction only returns space (BerkeleyDB reuses freed pages anyway): a failure is reported + // and never lets an exception terminate the node. + constexpr u_int32_t kPagesPerStep = 64; + constexpr int kMaxSteps = 1'000'000; + try { + for (Db* db : { db_blocks_.get(), db_seq_no_.get() }) { + cs::Bytes resume; + for (int step = 0; step < kMaxSteps; ++step) { + DB_COMPACT progress{}; + progress.compact_pages = kPagesPerStep; + Dbt from(resume.data(), static_cast(resume.size())); + Dbt_safe stopped; + int status = db->compact(nullptr, resume.empty() ? nullptr : &from, nullptr, &progress, DB_FREE_SPACE, &stopped); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } + // pages left over in the budget: the pass reached the end of the database + if (progress.compact_pages > 0 || stopped.get_size() == 0) { + break; + } + auto begin = static_cast(stopped.get_data()); + resume.assign(begin, begin + stopped.get_size()); + } } } + catch (const std::exception& e) { + set_last_error(UnknownError, "compact: %s", e.what()); + return false; + } set_last_error(); return true; From b96c5f8a4fec26e7021207bdeed75ba9d5959334 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 12 Oct 2026 07:20:38 +0200 Subject: [PATCH 2/4] csdb: a failed compaction is a warning, not a storage error BerkeleyDB's default buffer cache (about 64 pages of 4 KB) is usually full of freshly written pages while a node syncs, and Db::compact then cannot allocate pages even in small steps (seen on mainnet at the 300'000 checkpoint). Nothing is wrong with the storage: compaction only returns space, and the pages freed by pruning are reused for new blocks (blockchain.db grew from 207 to 232 MB while 100'000 blocks were added and 100'000 removed). Log it as a warning instead of setting the storage error. Co-Authored-By: Claude Opus 5.5 --- csdb/src/storage.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index c86fa80de..28ba08580 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -1109,7 +1109,8 @@ bool Storage::compact() { return false; } if (!d->db->compact()) { - d->set_last_error(DatabaseError, "%s: %s", funcName(), d->db->last_error_message().c_str()); + // not a fault: compaction only returns space, and freed pages are reused for new blocks anyway + cswarning() << "Storage> compaction skipped: " << d->db->last_error_message(); return false; } return true; From 5e8905c3709472b2b1a9e19964c8670fae80e27a Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 12 Oct 2026 09:53:09 +0200 Subject: [PATCH 3/4] csdb, csnode: removing a pruned block can no longer terminate the node On mainnet a pruned node syncing fast terminated in Db::del: BerkeleyDB's default buffer cache was exhausted (BDB3017) and the exception escaped DatabaseBerkeleyDB::remove. remove now turns any exception into an error result, and pruneStep keeps a block it could not remove and retries it on the next step instead of counting it as pruned. Co-Authored-By: Claude Opus 5.5 --- csdb/src/database_berkeleydb.cpp | 38 +++++++++++++++++++------------- csnode/src/blockchain.cpp | 8 ++++++- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/csdb/src/database_berkeleydb.cpp b/csdb/src/database_berkeleydb.cpp index c265e7e4a..6f7689e26 100644 --- a/csdb/src/database_berkeleydb.cpp +++ b/csdb/src/database_berkeleydb.cpp @@ -332,23 +332,31 @@ bool DatabaseBerkeleyDB::remove(const cs::Bytes &key) { return false; } - Dbt_copy db_key(key); - Dbt_copy db_seq_no; - int status = db_seq_no_->get(nullptr, &db_key, &db_seq_no, 0); - if (status != 0) { - set_last_error_from_berkeleydb(status); - return false; - } + // BerkeleyDB throws when its buffer cache is exhausted (BDB3017, seen on mainnet while a pruned + // node removed old blocks during a fast sync): report it, the caller may retry later + try { + Dbt_copy db_key(key); + Dbt_copy db_seq_no; + int status = db_seq_no_->get(nullptr, &db_key, &db_seq_no, 0); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } - status = db_seq_no_->del(nullptr, &db_key, 0); - if (status != 0) { - set_last_error_from_berkeleydb(status); - return false; - } + status = db_seq_no_->del(nullptr, &db_key, 0); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } - status = db_blocks_->del(nullptr, &db_seq_no, 0); - if (status != 0) { - set_last_error_from_berkeleydb(status); + status = db_blocks_->del(nullptr, &db_seq_no, 0); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } + } + catch (const std::exception& e) { + set_last_error(UnknownError, "remove: %s", e.what()); return false; } diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index e1dfcb3db..16fcd47be 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -270,9 +270,15 @@ void BlockChain::pruneStep() { if (pool.is_valid() && trxIndex_) { trxIndex_->onPruneBlock(pool); } - if (hash.is_empty() || !storage_.pool_prune(hash)) { + if (hash.is_empty()) { csdebug() << kLogPrefix << "pruned storage: block " << from << " not found"; } + else if (!storage_.pool_prune(hash) && pool.is_valid()) { + // the block is there but could not be removed now (e.g. the storage is short of buffer + // cache during a fast sync): keep it and retry from here on the next step + cswarning() << kLogPrefix << "pruned storage: block " << from << " not removed yet, will retry"; + break; + } // getHashBySequence() then answers an empty hash, as for any block not stored blockHashes_->remove(from); } From ec939dbd9a0ed96ef00b85cf2e239495ce2d8597 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Mon, 12 Oct 2026 10:18:26 +0200 Subject: [PATCH 4/4] csdb: LMDB block database for pruned storage On mainnet a pruned node's blockchain.db kept growing even though the blocks it keeps stayed the same: BerkeleyDB's Recno keeps a placeholder for every removed record, so the pages of pruned blocks are never freed (db_stat on a copy: 104'123 records on 284'131 pages, 8% full, nothing on the free list), and compacting such a file exhausts the environment's locks and log region even with larger limits. DatabaseLmdb implements the same csdb::Database contract on LMDB, which the node already uses for its caches: blocks by 0-based sequence (MDB_INTEGERKEY), a hash -> sequence index and contract data, one transaction per operation, MDB_NOTLS (transactions are not tied to threads) and MDB_NOSYNC (as BerkeleyDB's DB_TXN_NOSYNC). LMDB frees the pages of removed records and reuses them, so the file stays the size of the blocks kept. [storage] block_db=lmdb selects it (default berkeleydb); the database lives in /lmdb, so an existing BerkeleyDB database is left alone. Nodes without the option are unchanged. Tests DatabaseLmdb.StoresFindsRemovesAndIterates and DatabaseLmdb.ReusesTheSpaceOfRemovedBlocks (ten windows of blocks written and pruned keep the file under twice one window); nodetests 136/136. Co-Authored-By: Claude Opus 5.5 --- client/config/config.cpp | 10 + client/config/config.hpp | 1 + csdb/CMakeLists.txt | 2 + csdb/include/csdb/database_lmdb.hpp | 50 ++++ csdb/include/csdb/storage.hpp | 10 +- csdb/src/database_lmdb.cpp | 367 +++++++++++++++++++++++++++ csdb/src/storage.cpp | 17 +- csnode/src/blockchain.cpp | 8 +- tests/csnode/database_lmdb_tests.cpp | 102 ++++++++ 9 files changed, 562 insertions(+), 5 deletions(-) create mode 100644 csdb/include/csdb/database_lmdb.hpp create mode 100644 csdb/src/database_lmdb.cpp create mode 100644 tests/csnode/database_lmdb_tests.cpp diff --git a/client/config/config.cpp b/client/config/config.cpp index e5533d2ad..75c2c7ad6 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -74,6 +74,7 @@ const std::string PARAM_NAME_STORAGE_CHECKPOINT_KEEP = "checkpoint_keep"; const std::string PARAM_NAME_STORAGE_CHECKPOINT_EVERY = "checkpoint_every"; const std::string PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES = "checkpoint_every_minutes"; const std::string PARAM_NAME_STORAGE_PRUNE_KEEP_BLOCKS = "prune_keep_blocks"; +const std::string PARAM_NAME_STORAGE_BLOCK_DB = "block_db"; const std::string PARAM_NAME_API_PORT = "port"; const std::string PARAM_NAME_AJAX_PORT = "ajax_port"; @@ -964,6 +965,15 @@ void Config::readStorageData(const boost::property_tree::ptree& config) { } checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes); checkAndSaveValue(data, block, PARAM_NAME_STORAGE_PRUNE_KEEP_BLOCKS, storageData_.pruneKeepBlocks); + if (data.count(PARAM_NAME_STORAGE_BLOCK_DB)) { + const auto engine = data.get(PARAM_NAME_STORAGE_BLOCK_DB); + if (engine == "lmdb") { + storageData_.lmdbBlocks = true; + } + else if (engine != "berkeleydb") { + cswarning() << "config: unknown block_db=" << engine << ", using berkeleydb"; + } + } if (storageData_.pruneKeepBlocks > 0) { // contracts refer to blocks up to 100 rounds back; keep a margin above that if (storageData_.pruneKeepBlocks < 200) { diff --git a/client/config/config.hpp b/client/config/config.hpp index 4eac85f56..6489598fd 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -63,6 +63,7 @@ struct StorageData { size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep) size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks) size_t pruneKeepBlocks = 0; // pruned storage: blocks kept before the previous periodic checkpoint (0 = keep every block) + bool lmdbBlocks = false; // block database on LMDB (block_db=lmdb) instead of BerkeleyDB; meant for pruned storage }; struct ApiData { diff --git a/csdb/CMakeLists.txt b/csdb/CMakeLists.txt index ca0968ef7..4933ab97d 100644 --- a/csdb/CMakeLists.txt +++ b/csdb/CMakeLists.txt @@ -30,6 +30,7 @@ add_library(${PROJECT_NAME} STATIC src/priv_crypto.hpp src/database.cpp src/database_berkeleydb.cpp + src/database_lmdb.cpp src/user_field.cpp include/csdb/internal/shared_data.hpp include/csdb/internal/shared_data_ptr_implementation.hpp @@ -79,6 +80,7 @@ target_link_libraries( Boost::filesystem Boost::disable_autolinking BerkeleyDB + lmdb lz4 lib ) diff --git a/csdb/include/csdb/database_lmdb.hpp b/csdb/include/csdb/database_lmdb.hpp new file mode 100644 index 000000000..38b0e27aa --- /dev/null +++ b/csdb/include/csdb/database_lmdb.hpp @@ -0,0 +1,50 @@ +/** + * @file database_lmdb.hpp + * Block storage on LMDB, for pruned nodes ([storage] block_db=lmdb). + */ + +#ifndef _CREDITS_CSDB_DATABASE_LMDB_H_INCLUDED_ +#define _CREDITS_CSDB_DATABASE_LMDB_H_INCLUDED_ + +#include + +struct MDB_env; + +namespace csdb { + +// Same contract as DatabaseBerkeleyDB: blocks by 0-based sequence, a hash -> sequence index and +// contract data. BerkeleyDB's Recno keeps a placeholder for every removed record, so the pages of +// pruned blocks are never reused and its file keeps growing; LMDB frees the pages of removed +// records and reuses them, so a pruned node's storage stays the size of the blocks it keeps. +class DatabaseLmdb : public Database { +public: + DatabaseLmdb(); + ~DatabaseLmdb() override; + + bool open(const std::string& path); + +private: + bool is_open() const final; + bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final; + bool get(const cs::Bytes& key, cs::Bytes* value) final; + bool get(const uint32_t seq_no, cs::Bytes* value) final; + bool remove(const cs::Bytes& key) final; + bool seq_no(const cs::Bytes& key, uint32_t* value) final; + bool write_batch(const ItemList&) final; + IteratorPtr new_iterator() final; + + bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) final; + bool getContractData(const cs::Bytes& key, cs::Bytes& data) final; + + class Iterator; + + bool fail(int status, const char* what); + + MDB_env* env_ = nullptr; + unsigned int blocks_ = 0; // MDB_dbi: sequence (uint32, MDB_INTEGERKEY) -> block + unsigned int seqNo_ = 0; // MDB_dbi: block hash -> sequence + unsigned int contracts_ = 0; // MDB_dbi: contract address -> contract data +}; + +} // namespace csdb +#endif // _CREDITS_CSDB_DATABASE_LMDB_H_INCLUDED_ diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 8a76c1731..5399a7145 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -137,10 +137,18 @@ class Storage final { * In case of fail the error info can be received with methods: \ref last_error, * \ref last_error_message, \ref db_last_error() and \ref db_last_error_message() */ + // block database engine: BerkeleyDB (every node so far), or LMDB in /lmdb for pruned + // storage, whose removed blocks LMDB reuses the space of + enum class Engine { + BerkeleyDB, + Lmdb + }; + bool open(const ::std::string& path_to_base = ::std::string{}, OpenCallback callback = nullptr, cs::Sequence newBlockchainTop = cs::kWrongSequence, - cs::Sequence startReadFrom = 0); + cs::Sequence startReadFrom = 0, + Engine engine = Engine::BerkeleyDB); /** * @brief Creating the storage using the parameters set. diff --git a/csdb/src/database_lmdb.cpp b/csdb/src/database_lmdb.cpp new file mode 100644 index 000000000..5373561ad --- /dev/null +++ b/csdb/src/database_lmdb.cpp @@ -0,0 +1,367 @@ +#include + +#include + +#include + +#include + +namespace csdb { + +namespace { +// a sparse file on 64-bit systems: only the pages in use take disk space +constexpr size_t kMapSize = size_t(1) << 40; + +MDB_val toVal(const cs::Bytes& bytes) { + return MDB_val{bytes.size(), const_cast(bytes.data())}; +} + +MDB_val toVal(uint32_t& seq) { + return MDB_val{sizeof(seq), &seq}; +} + +void assign(cs::Bytes* out, const MDB_val& val) { + auto begin = static_cast(val.mv_data); + out->assign(begin, begin + val.mv_size); +} + +// a transaction aborted on scope exit unless committed +class Txn { +public: + Txn(MDB_env* env, bool readOnly) { + status_ = mdb_txn_begin(env, nullptr, readOnly ? MDB_RDONLY : 0, &txn_); + } + ~Txn() { + if (txn_ != nullptr) { + mdb_txn_abort(txn_); + } + } + int status() const { + return status_; + } + MDB_txn* get() const { + return txn_; + } + int commit() { + int status = mdb_txn_commit(txn_); + txn_ = nullptr; + return status; + } + +private: + MDB_txn* txn_ = nullptr; + int status_ = 0; +}; +} // namespace + +DatabaseLmdb::DatabaseLmdb() = default; + +DatabaseLmdb::~DatabaseLmdb() { + if (env_ != nullptr) { + mdb_env_sync(env_, 1); + mdb_env_close(env_); + } +} + +bool DatabaseLmdb::fail(int status, const char* what) { + set_last_error(status == MDB_NOTFOUND ? NotFound : UnknownError, "LMDB %s: %s", what, mdb_strerror(status)); + return false; +} + +bool DatabaseLmdb::open(const std::string& path) { + boost::system::error_code ec; + boost::filesystem::create_directories(path, ec); + + int status = mdb_env_create(&env_); + if (status != 0) { + env_ = nullptr; + return fail(status, "env_create"); + } + status = mdb_env_set_mapsize(env_, kMapSize); + status = status ? status : mdb_env_set_maxdbs(env_, 3); + // MDB_NOTLS: transactions are not bound to threads (sync, API and node threads all read); + // MDB_NOSYNC as BerkeleyDB's DB_TXN_NOSYNC: a system crash may lose the last blocks, which the + // node fetches again, but not the database's integrity + status = status ? status : mdb_env_open(env_, path.c_str(), MDB_NOTLS | MDB_NOSYNC, 0644); + if (status != 0) { + mdb_env_close(env_); + env_ = nullptr; + return fail(status, "env_open"); + } + + Txn txn(env_, false); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + status = mdb_dbi_open(txn.get(), "blocks", MDB_CREATE | MDB_INTEGERKEY, &blocks_); + status = status ? status : mdb_dbi_open(txn.get(), "seq_no", MDB_CREATE, &seqNo_); + status = status ? status : mdb_dbi_open(txn.get(), "contracts", MDB_CREATE, &contracts_); + status = status ? status : txn.commit(); + if (status != 0) { + return fail(status, "dbi_open"); + } + + set_last_error(); + return true; +} + +bool DatabaseLmdb::is_open() const { + return env_ != nullptr; +} + +bool DatabaseLmdb::put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) { + if (!is_open()) { + set_last_error(NotOpen); + return false; + } + Txn txn(env_, false); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + MDB_val seq = toVal(seq_no); + MDB_val block = toVal(value); + MDB_val hash = toVal(key); + int status = mdb_put(txn.get(), blocks_, &seq, &block, 0); + status = status ? status : mdb_put(txn.get(), seqNo_, &hash, &seq, 0); + status = status ? status : txn.commit(); + if (status != 0) { + return fail(status, "put"); + } + set_last_error(); + return true; +} + +bool DatabaseLmdb::get(const cs::Bytes& key, cs::Bytes* value) { + if (!is_open()) { + set_last_error(NotOpen); + return false; + } + Txn txn(env_, true); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + MDB_val hash = toVal(key); + MDB_val seq; + int status = mdb_get(txn.get(), seqNo_, &hash, &seq); + if (status != 0) { + return fail(status, "get"); + } + if (value == nullptr) { + return true; // only asked whether the block is stored + } + MDB_val block; + status = mdb_get(txn.get(), blocks_, &seq, &block); + if (status != 0) { + return fail(status, "get"); + } + assign(value, block); + set_last_error(); + return true; +} + +bool DatabaseLmdb::get(const uint32_t seq_no, cs::Bytes* value) { + if (!is_open()) { + set_last_error(NotOpen); + return false; + } + if (value == nullptr) { + return false; + } + Txn txn(env_, true); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + uint32_t key = seq_no; + MDB_val seq = toVal(key); + MDB_val block; + int status = mdb_get(txn.get(), blocks_, &seq, &block); + if (status != 0) { + return fail(status, "get"); + } + assign(value, block); + set_last_error(); + return true; +} + +bool DatabaseLmdb::remove(const cs::Bytes& key) { + if (!is_open()) { + set_last_error(NotOpen); + return false; + } + Txn txn(env_, false); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + MDB_val hash = toVal(key); + MDB_val found; + int status = mdb_get(txn.get(), seqNo_, &hash, &found); + if (status != 0) { + return fail(status, "remove"); + } + uint32_t seqNo = *static_cast(found.mv_data); + MDB_val seq = toVal(seqNo); + status = mdb_del(txn.get(), seqNo_, &hash, nullptr); + status = status ? status : mdb_del(txn.get(), blocks_, &seq, nullptr); + status = status ? status : txn.commit(); + if (status != 0) { + return fail(status, "remove"); + } + set_last_error(); + return true; +} + +bool DatabaseLmdb::seq_no(const cs::Bytes& key, uint32_t* value) { + if (value == nullptr) { + set_last_error(InvalidArgument); + return false; + } + if (!is_open()) { + set_last_error(NotOpen); + return false; + } + Txn txn(env_, true); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + MDB_val hash = toVal(key); + MDB_val seq; + int status = mdb_get(txn.get(), seqNo_, &hash, &seq); + if (status != 0) { + return fail(status, "seq_no"); + } + *value = *static_cast(seq.mv_data); + set_last_error(); + return true; +} + +bool DatabaseLmdb::write_batch(const ItemList&) { + set_last_error(NotSupported); + return false; +} + +bool DatabaseLmdb::updateContractData(const cs::Bytes& key, const cs::Bytes& data) { + if (!is_open()) { + set_last_error(NotOpen); + return false; + } + Txn txn(env_, false); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + MDB_val k = toVal(key); + MDB_val v = toVal(data); + int status = mdb_put(txn.get(), contracts_, &k, &v, 0); + status = status ? status : txn.commit(); + if (status != 0) { + return fail(status, "updateContractData"); + } + set_last_error(); + return true; +} + +bool DatabaseLmdb::getContractData(const cs::Bytes& key, cs::Bytes& data) { + if (!is_open()) { + set_last_error(NotOpen); + return false; + } + Txn txn(env_, true); + if (txn.status() != 0) { + return fail(txn.status(), "txn_begin"); + } + MDB_val k = toVal(key); + MDB_val v; + int status = mdb_get(txn.get(), contracts_, &k, &v); + if (status != 0) { + return fail(status, "getContractData"); + } + assign(&data, v); + set_last_error(); + return true; +} + +// Walks the blocks in sequence order. It holds a read transaction, which keeps LMDB from reusing +// the pages it sees: iterators are short-lived (the storage scan at start-up). +class DatabaseLmdb::Iterator final : public Database::Iterator { +public: + Iterator(MDB_env* env, unsigned int blocks) { + if (mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn_) != 0) { + txn_ = nullptr; + return; + } + if (mdb_cursor_open(txn_, blocks, &cursor_) != 0) { + cursor_ = nullptr; + } + } + ~Iterator() final { + if (cursor_ != nullptr) { + mdb_cursor_close(cursor_); + } + if (txn_ != nullptr) { + mdb_txn_abort(txn_); + } + } + + bool is_valid() const final { + return valid_; + } + void seek_to_first() final { + move(nullptr, MDB_FIRST); + } + void seek_to_last() final { + move(nullptr, MDB_LAST); + } + void seek(const cs::Bytes&) final { + valid_ = false; // by hash is not used, as with BerkeleyDB + } + void seek(const uint32_t sec_no) final { + uint32_t key = sec_no; + MDB_val k = toVal(key); + move(&k, MDB_SET_KEY); + } + void next() final { + move(nullptr, MDB_NEXT); + } + void prev() final { + move(nullptr, MDB_PREV); + } + uint32_t key() const final { + return valid_ ? key_ : std::numeric_limits::max(); + } + cs::Bytes value() const final { + return valid_ ? value_ : cs::Bytes{}; + } + +private: + void move(MDB_val* key, MDB_cursor_op op) { + valid_ = false; + if (cursor_ == nullptr) { + return; + } + MDB_val k{}; + MDB_val v{}; + if (key != nullptr) { + k = *key; + } + if (mdb_cursor_get(cursor_, &k, &v, op) != 0) { + return; + } + key_ = *static_cast(k.mv_data); + assign(&value_, v); + valid_ = true; + } + + MDB_txn* txn_ = nullptr; + MDB_cursor* cursor_ = nullptr; + bool valid_ = false; + uint32_t key_ = 0; + cs::Bytes value_; +}; + +Database::IteratorPtr DatabaseLmdb::new_iterator() { + if (!is_open()) { + return nullptr; + } + return std::make_shared(env_, blocks_); +} + +} // namespace csdb diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index 28ba08580..7b2250b46 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -506,15 +507,25 @@ bool Storage::open( const ::std::string& path_to_base, OpenCallback callback, cs::Sequence newBlockchainTop, - cs::Sequence startReadFrom + cs::Sequence startReadFrom, + Engine engine ) { ::std::string path{path_to_base}; if (path.empty()) { path = ::csdb::internal::app_data_path() + "/CREDITS"; } - auto db{::std::make_shared<::csdb::DatabaseBerkeleyDB>()}; - db->open(path); + ::std::shared_ptr db; + if (engine == Engine::Lmdb) { + auto lmdb{::std::make_shared<::csdb::DatabaseLmdb>()}; + lmdb->open(path + "/lmdb"); + db = lmdb; + } + else { + auto berkeley{::std::make_shared<::csdb::DatabaseBerkeleyDB>()}; + berkeley->open(path); + db = berkeley; + } //d->write_thread = std::thread(&Storage::priv::write_routine, d.get()); diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index 16fcd47be..700c9c681 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -181,7 +181,13 @@ bool BlockChain::init( return false; }; - if (!storage_.open(path, progress, newBlockchainTop, firstBlockToReadInDatabase)) { + // LMDB reuses the space of pruned blocks; BerkeleyDB's Recno never does ([storage] block_db) + const bool lmdbBlocks = cs::ConfigHolder::instance().config()->getStorageSettings().lmdbBlocks; + if (lmdbBlocks) { + cslog() << kLogPrefix << "block database: LMDB in " << path << "/lmdb"; + } + if (!storage_.open(path, progress, newBlockchainTop, firstBlockToReadInDatabase, + lmdbBlocks ? csdb::Storage::Engine::Lmdb : csdb::Storage::Engine::BerkeleyDB)) { cserror() << kLogPrefix << "Couldn't open database at " << path; return false; } diff --git a/tests/csnode/database_lmdb_tests.cpp b/tests/csnode/database_lmdb_tests.cpp new file mode 100644 index 000000000..a38e0e4a6 --- /dev/null +++ b/tests/csnode/database_lmdb_tests.cpp @@ -0,0 +1,102 @@ +#include "gtest/gtest.h" + +#include +#include + +#include + +namespace { +cs::Bytes hashOf(uint32_t seq) { + cs::Bytes hash(32, 0); + hash[0] = static_cast(seq); + hash[1] = static_cast(seq >> 8); + hash[2] = static_cast(seq >> 16); + return hash; +} + +cs::Bytes blockOf(uint32_t seq, size_t size = 3000) { + return cs::Bytes(size, static_cast(seq)); +} + +std::shared_ptr openAt(const std::filesystem::path& dir) { + auto lmdb = std::make_shared(); + lmdb->open(dir.string()); + return lmdb; +} +} // namespace + +// the same contract the storage relies on with BerkeleyDB: blocks by sequence and by hash, +// removal by hash, sequence order when iterating, contract data +TEST(DatabaseLmdb, StoresFindsRemovesAndIterates) { + const auto dir = std::filesystem::temp_directory_path() / "cs_lmdb_contract_test"; + std::filesystem::remove_all(dir); + { + auto db = openAt(dir); + ASSERT_TRUE(db->is_open()); + for (uint32_t seq = 0; seq < 10; ++seq) { + ASSERT_TRUE(db->put(hashOf(seq), seq, blockOf(seq))); + } + cs::Bytes value; + ASSERT_TRUE(db->get(uint32_t(7), &value)); + ASSERT_EQ(value, blockOf(7)); + ASSERT_TRUE(db->get(hashOf(3), &value)); + ASSERT_EQ(value, blockOf(3)); + ASSERT_TRUE(db->get(hashOf(3), nullptr)); + uint32_t seq = 0; + ASSERT_TRUE(db->seq_no(hashOf(5), &seq)); + ASSERT_EQ(seq, 5u); + + ASSERT_TRUE(db->remove(hashOf(0))); + ASSERT_TRUE(db->remove(hashOf(1))); + ASSERT_FALSE(db->get(uint32_t(0), &value)); + ASSERT_FALSE(db->get(hashOf(1), nullptr)); + ASSERT_FALSE(db->remove(hashOf(1))); + + auto it = db->new_iterator(); + it->seek_to_first(); + ASSERT_TRUE(it->is_valid()); + ASSERT_EQ(it->key(), 2u); // a pruned storage starts after block 0 + it->seek_to_last(); + ASSERT_EQ(it->key(), 9u); + it->seek(uint32_t(4)); + ASSERT_TRUE(it->is_valid()); + it->next(); + ASSERT_EQ(it->key(), 5u); + ASSERT_EQ(it->value(), blockOf(5)); + it->seek(uint32_t(1)); + ASSERT_FALSE(it->is_valid()); + + ASSERT_TRUE(db->updateContractData(hashOf(100), blockOf(1, 10))); + ASSERT_TRUE(db->getContractData(hashOf(100), value)); + ASSERT_EQ(value, blockOf(1, 10)); + } + // reopened, the data is still there + auto db = openAt(dir); + cs::Bytes value; + ASSERT_TRUE(db->get(uint32_t(9), &value)); + ASSERT_EQ(value, blockOf(9)); + std::filesystem::remove_all(dir); +} + +// the reason for LMDB: a rolling window of blocks keeps the file at the window's size, where +// BerkeleyDB's Recno never reuses the pages of removed records +TEST(DatabaseLmdb, ReusesTheSpaceOfRemovedBlocks) { + const auto dir = std::filesystem::temp_directory_path() / "cs_lmdb_reuse_test"; + std::filesystem::remove_all(dir); + auto db = openAt(dir); + constexpr uint32_t kWindow = 2000; + auto fileSize = [&dir] { return std::filesystem::file_size(dir / "data.mdb"); }; + + for (uint32_t seq = 0; seq < kWindow; ++seq) { + ASSERT_TRUE(db->put(hashOf(seq), seq, blockOf(seq))); + } + const auto windowSize = fileSize(); + + // ten more windows of blocks, each one pruning the blocks a window behind + for (uint32_t seq = kWindow; seq < 11 * kWindow; ++seq) { + ASSERT_TRUE(db->put(hashOf(seq), seq, blockOf(seq))); + ASSERT_TRUE(db->remove(hashOf(seq - kWindow))); + } + EXPECT_LT(fileSize(), 2 * windowSize) << "window " << windowSize << " bytes"; + std::filesystem::remove_all(dir); +}