diff --git a/client/config/config.cpp b/client/config/config.cpp index f1325669f..e5533d2ad 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -73,6 +73,7 @@ const std::string PARAM_NAME_POOL_SYNC_SEQ_VERIF_FREQ = "sequences_verification_ const std::string PARAM_NAME_STORAGE_CHECKPOINT_KEEP = "checkpoint_keep"; const std::string PARAM_NAME_STORAGE_CHECKPOINT_EVERY = "checkpoint_every"; const std::string PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES = "checkpoint_every_minutes"; +const std::string PARAM_NAME_STORAGE_PRUNE_KEEP_BLOCKS = "prune_keep_blocks"; const std::string PARAM_NAME_API_PORT = "port"; const std::string PARAM_NAME_AJAX_PORT = "ajax_port"; @@ -962,6 +963,23 @@ void Config::readStorageData(const boost::property_tree::ptree& config) { storageData_.checkpointEvery = 1000; } checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes); + checkAndSaveValue(data, block, PARAM_NAME_STORAGE_PRUNE_KEEP_BLOCKS, storageData_.pruneKeepBlocks); + if (storageData_.pruneKeepBlocks > 0) { + // contracts refer to blocks up to 100 rounds back; keep a margin above that + if (storageData_.pruneKeepBlocks < 200) { + cswarning() << "config: prune_keep_blocks=" << storageData_.pruneKeepBlocks << " is too low, raising to 200"; + storageData_.pruneKeepBlocks = 200; + } + // pruning follows the periodic checkpoints and keeps the previous one usable + if (storageData_.checkpointEvery == 0) { + cswarning() << "config: prune_keep_blocks needs checkpoint_every, pruning is off"; + storageData_.pruneKeepBlocks = 0; + } + if (storageData_.checkpointKeep < 2) { + cswarning() << "config: prune_keep_blocks keeps at least 2 checkpoints, raising checkpoint_keep to 2"; + storageData_.checkpointKeep = 2; + } + } } void Config::readApiData(const boost::property_tree::ptree& config) { diff --git a/client/config/config.hpp b/client/config/config.hpp index b014d2366..4eac85f56 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -62,6 +62,7 @@ struct StorageData { size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top) size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep) size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks) + size_t pruneKeepBlocks = 0; // pruned storage: blocks kept before the previous periodic checkpoint (0 = keep every block) }; struct ApiData { diff --git a/csdb/include/csdb/database.hpp b/csdb/include/csdb/database.hpp index 498a6b542..11ea232f2 100644 --- a/csdb/include/csdb/database.hpp +++ b/csdb/include/csdb/database.hpp @@ -48,6 +48,11 @@ class Database { virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0; virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0; + // returns the space of removed records to the file system; a no-op where not supported + virtual bool compact() { + return true; + } + class Iterator { protected: Iterator(); diff --git a/csdb/include/csdb/database_berkeleydb.hpp b/csdb/include/csdb/database_berkeleydb.hpp index 00af43220..1f6f57d6b 100644 --- a/csdb/include/csdb/database_berkeleydb.hpp +++ b/csdb/include/csdb/database_berkeleydb.hpp @@ -40,6 +40,7 @@ class DatabaseBerkeleyDB : public Database { bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override; bool getContractData(const cs::Bytes& key, cs::Bytes& data) override; + bool compact() override; void logfile_routine(); diff --git a/csdb/include/csdb/storage.hpp b/csdb/include/csdb/storage.hpp index 192d1764c..8a76c1731 100644 --- a/csdb/include/csdb/storage.hpp +++ b/csdb/include/csdb/storage.hpp @@ -316,6 +316,28 @@ class Storage final { csdb::PoolHash pool_hash(cs::Sequence sequence) const; + /** + * Pruned storage: removes an old pool, which the node no longer needs. Not meant for the last + * pools, whose bookkeeping (count, last hash, write queue) it leaves alone. + * + * @param hash The pool hash. + * + * @returns True if the pool was removed. + */ + bool pool_prune(const PoolHash& hash); + + /** + * Sequence of the oldest pool stored: 0 unless the storage was pruned. + * + * @returns The sequence, or std::numeric_limits::max() if the storage is empty. + */ + cs::Sequence first_sequence() const; + + /** + * Returns the space of pruned pools to the file system. + */ + bool compact(); + public signals: const ReadBlockSignal& readBlockEvent() const; const BlockReadingStartedSingal& readingStartedEvent() const; diff --git a/csdb/src/database_berkeleydb.cpp b/csdb/src/database_berkeleydb.cpp index e06282f19..a149843b9 100644 --- a/csdb/src/database_berkeleydb.cpp +++ b/csdb/src/database_berkeleydb.cpp @@ -356,6 +356,25 @@ bool DatabaseBerkeleyDB::remove(const cs::Bytes &key) { return true; } +bool DatabaseBerkeleyDB::compact() { + if (!db_blocks_) { + set_last_error(NotOpen); + return false; + } + + // DB_FREE_SPACE gives emptied pages back to the file system + for (Db* db : { db_blocks_.get(), db_seq_no_.get() }) { + int status = db->compact(nullptr, nullptr, nullptr, nullptr, DB_FREE_SPACE, nullptr); + if (status != 0) { + set_last_error_from_berkeleydb(status); + return false; + } + } + + set_last_error(); + return true; +} + bool DatabaseBerkeleyDB::write_batch(const ItemList &) { assert(false); diff --git a/csdb/src/storage.cpp b/csdb/src/storage.cpp index 442f4c437..c86fa80de 100644 --- a/csdb/src/storage.cpp +++ b/csdb/src/storage.cpp @@ -272,6 +272,16 @@ bool Storage::priv::rescan(Storage::OpenCallback callback) { csinfo() << "Storage: slow start"; it->seek_to_first(); slowStart = true; + // a pruned storage starts after block 0: replaying it onto empty caches would build a wrong + // state, so such a node can only start from a quick-start checkpoint + if (it->is_valid() && it->key() != 0) { + set_last_error(Storage::DataIntegrityError, + "Pruned storage starts at block %u: it needs a quick-start checkpoint, a slow start is not possible", + it->key()); + cserror() << "Storage: pruned storage starts at block " << it->key() + << ", no quick-start checkpoint covers it; restore one in qs/ or resync from scratch"; + return false; + } } if (count_pool == last_seq_in_db && !slowStart) { @@ -1070,6 +1080,41 @@ bool Storage::update_contract_data(const Address& abs_addr /*input*/, const cs:: return d->db->updateContractData(bytes, data); } +bool Storage::pool_prune(const PoolHash& hash) { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + // the pool cache is left alone: it is not locked, and an old pool it may still hold stays valid + if (!d->db->remove(hash.to_binary())) { + d->set_last_error(DatabaseError, "%s: cannot remove pool %s", funcName(), hash.to_string().c_str()); + return false; + } + return true; +} + +cs::Sequence Storage::first_sequence() const { + if (!isOpen()) { + d->set_last_error(NotOpen); + return std::numeric_limits::max(); + } + auto it = d->db->new_iterator(); + it->seek_to_first(); + return it->is_valid() ? it->key() : std::numeric_limits::max(); +} + +bool Storage::compact() { + if (!isOpen()) { + d->set_last_error(NotOpen); + return false; + } + if (!d->db->compact()) { + d->set_last_error(DatabaseError, "%s: %s", funcName(), d->db->last_error_message().c_str()); + return false; + } + return true; +} + cs::Sequence Storage::pool_sequence(const PoolHash& hash) const { cs::Sequence seq = std::numeric_limits::max(); if (!isOpen()) { diff --git a/csnode/include/csnode/blockchain.hpp b/csnode/include/csnode/blockchain.hpp index 768a8df4a..c8ccb4b81 100644 --- a/csnode/include/csnode/blockchain.hpp +++ b/csnode/include/csnode/blockchain.hpp @@ -92,6 +92,10 @@ class BlockChain { // false while trxIndex has an unwalked floor gap; consensus uses this to gate Trusted. bool isTrxIndexReady() const; + // pruned storage ([storage] prune_keep_blocks): the blocks before getFirstStoredSequence() were + // removed once a later periodic checkpoint made them unnecessary + cs::Sequence getFirstStoredSequence() const; + // return unique id of database if at least one unique block has written, otherwise (only genesis block) 0 uint64_t uuid() const; @@ -577,6 +581,13 @@ public slots: static constexpr size_t kQuickStartSaveCachesInterval = 500'000; std::chrono::steady_clock::time_point lastCheckpointWallClock_ = std::chrono::steady_clock::now(); + + // pruned storage: after the periodic checkpoint C, the blocks before C - checkpoint_every - + // prune_keep_blocks go; they are removed a few at a time as new blocks are applied + void schedulePruning(cs::Sequence checkpointSeq); + void pruneStep(); + std::atomic firstStoredSeq_ = 0; + cs::Sequence pruneUpTo_ = 0; // consensus settings persisted in quick-start caches; same defaults as BlockChain_Serializer::clear() int32_t blockRewardIntegral_ = 0; uint64_t blockRewardFraction_ = 0; diff --git a/csnode/include/csnode/transactionsindex.hpp b/csnode/include/csnode/transactionsindex.hpp index 19562749a..6518aefaf 100644 --- a/csnode/include/csnode/transactionsindex.hpp +++ b/csnode/include/csnode/transactionsindex.hpp @@ -45,6 +45,10 @@ class TransactionsIndex { Sequence getPrevTransBlock(const csdb::Address& _addr, Sequence _curr) const; + // pruned storage: forget an old block's entries; a walk back through an address's history + // then ends at the oldest block still stored (getPrevTransBlock answers kWrongSequence) + void onPruneBlock(const csdb::Pool&); + public slots: void onStartReadFromDb(Sequence _lastWrittenPoolSeq); void onReadFromDb(const csdb::Pool&); diff --git a/csnode/src/blockchain.cpp b/csnode/src/blockchain.cpp index e2cccfb44..e1dfcb3db 100644 --- a/csnode/src/blockchain.cpp +++ b/csnode/src/blockchain.cpp @@ -185,6 +185,12 @@ bool BlockChain::init( cserror() << kLogPrefix << "Couldn't open database at " << path; return false; } + if (const auto first = storage_.first_sequence(); first != std::numeric_limits::max()) { + firstStoredSeq_ = first; + if (first > 0) { + cslog() << kLogPrefix << "pruned storage: oldest block stored is " << WithDelimiters(first); + } + } if (newBlockchainTop != cs::kWrongSequence) { if (trxIndex_) trxIndex_->trimToFloor(newBlockchainTop); @@ -233,6 +239,51 @@ bool BlockChain::isTrxIndexReady() const { return !trxIndex_ || trxIndex_->isReady(); } +cs::Sequence BlockChain::getFirstStoredSequence() const { + return firstStoredSeq_; +} + +void BlockChain::schedulePruning(cs::Sequence checkpointSeq) { + const auto& sto = cs::ConfigHolder::instance().config()->getStorageSettings(); + if (sto.pruneKeepBlocks == 0 || checkpointSeq < sto.checkpointEvery + sto.pruneKeepBlocks) { + return; + } + // the previous periodic checkpoint and the blocks just before it stay, should this one be unusable + const cs::Sequence target = checkpointSeq - sto.checkpointEvery - sto.pruneKeepBlocks; + if (target > pruneUpTo_) { + pruneUpTo_ = target; + cslog() << kLogPrefix << "pruned storage: blocks before " << WithDelimiters(target) << " will be removed"; + } +} + +void BlockChain::pruneStep() { + constexpr cs::Sequence kBlocksPerStep = 64; + cs::Sequence from = firstStoredSeq_; + if (from >= pruneUpTo_) { + return; + } + const cs::Sequence end = std::min(pruneUpTo_, from + kBlocksPerStep); + for (; from < end; ++from) { + // the block itself tells which transaction index entries it owns + const csdb::Pool pool = storage_.pool_load(from); + csdb::PoolHash hash = pool.is_valid() ? pool.hash() : blockHashes_->find(from); + if (pool.is_valid() && trxIndex_) { + trxIndex_->onPruneBlock(pool); + } + if (hash.is_empty() || !storage_.pool_prune(hash)) { + csdebug() << kLogPrefix << "pruned storage: block " << from << " not found"; + } + // getHashBySequence() then answers an empty hash, as for any block not stored + blockHashes_->remove(from); + } + firstStoredSeq_ = from; + if (from == pruneUpTo_) { + const bool compacted = storage_.compact(); + cslog() << kLogPrefix << "pruned storage: blocks before " << WithDelimiters(from) << " removed" + << (compacted ? ", storage compacted" : ", storage compaction failed"); + } +} + uint64_t BlockChain::uuid() const { return uuid_; } @@ -292,6 +343,9 @@ void BlockChain::onReadFromDB(csdb::Pool block, bool* shouldStop) { if (serializationManPtr_->save(blockSeq, head)) { serializationManPtr_->pruneCheckpoints(sto.checkpointKeep); lastCheckpointWallClock_ = now; + if (byCount) { + schedulePruning(blockSeq); // carried out once blocks are applied live, not while the storage is read + } } else { cserror() << kLogPrefix << "slow start: cannot save checkpoint at " << blockSeq; @@ -869,12 +923,16 @@ bool BlockChain::applyBlockToCaches(const csdb::Pool& pool) { if (serializationManPtr_->save(pool.sequence(), head)) { serializationManPtr_->pruneCheckpoints(sto.checkpointKeep); lastCheckpointWallClock_ = now; + if (byCount) { + schedulePruning(pool.sequence()); + } } else { cserror() << "Cannot save caches with version " << pool.sequence(); } } } + pruneStep(); return true; } diff --git a/csnode/src/transactionsindex.cpp b/csnode/src/transactionsindex.cpp index f5064c1f2..be0324b9a 100644 --- a/csnode/src/transactionsindex.cpp +++ b/csnode/src/transactionsindex.cpp @@ -369,6 +369,20 @@ void TransactionsIndex::setPrevTransBlock(const PublicKey& _pubKey, cs::Sequence db_->insert(getTrxIndexKey(_pubKey, _curr), _prev); } +void TransactionsIndex::onPruneBlock(const csdb::Pool& _pool) { + std::set removed; + auto lbd = [&removed, &_pool, this](const csdb::Address& _addr) { + auto key = bc_.getAddressByType(_addr, BlockChain::AddressType::PublicKey); + if (removed.insert(key).second) { + removeLastTransBlock(key.public_key(), _pool.sequence()); + } + }; + for (auto& tr : _pool.transactions()) { + lbd(tr.source()); + lbd(tr.target()); + } +} + void TransactionsIndex::removeLastTransBlock(const PublicKey& _pubKey, cs::Sequence _curr) { db_->remove(getTrxIndexKey(_pubKey, _curr)); } diff --git a/solver/src/smartcontracts.cpp b/solver/src/smartcontracts.cpp index 5eb47514b..f11b683bb 100644 --- a/solver/src/smartcontracts.cpp +++ b/solver/src/smartcontracts.cpp @@ -3178,6 +3178,11 @@ size_t SmartContracts::validateRestoredStatesAgainstChain(BlockChain& blockchain const SmartContractRef& chain_ref = item.ref_state.is_valid() ? item.ref_state : item.ref_execute; if (!chain_ref.is_valid()) { ++unverifiable; continue; } csdb::Transaction t = get_transaction(chain_ref, abs_addr); + if (!t.is_valid() && chain_ref.sequence < blockchain.getFirstStoredSequence()) { + // pruned storage: the block is gone, the checkpoint is the only record of this state + ++unverifiable; + continue; + } if (!t.is_valid()) { cswarning() << kLogPrefix << to_base58(abs_addr) << ": qs-restored new_state ref " << chain_ref