Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions csnode/include/csnode/blockchain.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,14 @@ public slots:
void tryFlushDeferredBlock();
void addIncorrectBlockNumber(cs::Sequence seq);

// the state the caches hold now: wallets digest, wallet count and consensus settings hash, as a
// checkpoint saves it (state_digest.bin)
cs::CheckpointDigest checkpointDigest() const;
// hash of the consensus settings a checkpoint carries: block reward, mining coefficient, staking
// and mining flags, minimal stage-1 time, pending settings, DPOS start and the recorded special
// orders (the initial confidants come from the node's configuration and are not included)
std::array<uint8_t, 32> consensusSettingsDigest() const;

csdb::Amount getBlockReward() {
return csdb::Amount(blockRewardIntegral_, blockRewardFraction_);
}
Expand Down
7 changes: 6 additions & 1 deletion csnode/include/csnode/caches_serialization_manager.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <set>

#include <client/params.hpp>
#include <csnode/statedigest.hpp>
#include <lib/system/common.hpp>

class BlockChain;
Expand Down Expand Up @@ -51,7 +52,8 @@ class CachesSerializationManager {

void clear(size_t version = 0);

bool save(size_t version, const CheckpointHead& head);
// digest: the state the caches hold, written as state_digest.bin next to them
bool save(size_t version, const CheckpointHead& head, const std::optional<CheckpointDigest>& digest = std::nullopt);
bool save(size_t version = 0); // legacy: no chain-binding (qs/0 from pre-A nodes)
// Optional verifier is called after each candidate's hashes pass; if it
// returns false (e.g. chain-binding mismatch), the candidate is
Expand All @@ -64,6 +66,9 @@ class CachesSerializationManager {
std::optional<CheckpointHead> getLoadedHead() const;
// the version load() took the caches from, if any
std::optional<size_t> getLoadedVersion() const;
// the state_digest.bin of the version being loaded or loaded, nullopt when it has none; set
// before the ChainVerifier is called, so that the verifier can compare it with the caches
std::optional<CheckpointDigest> getLoadedDigest() const;
// moves a version aside to qs/<version>.bad, so that load() no longer takes it
void quarantine(size_t version);
// Sticky bit indicating the caches have been validated by a full
Expand Down
2 changes: 2 additions & 0 deletions csnode/include/csnode/multiwallets.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ class MultiWallets {
// order-independent digest of every wallet's balance-relevant state (ECMH, see statedigest.hpp);
// computed in full on the first call, then kept up to date on every wallet update
MultisetHash::Digest stateDigest() const;
// the wallets were replaced as a whole (a checkpoint loaded): the next stateDigest() recomputes
void invalidateStateDigest();
void iterate(std::function<bool(const PublicKey& key, const InternalData& data)> func);

protected:
Expand Down
31 changes: 31 additions & 0 deletions csnode/include/csnode/statedigest.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@

#include <array>
#include <cstdint>
#include <filesystem>
#include <optional>
#include <string>

#include <csnode/walletscache.hpp>
#include <lib/system/common.hpp>
Expand Down Expand Up @@ -41,6 +44,34 @@ class MultisetHash {
// records do not change the digest.
cs::Bytes serializeWalletState(const WalletsCache::WalletData& wallet);

// The state a checkpoint holds, saved with it as state_digest.bin: the digest of the wallets
// (MultiWallets::stateDigest), their number, and a hash of the consensus settings the checkpoint
// carries (BlockChain::consensusSettingsDigest). A snapshot made from the checkpoint declares the
// same values, and a node that loads the checkpoint recomputes and compares them.
struct CheckpointDigest {
static constexpr uint8_t kVersion = 1;
static constexpr const char* kFileName = "state_digest.bin";

MultisetHash::Digest digest{};
uint64_t wallets = 0;
std::array<uint8_t, 32> settings{};

bool write(const std::filesystem::path& file) const;
// nullopt for a missing, truncated or unknown-version file
static std::optional<CheckpointDigest> read(const std::filesystem::path& file);

bool operator==(const CheckpointDigest& other) const {
return digest == other.digest && wallets == other.wallets && settings == other.settings;
}
bool operator!=(const CheckpointDigest& other) const {
return !(*this == other);
}
};

// "STATE DIGEST #<sequence> <hex> wallets <n>": the form every digest line of the log has, so that
// scripts read the same fields from the periodic, the start-up and the checkpoint lines
std::string stateDigestLine(cs::Sequence sequence, const MultisetHash::Digest& digest, uint64_t wallets);

} // namespace cs

#endif // STATEDIGEST_HPP
3 changes: 3 additions & 0 deletions csnode/include/csnode/walletscache_serializer.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ using namespace boost::multi_index;

namespace cs {
class WalletsCache;
class MultiWallets;

class WalletsCache_Serializer {
public:
Expand Down Expand Up @@ -218,6 +219,8 @@ class WalletsCache_Serializer {
#endif
CurrentDelegations *currentDelegations_ = nullptr;
MiningDelegations *miningDelegations_ = nullptr;
// its state digest is kept up to date incrementally: replacing the wallets invalidates it
MultiWallets *multiWallets_ = nullptr;
};
} // namespace cs
#endif // WALLETS_CACHE_SERIALIZER_HPP
68 changes: 60 additions & 8 deletions csnode/src/blockchain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -97,16 +97,33 @@ bool BlockChain::tryQuickStart(
// trust the caches only if their head is still the block this node has at that sequence
cs::Sequence loadedHead = 0;
bool ok = serializationManPtr_->load([this, &loadedHead](const cs::CheckpointHead& head) {
if (cs::isCheckpointOnChain(head, [this](cs::Sequence seq) { return blockHashes_->find(seq).to_binary(); })) {
loadedHead = head.sequence;
return true;
if (!cs::isCheckpointOnChain(head, [this](cs::Sequence seq) { return blockHashes_->find(seq).to_binary(); })) {
cswarning() << "QUICK START: checkpoint head #" << head.sequence << " is not on this chain, skip it";
return false;
}
cswarning() << "QUICK START: checkpoint head #" << head.sequence << " is not on this chain, skip it";
return false;
// the caches must hold the state the checkpoint was saved with (no file: saved by an earlier build)
if (const auto saved = serializationManPtr_->getLoadedDigest(); saved.has_value()) {
const auto loaded = checkpointDigest();
if (loaded != *saved) {
// no "STATE DIGEST #" here: scripts take the digest of a block from lines with that prefix
cserror() << "QUICK START: checkpoint #" << head.sequence << " declares state "
<< cs::Utils::byteStreamToHex(saved->digest.data(), saved->digest.size()) << " (" << saved->wallets << " wallets)"
<< ", its caches hold " << cs::Utils::byteStreamToHex(loaded.digest.data(), loaded.digest.size())
<< " (" << loaded.wallets << " wallets)"
<< (loaded.settings != saved->settings ? " and other consensus settings" : "") << ", skip it";
return false;
}
}
loadedHead = head.sequence;
return true;
});

if (ok) {
cslog() << "Caches for QUICK START loaded successfully!";
if (loadedHead != 0) {
const auto loaded = checkpointDigest();
cslog() << cs::stateDigestLine(loadedHead, loaded.digest, loaded.wallets) << " checkpoint loaded";
}
// The caches hold the head block. A live checkpoint of an earlier build was saved before
// recordBlock advanced lastSequence_ to that block, so it reads one less: reading the database
// from there would apply the head block a second time and change the balances.
Expand Down Expand Up @@ -422,7 +439,9 @@ void BlockChain::onReadFromDB(csdb::Pool block, bool* shouldStop) {
head.sequence = blockSeq;
head.head_hash = block.hash().to_binary();
head.prev_hash = block.previous_hash().to_binary();
if (serializationManPtr_->save(blockSeq, head)) {
const auto digest = checkpointDigest();
if (serializationManPtr_->save(blockSeq, head, digest)) {
cslog() << cs::stateDigestLine(blockSeq, digest.digest, digest.wallets) << " checkpoint";
serializationManPtr_->pruneCheckpoints(sto.checkpointKeep);
lastCheckpointWallClock_ = now;
if (byCount) {
Expand Down Expand Up @@ -1030,7 +1049,9 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) {
head.sequence = pool.sequence();
head.head_hash = pool.hash().to_binary();
head.prev_hash = pool.previous_hash().to_binary();
if (serializationManPtr_->save(pool.sequence(), head)) {
const auto digest = checkpointDigest();
if (serializationManPtr_->save(pool.sequence(), head, digest)) {
cslog() << cs::stateDigestLine(pool.sequence(), digest.digest, digest.wallets) << " checkpoint";
serializationManPtr_->pruneCheckpoints(sto.checkpointKeep);
lastCheckpointWallClock_ = now;
if (byCount) {
Expand All @@ -1043,6 +1064,35 @@ void BlockChain::saveLiveCheckpoint(const csdb::Pool& pool) {
}
}

cs::CheckpointDigest BlockChain::checkpointDigest() const {
cs::CheckpointDigest result;
const auto& wallets = multiWallets();
result.digest = wallets.stateDigest();
result.wallets = wallets.size();
result.settings = consensusSettingsDigest();
return result;
}

std::array<uint8_t, 32> BlockChain::consensusSettingsDigest() const {
cs::Bytes data;
cs::ODataStream<cs::Bytes> stream(data);
stream << blockRewardIntegral_ << blockRewardFraction_ << miningCoefficientIntegral_ << miningCoefficientFraction_;
stream << static_cast<uint8_t>(stakingOn_) << static_cast<uint8_t>(miningOn_) << TimeMinStage1_;
const auto& pending = pendingConsensusSettings_;
stream << pending.round << static_cast<uint8_t>(pending.stakingOn) << static_cast<uint8_t>(pending.miningOn);
stream << pending.blockReward.integral() << pending.blockReward.fraction();
stream << pending.miningCoefficient.integral() << pending.miningCoefficient.fraction();
stream << startingDPOS_;
stream << static_cast<uint32_t>(specialOrders_.size());
for (const auto& [kind, payload] : specialOrders_) {
stream << kind << payload;
}
const cs::Hash hash = cscrypto::calculateHash(data.data(), data.size());
std::array<uint8_t, 32> result{};
std::copy(hash.begin(), hash.end(), result.begin());
return result;
}

csdb::PoolHash BlockChain::getHashBySequence(cs::Sequence seq) const {
std::lock_guard lock(dbLock_);

Expand Down Expand Up @@ -1343,8 +1393,10 @@ void BlockChain::close() {

csinfo() << "Blockchain: try to save caches for QUICK START.";

if (serializationManPtr_->save(0, headInfo)) {
const auto digest = checkpointDigest();
if (serializationManPtr_->save(0, headInfo, digest)) {
csinfo() << "Blockchain: caches for QUICK START saved successfully.";
cslog() << cs::stateDigestLine(headInfo.sequence, digest.digest, digest.wallets) << " checkpoint";
}
else {
csinfo() << "~Blockchain: couldn't save caches for QUICK START.";
Expand Down
15 changes: 14 additions & 1 deletion csnode/src/caches_serialization_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <exception>
#include <filesystem>
#include <fstream>
#include <stdexcept>
#include <optional>
#include <sstream>
#include <vector>
Expand Down Expand Up @@ -92,6 +93,7 @@ struct CachesSerializationManager::Impl {
const std::vector <std::string> hashesDivisions = {"blockchain", "smartcontracts","walletscache","walletsIds","roundstat","tokensmaster","apihandler"};

std::optional<CheckpointHead> loadedHead_{};
std::optional<CheckpointDigest> loadedDigest_{};
std::optional<size_t> loadedVersion_{};
bool completedFromGenesis_ = false;
bool loadedFromCompleted_ = false;
Expand Down Expand Up @@ -391,6 +393,7 @@ struct CachesSerializationManager::Impl {

bool loadVersion(size_t version, const std::function<bool(const CheckpointHead&)>& verify) {
csinfo() << "CachesSerializationManager: try to load version " << version;
loadedDigest_.reset();
try {
std::filesystem::path p(kQuickStartRoot);
p /= std::to_string(version);
Expand Down Expand Up @@ -429,13 +432,15 @@ struct CachesSerializationManager::Impl {
return false;
}
loadedHead_ = loadHead(p); // nullopt for legacy (no head.bin)
loadedDigest_ = CheckpointDigest::read(p / CheckpointDigest::kFileName); // nullopt before it existed
loadedFromCompleted_ = loadSentinel(p);
// a completed snapshot remains completed after a successful reload
if (loadedFromCompleted_) completedFromGenesis_ = true;
if (verify && loadedHead_.has_value() && !verify(*loadedHead_)) {
cserror() << "CachesSerializationManager: chain-binding verification failed for version "
<< version << "; quarantining";
loadedHead_.reset();
loadedDigest_.reset();
loadedFromCompleted_ = false;
clearInMem();
quarantineVersion(version);
Expand Down Expand Up @@ -544,7 +549,7 @@ bool CachesSerializationManager::save(size_t version) {
return save(version, CheckpointHead{});
}

bool CachesSerializationManager::save(size_t version, const CheckpointHead& head) {
bool CachesSerializationManager::save(size_t version, const CheckpointHead& head, const std::optional<CheckpointDigest>& digest) {
if (!pImpl_->bindingsReady()) {
cserror() << "CachesSerializationManager: save error: "
<< "bindings are not ready";
Expand Down Expand Up @@ -579,6 +584,9 @@ bool CachesSerializationManager::save(size_t version, const CheckpointHead& head
if (!head.head_hash.empty()) {
pImpl_->saveHead(tmp_path, head);
}
if (digest.has_value() && !digest->write(tmp_path / CheckpointDigest::kFileName)) {
throw std::runtime_error("cannot write state digest");
}
} catch (const std::exception& e) {
cserror() << "CachesSerializationManager: error on save: " << e.what();
std::filesystem::remove_all(tmp_path, ec);
Expand Down Expand Up @@ -651,6 +659,7 @@ bool CachesSerializationManager::load(const ChainVerifier& verify) {
}

pImpl_->loadedHead_.reset();
pImpl_->loadedDigest_.reset();
pImpl_->loadedVersion_.reset();
pImpl_->loadedFromCompleted_ = false;

Expand Down Expand Up @@ -721,6 +730,10 @@ std::optional<CheckpointHead> CachesSerializationManager::getLoadedHead() const
return pImpl_->loadedHead_;
}

std::optional<CheckpointDigest> CachesSerializationManager::getLoadedDigest() const {
return pImpl_->loadedDigest_;
}

std::optional<size_t> CachesSerializationManager::getLoadedVersion() const {
return pImpl_->loadedVersion_;
}
Expand Down
5 changes: 5 additions & 0 deletions csnode/src/multiwallets.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ void cs::MultiWallets::onWalletCacheUpdated(const cs::WalletsCache::WalletData&
}
}

void cs::MultiWallets::invalidateStateDigest() {
cs::Lock lock(mutex_);
digestReady_ = false;
}

cs::MultisetHash::Digest cs::MultiWallets::stateDigest() const {
cs::Lock lock(mutex_);

Expand Down
4 changes: 1 addition & 3 deletions csnode/src/node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4475,9 +4475,7 @@ void Node::logStateDigest(const csdb::Pool& pool) {

void Node::logStateDigest(cs::Sequence sequence) {
const auto& wallets = blockChain_.multiWallets();
const auto digest = wallets.stateDigest();
cslog() << "STATE DIGEST #" << sequence << " " << cs::Utils::byteStreamToHex(digest.data(), digest.size())
<< " wallets " << wallets.size();
cslog() << cs::stateDigestLine(sequence, wallets.stateDigest(), wallets.size());
}

void Node::validateBlock(const csdb::Pool& block, bool* shouldStop) {
Expand Down
43 changes: 43 additions & 0 deletions csnode/src/statedigest.cpp
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
#include <csnode/statedigest.hpp>

#include <algorithm>
#include <fstream>
#include <sstream>
#include <tuple>
#include <vector>

#include <sodium.h>

#include <csnode/nodecore.hpp>
#include <lib/system/utils.hpp>

namespace {
static_assert(crypto_core_ristretto255_BYTES == 32, "ristretto255 element size");
Expand Down Expand Up @@ -125,4 +128,44 @@ cs::Bytes serializeWalletState(const WalletsCache::WalletData& wallet) {
return out;
}

bool CheckpointDigest::write(const std::filesystem::path& file) const {
cs::Bytes out;
putU8(out, kVersion);
out.insert(out.end(), digest.begin(), digest.end());
putU64(out, wallets);
out.insert(out.end(), settings.begin(), settings.end());
std::ofstream f(file, std::ios::binary | std::ios::trunc);
f.write(reinterpret_cast<const char*>(out.data()), static_cast<std::streamsize>(out.size()));
f.flush();
return static_cast<bool>(f);
}

std::optional<CheckpointDigest> CheckpointDigest::read(const std::filesystem::path& file) {
std::ifstream f(file, std::ios::binary);
if (!f.is_open()) {
return std::nullopt;
}
constexpr size_t kSize = 1 + 32 + 8 + 32;
std::array<uint8_t, kSize + 1> in{};
f.read(reinterpret_cast<char*>(in.data()), static_cast<std::streamsize>(in.size()));
if (static_cast<size_t>(f.gcount()) != kSize || in[0] != kVersion) {
return std::nullopt;
}
CheckpointDigest result;
std::copy(in.begin() + 1, in.begin() + 33, result.digest.begin());
for (int i = 0; i < 8; ++i) {
result.wallets |= static_cast<uint64_t>(in[33 + i]) << (8 * i);
}
std::copy(in.begin() + 41, in.begin() + 73, result.settings.begin());
return result;
}

std::string stateDigestLine(cs::Sequence sequence, const MultisetHash::Digest& digest, uint64_t wallets) {
std::ostringstream os;
os << "STATE DIGEST #" << sequence << " " << cs::Utils::byteStreamToHex(digest.data(), digest.size())
<< " wallets " << wallets;
return os.str();
}

} // namespace cs

Loading
Loading