diff --git a/csnode/CMakeLists.txt b/csnode/CMakeLists.txt index c100d398b..c016cf100 100644 --- a/csnode/CMakeLists.txt +++ b/csnode/CMakeLists.txt @@ -68,6 +68,7 @@ add_library(csnode include/csnode/compressor.hpp include/csnode/configholder.hpp include/csnode/multiwallets.hpp + include/csnode/statedigest.hpp include/csnode/sendcachedata.hpp include/csnode/eventreport.hpp include/csnode/idatastream.hpp @@ -107,6 +108,7 @@ add_library(csnode src/roundpackage.cpp src/configholder.cpp src/multiwallets.cpp + src/statedigest.cpp src/sendcachedata.cpp src/eventreport.cpp src/poolcache.cpp diff --git a/csnode/include/csnode/multiwallets.hpp b/csnode/include/csnode/multiwallets.hpp index 0eb3dc7c3..0719948de 100644 --- a/csnode/include/csnode/multiwallets.hpp +++ b/csnode/include/csnode/multiwallets.hpp @@ -9,6 +9,7 @@ #include #include +#include #include using namespace boost::multi_index; @@ -57,6 +58,10 @@ class MultiWallets { } csdb::Amount checkWallets(); + + // order-independent digest of every wallet's balance-relevant state (ECMH, see statedigest.hpp); + // computed in full on the first call, then kept up to date on every wallet update + MultisetHash::Digest stateDigest() const; void iterate(std::function func); protected: @@ -111,6 +116,9 @@ class MultiWallets { mutable std::mutex mutex_; Container indexes_; + + mutable MultisetHash digest_; + mutable bool digestReady_ = false; }; } diff --git a/csnode/include/csnode/node.hpp b/csnode/include/csnode/node.hpp index be20c8107..d26053670 100644 --- a/csnode/include/csnode/node.hpp +++ b/csnode/include/csnode/node.hpp @@ -385,6 +385,10 @@ public slots: // request current trusted nodes for block with specific sequence void sendBlockRequestToConfidants(cs::Sequence sequence); void processSpecialInfo(const csdb::Pool& pool); + // logs the wallet state digest every kStateDigestInterval blocks, comparable between nodes + void logStateDigest(const csdb::Pool& pool); + void logStateDigest(cs::Sequence sequence); + static constexpr cs::Sequence kStateDigestInterval = 1000; void checkConsensusSettings(cs::Sequence seq, std::string& msg); void validateBlock(const csdb::Pool& block, bool* shouldStop); diff --git a/csnode/include/csnode/statedigest.hpp b/csnode/include/csnode/statedigest.hpp new file mode 100644 index 000000000..9e69608f1 --- /dev/null +++ b/csnode/include/csnode/statedigest.hpp @@ -0,0 +1,46 @@ +#ifndef STATEDIGEST_HPP +#define STATEDIGEST_HPP + +#include +#include + +#include +#include + +namespace cs { + +// Elliptic-curve multiset hash (ECMH) over ristretto255: an order-independent digest of a set of +// items, updated in O(1) per added or removed item. Two nodes holding the same set get the same +// digest regardless of the order in which the items were added. +class MultisetHash { +public: + using Digest = std::array; + using Element = std::array; + + MultisetHash(); + + // the group element an item contributes; callers may keep it to remove the item later + static Element toElement(const cs::Bytes& item); + + void add(const cs::Bytes& item); + void remove(const cs::Bytes& item); + void add(const Element& element); + void remove(const Element& element); + void reset(); + + const Digest& digest() const { + return sum_; + } + +private: + Digest sum_; +}; + +// Canonical serialisation of the balance-relevant state of a wallet: key, balance, delegated amount +// and delegations in both directions, sorted. Empty for a wallet without such state, so that empty +// records do not change the digest. +cs::Bytes serializeWalletState(const WalletsCache::WalletData& wallet); + +} // namespace cs + +#endif // STATEDIGEST_HPP diff --git a/csnode/include/csnode/walletscache.hpp b/csnode/include/csnode/walletscache.hpp index 16354ca0a..381ba1e78 100644 --- a/csnode/include/csnode/walletscache.hpp +++ b/csnode/include/csnode/walletscache.hpp @@ -1,6 +1,7 @@ #ifndef WALLETS_CACHE_HPP #define WALLETS_CACHE_HPP +#include #include #include #include @@ -56,6 +57,9 @@ class WalletsCache { #ifdef MONITOR_NODE uint64_t createTime_ = 0; #endif + // ECMH element of this wallet's state, kept by MultiWallets for the state digest (not persisted) + std::array stateElement_{}; + bool hasStateElement_ = false; }; struct TrustedData { diff --git a/csnode/include/csnode/walletscache_serializer.hpp b/csnode/include/csnode/walletscache_serializer.hpp index b06c3e8da..07d967e1d 100644 --- a/csnode/include/csnode/walletscache_serializer.hpp +++ b/csnode/include/csnode/walletscache_serializer.hpp @@ -1,5 +1,6 @@ #ifndef WALLETS_CACHE_SERIALIZER_HPP #define WALLETS_CACHE_SERIALIZER_HPP +#include #include #include @@ -161,6 +162,10 @@ class WalletsCache_Serializer { #ifdef MONITOR_NODE uint64_t createTime_; #endif + // keep the layout identical to WalletsCache::WalletData (wallets_ aliases its container); + // the state-digest element is rebuilt at start and never serialised + std::array stateElement_{}; + bool hasStateElement_ = false; }; #ifdef MONITOR_NODE class TrustedData { diff --git a/csnode/src/multiwallets.cpp b/csnode/src/multiwallets.cpp index 4ddcca564..a2d6f122e 100644 --- a/csnode/src/multiwallets.cpp +++ b/csnode/src/multiwallets.cpp @@ -69,12 +69,53 @@ void cs::MultiWallets::onWalletCacheUpdated(const cs::WalletsCache::WalletData& << ", delegated: " << data.delegated_.to_string(); } + InternalData stored = data; + stored.hasStateElement_ = false; + if (digestReady_) { + const auto state = cs::serializeWalletState(stored); + if (!state.empty()) { + stored.stateElement_ = cs::MultisetHash::toElement(state); + stored.hasStateElement_ = true; + } + } + if (auto iter = byKey.find(data.key_); iter != byKey.end()) { - byKey.replace(iter, data); + // the stored element, not the stored record: delegation maps are shared and already updated in place + if (digestReady_ && iter->hasStateElement_) { + digest_.remove(iter->stateElement_); + } + byKey.replace(iter, stored); } else { - indexes_.insert(data); + indexes_.insert(stored); } + + if (stored.hasStateElement_) { + digest_.add(stored.stateElement_); + } +} + +cs::MultisetHash::Digest cs::MultiWallets::stateDigest() const { + cs::Lock lock(mutex_); + + if (!digestReady_) { + digest_.reset(); + auto& byKey = indexes_.get(); + for (auto it = byKey.begin(); it != byKey.end(); ++it) { + const auto state = cs::serializeWalletState(*it); + const bool hasState = !state.empty(); + const auto element = hasState ? cs::MultisetHash::toElement(state) : cs::MultisetHash::Element{}; + if (hasState) { + digest_.add(element); + } + // the element is not a key of any index, so updating it in place keeps the container valid + const_cast(*it).stateElement_ = element; + const_cast(*it).hasStateElement_ = hasState; + } + digestReady_ = true; + } + + return digest_.digest(); } void cs::MultiWallets::iterate(std::function func) { diff --git a/csnode/src/node.cpp b/csnode/src/node.cpp index 00b01aa0e..6f1f5d7da 100644 --- a/csnode/src/node.cpp +++ b/csnode/src/node.cpp @@ -107,6 +107,7 @@ Node::Node(cs::config::Observer& observer) cs::Connector::connect(&blockChain_.tryToStoreBlockEvent, this, &Node::deepBlockValidation); cs::Connector::connect(&blockChain_.stopNode, this, &Node::stop); cs::Connector::connect(&blockChain_.storeBlockEvent, this, &Node::processSpecialInfo); + cs::Connector::connect(&blockChain_.storeBlockEvent, this, static_cast(&Node::logStateDigest)); cs::Connector::connect(&blockChain_.uncertainBlock, this, &Node::sendBlockRequestToConfidants); cs::Connector::connect(&blockChain_.orderNecessaryBlock, this, &Node::sendNecessaryBlockRequest); cs::Connector::connect(&stat_.accountInitiationRequest, this, &Node::accountInitiationRequest); @@ -248,6 +249,8 @@ bool Node::init() { } cslog() << "Blockchain is ready, contains " << WithDelimiters(stat_.totalTransactions()) << " transactions"; + // full digest once here, before taking part in the network; incremental from now on + logStateDigest(blockChain_.getLastSeq()); #ifdef NODE_API api_->run(); @@ -4300,6 +4303,19 @@ void Node::checkConsensusSettings(cs::Sequence seq, std::string& msg){ saveConsensusSettingsToChain(); } +void Node::logStateDigest(const csdb::Pool& pool) { + if (pool.sequence() % kStateDigestInterval == 0) { + logStateDigest(pool.sequence()); + } +} + +void Node::logStateDigest(cs::Sequence sequence) { + const auto& wallets = blockChain_.multiWallets(); + const auto digest = wallets.stateDigest(); + cslog() << "STATE DIGEST #" << sequence << " " << cs::Utils::byteStreamToHex(digest.data(), digest.size()) + << " wallets " << wallets.size(); +} + void Node::validateBlock(const csdb::Pool& block, bool* shouldStop) { // user-cancel is honoured via BlockChain::init's OpenCallback; *shouldStop here means DataIntegrityError if (!blockValidator_->validateBlock(block, diff --git a/csnode/src/statedigest.cpp b/csnode/src/statedigest.cpp new file mode 100644 index 000000000..a2a75db8b --- /dev/null +++ b/csnode/src/statedigest.cpp @@ -0,0 +1,128 @@ +#include + +#include +#include +#include + +#include + +#include + +namespace { +static_assert(crypto_core_ristretto255_BYTES == 32, "ristretto255 element size"); + +void putU8(cs::Bytes& out, uint8_t value) { + out.push_back(value); +} + +void putU32(cs::Bytes& out, uint32_t value) { + for (int i = 0; i < 4; ++i) { + out.push_back(static_cast(value >> (8 * i))); + } +} + +void putU64(cs::Bytes& out, uint64_t value) { + for (int i = 0; i < 8; ++i) { + out.push_back(static_cast(value >> (8 * i))); + } +} + +void putAmount(cs::Bytes& out, const csdb::Amount& amount) { + putU32(out, static_cast(amount.integral())); + putU64(out, amount.fraction()); +} + +void putKey(cs::Bytes& out, const cs::PublicKey& key) { + out.insert(out.end(), key.begin(), key.end()); +} + +using Delegations = std::shared_ptr>>; + +// std::map keeps the counterparts sorted by key; the entries of each counterpart are sorted here, +// since their order depends on how blocks were applied and reverted +void putDelegations(cs::Bytes& out, const Delegations& delegations) { + if (!delegations) { + putU32(out, 0); + return; + } + + putU32(out, static_cast(delegations->size())); + for (const auto& [counterpart, entries] : *delegations) { + auto sorted = entries; + std::sort(sorted.begin(), sorted.end(), [](const cs::TimeMoney& lhs, const cs::TimeMoney& rhs) { + return std::make_tuple(lhs.time, lhs.initialTime, lhs.amount.integral(), lhs.amount.fraction(), static_cast(lhs.coeff)) < + std::make_tuple(rhs.time, rhs.initialTime, rhs.amount.integral(), rhs.amount.fraction(), static_cast(rhs.coeff)); + }); + + putKey(out, counterpart); + putU32(out, static_cast(sorted.size())); + for (const auto& entry : sorted) { + putU64(out, entry.initialTime); + putU64(out, entry.time); + putAmount(out, entry.amount); + putU8(out, static_cast(entry.coeff)); + } + } +} + +bool hasEntries(const Delegations& delegations) { + if (!delegations) { + return false; + } + return std::any_of(delegations->begin(), delegations->end(), [](const auto& item) { return !item.second.empty(); }); +} +} // namespace + +namespace cs { + +MultisetHash::MultisetHash() { + reset(); +} + +MultisetHash::Element MultisetHash::toElement(const cs::Bytes& item) { + std::array hash; + crypto_generichash(hash.data(), hash.size(), item.data(), item.size(), nullptr, 0); + + Element element; + crypto_core_ristretto255_from_hash(element.data(), hash.data()); + return element; +} + +void MultisetHash::add(const cs::Bytes& item) { + add(toElement(item)); +} + +void MultisetHash::remove(const cs::Bytes& item) { + remove(toElement(item)); +} + +void MultisetHash::add(const Element& element) { + crypto_core_ristretto255_add(sum_.data(), sum_.data(), element.data()); +} + +void MultisetHash::remove(const Element& element) { + crypto_core_ristretto255_sub(sum_.data(), sum_.data(), element.data()); +} + +void MultisetHash::reset() { + sum_.fill(0); // the identity element of ristretto255 +} + +cs::Bytes serializeWalletState(const WalletsCache::WalletData& wallet) { + const bool hasState = wallet.balance_ != csdb::Amount{0} || wallet.delegated_ != csdb::Amount{0} || + hasEntries(wallet.delegateSources_) || hasEntries(wallet.delegateTargets_); + if (!hasState) { + return {}; + } + + cs::Bytes out; + putU8(out, 1); // format version + putKey(out, wallet.key_); + putAmount(out, wallet.balance_); + putAmount(out, wallet.delegated_); + putDelegations(out, wallet.delegateSources_); + putDelegations(out, wallet.delegateTargets_); + return out; +} + +} // namespace cs diff --git a/csnode/src/walletscache_serializer.cpp b/csnode/src/walletscache_serializer.cpp index b4e0ff2fb..609d67659 100644 --- a/csnode/src/walletscache_serializer.cpp +++ b/csnode/src/walletscache_serializer.cpp @@ -24,6 +24,7 @@ void WalletsCache_Serializer::bind(WalletsCache& wCache) { smartPayableTransactions_ = reinterpret_cast(&wCache.smartPayableTransactions_); canceledSmarts_ = reinterpret_cast(&wCache.canceledSmarts_); + static_assert(sizeof(WalletData) == sizeof(WalletsCache::WalletData), "WalletData mirror must match WalletsCache::WalletData"); wallets_ = reinterpret_cast(&wCache.multiWallets_->indexes_); #ifdef MONITOR_NODE trusted_info_ = reinterpret_cast(&wCache.trusted_info_); diff --git a/tests/csnode/statedigest_tests.cpp b/tests/csnode/statedigest_tests.cpp new file mode 100644 index 000000000..45d4410a0 --- /dev/null +++ b/tests/csnode/statedigest_tests.cpp @@ -0,0 +1,79 @@ +#include + +#include +#include + +namespace { +cs::Bytes item(uint8_t b) { + return cs::Bytes(16, b); +} + +cs::WalletsCache::WalletData wallet(uint8_t keyByte, int32_t balance) { + cs::WalletsCache::WalletData w; + w.key_.fill(keyByte); + w.balance_ = csdb::Amount{ balance }; + return w; +} +} // namespace + +TEST(MultisetHash, OrderIndependent) { + cs::MultisetHash ab; + ab.add(item(1)); + ab.add(item(2)); + + cs::MultisetHash ba; + ba.add(item(2)); + ba.add(item(1)); + + ASSERT_EQ(ab.digest(), ba.digest()); +} + +TEST(MultisetHash, RemoveUndoesAdd) { + cs::MultisetHash empty; + cs::MultisetHash h; + h.add(item(1)); + ASSERT_NE(h.digest(), empty.digest()); + + h.add(item(2)); + h.remove(item(2)); + cs::MultisetHash one; + one.add(item(1)); + ASSERT_EQ(h.digest(), one.digest()); + + h.remove(item(1)); + ASSERT_EQ(h.digest(), empty.digest()); +} + +TEST(MultisetHash, DifferentSetsDiffer) { + cs::MultisetHash a; + a.add(item(1)); + cs::MultisetHash b; + b.add(item(2)); + ASSERT_NE(a.digest(), b.digest()); +} + +TEST(WalletState, EmptyWalletHasNoState) { + ASSERT_TRUE(cs::serializeWalletState(wallet(1, 0)).empty()); + ASSERT_FALSE(cs::serializeWalletState(wallet(1, 5)).empty()); +} + +TEST(WalletState, BalanceChangesSerialisation) { + ASSERT_NE(cs::serializeWalletState(wallet(1, 5)), cs::serializeWalletState(wallet(1, 6))); + ASSERT_NE(cs::serializeWalletState(wallet(1, 5)), cs::serializeWalletState(wallet(2, 5))); +} + +// the order of delegation entries depends on apply/revert history, the digest must not +TEST(WalletState, DelegationOrderDoesNotMatter) { + cs::PublicKey other{}; + other.fill(9); + + auto w1 = wallet(1, 5); + w1.delegateTargets_ = std::make_shared>>(); + (*w1.delegateTargets_)[other] = { cs::TimeMoney(1'000'000, 2'000, csdb::Amount{ 1 }), cs::TimeMoney(1'000'000, 3'000, csdb::Amount{ 2 }) }; + + auto w2 = wallet(1, 5); + w2.delegateTargets_ = std::make_shared>>(); + (*w2.delegateTargets_)[other] = { cs::TimeMoney(1'000'000, 3'000, csdb::Amount{ 2 }), cs::TimeMoney(1'000'000, 2'000, csdb::Amount{ 1 }) }; + + ASSERT_EQ(cs::serializeWalletState(w1), cs::serializeWalletState(w2)); +}