diff --git a/README.md b/README.md index 71bfa47c6..0bc108ca5 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ This repository is the community continuation of [CREDITSCOM/node](https://githu | What changes from the official node | [docs/differences.md](docs/differences.md) | | Every change, pull request by pull request | [docs/CHANGES.md](docs/CHANGES.md) | | Block database, pruned storage, conversion | [docs/storage.md](docs/storage.md) | +| Tokens, the token API and the token tests | [docs/tokens.md](docs/tokens.md) | | Docker (pruned node from a snapshot) | [packaging/docker/README.md](packaging/docker/README.md) | | Packages (Linux x86-64, Windows x64, macOS Apple Silicon; `config.ini` full and `config.pruned.ini` pruned, one package per system) | [Releases](https://github.com/EnzinoBB/node/releases) | diff --git a/api/include/apihandler.hpp b/api/include/apihandler.hpp index ecba170a8..f112a28b4 100644 --- a/api/include/apihandler.hpp +++ b/api/include/apihandler.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -184,6 +185,13 @@ class APIHandler : public APIHandlerInterface { cs::DumbCv dumbCv_; std::atomic isBDLoaded_{ false }; + // the stored blocks are read (the last one, or run() after a quick start that read none): the + // token cache is refreshed once + void onBaseLoaded(bool refreshInBackground); + // an empty token cache on a node that has contracts (the cache was off when its checkpoint + // was saved): find the tokens among the contracts and their holders in the chain + void rebuildTokenCache(); + std::thread tokensRefresh_; struct smart_trxns_queue { cs::SpinLock lock{ATOMIC_FLAG_INIT}; diff --git a/api/include/tokens.hpp b/api/include/tokens.hpp index 05b89154a..bb5abdf96 100644 --- a/api/include/tokens.hpp +++ b/api/include/tokens.hpp @@ -23,10 +23,13 @@ class SmartContractInvocation; using TokenId = csdb::Address; using HolderKey = csdb::Address; +// the ids the executor reports (com.credits.scapi.misc.TokenStandardId) enum TokenStandard { NotAToken = 0, - CreditsBasic = 1, - CreditsExtended = 2 + CreditsBasic = 1, // v0 BasicStandard: amounts as strings + CreditsExtended = 2, // v0 ExtensionStandard + CreditsBasicToken = 3, // v1 (and v2) BasicTokenStandard: amounts as BigDecimal + CreditsExtendedToken = 4 // v1 (and v2) ExtensionTokenStandard }; struct Token { @@ -60,9 +63,26 @@ class TokensMaster { TokensMaster(api::APIHandler*); ~TokensMaster(); + // [api] tokens_cache, or a RelMonitor/RelWallet build: the token cache is kept and served + bool enabled() const { + return enabled_; + } + void checkNewDeploy(const csdb::Address& sc, const csdb::Address& deployer, const api::SmartContractInvocation&); - void checkNewState(const csdb::Address& sc, const csdb::Address& initiator, const api::SmartContractInvocation&, const std::string& newState); + // refresh = false: holders and counters only, the balances are read later (rebuild) + void checkNewState(const csdb::Address& sc, const csdb::Address& initiator, const api::SmartContractInvocation&, const std::string& newState, + bool refresh = true); + + bool isToken(const csdb::Address& addr) { + std::lock_guard l(dataMut_); + return tokens_.count(addr) != 0; + } + + bool empty() { + std::lock_guard l(dataMut_); + return tokens_.empty(); + } void loadTokenInfo(const std::function); @@ -87,14 +107,24 @@ class TokensMaster { std::list invocations; }; - void updateTokenChaches(const csdb::Address& addr, const std::string& newState, const TokenInvocationData::Params& params); + void updateTokenChaches(const csdb::Address& addr, const std::string& newState, const TokenInvocationData::Params& params, bool refresh = true); - void refreshTokenState(const csdb::Address& token, const std::string& newState, bool checkBalance = false); + // name, symbol and total supply; with checkBalance the balances of `only` (every holder when empty) + void refreshTokenState(const csdb::Address& token, const std::string& newState, bool checkBalance = false, + const std::set& only = {}); + + // every token's name, symbol, supply and balances, one token at a time under the lock + void refreshAll(); + + static bool isExtended(int64_t tokenStandard) { + return tokenStandard == CreditsExtended || tokenStandard == CreditsExtendedToken; + } private: void initiateHolder(Token&, const csdb::Address& token, const csdb::Address& holder, bool increaseTransfers = false); api::APIHandler* api_; + bool enabled_ = false; std::mutex cvMut_; diff --git a/api/src/apihandler.cpp b/api/src/apihandler.cpp index 2d12664d8..c9a2c9c23 100644 --- a/api/src/apihandler.cpp +++ b/api/src/apihandler.cpp @@ -71,9 +71,8 @@ void APIHandler::run() { #endif state_updater_running.test_and_set(std::memory_order_acquire); // the node runs once its stored blocks are read; a quick start at the last stored block reads - // none, so baseLoaded() never marks them read, and TransactionFlow would never get the result - // of a contract call (the signal at the new_state of the call needs isBDLoaded_) - isBDLoaded_.store(true, std::memory_order_release); + // none, so baseLoaded() is never called then + onBaseLoaded(true); } APIHandler::~APIHandler() { @@ -82,6 +81,9 @@ APIHandler::~APIHandler() { if (state_updater.joinable()) { state_updater.join(); } + if (tokensRefresh_.joinable()) { + tokensRefresh_.join(); + } } template @@ -1874,21 +1876,78 @@ void APIHandler::store_block_slot(const csdb::Pool& pool) { void APIHandler::baseLoaded(const csdb::Pool& pool) { if (maxReadSequence && pool.sequence() == maxReadSequence) { - isBDLoaded_.store(true, std::memory_order_release); -#ifdef TOKENS_CACHE - tm_.loadTokenInfo([&](const TokensMap& tokens, const HoldersMap&) { - int count{}, i{}; - size_t tokenSize = tokens.size(); - cslog() << "tokens are loading(" << tokenSize << ")..."; - for (auto& tk : tokens) { - if (tokenSize > static_cast(100) && !(i++ % (tokenSize / 10))) - cslog() << "loading tokens: " << 10 * (count++) << "%"; - tm_.refreshTokenState(tk.first, cs::SmartContracts::get_contract_state(get_s_blockchain(), tk.first), true); - } - cslog() << "tokens loaded!"; - }); + onBaseLoaded(false); + } +} + +void APIHandler::onBaseLoaded(bool refreshInBackground) { + if (isBDLoaded_.exchange(true)) { + return; + } +#ifdef TOKENS_CACHE + auto refresh = [this]() { + if (tm_.enabled() && tm_.empty()) { + rebuildTokenCache(); + } + tm_.refreshAll(); + }; + if (refreshInBackground) { + tokensRefresh_ = std::thread(refresh); + } + else { + refresh(); + } +#else + csunused(refreshInBackground); #endif +} + +void APIHandler::rebuildTokenCache() { + std::vector contracts; + { + auto origins = lockedReference(this->smart_origin); + for (const auto& item : *origins) { + contracts.push_back(item.first); + } } + if (contracts.empty()) { + return; + } + cslog() << "API: token cache is empty, looking for tokens among " << contracts.size() << " contracts"; + size_t found = 0; + for (const auto& addr : contracts) { + const auto deploy = solver_.smart_contracts().get_contract_deploy(addr); + if (!deploy.is_valid()) { + continue; + } + const auto deployer = blockchain_.getAddressByType(deploy.source(), BlockChain::AddressType::PublicKey); + tm_.checkNewDeploy(addr, deployer, fetch_smart(deploy)); + if (!tm_.isToken(addr)) { + continue; + } + ++found; + // newest first: a call's new_state comes after the call + std::set succeeded; + std::vector calls; + for (auto it = cs::TransactionsIterator(blockchain_, addr); it.isValid(); it.next()) { + const auto& t = *it; + if (is_smart_state(t)) { + if (blockchain_.getAddressByType(t.target(), BlockChain::AddressType::PublicKey) == addr && cs::SmartContracts::is_state_updated(t)) { + cs::SmartContractRef ref(t.user_field(cs::trx_uf::new_state::RefStart)); + succeeded.insert(csdb::TransactionID(ref.sequence, ref.transaction)); + } + } + else if (succeeded.erase(t.id()) != 0) { + calls.push_back(t); + } + } + for (auto it = calls.rbegin(); it != calls.rend(); ++it) { + const auto caller = blockchain_.getAddressByType(it->source(), BlockChain::AddressType::PublicKey); + const auto sci = is_smart(*it) ? fetch_smart(*it) : api::SmartContractInvocation{}; + tm_.checkNewState(addr, caller, is_smart_deploy(sci) ? api::SmartContractInvocation{} : sci, std::string{}, false); + } + } + cslog() << "API: " << found << " tokens found"; } void APIHandler::maxBlocksCount(cs::Sequence lastBlockNum) { @@ -2099,7 +2158,11 @@ bool APIHandler::updateSmartCachesTransaction(csdb::Transaction trxn, cs::Sequen } } - if (!newStateStr.empty() || !newHashStr.empty()) { // update tokens + // update tokens: only from a successful execution (a failed one leaves the state as it + // was), and only for the contract called, whose method and arguments these are (the + // state of a contract it called through invokeExternalContract comes with the same call) + const bool primaryState = blockchain_.getAddressByType(execTrans.target(), BlockChain::AddressType::PublicKey) == target_pk; + if ((!newStateStr.empty() || !newHashStr.empty()) && primaryState && cs::SmartContracts::is_state_updated(trxn)) { auto caller_pk = blockchain_.getAddressByType(execTrans.source(), BlockChain::AddressType::PublicKey); if (is_smart_deploy(smart)) { @@ -2684,13 +2747,11 @@ void APIHandler::ExecuteCountGet(ExecuteCountGetResult& _return, const std::stri } void APIHandler::TokenBalancesGet(api::TokenBalancesResult& _return, const general::Address& address) { - #ifndef MONITOR_NODE - #ifndef WEB_WALLET_NODE - _return.status.code = int8_t(ERROR_CODE); - _return.status.message = "This node doesn't provide such info"; - return; - #endif - #endif + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } const csdb::Address addr = BlockChain::getAddressFromKey(address); tm_.loadTokenInfo([&_return, &addr](const TokensMap& tokens, const HoldersMap& holders) { auto holderIt = holders.find(addr); @@ -2722,17 +2783,20 @@ void APIHandler::TokenBalancesGet(api::TokenBalancesResult& _return, const gener } void APIHandler::TokenTransfersGet(api::TokenTransfersResult& _return, const general::Address& token, int64_t offset, int64_t limit) { - #ifndef MONITOR_NODE - #ifndef WEB_WALLET_NODE - _return.status.code = int8_t(ERROR_CODE); - _return.status.message = "This node doesn't provide such info"; - return; - #endif - #endif + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } tokenTransactionsInternal(_return, *this, tm_, token, true, false, offset, limit); } void APIHandler::TokenTransferGet(api::TokenTransfersResult& _return, const general::Address& token, const TransactionId& id) { + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } const csdb::TransactionID trxn_id = csdb::TransactionID(cs::Sequence(id.poolSeq), cs::Sequence(id.index)); const csdb::Transaction trxn = executor_.loadTransactionApi(trxn_id); const csdb::Address addr = BlockChain::getAddressFromKey(token); @@ -2806,21 +2870,35 @@ void APIHandler::TransactionsListGet(api::TransactionsGetResult& _return, int64_ } void APIHandler::TokenTransfersListGet(api::TokenTransfersResult& _return, int64_t offset, int64_t limit) { + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } if (!validatePagination(_return, *this, offset, limit)) { return; } uint64_t totalTransfers = 0; std::multimap tokenTransPools; + std::vector tokenAddresses; - tm_.loadTokenInfo([&totalTransfers, &tokenTransPools, this](const TokensMap& tm_, const HoldersMap&) { + tm_.loadTokenInfo([&totalTransfers, &tokenAddresses](const TokensMap& tm_, const HoldersMap&) { for (auto& t : tm_) { totalTransfers += t.second.transfersCount; - tokenTransPools.insert(std::make_pair(blockchain_.getLastTransaction(t.first).pool_seq(), t.first)); + tokenAddresses.push_back(t.first); } }); + // chain reads outside the token cache's lock + const std::set tokenSet(tokenAddresses.begin(), tokenAddresses.end()); + for (const auto& token : tokenAddresses) { + const auto lastSeq = blockchain_.getLastTransaction(token).pool_seq(); + if (lastSeq != cs::kWrongSequence) { + tokenTransPools.insert(std::make_pair(lastSeq, token)); + } + } - _return.count = int32_t(totalTransfers); + _return.count = int32_t(std::min(totalTransfers, std::numeric_limits::max())); cs::Sequence seq = blockchain_.getLastNonEmptyBlock().first; while (limit && seq != cs::kWrongSequence && tokenTransPools.size()) { @@ -2832,6 +2910,10 @@ void APIHandler::TokenTransfersListGet(api::TokenTransfersResult& _return, int64 if (!is_smart(t)) { continue; } + csdb::Address target_pk = blockchain_.getAddressByType(t.target(), BlockChain::AddressType::PublicKey); + if (tokenSet.count(target_pk) == 0) { + continue; + } const auto smart = fetch_smart(t); if (!TokensMaster::isTransfer(smart.method, smart.params)) { continue; @@ -2839,8 +2921,9 @@ void APIHandler::TokenTransfersListGet(api::TokenTransfersResult& _return, int64 if (--offset >= 0) { continue; } - csdb::Address target_pk = blockchain_.getAddressByType(t.target(), BlockChain::AddressType::PublicKey); - auto addrPair = TokensMaster::getTransferData(target_pk, smart.method, smart.params); + // the sender of a transfer() is the wallet that signed it + csdb::Address source_pk = blockchain_.getAddressByType(t.source(), BlockChain::AddressType::PublicKey); + auto addrPair = TokensMaster::getTransferData(source_pk, smart.method, smart.params); addTokenResult(_return, target_pk, "", pool, t, smart, addrPair); if (--limit == 0) { break; @@ -2875,29 +2958,30 @@ void APIHandler::TokenTransfersListGet(api::TokenTransfersResult& _return, int64 } void APIHandler::TokenWalletTransfersGet(api::TokenTransfersResult& _return, const general::Address& token, const general::Address& address, int64_t offset, int64_t limit) { - #ifndef MONITOR_NODE - #ifndef WEB_WALLET_NODE - _return.status.code = int8_t(ERROR_CODE); - _return.status.message = "This node doesn't provide such info"; - return; - #endif - #endif + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } const csdb::Address wallet = BlockChain::getAddressFromKey(address); tokenTransactionsInternal(_return, *this, tm_, token, true, true, offset, limit, wallet); } void APIHandler::TokenTransactionsGet(api::TokenTransactionsResult& _return, const general::Address& token, int64_t offset, int64_t limit) { + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } tokenTransactionsInternal(_return, *this, tm_, token, false, false, offset, limit); } void APIHandler::TokenInfoGet(api::TokenInfoResult& _return, const general::Address& token) { - #ifndef MONITOR_NODE - #ifndef WEB_WALLET_NODE - _return.status.code = int8_t(ERROR_CODE); - _return.status.message = "This node doesn't provide such info"; - return; - #endif - #endif + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } bool found = false; const csdb::Address addr = BlockChain::getAddressFromKey(token); tm_.loadTokenInfo([&token, &addr, &found, &_return](const TokensMap& tm_, const HoldersMap&) { @@ -2931,15 +3015,24 @@ static std::function getComparator(const FieldType fie return [field, desc](const T& lhs, const T& rhs) { return desc ? (lhs.second.*field > rhs.second.*field) : (lhs.second.*field < rhs.second.*field); }; } +// an amount the token cache holds ("12.5", "0", or "" when the getter failed); never throws +static double amountValue(const std::string& amount) { + return amount.empty() ? 0.0 : std::strtod(amount.c_str(), nullptr); +} + +// a strict weak ordering in both directions (desc ^ (a < b) is not one when a == b) +template +static std::function orderBy(F value, const bool desc) { + return [value, desc](const T& lhs, const T& rhs) { return desc ? value(rhs) < value(lhs) : value(lhs) < value(rhs); }; +} + void APIHandler::TokenHoldersGet(api::TokenHoldersResult& _return, const general::Address& token, int64_t offset, int64_t limit, const TokenHoldersSortField order, const bool desc) { - #ifndef MONITOR_NODE - #ifndef WEB_WALLET_NODE - _return.status.code = int8_t(ERROR_CODE); - _return.status.message = "This node doesn't provide such info"; - return; - #endif - #endif + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } if (!validatePagination(_return, *this, offset, limit)) { return; } @@ -2952,12 +3045,13 @@ void APIHandler::TokenHoldersGet(api::TokenHoldersResult& _return, const general std::function comparator; switch (order) { - case TH_Balance: - comparator = [desc](const HT& lhs, const HT& rhs) { return desc ^ (stod(lhs.second.balance) < stod(rhs.second.balance)); }; - break; case TH_TransfersCount: comparator = getComparator(&Token::HolderInfo::transfersCount, desc); break; + case TH_Balance: + default: + comparator = orderBy([](const HT& h) { return amountValue(h.second.balance); }, desc); + break; } const csdb::Address addr = BlockChain::getAddressFromKey(token); @@ -2997,13 +3091,11 @@ void APIHandler::TokenHoldersGet(api::TokenHoldersResult& _return, const general } void APIHandler::TokensListGet(api::TokensListResult& _return, int64_t offset, int64_t limit, const TokensListSortField order, const bool desc, const TokenFilters& filters) { - #ifndef MONITOR_NODE - #ifndef WEB_WALLET_NODE - _return.status.code = int8_t(ERROR_CODE); - _return.status.message = "This node doesn't provide such info"; - return; - #endif - #endif + if (!tm_.enabled()) { + _return.status.code = int8_t(ERROR_CODE); + _return.status.message = "This node doesn't provide such info (the token cache is off: [api] tokens_cache=true)"; + return; + } if (!validatePagination(_return, *this, offset, limit)) { return; } @@ -3019,10 +3111,10 @@ void APIHandler::TokensListGet(api::TokensListResult& _return, int64_t offset, i comparator = getComparator(&Token::name, desc); break; case TL_TotalSupply: - comparator = [desc](const VT& lhs, const VT& rhs) { return desc ^ (stod(lhs.second.totalSupply) < stod(rhs.second.totalSupply)); }; + comparator = orderBy([](const VT& t) { return amountValue(t.second.totalSupply); }, desc); break; case TL_Address: - comparator = [desc](const VT& lhs, const VT& rhs) { return desc ^ (lhs.first < rhs.first); }; + comparator = [desc](const VT& lhs, const VT& rhs) { return desc ? rhs.first < lhs.first : lhs.first < rhs.first; }; break; case TL_HoldersCount: comparator = getComparator(&Token::realHoldersCount, desc); @@ -3033,36 +3125,38 @@ void APIHandler::TokensListGet(api::TokensListResult& _return, int64_t offset, i case TL_TransactionsCount: comparator = getComparator(&Token::transactionsCount, desc); break; + default: + comparator = getComparator(&Token::symbol, desc); + break; } + // a filter field left empty (0 for the standard) matches every token; offset, limit and count + // apply to the tokens the filters keep + auto matches = [&filters](const Token& t) { + return (filters.name.empty() || t.name.find(filters.name) != std::string::npos) && + (filters.code.empty() || t.symbol.find(filters.code) != std::string::npos) && + (filters.tokenStandard == 0 || t.tokenStandard == filters.tokenStandard); + }; + tm_.loadTokenInfo([&, comparator](const TokensMap& tm_, const HoldersMap&) { - _return.count = static_cast(tm_.size()); + int64_t matching = 0; applyToSortedMap(tm_, comparator, [&](const TokensMap::value_type& t) { - if (--offset >= 0) { + if (!matches(t.second)) { + return true; + } + ++matching; + if (--offset >= 0 || limit <= 0) { return true; } api::TokenInfo tok; putTokenInfo(tok, fromByteArray(t.first.public_key()), t.second); - - // filters - auto posName = tok.name.find(filters.name); - auto posCode = tok.code.find(filters.code); - if ((posName != std::string::npos && posCode != std::string::npos && tok.tokenStandard == filters.tokenStandard) || - (posName && posCode && !tok.tokenStandard) || - (posName && filters.code.empty() && !tok.tokenStandard) || - (filters.name.empty() && posCode && tok.tokenStandard) || - (filters.name.empty() && posCode && !tok.tokenStandard) || - (filters.name.empty() && filters.code.empty() && tok.tokenStandard) || - (filters.name.empty() && filters.code.empty() && !tok.tokenStandard)) - _return.tokens.push_back(tok); - - if (--limit == 0) - return false; - + _return.tokens.push_back(tok); + --limit; return true; - }); }); + _return.count = static_cast(std::min(matching, std::numeric_limits::max())); + }); SetResponseStatus(_return.status, APIRequestStatusType::SUCCESS); } diff --git a/api/src/tokens.cpp b/api/src/tokens.cpp index 5f31b7b4c..f4810284b 100644 --- a/api/src/tokens.cpp +++ b/api/src/tokens.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -9,34 +10,31 @@ #ifdef TOKENS_CACHE -static inline bool isStringParam(const std::string& param) { - return param.size() >= 2 && param[0] == '"' && param.back() == '"'; +// An amount argument: v0 tokens take it as a string, v1 and v2 tokens as a BigDecimal +static inline bool isAmountParam(const general::Variant& v) { + return v.__isset.v_string || v.__isset.v_big_decimal; } -/*static inline bool isNormalTransfer(const std::string& method, - const std::vector& params) { - return method == "transfer" && params.size() == 2 && isStringParam(params[0]) && isStringParam(params[1]); -}*/ +static inline const std::string& amountParam(const general::Variant& v) { + return v.__isset.v_big_decimal ? v.v_big_decimal : v.v_string; +} static inline bool isNormalTransfer(const std::string& method, const std::vector& params) { - return method == "transfer" && params.size() == 2 && params[0].__isset.v_string && - params[1].__isset.v_string; /*&& isStringParam(params[0].v_string) && isStringParam(params[1].v_string);*/ + return method == "transfer" && params.size() == 2 && params[0].__isset.v_string && isAmountParam(params[1]); } -/*static inline bool isTransferFrom(const std::string& method, - const std::vector& params) { - return method == "transferFrom" && params.size() == 3 && isStringParam(params[0]) && isStringParam(params[1]) && isStringParam(params[1]); -}*/ - static inline bool isTransferFrom(const std::string& method, const std::vector& params) { return method == "transferFrom" && params.size() == 3 && params[0].__isset.v_string && params[1].__isset.v_string && - params[2].__isset.v_string; /*&& isStringParam(params[0].v_string) && isStringParam(params[1].v_string) && isStringParam(params[2].v_string);*/ + isAmountParam(params[2]); } static csdb::Address tryExtractPublicKey(const std::string& str) { // First try to decipher csdb::Address result; std::vector vc; + if (str.size() > 64) { + return result; + } bool decodeSucc = DecodeBase58(str, vc); if (!decodeSucc || vc.size() != cscrypto::kPublicKeySize) return result; @@ -45,102 +43,100 @@ static csdb::Address tryExtractPublicKey(const std::string& str) { } static csdb::Address tryGetRegisterData(const std::string& method, const std::vector& params) { - if (method == "register" && params.size() == 1) + if (method == "register" && params.size() == 1 && params[0].__isset.v_string) return tryExtractPublicKey(params[0].v_string); return csdb::Address(); } -static inline bool isValidAmountChar(const char c) { - return std::isdigit(c) || c == '.' || c == ','; -} - +// A plain decimal ("12.5") of a decimal string or of a Java BigDecimal.toString() ("1.5E+3"), +// without leading or trailing zeros; "0" for anything else, a negative amount included static std::string tryExtractAmount(const std::string& str) { - if (str.size() > 256) + if (str.empty() || str.size() > 256) return "0"; - const auto end = str.end(); - bool hasSep = false; - - uint8_t leadingZeros = 0; // From beginning - uint8_t trailingZeros = 0; // After . - - bool hasNonZero = false; - - std::string result; - for (auto cIt = str.begin(); cIt != end; ++cIt) { - if (!isValidAmountChar(*cIt)) { - result.clear(); - break; + std::string digits; + size_t pointPos = std::string::npos; // number of digits before the separator + size_t i = 0; + if (str[0] == '+') + ++i; + for (; i < str.size(); ++i) { + const char c = str[i]; + if (std::isdigit(static_cast(c))) { + digits.push_back(c); } - - if (!std::isdigit(*cIt)) { // Found a separator - if (hasSep) { // double separator - result.clear(); - break; - } - - hasSep = true; - hasNonZero = true; - result.push_back('.'); + else if ((c == '.' || c == ',') && pointPos == std::string::npos) { + pointPos = digits.size(); } else { - if (*cIt == '0') { - if (!hasNonZero) - ++leadingZeros; - else if (hasSep) - ++trailingZeros; - } - else { - if (!hasSep) - hasNonZero = true; - else - trailingZeros = 0; - } - - result.push_back(*cIt); + break; } } - - if (result.empty()) + if (digits.empty()) return "0"; - result = result.substr(leadingZeros, result.size() - leadingZeros - trailingZeros); - if (result.empty()) - return "0"; - - if (result.front() == '.') - result.insert(result.begin(), '0'); - if (result.back() == '.') - result.pop_back(); - return result.empty() ? "0" : result; -} + long exponent = 0; + if (i < str.size()) { + if (str[i] != 'e' && str[i] != 'E') + return "0"; + ++i; + bool negative = false; + if (i < str.size() && (str[i] == '+' || str[i] == '-')) { + negative = str[i] == '-'; + ++i; + } + if (i == str.size() || str.size() - i > 4) + return "0"; + for (; i < str.size(); ++i) { + if (!std::isdigit(static_cast(str[i]))) + return "0"; + exponent = exponent * 10 + (str[i] - '0'); + } + if (negative) + exponent = -exponent; + } -static bool javaTypesEqual(const std::string& goodType, const std::string& questionType) { - if (goodType.size() != questionType.size()) - return false; + long intLen = static_cast(pointPos == std::string::npos ? digits.size() : pointPos) + exponent; + if (intLen > 512 || intLen < -512) + return "0"; - for (uint32_t i = 0; i < goodType.size(); ++i) { - if (goodType[i] != static_cast(std::tolower(questionType[i]))) - return false; + std::string integral, fraction; + if (intLen <= 0) { + fraction = std::string(static_cast(-intLen), '0') + digits; + } + else if (static_cast(intLen) >= digits.size()) { + integral = digits + std::string(static_cast(intLen) - digits.size(), '0'); + } + else { + integral = digits.substr(0, static_cast(intLen)); + fraction = digits.substr(static_cast(intLen)); } - return true; + const auto firstNonZero = integral.find_first_not_of('0'); + integral = firstNonZero == std::string::npos ? "0" : integral.substr(firstNonZero); + const auto lastNonZero = fraction.find_last_not_of('0'); + fraction = lastNonZero == std::string::npos ? "" : fraction.substr(0, lastNonZero + 1); + + return fraction.empty() ? integral : integral + "." + fraction; } template T getVariantAs(const general::Variant&); template <> std::string getVariantAs(const general::Variant& var) { - if (var.__isset.v_big_decimal) { - try { - if (stof(var.v_big_decimal) == 0) - return "0"; - } - catch (...) {} + if (var.__isset.v_big_decimal) return var.v_big_decimal; - } - return var.v_string; + if (var.__isset.v_string) + return var.v_string; + if (var.__isset.v_int) + return std::to_string(var.v_int); + if (var.__isset.v_int_box) + return std::to_string(var.v_int_box); + if (var.__isset.v_long) + return std::to_string(var.v_long); + if (var.__isset.v_long_box) + return std::to_string(var.v_long_box); + return std::string{}; } template @@ -159,90 +155,101 @@ void executeAndCall(api::APIHandler* p_api, const general::Address& addr, const } p_api->getExecutor().executeByteCode(result, addr, addr_smart, byteCodeObjects, state, methodHeader, true /*isGetter*/, cs::Executor::kUseLastSequence); - if (!result.status.code && !result.results.empty()) + if (!result.status.code && !result.results.empty() && !result.results[0].status.code) handler(getVariantAs(result.results[0].ret_val)); } -void TokensMaster::refreshTokenState(const csdb::Address& token, const std::string& newState, bool checkBalance) { +// balanceOf asked for this many holders at once: the node waits executor_receive_timeout for the +// whole answer, and the executor deserializes the state once per holder +static constexpr size_t kBalancesPerRequest = 50; + +/* Call under data lock only */ +void TokensMaster::refreshTokenState(const csdb::Address& token, const std::string& newState, bool checkBalance, const std::set& only) { bool present = false; auto byteCodeObjects = api_->getSmartByteCode(token, present); - if (!present || byteCodeObjects.empty()) return; + if (!present || byteCodeObjects.empty() || newState.empty()) return; - std::string name, symbol, totalSupply; - csdb::Address deployer = tokens_[token].owner; + auto& t = tokens_[token]; + const csdb::Address deployer = t.owner; general::Address addr = std::string((char*)token.public_key().data(), token.public_key().size()); general::Address dpAddr = std::string((char*)deployer.public_key().data(), deployer.public_key().size()); + // a getter that fails keeps the value known before executeAndCall(api_, dpAddr, addr, byteCodeObjects, newState, "getName", std::vector(), - [&name](const std::string& newName) { name = newName.substr(0, 255); }); + [&t](const std::string& newName) { t.name = newName.substr(0, 255); }); executeAndCall(api_, dpAddr, addr, byteCodeObjects, newState, "totalSupply", std::vector(), - [&totalSupply](const std::string& newSupp) { totalSupply = tryExtractAmount(newSupp); }); + [&t](const std::string& newSupp) { t.totalSupply = tryExtractAmount(newSupp); }); executeAndCall(api_, dpAddr, addr, byteCodeObjects, newState, "getSymbol", std::vector(), - [&symbol](const std::string& newSymb) { - symbol.clear(); + [&t](const std::string& newSymb) { + t.symbol.clear(); for (uint32_t i = 0; i < newSymb.size(); ++i) { if (i >= 4) break; - symbol.push_back((char)std::toupper(newSymb[i])); + t.symbol.push_back((char)std::toupper(newSymb[i])); } - }); - - auto& t = tokens_[token]; - t.name = name; - t.symbol = symbol; - t.totalSupply = totalSupply; + }); - // balance - if (checkBalance) { - executor::ExecuteByteCodeMultipleResult result; - if (byteCodeObjects.empty()) - return; - - executor::SmartContractBinary smartContractBinary; - smartContractBinary.contractAddress = addr; - smartContractBinary.object.byteCodeObjects = byteCodeObjects; - smartContractBinary.object.instance = newState; - smartContractBinary.stateCanModify = 0; + if (!checkBalance) { + return; + } - std::vector holders; + std::vector holders; + if (only.empty()) { holders.reserve(t.holders.size()); for (auto& h : t.holders) holders.push_back(h.first); + } + else { + for (auto& h : only) { + if (t.holders.count(h)) + holders.push_back(h); + } + } + executor::SmartContractBinary smartContractBinary; + smartContractBinary.contractAddress = addr; + smartContractBinary.object.byteCodeObjects = byteCodeObjects; + smartContractBinary.object.instance = newState; + smartContractBinary.stateCanModify = 0; + + for (size_t from = 0; from < holders.size(); from += kBalancesPerRequest) { + const size_t to = std::min(holders.size(), from + kBalancesPerRequest); std::vector> holderKeysParams; - holderKeysParams.reserve(holders.size()); - for (auto& h : holders) { + holderKeysParams.reserve(to - from); + for (size_t i = from; i < to; ++i) { general::Variant var; - auto key = h.public_key(); + auto key = holders[i].public_key(); var.__set_v_string(EncodeBase58(cs::Bytes(key.begin(), key.end()))); holderKeysParams.push_back(std::vector(1, var)); } + executor::ExecuteByteCodeMultipleResult result; api_->getExecutor().executeByteCodeMultiple(result, dpAddr, smartContractBinary, "balanceOf", holderKeysParams, 100, cs::Executor::kUseLastSequence); - - if (!result.status.code && (result.results.size() == holders.size())) { - for (uint32_t i = 0; i < holders.size(); ++i) { - const auto& res = result.results[i]; - if (!res.status.code) { - bool zeroBalanceFlg = true; - if (!isZeroAmount(t.holders[holders[i]].balance)) - zeroBalanceFlg = false; - - t.holders[holders[i]].balance = tryExtractAmount(getVariantAs(res.ret_val)); - if (zeroBalanceFlg && !isZeroAmount(t.holders[holders[i]].balance)) - ++t.realHoldersCount; - else if (!zeroBalanceFlg && isZeroAmount(t.holders[holders[i]].balance)) - --t.realHoldersCount; - } + if (result.status.code || result.results.size() != to - from) { + continue; + } + for (size_t i = from; i < to; ++i) { + const auto& res = result.results[i - from]; + if (!res.status.code) { + t.holders[holders[i]].balance = tryExtractAmount(getVariantAs(res.ret_val)); } } } + + t.realHoldersCount = 0; + for (const auto& h : t.holders) { + if (!isZeroAmount(h.second.balance)) + ++t.realHoldersCount; + } } /* Call under data lock only */ void TokensMaster::initiateHolder(Token& token, const csdb::Address& address, const csdb::Address& holder, bool increaseTransfers /* = false*/) { + if (!holder.is_valid()) { + return; + } if (increaseTransfers) ++token.holders[holder].transfersCount; else @@ -253,18 +260,32 @@ void TokensMaster::initiateHolder(Token& token, const csdb::Address& address, co TokensMaster::TokensMaster(api::APIHandler* api) : api_(api) { +#if defined(MONITOR_NODE) || defined(WEB_WALLET_NODE) + enabled_ = true; +#else + enabled_ = cs::ConfigHolder::instance().config()->getApiSettings().tokensCache; +#endif + if (enabled_) { + cslog() << "API: token cache on"; + } } TokensMaster::~TokensMaster() { } -void TokensMaster::updateTokenChaches(const csdb::Address& addr, const std::string& newState, const TokenInvocationData::Params& ps) { - csunused(newState); +void TokensMaster::updateTokenChaches(const csdb::Address& addr, const std::string& newState, const TokenInvocationData::Params& ps, bool refresh) { + if (!enabled_) + return; std::lock_guard lInt(dataMut_); auto tIt = tokens_.find(addr); if (tIt == tokens_.end()) return; // Ignore if not-a-token + // the wallets whose balance the call may have changed: the initiator and every address among + // the arguments (the recipient of a transfer, the spender of an approval, a registered + // wallet, or the arguments of a method outside the standard such as mint or airdrop) + std::set touched; + touched.insert(ps.initiator); initiateHolder(tIt->second, tIt->first, ps.initiator); ++tIt->second.transactionsCount; @@ -272,97 +293,36 @@ void TokensMaster::updateTokenChaches(const csdb::Address& addr, const std::stri if (isTransfer(ps.method, ps.params)) { ++tIt->second.transfersCount; auto trPair = getTransferData(ps.initiator, ps.method, ps.params); - if (trPair.first.is_valid()) - initiateHolder(tIt->second, tIt->first, trPair.first, true); - if (trPair.second.is_valid()) - initiateHolder(tIt->second, tIt->first, trPair.second, true); + initiateHolder(tIt->second, tIt->first, trPair.first, true); + initiateHolder(tIt->second, tIt->first, trPair.second, true); } - else if (tIt->second.tokenStandard == TokenStandard::CreditsExtended) { + else if (isExtended(tIt->second.tokenStandard)) { csdb::Address regDude = tryGetRegisterData(ps.method, ps.params); - if (regDude.is_valid()) - initiateHolder(tIt->second, tIt->first, regDude); + initiateHolder(tIt->second, tIt->first, regDude); } - else if (ps.method == "approve") { - initiateHolder(tIt->second, tIt->first, tryExtractPublicKey(ps.params[0].v_string)); - } - } - -#ifdef MANUAL_BALANCE - // Balance update - auto refreshBalance = [&](const csdb::Address& addrFrom, const csdb::Address& addrTo = csdb::Address{}, const std::string& amount = "") { - auto getCurrBalance = [&](const csdb::Address& addrOwner) -> std::string { - bool present = false; - auto byteCodeObjects = api_->getSmartByteCode(addr, present); - if (!present || byteCodeObjects.empty()) return "0"; - - general::Address addrToken{ addr.public_key().begin(), addr.public_key().end() }; - - auto dpAddrPK = tokens_[addr].owner.public_key(); - general::Address dpAddr{ dpAddrPK.begin(), dpAddrPK.end() }; - - // param: owner - std::vector param(1); - std::string addrOwnerStr = addrOwner.to_api_addr(); - param[0].__set_v_string(EncodeBase58({ addrOwnerStr.begin(), addrOwnerStr.end() })); - // - std::string retBalance{ "0" }; - executeAndCall(api_, dpAddr, addrToken, byteCodeObjects, newState, "balanceOf", param, - [&retBalance](const std::string& balance) { retBalance = balance; }); - - if (!std::all_of(retBalance.begin(), retBalance.end(), [](char ch) { return (isdigit(ch) || ch == '.'); })) { - csdebug() << "executor returns balance as " << retBalance; - retBalance = "0"; + for (const auto& p : ps.params) { + if (p.__isset.v_string) { + const csdb::Address a = tryExtractPublicKey(p.v_string); + if (a.is_valid()) { + initiateHolder(tIt->second, tIt->first, a); + touched.insert(a); + } } - return retBalance; - }; - - auto& t = tokens_[addr]; - if (addrTo == csdb::Address{}) { // for deploy token - t.holders[addrFrom].balance = getCurrBalance(addrFrom); - ++t.realHoldersCount; - return; } + } - // from - auto& currFromBalance = t.holders[addrFrom].balance; - auto newFromBalance = [&] { - if (isZeroAmount(currFromBalance)) - return getCurrBalance(addrFrom); - else - return std::to_string(stof(currFromBalance) - stof(amount)); - }(); - if (isZeroAmount(newFromBalance)) - --t.realHoldersCount; - currFromBalance = newFromBalance; - - // to - auto& currToBalance = t.holders[addrTo].balance; - auto newToBalance = [&] { - if (isZeroAmount(currToBalance)) - return getCurrBalance(addrTo); - else - return std::to_string(stof(currToBalance) + stof(amount)); - }(); - if (isZeroAmount(currToBalance) && !isZeroAmount(newToBalance)) - ++t.realHoldersCount; - currToBalance = newToBalance; - }; -#endif - - if(api_->isBDLoaded()) { -#ifdef MANUAL_BALANCE - if (ps.method == "transfer" && ps.params.size() == 2) - refreshBalance(ps.initiator, tryExtractPublicKey(ps.params[0].v_string), ps.params[1].v_string); - else if (ps.method == "transferFrom" && ps.params.size() == 3) - refreshBalance(tryExtractPublicKey(ps.params[0].v_string), tryExtractPublicKey(ps.params[1].v_string), ps.params[2].v_string); - else if (ps.method.empty()) // deploy token - refreshBalance(tokens_[addr].owner); -#endif - refreshTokenState(addr, cs::SmartContracts::get_contract_state(api_->get_s_blockchain(), addr), true); + if (refresh && api_->isBDLoaded()) { + const std::string state = newState.empty() ? cs::SmartContracts::get_contract_state(api_->get_s_blockchain(), addr) : newState; + // the token's own rules decide the balances: ask the contract for those of the wallets the + // call involves; a method outside the standard may move other balances too, so a token + // with few holders is read in full + refreshTokenState(addr, state, true, tIt->second.holders.size() <= kBalancesPerRequest ? std::set{} : touched); } } void TokensMaster::checkNewDeploy(const csdb::Address& sc, const csdb::Address& deployer, const api::SmartContractInvocation& sci) { + if (!enabled_) + return; DeployTask dt; dt.address = sc; dt.deployer = deployer; @@ -373,12 +333,12 @@ void TokensMaster::checkNewDeploy(const csdb::Address& sc, const csdb::Address& api_->getExecutor().getContractMethods(methodsResult, dt.byteCodeObjects); if (!methodsResult.status.code) { if (methodsResult.tokenStandard != TokenStandard::NotAToken) { - Token t; - t.tokenStandard = methodsResult.tokenStandard; - t.owner = dt.deployer; - - { - std::lock_guard lInt(dataMut_); + std::lock_guard lInt(dataMut_); + // a deploy is seen once; never reset a token known already + if (tokens_.count(dt.address) == 0) { + Token t; + t.tokenStandard = methodsResult.tokenStandard; + t.owner = dt.deployer; tokens_[dt.address] = t; } } @@ -386,12 +346,13 @@ void TokensMaster::checkNewDeploy(const csdb::Address& sc, const csdb::Address& } } -void TokensMaster::checkNewState(const csdb::Address& sc, const csdb::Address& initiator, const api::SmartContractInvocation& sci, const std::string& newState) { +void TokensMaster::checkNewState(const csdb::Address& sc, const csdb::Address& initiator, const api::SmartContractInvocation& sci, const std::string& newState, + bool refresh) { TokenInvocationData::Params ps; ps.initiator = initiator; ps.method = sci.method; ps.params = sci.params; - updateTokenChaches(sc, newState, ps); + updateTokenChaches(sc, newState, ps, refresh); } void TokensMaster::loadTokenInfo(const std::function func) { @@ -399,6 +360,30 @@ void TokensMaster::loadTokenInfo(const std::function tokens; + { + std::lock_guard l(dataMut_); + tokens.reserve(tokens_.size()); + for (const auto& t : tokens_) + tokens.push_back(t.first); + } + cslog() << "tokens are loading(" << tokens.size() << ")..."; + size_t done = 0; + for (const auto& token : tokens) { + const std::string state = cs::SmartContracts::get_contract_state(api_->get_s_blockchain(), token); + { + std::lock_guard l(dataMut_); + refreshTokenState(token, state, true); + } + if (tokens.size() > 100 && ++done % (tokens.size() / 10) == 0) + cslog() << "loading tokens: " << 100 * done / tokens.size() << "%"; + } + cslog() << "tokens loaded!"; +} + bool TokensMaster::isTransfer(const std::string& method, const std::vector& params) { return isNormalTransfer(method, params) || isTransferFrom(method, params); } @@ -422,9 +407,9 @@ AddrPair TokensMaster::getTransferData(const csdb::Address& initiator, const std std::string TokensMaster::getAmount(const api::SmartContractInvocation& sci) { if (isNormalTransfer(sci.method, sci.params)) - return tryExtractAmount(sci.params[1].v_string); + return tryExtractAmount(amountParam(sci.params[1])); else if (isTransferFrom(sci.method, sci.params)) - return tryExtractAmount(sci.params[2].v_string); + return tryExtractAmount(amountParam(sci.params[2])); return "0"; } @@ -437,10 +422,12 @@ TokensMaster::~TokensMaster() { } void TokensMaster::checkNewDeploy(const csdb::Address&, const csdb::Address&, const api::SmartContractInvocation&) { } -void TokensMaster::checkNewState(const csdb::Address&, const csdb::Address&, const api::SmartContractInvocation&, const std::string&) { +void TokensMaster::checkNewState(const csdb::Address&, const csdb::Address&, const api::SmartContractInvocation&, const std::string&, bool) { } void TokensMaster::loadTokenInfo(const std::function) { } +void TokensMaster::refreshAll() { +} bool TokensMaster::isTransfer(const std::string&, const std::vector&) { return false; } diff --git a/client/config/config.cpp b/client/config/config.cpp index c8ec6fe4c..84c35e3bf 100644 --- a/client/config/config.cpp +++ b/client/config/config.cpp @@ -100,6 +100,7 @@ const std::string PARAM_NAME_EXECUTOR_RUN_DELAY = "executor_run_delay"; const std::string PARAM_NAME_EXECUTOR_BACKGROUND_THREAD_DELAY = "executor_background_thread_delay"; const std::string PARAM_NAME_EXECUTOR_CHECK_VERSION_DELAY = "executor_check_version_delay"; const std::string PARAM_NAME_EXECUTOR_MULTI_INSTANCE = "executor_multi_instance"; +const std::string PARAM_NAME_TOKENS_CACHE = "tokens_cache"; const std::string PARAM_NAME_EXECUTOR_VERSION_COMMIT_MIN = "executor_commit_min"; const std::string PARAM_NAME_EXECUTOR_VERSION_COMMIT_MAX = "executor_commit_max"; const std::string PARAM_NAME_JPS_COMMAND_LINE = "jps_command"; @@ -1150,6 +1151,7 @@ void Config::readApiData(const boost::property_tree::ptree& config) { checkAndSaveValue(data, BLOCK_NAME_API, PARAM_NAME_EXECUTOR_BACKGROUND_THREAD_DELAY, apiData_.executorBackgroundThreadDelay); checkAndSaveValue(data, BLOCK_NAME_API, PARAM_NAME_EXECUTOR_CHECK_VERSION_DELAY, apiData_.executorCheckVersionDelay); checkAndSaveValue(data, BLOCK_NAME_API, PARAM_NAME_EXECUTOR_MULTI_INSTANCE, apiData_.executorMultiInstance); + checkAndSaveValue(data, BLOCK_NAME_API, PARAM_NAME_TOKENS_CACHE, apiData_.tokensCache); checkAndSaveValue(data, BLOCK_NAME_API, PARAM_NAME_SERVER_SEND_TIMEOUT, apiData_.serverSendTimeout); checkAndSaveValue(data, BLOCK_NAME_API, PARAM_NAME_SERVER_RECEIVE_TIMEOUT, apiData_.serverReceiveTimeout); checkAndSaveValue(data, BLOCK_NAME_API, PARAM_NAME_AJAX_SERVER_SEND_TIMEOUT, apiData_.ajaxServerSendTimeout); @@ -1298,6 +1300,7 @@ bool operator==(const ApiData& lhs, const ApiData& rhs) { lhs.executorBackgroundThreadDelay == rhs.executorBackgroundThreadDelay && lhs.executorCheckVersionDelay == rhs.executorCheckVersionDelay && lhs.executorMultiInstance == rhs.executorMultiInstance && + lhs.tokensCache == rhs.tokensCache && lhs.executorCommitMin == rhs.executorCommitMin && lhs.executorCommitMax == rhs.executorCommitMax && lhs.jpsCmdLine == rhs.jpsCmdLine; diff --git a/client/config/config.hpp b/client/config/config.hpp index 386869696..75a249235 100644 --- a/client/config/config.hpp +++ b/client/config/config.hpp @@ -110,6 +110,8 @@ struct ApiData { int executorBackgroundThreadDelay = 100; int executorCheckVersionDelay = 5000; bool executorMultiInstance = false; + // the token cache behind the Token* API methods (always on in RelMonitor and RelWallet builds) + bool tokensCache = false; int executorCommitMin = 1506; // first commit with support of checking int executorCommitMax{-1}; // unlimited range on the right std::string jpsCmdLine = "jps"; diff --git a/client/include/client/params.hpp b/client/include/client/params.hpp index 45c191607..84c32458e 100644 --- a/client/include/client/params.hpp +++ b/client/include/client/params.hpp @@ -18,9 +18,11 @@ #if defined(MONITOR_NODE) || defined(WEB_WALLET_NODE) #define PROFILE_API -#define TOKENS_CACHE #endif +// the token cache is built in every node; [api] tokens_cache turns it on (TokensMaster::enabled) +#define TOKENS_CACHE + #ifdef MONITOR_NODE #define STATS #endif diff --git a/docs/CHANGES.md b/docs/CHANGES.md index 2831a0139..f4c2f4cb2 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -43,6 +43,7 @@ lines under the new release. | #120 | ci: a private network of 7 nodes (with the restart of one at block 300) runs on macOS and on Windows every night and on demand (`testnet-platforms.yml`): the nodes reach the same state digests as on Linux and a stopped node quick-starts again; Linux-only scenarios (network faults, slow nodes, full disks) stay on Linux | | #126 | api: after a quick start at the last stored block, contract calls through the API returned "Transaction in progress" and never their result; the node now marks its stored blocks as read when it runs | | #127 | ci: token tests: `ci/contracts/executor_harness.py` compares contract executors on token contracts without a node, and `TOKENS=1` runs `ci/testnet/tokens.py` (v1 and v0 tokens, balances and states on every node, the token API; `TOKENS_VIA` through another node) on the private network | +| #128 | api: the token cache (`[api] tokens_cache=true`, any build) follows v1/v2 tokens (BigDecimal amounts, standards 3 and 4) and only successful calls. It reads balances from the contract for the wallets a call involves, refreshes them in the background at start, and is rebuilt from the chain when the checkpoint holds none. `approve()` without arguments no longer reads out of bounds. `TokensListGet` filters before paging, and the token lists sort without exceptions or invalid orderings. `TokenTransfersListGet` reports the signer as the sender. See [tokens.md](tokens.md) | ## community.14 — 2026-10-27 diff --git a/docs/differences.md b/docs/differences.md index 11e93487c..6fd396c48 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -94,6 +94,13 @@ What an operator sees and sets is in [snapshots.md](snapshots.md). `snapshot.mismatch` exists. An import interrupted by a kill leaves `db/.importing`; the next start removes what that import wrote before it looks at `db/`. +### Token cache + +`[api] tokens_cache=true` turns on the token cache behind the `Token*` API methods in any build. +The official node has it only in its monitor and wallet builds. The cache follows tokens of every +standard, including v1/v2 tokens whose amounts are BigDecimals, and it skips failed calls. It is +rebuilt from the chain when the checkpoint holds none. See [tokens.md](tokens.md). + ## Start-up behaviour - **Database check.** Before opening the block database, the node looks at `db/`. It writes the diff --git a/docs/tokens.md b/docs/tokens.md new file mode 100644 index 000000000..580ff80db --- /dev/null +++ b/docs/tokens.md @@ -0,0 +1,76 @@ +# Tokens and the token API + +Credits tokens are smart contracts that implement one of the token standards of the contract API. +The chain only knows contracts and their states. The token methods of the Thrift API +(`TokensListGet`, `TokenInfoGet`, `TokenBalancesGet`, `TokenHoldersGet`, `TokenTransfersGet`, +`TokenWalletTransfersGet`, `TokenTransfersListGet`, `TokenTransferGet`, `TokenTransactionsGet`) +answer from a token cache that the node builds next to the chain. This cache affects only the +API, never consensus. + +## Token standards + +The executor reports a standard for every contract (`getContractMethods`, `tokenStandard`): + +| Id | Interface | Amounts | +|---|---|---| +| 0 | not a token | | +| 1 | `com.credits.scapi.v0.BasicStandard` | `String` | +| 2 | `com.credits.scapi.v0.ExtensionStandard` (adds `register()`, `buyTokens(String)`) | `String` | +| 3 | `com.credits.scapi.v1.BasicTokenStandard`, and the v2 interface that extends it | `BigDecimal` | +| 4 | `com.credits.scapi.v1.ExtensionTokenStandard` (adds `register()`), and its v2 interface | `BigDecimal` | + +The executor of the mainnet release (build 1518) reports a token that implements the v2 +interfaces as 0, not a token. The community executor reports it as 3 or 4. For every other +contract the two executors report the same id. + +A call must pass each argument in the type the method declares. A v1 token's +`transfer(String, BigDecimal)` takes the amount as `v_big_decimal`. Sent as `v_string`, the +call fails with "Cannot find a method by name and parameters specified", and the transaction +fee is still charged. The reverse applies to v0 tokens. A wallet reads the types from +`ContractMethodsGet` (or `SmartContractGet`) before it builds the call. + +## The token cache + +- `[api] tokens_cache=true` turns the cache on. It is off by default. The RelMonitor and + RelWallet builds always have it on, as the official monitor and wallet nodes do. With the + cache off, the token methods answer "This node doesn't provide such info". +- **What it records.** A token is recorded at its deploy. After each successful call, the node + records: + - the initiator, and every argument that is a wallet address, as holders of the token; + - `transfer` and `transferFrom` calls, with the amount as a string or a BigDecimal, as + transfers. + + The node then asks the contract for the balances of those wallets (`balanceOf`), and for its + name, symbol and total supply. A token with up to 50 holders is read in full. A balance is + always the one the contract reports, including after methods outside the standard such as + mint, airdrop or a sale for CS. Failed calls change nothing. +- **Starting.** When the node runs, it reads every token's balances again. This includes a quick + start at the last stored block, which reads no block. If the checkpoint holds no tokens, for + example because it was saved with the cache off, the node first looks for tokens among all + deployed contracts and their transactions. The log has `API: token cache is empty, looking for + tokens among N contracts`, then `tokens loaded!`. + +## Contract calls through the API after a restart + +`TransactionFlow` waits for the result of a contract call and returns its return value. A node +that quick-started from a checkpoint at its last stored block never marked its stored blocks as +read. On such a node every contract call through its API ended with "Transaction in progress" +and no result, even though the network executed the call. The node now marks the blocks as read +once it runs. + +## Tests + +- `ci/contracts/executor_harness.py` runs the token scenarios of `ci/contracts/token_scenarios.py` + on one or more contract executor jars, with no node. With two jars, any answer that differs + from the first jar's fails the run: + + ```sh + python3 ci/contracts/executor_harness.py --jar csnode/contract-executor.jar \ + --jar other/contract-executor.jar --jdk csnode/ojdkbuild/java-11-openjdk-11.0.4 + ``` + +- `ci/testnet/run.sh` with `CONTRACTS=1 TOKENS=1` runs `ci/testnet/tokens.py` on the private + network. The script deploys a v1 token and a v0 token, then makes transfers, approvals, a CS + payment to the token and a sale back for CS. It checks the balances and the states on every + node, and the token API. `TOKENS_VIA=5` with `RESTART_NODES=5` sends everything through the + node of the restart test after it quick-started.