Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ jobs:
with:
name: testnet-node-${{ github.sha }}
path: build/client/node
retention-days: 7
retention-days: ${{ github.ref_name == 'master' && 7 || 3 }}

# mixed nodes, as in testnet.yml: the even nodes run a node built from another ref
- name: Build the node under comparison
Expand Down Expand Up @@ -182,7 +182,7 @@ jobs:
with:
name: testnet-node-alt-${{ github.sha }}
path: build-alt/client/node
retention-days: 7
retention-days: ${{ github.ref_name == 'master' && 7 || 3 }}

scenario:
needs: build
Expand Down Expand Up @@ -608,4 +608,4 @@ jobs:
testnet-run/resources.tsv
testnet-run/resources-summary.txt
testnet-run/sampler.log
retention-days: 14
retention-days: ${{ github.ref_name == 'master' && 14 || 3 }}
6 changes: 3 additions & 3 deletions .github/workflows/scale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ jobs:
with:
name: testnet-node-${{ github.sha }}
path: build/client/node
retention-days: 7
retention-days: ${{ github.ref_name == 'master' && 7 || 3 }}

# the official 4.2.532 node (CREDITSCOM 4a95aaaf) with only the two hooks a CI test network
# needs (its starter key and genesis recipient), for the even nodes
Expand Down Expand Up @@ -223,7 +223,7 @@ jobs:
with:
name: testnet-node-official-${{ github.sha }}
path: build-alt/client/node
retention-days: 7
retention-days: ${{ github.ref_name == 'master' && 7 || 3 }}

network:
needs: [plan, build]
Expand Down Expand Up @@ -356,4 +356,4 @@ jobs:
testnet-run/n*/config.ini
testnet-run/resources.tsv
testnet-run/resources-summary.txt
retention-days: 14
retention-days: ${{ github.ref_name == 'master' && 14 || 3 }}
24 changes: 24 additions & 0 deletions ci/testnet/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2270,6 +2270,30 @@ if [ -n "$forks" ]; then
exit 1
fi

# chain reorganisations: how deep each node unwound on its own (community nodes log one line per
# finished rollback episode). In a normal run the guard must never stop a reorg and no competing block
# may be refused (a refusal leaves a lagging node waiting for the operator, FORK HELD).
# DEEP_ROLLBACK_EXPECTED=1 is for a scenario that provokes either on purpose.
reorgs=$(grep -ah 'rollback episode ended: unwound' "$WORK"/n*/node.log 2>/dev/null | sed -E 's/.*unwound ([0-9]+) .*/\1/' | sort -n || true)
echo "REORG DEPTHS: $(echo "$reorgs" | grep -c . || true) episode(s), deepest $(echo "$reorgs" | tail -1 | grep . || echo 0) block(s)"
refused=$({ grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log 2>/dev/null || true; } | wc -l)
echo "UNWINDS REFUSED: $refused competing block(s) not vouched for by the node's confidants"
held=$(grep -al 'FORK HELD since' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true)
[ -z "$held" ] || echo "FORK HELD on: $held"
guarded=$(grep -al 'LONGEST-CHAIN GUARD: a competing chain' "$WORK"/n*/node.log 2>/dev/null | sed -E 's#.*/(n[0-9]+)/node.log#\1#' | tr '\n' ' ' || true)
if [ "${DEEP_ROLLBACK_EXPECTED:-0}" != 1 ]; then
if [ -n "$guarded" ]; then
grep -ah 'LONGEST-CHAIN GUARD' "$WORK"/n*/node.log | head -3
echo "FAIL: the automatic rollback bound stopped a reorg in a normal run on: $guarded"
exit 1
fi
if [ "$refused" -gt 0 ]; then
grep -ah 'is not authenticated by this node.s confidants' "$WORK"/n*/node.log | head -3
echo "FAIL: $refused competing block(s) refused in a normal run (set DEEP_ROLLBACK_EXPECTED=1 for a scenario that provokes it)"
exit 1
fi
fi

# a network still restarting at the end is stuck, even if it reached the target before (a bootstrap
# round, R-x.1 and up, that never gives way to R-(x+1).0): the last three round headers of n1
stuck=$(grep -a ' R-[0-9'"'"']*\.[0-9]* ' "$WORK/n1/node.log" | tail -3 | grep -vc '\.0 ' || true)
Expand Down
10 changes: 10 additions & 0 deletions client/config/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ const std::string PARAM_NAME_COMPATIBLE_VERSION = "compatible_version";
const std::string PARAM_NAME_TRAVERSE_NAT = "traverse_nat";
const std::string PARAM_NAME_SYNC_ON = "sync_on";
const std::string PARAM_NAME_MAX_UNCORRECTED_BLOCK = "max_uncorrected_block";
const std::string PARAM_NAME_MAX_AUTO_ROLLBACK = "max_auto_rollback";
const std::string PARAM_NAME_ALLOW_DEEP_ROLLBACK = "allow_deep_rollback";
const std::string PARAM_NAME_GENERATE_FORK = "generate_fork";
const std::string PARAM_NAME_IDLE_MODE = "idle_mode";
const std::string PARAM_NAME_DAEMON_MODE = "daemon_mode";
Expand Down Expand Up @@ -133,6 +135,7 @@ const std::string ARG_NAME_PRIVATE_KEY_FILE = "private-key-file";
const std::string ARG_NAME_ENCRYPT_KEY_FILE = "encryptkey";
const std::string ARG_NAME_RECREATE_INDEX = "recreate-index";
const std::string ARG_NAME_NEW_BC_TOP = "set-bc-top";
const std::string ARG_NAME_RESOLVE_FORK = "resolve-fork";
const std::string ARG_NAME_BAL_CHANGE = "bal-change";
const std::string ARG_NAME_DISABLE_AUTO_SHUTDOWN = "disable-auto-shutdown";

Expand Down Expand Up @@ -277,6 +280,7 @@ Config Config::read(po::variables_map& vm, bool atStart) {
result.pathToDb_ = getArgFromCmdLine(vm, ARG_NAME_DB_PATH, DEFAULT_PATH_TO_DB);

if (atStart) {
result.resolveFork_ = vm.count(ARG_NAME_RESOLVE_FORK) > 0;
if (vm.count(ARG_NAME_NEW_BC_TOP)) {
result.newBlockchainTop_ = true;
result.newBlockchainTopSeq_ =
Expand Down Expand Up @@ -997,6 +1001,7 @@ Config Config::readFromFile(const std::string& fileName) {
}

result.maxUncorrectedBlock_ = params.count(PARAM_NAME_MAX_UNCORRECTED_BLOCK) ? params.get<uint64_t>(PARAM_NAME_MAX_UNCORRECTED_BLOCK) : DEFAULT_MAX_UNCORRECTED_BLOCK;
readRollbackGuardSettings(params, result.maxAutoRollbackDepth_, result.allowDeepRollback_);
result.generateFork_ = params.count(PARAM_NAME_GENERATE_FORK) ? params.get<bool>(PARAM_NAME_GENERATE_FORK) : false;

if (params.count(PARAM_NAME_SYNC_ON)) {
Expand Down Expand Up @@ -1201,6 +1206,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
}
}

void Config::readRollbackGuardSettings(const boost::property_tree::ptree& params, uint64_t& maxDepth, bool& allowDeep) {
maxDepth = params.count(PARAM_NAME_MAX_AUTO_ROLLBACK) ? params.get<uint64_t>(PARAM_NAME_MAX_AUTO_ROLLBACK) : DEFAULT_MAX_AUTO_ROLLBACK;
allowDeep = params.count(PARAM_NAME_ALLOW_DEEP_ROLLBACK) ? params.get<bool>(PARAM_NAME_ALLOW_DEEP_ROLLBACK) : false;
}

void Config::readBackfillSettings(const boost::property_tree::ptree& data, StorageData& storage) {
if (data.count("backfill")) {
const auto text = data.get<std::string>("backfill");
Expand Down
25 changes: 25 additions & 0 deletions client/config/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ const std::string DEFAULT_PATH_TO_PRIVATE_KEY = "NodePrivate.txt";
const uint32_t DEFAULT_MIN_NEIGHBOURS = Neighbourhood::kMinNeighbours;
const uint32_t DEFAULT_MAX_NEIGHBOURS = Neighbourhood::kMaxNeighbours;
const uint64_t DEFAULT_MAX_UNCORRECTED_BLOCK = 0;
// keep in step with Consensus::MaxAutoRollbackDepth (solver/include/solver/consensus.hpp)
const uint64_t DEFAULT_MAX_AUTO_ROLLBACK = 100;
const uint32_t DEFAULT_CONNECTION_BANDWIDTH = 1 << 19;
const uint32_t DEFAULT_OBSERVER_WAIT_TIME = 5 * 60 * 1000; // ms
const uint32_t DEFAULT_ROUND_ELAPSE_TIME = 1000 * 60; // ms
Expand Down Expand Up @@ -327,6 +329,23 @@ class Config {
return maxUncorrectedBlock_;
}

// longest-chain guard: the most own confirmed blocks the node will roll back automatically when it
// adopts a competing chain during sync, before it stops and waits for the operator.
uint64_t maxAutoRollbackDepth() const {
return maxAutoRollbackDepth_;
}

// longest-chain guard override: accept a deliberate deep switch without the depth bound (for one
// completed rollback episode; then it is off until the node restarts).
bool allowDeepRollback() const {
return allowDeepRollback_;
}

// --resolve-fork: truncate the chain to the fork point recorded in fork-held.json, then stop
bool resolveFork() const {
return resolveFork_;
}

bool traverseNAT() const {
return traverseNAT_;
}
Expand Down Expand Up @@ -395,6 +414,9 @@ class Config {
static SnapshotData parseSnapshotData(const boost::property_tree::ptree& config);
// the backfill keys of a [storage] section (backfill, backfill_batch, backfill_rate)
static void readBackfillSettings(const boost::property_tree::ptree& storage, StorageData& data);
// the [params] keys of the bound on automatic chain reorganisations: max_auto_rollback (default
// DEFAULT_MAX_AUTO_ROLLBACK, 0 = no depth bound) and allow_deep_rollback (default false)
static void readRollbackGuardSettings(const boost::property_tree::ptree& params, uint64_t& maxDepth, bool& allowDeep);
private:

void setLoggerSettings(const boost::property_tree::ptree& config);
Expand Down Expand Up @@ -430,6 +452,9 @@ class Config {
bool restrictNeighbours_ = false;
uint64_t connectionBandwidth_ = DEFAULT_CONNECTION_BANDWIDTH;
uint64_t maxUncorrectedBlock_ = DEFAULT_MAX_UNCORRECTED_BLOCK;
uint64_t maxAutoRollbackDepth_ = DEFAULT_MAX_AUTO_ROLLBACK;
bool allowDeepRollback_ = false;
bool resolveFork_ = false;
bool symmetric_ = false;

EndpointData hostAddressEp_;
Expand Down
1 change: 1 addition & 0 deletions client/include/client/cmdlineargs.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ struct cmdline {
static constexpr const char* argSeed = "seed";
static constexpr const char* argDumpKeys = "dumpkeys";
static constexpr const char* argSetBCTop = "set-bc-top";
static constexpr const char* argResolveFork = "resolve-fork";
static constexpr const char* argBalChange = "bal-change";
#ifdef _WIN32
static constexpr const char* argInstall = "install";
Expand Down
1 change: 1 addition & 0 deletions client/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ int main(int argc, char* argv[]) {
("recreate-index", "recreate index.db")
(cmdline::argSeed, "enter with seed instead of keys")
(cmdline::argSetBCTop, po::value<uint64_t>(), "all blocks in blockchain with higher sequence will be removed")
(cmdline::argResolveFork, "a node that reports FORK HELD: remove the blocks above the fork point recorded in fork-held.json (only if the block there is still the recorded one), then exit; at the next start the node follows the other chain. Check with other nodes first (the node must be stopped)")
(cmdline::argBalChange, po::value<std::string>(), "prints all changes in account balance in log file")
("disable-auto-shutdown", "node will be prohibited to shutdown in case of fatal errors")
("version", "show node version")
Expand Down
2 changes: 1 addition & 1 deletion client/src/peer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ bool Peer::onInit(const char*) {
}

bool Peer::onRun(const char*) {
if (vm_.count(cmdline::argSetBCTop) == 0) {
if (vm_.count(cmdline::argSetBCTop) == 0 && vm_.count(cmdline::argResolveFork) == 0) {
cslog() << "Running Node";
node_->run();
}
Expand Down
7 changes: 7 additions & 0 deletions csdb/include/csdb/storage.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ class Storage final {
// a quick start asked to read from a block after the last one stored: its checkpoint is
// ahead of the blocks on disk
StartAfterLastBlock = 7,
// --resolve-fork: the block at the requested top is not the one recorded (set_truncate_check)
TruncateMismatch = 8,
UnknownError = 255,
};

Expand Down Expand Up @@ -152,6 +154,11 @@ class Storage final {
* In case of fail the error info can be received with methods: \ref last_error,
* \ref last_error_message, \ref db_last_error() and \ref db_last_error_message()
*/
// The next open() with a new blockchain top (--set-bc-top) cuts the chain only if the stored block
// at that top has this hash; otherwise it removes nothing and fails with TruncateMismatch. An
// empty hash (the default) checks nothing. Used by --resolve-fork.
void set_truncate_check(const PoolHash& expectedTopHash);

// dbBackend: "berkeleydb" (default), "rocksdb", or "lmdb" in <path>/lmdb, whose removed blocks
// LMDB reuses the space of (pruned storage)
bool open(const ::std::string& path_to_base = ::std::string{},
Expand Down
40 changes: 39 additions & 1 deletion csdb/src/storage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -218,15 +218,29 @@ class Storage::priv {
> PoolCache;
PoolCache pools_cache;
static const size_t cacheSize = 50;
// set_truncate_check(): the hash the block at the new blockchain top must have
PoolHash truncate_expected_hash;

// the cache keeps one pool per sequence and per hash: a pool saved at a sequence whose previous
// pool was removed (a chain reorganisation) replaces the cached one, instead of being refused by
// the unique index and leaving the removed pool to be returned by pool_load(sequence)
void pools_cache_insert(const cs::Sequence& seq, const PoolHash &hash, const Pool &pool) {
if (pools_cache.size() == cacheSize) {
pools_cache_erase(seq, hash);
if (pools_cache.size() >= cacheSize) {
auto random = pools_cache.begin();
pools_cache.erase(random);
}
pools_cache.insert({seq, hash, pool});
}

// forget a pool removed from the top of the chain (by sequence and by hash)
void pools_cache_erase(const cs::Sequence& seq, const PoolHash &hash) {
pools_cache.get<PoolElement::bySequence>().erase(seq);
if (!hash.is_empty()) {
pools_cache.get<PoolElement::byHash>().erase(hash);
}
}

friend class ::csdb::Storage;

private signals:
Expand Down Expand Up @@ -465,6 +479,10 @@ Storage::WeakPtr Storage::weak_ptr() const noexcept {
return d;
}

void Storage::set_truncate_check(const PoolHash& expectedTopHash) {
d->truncate_expected_hash = expectedTopHash;
}

Storage::Error Storage::last_error() const {
return last_error_map(d.get()).last_error_;
}
Expand Down Expand Up @@ -537,6 +555,22 @@ bool Storage::open(const OpenOptions& opt, OpenCallback callback) {
}

if (opt.newBlockchainTop != cs::kWrongSequence) {
if (!d->truncate_expected_hash.is_empty()) {
const PoolHash expected = d->truncate_expected_hash;
d->truncate_expected_hash = PoolHash{};
cs::Bytes topBinary;
PoolHash stored;
if (d->db->get(static_cast<uint32_t>(opt.newBlockchainTop), &topBinary)) {
stored = csdb::Pool::hash_from_binary(std::move(topBinary));
}
if (stored != expected) {
d->set_last_error(TruncateMismatch, "block %llu is %s, not the recorded %s: nothing removed",
static_cast<unsigned long long>(opt.newBlockchainTop),
stored.is_empty() ? "not stored" : stored.to_string().c_str(),
expected.to_string().c_str());
return false;
}
}
auto seqToRemove = static_cast<uint32_t>(opt.newBlockchainTop + 1);
auto seqLast = seqToRemove;
{
Expand Down Expand Up @@ -1068,6 +1102,7 @@ Pool Storage::pool_remove_last() {
bool found = write_queue_pop(res);

if (found) {
d->pools_cache_erase(res.sequence(), res.hash());
std::unique_lock<std::mutex> lock(d->data_lock);
--d->count_pool;
d->last_hash = res.previous_hash();
Expand Down Expand Up @@ -1099,6 +1134,7 @@ Pool Storage::pool_remove_last() {

// error nearly impossible
/*bool ok =*/ d->db->remove(last_hash().to_binary());
d->pools_cache_erase(res.sequence(), res.hash());

--d->count_pool;
d->last_hash = res.previous_hash();
Expand Down Expand Up @@ -1134,6 +1170,7 @@ bool Storage::pool_remove_last_repair(cs::Sequence test_sequence, const csdb::Po
if (!write_queue_pop_last(test_hash, queued)) {
return false;
}
d->pools_cache_erase(test_sequence, test_hash);
if (queued) {
return true;
}
Expand Down Expand Up @@ -1198,6 +1235,7 @@ bool Storage::pool_remove_(cs::Sequence testSequence) {
if (!write_queue_pop_last(testHash, queued)) {
return false;
}
d->pools_cache_erase(testSequence, testHash);
if (queued) {
return true;
}
Expand Down
2 changes: 2 additions & 0 deletions csnode/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ add_library(csnode
include/csnode/snapshot_bootstrap.hpp
include/csnode/snapshot_fetcher.hpp
include/csnode/blockverifier.hpp
include/csnode/forkguardstate.hpp
include/csnode/backwardsync.hpp
include/csnode/sendcachedata.hpp
include/csnode/eventreport.hpp
Expand Down Expand Up @@ -134,6 +135,7 @@ add_library(csnode
src/snapshot_bootstrap.cpp
src/snapshot_fetcher.cpp
src/blockverifier.cpp
src/forkguardstate.cpp
src/backwardsync.cpp
src/sendcachedata.cpp
src/eventreport.cpp
Expand Down
Loading
Loading