Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions client/config/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const std::string BLOCK_NAME_HOST_OUTPUT = "host_output";
const std::string BLOCK_NAME_HOST_ADDRESS = "host_address";
const std::string BLOCK_NAME_POOL_SYNC = "pool_sync";
const std::string BLOCK_NAME_STORAGE = "storage";
const std::string BLOCK_NAME_SNAPSHOT = "snapshot";
const std::string BLOCK_NAME_API = "api";
const std::string BLOCK_NAME_CONVEYER = "conveyer";
const std::string BLOCK_NAME_EVENT_REPORTER = "event_report";
Expand Down Expand Up @@ -882,6 +883,7 @@ Config Config::readFromFile(const std::string& fileName) {
result.setLoggerSettings(config);
result.readPoolSynchronizerData(config);
result.readStorageData(config);
result.snapshotData_ = parseSnapshotData(config);
result.readApiData(config);
result.readConveyerData(config);
result.readEventsReportData(config);
Expand Down Expand Up @@ -999,6 +1001,91 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
}
}

namespace {
// a [snapshot] number: the default and a warning when it is not a number or out of [min, max]
template <typename T>
void readSnapshotNumber(const boost::property_tree::ptree& data, const std::string& key, T& value, T min, T max) {
if (!data.count(key)) {
return;
}
const auto text = data.get<std::string>(key);
try {
size_t used = 0;
const unsigned long long parsed = std::stoull(text, &used);
if (used == text.size() && text.find('-') == std::string::npos && parsed >= min && parsed <= max) {
value = static_cast<T>(parsed);
return;
}
}
catch (const std::exception&) {
}
cswarning() << "config: [snapshot] " << key << "=" << text << " is not in [" << min << ", " << max << "], using " << value;
}

void readSnapshotBool(const boost::property_tree::ptree& data, const std::string& key, bool& value) {
if (!data.count(key)) {
return;
}
const auto text = data.get<std::string>(key);
if (text == "true" || text == "1" || text == "on") {
value = true;
}
else if (text == "false" || text == "0" || text == "off") {
value = false;
}
else {
cswarning() << "config: [snapshot] " << key << "=" << text << " is not true or false, using " << (value ? "true" : "false");
}
}

void readSnapshotChoice(const boost::property_tree::ptree& data, const std::string& key, std::string& value,
std::initializer_list<const char*> choices) {
if (!data.count(key)) {
return;
}
const auto text = data.get<std::string>(key);
for (const char* choice : choices) {
if (text == choice) {
value = text;
return;
}
}
cswarning() << "config: [snapshot] " << key << "=" << text << " is not a known value, using " << value;
}
} // namespace

SnapshotData Config::parseSnapshotData(const boost::property_tree::ptree& config) {
SnapshotData result;
if (!config.count(BLOCK_NAME_SNAPSHOT)) {
return result;
}
const boost::property_tree::ptree& data = config.get_child(BLOCK_NAME_SNAPSHOT);
readSnapshotNumber<uint16_t>(data, "serve_port", result.servePort, 0, 65535);
if (data.count("serve_bind")) {
result.serveBind = data.get<std::string>("serve_bind");
}
readSnapshotNumber<size_t>(data, "serve_max_connections", result.serveMaxConnections, 1, 1024);
readSnapshotNumber<size_t>(data, "serve_rate_kbps", result.serveRateKbps, 0, 10'000'000);
readSnapshotNumber<size_t>(data, "snapshot_blocks", result.snapshotBlocks, 1000, 10'000'000);
readSnapshotChoice(data, "fetch", result.fetch, {"auto", "off"});
// empty values are meaningful here: no extra sources, no sources file, no GitHub source
if (data.count("sources")) {
result.sources = data.get<std::string>("sources");
}
if (data.count("sources_file")) {
result.sourcesFile = data.get<std::string>("sources_file");
}
readSnapshotBool(data, "probe_known_hosts", result.probeKnownHosts);
if (data.count("github_manifest")) {
result.githubManifest = data.get<std::string>("github_manifest");
}
readSnapshotNumber<size_t>(data, "min_sources", result.minSources, 1, 20);
readSnapshotChoice(data, "fallback", result.fallback, {"github", "none"});
readSnapshotNumber<size_t>(data, "fetch_connections", result.fetchConnections, 1, 16);
readSnapshotNumber<size_t>(data, "fetch_rate_kbps", result.fetchRateKbps, 0, 10'000'000);
return result;
}

void Config::readApiData(const boost::property_tree::ptree& config) {
if (!config.count(BLOCK_NAME_API)) {
return;
Expand Down Expand Up @@ -1173,6 +1260,27 @@ bool operator!=(const ApiData& lhs, const ApiData& rhs) {
return !(lhs == rhs);
}

bool operator==(const SnapshotData& lhs, const SnapshotData& rhs) {
return lhs.servePort == rhs.servePort &&
lhs.serveBind == rhs.serveBind &&
lhs.serveMaxConnections == rhs.serveMaxConnections &&
lhs.serveRateKbps == rhs.serveRateKbps &&
lhs.snapshotBlocks == rhs.snapshotBlocks &&
lhs.fetch == rhs.fetch &&
lhs.sources == rhs.sources &&
lhs.sourcesFile == rhs.sourcesFile &&
lhs.probeKnownHosts == rhs.probeKnownHosts &&
lhs.githubManifest == rhs.githubManifest &&
lhs.minSources == rhs.minSources &&
lhs.fallback == rhs.fallback &&
lhs.fetchConnections == rhs.fetchConnections &&
lhs.fetchRateKbps == rhs.fetchRateKbps;
}

bool operator!=(const SnapshotData& lhs, const SnapshotData& rhs) {
return !(lhs == rhs);
}

bool operator==(const ConveyerData& lhs, const ConveyerData& rhs) {
return lhs.maxPacketLifeTime == rhs.maxPacketLifeTime;
}
Expand Down Expand Up @@ -1233,6 +1341,7 @@ bool operator==(const Config& lhs, const Config& rhs) {
lhs.privateKey_ == rhs.privateKey_ &&
lhs.poolSyncData_ == rhs.poolSyncData_ &&
lhs.apiData_ == rhs.apiData_ &&
lhs.snapshotData_ == rhs.snapshotData_ &&
lhs.alwaysExecuteContracts_ == rhs.alwaysExecuteContracts_ &&
lhs.compatibleVersion_ == rhs.compatibleVersion_ &&
lhs.recreateIndex_ == rhs.recreateIndex_ &&
Expand Down
32 changes: 32 additions & 0 deletions client/config/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,26 @@ struct StorageData {
size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

// [snapshot]: snapshots served by this node and fetched from other nodes at a first start
// (docs/snapshots.md). Read again by the config observer: every key but fetch applies while the
// node runs.
struct SnapshotData {
uint16_t servePort = 0; // 0: no snapshot service
std::string serveBind; // empty: all interfaces
size_t serveMaxConnections = 8; // more get 503
size_t serveRateKbps = 0; // upload cap of all connections; 0: unlimited
size_t snapshotBlocks = 101'000; // blocks before the checkpoint in a snapshot
std::string fetch = "auto"; // auto: restore at a start on an empty database; off: never
std::string sources; // host:port,host:port
std::string sourcesFile = "snapshot_sources.txt";
bool probeKnownHosts = true; // also try every known host at the default port
std::string githubManifest = "https://github.com/EnzinoBB/node/releases/download/pruned-snapshot/latest.json";
size_t minSources = 3; // independent sources that must agree
std::string fallback = "github"; // github: GitHub alone when fewer agree; none: sync from block 0
size_t fetchConnections = 4;
size_t fetchRateKbps = 0; // 0: unlimited
};

struct ApiData {
// on by default:
uint16_t port = 9090;
Expand Down Expand Up @@ -226,6 +246,10 @@ class Config {
return poolSyncData_;
}

const SnapshotData& getSnapshotSettings() const {
return snapshotData_;
}

const StorageData& getStorageSettings() const {
return storageData_;
}
Expand Down Expand Up @@ -356,6 +380,10 @@ class Config {

private:
static Config readFromFile(const std::string& fileName);
public:
// the [snapshot] section, defaults for missing keys, a warning and the default for a bad value
static SnapshotData parseSnapshotData(const boost::property_tree::ptree& config);
private:

void setLoggerSettings(const boost::property_tree::ptree& config);
void readPoolSynchronizerData(const boost::property_tree::ptree& config);
Expand Down Expand Up @@ -408,6 +436,7 @@ class Config {

PoolSyncData poolSyncData_;
StorageData storageData_;
SnapshotData snapshotData_;
ApiData apiData_;
DbSQLData dbSQLData_;

Expand Down Expand Up @@ -454,6 +483,9 @@ bool operator!=(const PoolSyncData& lhs, const PoolSyncData& rhs);
bool operator==(const ApiData& lhs, const ApiData& rhs);
bool operator!=(const ApiData& lhs, const ApiData& rhs);

bool operator==(const SnapshotData& lhs, const SnapshotData& rhs);
bool operator!=(const SnapshotData& lhs, const SnapshotData& rhs);

bool operator==(const DbSQLData& lhs, const DbSQLData& rhs);
bool operator!=(const DbSQLData& lhs, const DbSQLData& rhs);

Expand Down
90 changes: 90 additions & 0 deletions tests/csnode/snapshotconfig_tests.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
#include <gtest/gtest.h>

#include <sstream>
#include <string>

#include <boost/property_tree/ini_parser.hpp>
#include <boost/property_tree/ptree.hpp>

#include <config.hpp>

namespace {
SnapshotData parse(const std::string& ini) {
std::istringstream in(ini);
boost::property_tree::ptree tree;
boost::property_tree::read_ini(in, tree);
return Config::parseSnapshotData(tree);
}
} // namespace

TEST(SnapshotConfig, DefaultsWithoutTheSection) {
const auto s = parse("[params]\nipv6=false\n");
EXPECT_EQ(s, SnapshotData{});
EXPECT_EQ(s.servePort, 0);
EXPECT_EQ(s.fetch, "auto");
EXPECT_EQ(s.minSources, 3u);
EXPECT_EQ(s.fallback, "github");
EXPECT_EQ(s.snapshotBlocks, 101000u);
EXPECT_TRUE(s.probeKnownHosts);
}

TEST(SnapshotConfig, EveryKeyIsRead) {
const auto s = parse(
"[snapshot]\n"
"serve_port=9511\n"
"serve_bind=127.0.0.1\n"
"serve_max_connections=3\n"
"serve_rate_kbps=2000\n"
"snapshot_blocks=50000\n"
"fetch=off\n"
"sources=10.0.0.1:9511,10.0.0.2:9511\n"
"sources_file=mine.txt\n"
"probe_known_hosts=false\n"
"github_manifest=https://example.org/latest.json\n"
"min_sources=5\n"
"fallback=none\n"
"fetch_connections=8\n"
"fetch_rate_kbps=1000\n");
EXPECT_EQ(s.servePort, 9511);
EXPECT_EQ(s.serveBind, "127.0.0.1");
EXPECT_EQ(s.serveMaxConnections, 3u);
EXPECT_EQ(s.serveRateKbps, 2000u);
EXPECT_EQ(s.snapshotBlocks, 50000u);
EXPECT_EQ(s.fetch, "off");
EXPECT_EQ(s.sources, "10.0.0.1:9511,10.0.0.2:9511");
EXPECT_EQ(s.sourcesFile, "mine.txt");
EXPECT_FALSE(s.probeKnownHosts);
EXPECT_EQ(s.githubManifest, "https://example.org/latest.json");
EXPECT_EQ(s.minSources, 5u);
EXPECT_EQ(s.fallback, "none");
EXPECT_EQ(s.fetchConnections, 8u);
EXPECT_EQ(s.fetchRateKbps, 1000u);
}

TEST(SnapshotConfig, BadValuesKeepTheDefaults) {
const auto s = parse(
"[snapshot]\n"
"serve_port=70000\n"
"serve_max_connections=0\n"
"snapshot_blocks=10\n"
"fetch=sometimes\n"
"probe_known_hosts=maybe\n"
"min_sources=-1\n"
"fallback=ask\n"
"fetch_connections=99\n"
"fetch_rate_kbps=fast\n");
EXPECT_EQ(s, SnapshotData{});
}

TEST(SnapshotConfig, EmptyValuesTurnSourcesOff) {
const auto s = parse("[snapshot]\ngithub_manifest=\nsources_file=\n");
EXPECT_TRUE(s.githubManifest.empty());
EXPECT_TRUE(s.sourcesFile.empty());
}

TEST(SnapshotConfig, ADifferentPortIsADifferentConfiguration) {
const auto a = parse("[snapshot]\nserve_port=9511\n");
const auto b = parse("[snapshot]\nserve_port=9512\n");
EXPECT_NE(a, b);
EXPECT_EQ(a, parse("[snapshot]\nserve_port=9511\n"));
}
Loading