Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions csnode/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ add_library(csnode
include/csnode/snapshot_fetcher.hpp
include/csnode/blockverifier.hpp
include/csnode/forkguardstate.hpp
include/csnode/peerstatus.hpp
include/csnode/backwardsync.hpp
include/csnode/sendcachedata.hpp
include/csnode/eventreport.hpp
Expand Down Expand Up @@ -136,6 +137,7 @@ add_library(csnode
src/snapshot_fetcher.cpp
src/blockverifier.cpp
src/forkguardstate.cpp
src/peerstatus.cpp
src/backwardsync.cpp
src/sendcachedata.cpp
src/eventreport.cpp
Expand Down
9 changes: 7 additions & 2 deletions csnode/include/csnode/forkguardstate.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ inline constexpr size_t kMaxBranchHashes = 256;
// the folder of these files for a database path: its parent folder ("." for a relative "db")
std::string stateDirFor(const std::string& dbPath);

// a small file in dir written whole: to a temporary name first, flushed to the disk, then renamed over
// the old one (the folder entry flushed too on POSIX), so a crash leaves either the old file or the new
// one. False (nothing replaced) on any failure or an empty dir.
bool writeFileAtomic(const std::string& dir, const char* name, const std::string& text);

struct Episode {
cs::Sequence highWater = 0;
};
Expand Down Expand Up @@ -121,7 +126,7 @@ std::string utcTime(int64_t unixSeconds);
std::string remedy(const ForkRecord& record);

// the periodic FORK HELD detail line: since when, where the chains diverge, the refused peer and the
// remedy
std::string summary(const ForkRecord& record);
// remedy. peerSide (optional): what the refused peer announces, e.g. "official 4.2.532"
std::string summary(const ForkRecord& record, const std::string& peerSide = std::string());

} // namespace cs::forkguard
14 changes: 14 additions & 0 deletions csnode/include/csnode/node.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <csnode/caches_serialization_manager.hpp>
#include <csnode/conveyer.hpp>
#include <csnode/compressor.hpp>
#include <csnode/peerstatus.hpp>

#include <lib/system/timer.hpp>

Expand Down Expand Up @@ -331,6 +332,12 @@ class Node {
return nodeIdKey_;
}

// no official node among the neighbours for 10 minutes, or among the trusted of the last 200
// rounds (log and status only; see csnode/peerstatus.hpp)
bool officialNodesGone() const {
return peerStatus_.officialGone();
}

Level getNodeLevel() const {
return myLevel_;
}
Expand Down Expand Up @@ -480,6 +487,13 @@ public slots:
// the fork guard's timer (every 5 s, queued to the processor thread): dismiss-fork, pending holds
cs::Timer forkGuardTimer_;
void onForkGuardTick();
// who keeps the network going (log and status only): fed by stored blocks, ticked by the fork
// guard's timer; the peers' kinds read again every 30 s
cs::peerstatus::Tracker peerStatus_;
std::chrono::steady_clock::time_point peerStatusRefreshed_{};
void onPeerStatusBlock(const csdb::Pool& pool);
void onPeerStatusBlockRemoved(const csdb::Pool& pool);
void onPeerStatusTick();
// snapshot.json of a restored folder against the digest of the loaded caches (design 6.8);
// false with exit code 4 on a mismatch
bool verifyAtStart();
Expand Down
149 changes: 149 additions & 0 deletions csnode/include/csnode/peerstatus.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#pragma once

#include <cstdint>
#include <deque>
#include <limits>
#include <map>
#include <mutex>
#include <string>
#include <vector>

#include <lib/system/common.hpp>

// Who keeps the network going: the official nodes (4.2.532, minor version 0 in the handshake) or the
// community releases (minor version N > 0). Log and status only: the node reads what peers announce
// (host_data) and the confidants of the blocks it stores; nothing here is sent, signed, hashed or
// validated.
//
// - every 10 minutes (and once at the start) a PEERS line: the neighbours by kind, the trusted seats
// of the last rounds by kind, how often this node was among them;
// - OFFICIAL NODES GONE when no official neighbour was seen for 10 minutes, or no official node was
// among the trusted of the last 200 rounds; OFFICIAL NODES BACK when they return;
// - a warning when the trusted seats of the last 1000 rounds were all community nodes;
// - peer-status.json next to the db folder (like fork-held.json), for the Docker healthcheck and
// operators.
namespace cs::peerstatus {

inline constexpr const char* kStatusFile = "peer-status.json";

enum class Kind : uint8_t {
Unknown,
Official,
Community
};

// from the announced host_data: version 0 means nothing announced yet
Kind classify(uint16_t version, uint16_t minorVersion);
const char* kindName(Kind kind);

struct Settings {
size_t window = 1000; // rounds in the PEERS line
int64_t goneSeconds = 600; // no official neighbour for this long: gone
size_t goneRounds = 200; // no official among the trusted of this many rounds: gone
size_t communityOnlyRounds = 1000; // all trusted seats community for this many rounds: warning
int64_t summaryEvery = 600; // the PEERS line
int64_t repeatEvery = 600; // an alert while its state holds
};

struct NeighbourCounts {
size_t total = 0;
size_t official = 0;
size_t community = 0;
size_t unknown = 0;
};

struct SeatCounts {
size_t rounds = 0;
size_t seats = 0;
size_t official = 0;
size_t community = 0;
size_t unknown = 0;
size_t selfTrusted = 0; // rounds with this node among the trusted
cs::Sequence first = 0;
cs::Sequence last = 0;
};

struct Line {
bool warning = false;
std::string text;
};

// integer percent, rounded; 0 for an empty total
unsigned percent(size_t part, size_t total);

class Tracker {
public:
explicit Tracker(Settings settings = Settings{});

// this node: its key and its own kind (a community build)
void setSelf(const cs::PublicKey& key, Kind kind);
// what the node knows of the peers' announced versions (replaces the previous map)
void setKinds(std::map<cs::PublicKey, Kind> kinds);
Kind kindOf(const cs::PublicKey& key) const;
// by the base58 key (the fork record's peer_key); Unknown when not known
Kind kindOfBase58(const std::string& key) const;

// a stored block: the confidants and the realTrusted mask (bit i: confidant i signed)
void addBlock(cs::Sequence seq, const std::vector<cs::PublicKey>& confidants, uint64_t realTrusted);
// a block removed from the top: forget it and anything above
void removeBlock(cs::Sequence seq);

// the trusted seats of the last `rounds` stored blocks (all of the window for 0)
SeatCounts seats(size_t rounds = 0) const;
NeighbourCounts countNeighbours(const std::vector<cs::PublicKey>& neighbours) const;

// every few seconds: the neighbours now; returns the lines to log (the PEERS line when due,
// alerts on transitions and at most every repeatEvery while they hold); nowSeconds: a monotonic clock
std::vector<Line> tick(int64_t nowSeconds, const NeighbourCounts& neighbours);

std::string peersLine(const NeighbourCounts& neighbours) const;

bool officialGone() const;
bool communityOnly() const;

// the status file's JSON text (as of the last tick)
std::string statusJson(int64_t wallUnix) const;

private:
struct Block {
cs::Sequence seq = 0;
std::vector<cs::PublicKey> trusted;
bool self = false;
};

SeatCounts seatsLocked(size_t rounds) const;
std::string peersLineLocked(const NeighbourCounts& n) const;

Settings settings_;
mutable std::mutex mutex_;
cs::PublicKey self_{};
Kind selfKind_ = Kind::Community;
std::map<cs::PublicKey, Kind> kinds_;
std::map<std::string, Kind> kindsBase58_;
std::deque<Block> blocks_;

// "long ago" for the throttles: the first alert or summary is always due
static constexpr int64_t kNever = (std::numeric_limits<int64_t>::min)() / 2;

bool started_ = false;
int64_t startedAt_ = 0;
int64_t lastSummary_ = kNever;
int64_t lastOfficialNeighbour_ = 0; // last tick with an official neighbour (startedAt_ before)
bool officialNeighbourSeen_ = false;
NeighbourCounts lastNeighbours_{};

bool gone_ = false;
std::string goneReason_;
int64_t goneSince_ = 0;
bool goneAlerted_ = false;
int64_t lastGoneAlert_ = kNever;
int64_t lastBackAlert_ = kNever;

bool communityOnly_ = false;
bool communityOnlyAlerted_ = false;
int64_t lastCommunityOnlyAlert_ = kNever;

std::string lastAlert_;
};

} // namespace cs::peerstatus
58 changes: 32 additions & 26 deletions csnode/src/forkguardstate.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,8 @@ void syncDir(const fs::path& dir) {
#endif
}

// a small file written whole: to a temporary name first, flushed to the disk, then renamed over the
// old one, so a crash leaves either the old file or the new one
// a small file written as JSON with writeFileAtomic
bool writeTree(const std::string& dir, const char* name, const ptree& tree) {
if (dir.empty()) {
return false;
}
std::error_code ec;
const fs::path target = fs::path(dir) / name;
const fs::path tmp = fs::path(dir) / (std::string(name) + ".tmp");
std::string text;
try {
std::ostringstream out;
Expand All @@ -66,23 +59,7 @@ bool writeTree(const std::string& dir, const char* name, const ptree& tree) {
catch (const std::exception&) {
return false;
}
std::FILE* f = std::fopen(tmp.string().c_str(), "wb");
if (f == nullptr) {
return false;
}
const bool written = std::fwrite(text.data(), 1, text.size(), f) == text.size() && syncFile(f);
const bool closed = std::fclose(f) == 0;
if (!written || !closed) {
fs::remove(tmp, ec);
return false;
}
fs::rename(tmp, target, ec);
if (ec) {
fs::remove(tmp, ec);
return false;
}
syncDir(fs::path(dir));
return true;
return writeFileAtomic(dir, name, text);
}

// the file's tree; nothing when it does not exist. A file that exists but cannot be read sets
Expand Down Expand Up @@ -129,6 +106,32 @@ void removeFile(const std::string& dir, const char* name) {

} // namespace

bool writeFileAtomic(const std::string& dir, const char* name, const std::string& text) {
if (dir.empty()) {
return false;
}
std::error_code ec;
const fs::path target = fs::path(dir) / name;
const fs::path tmp = fs::path(dir) / (std::string(name) + ".tmp");
std::FILE* f = std::fopen(tmp.string().c_str(), "wb");
if (f == nullptr) {
return false;
}
const bool written = std::fwrite(text.data(), 1, text.size(), f) == text.size() && syncFile(f);
const bool closed = std::fclose(f) == 0;
if (!written || !closed) {
fs::remove(tmp, ec);
return false;
}
fs::rename(tmp, target, ec);
if (ec) {
fs::remove(tmp, ec);
return false;
}
syncDir(fs::path(dir));
return true;
}

std::string stateDirFor(const std::string& dbPath) {
fs::path p(dbPath);
// "db/" and "db" name the same folder
Expand Down Expand Up @@ -377,7 +380,7 @@ std::string remedy(const ForkRecord& r) {
return os.str();
}

std::string summary(const ForkRecord& r) {
std::string summary(const ForkRecord& r, const std::string& peerSide) {
std::ostringstream os;
os << "FORK HELD since " << utcTime(r.sinceUnix) << ": ";
if (r.reason == "authority") {
Expand Down Expand Up @@ -407,6 +410,9 @@ std::string summary(const ForkRecord& r) {
if (r.peerHeight != 0) {
os << ", peer height #" << r.peerHeight;
}
if (!peerSide.empty()) {
os << "; the other side is " << peerSide;
}
os << ". The node keeps its chain. What to do: " << remedy(r);
return os.str();
}
Expand Down
Loading
Loading