Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/workflows/build-platforms.yml
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ jobs:
uses: actions/cache/restore@v4
with:
path: ${{ env.BOOST_PREFIX }}
key: boost-${{ env.BOOST_VERSION }}-static-macos-14-v1
key: boost-${{ env.BOOST_VERSION }}-static-cxx17-macos-14-v1

- name: Build Boost
if: steps.cache-boost.outputs.cache-hit != 'true'
Expand All @@ -168,8 +168,11 @@ jobs:
tar xjf boost.tar.bz2
cd "boost_${v}"
./bootstrap.sh --with-libraries=system,filesystem,program_options,date_time,regex,log,iostreams,thread,serialization,atomic,chrono
# cxxstd=17: Apple clang compiles C++ as C++98 unless told otherwise; Boost built that way
# holds definitions of header templates (Asio in Boost.Log's syslog backend) that differ from
# the node's C++17 ones, and the linker keeps one of each
./b2 -j"$(sysctl -n hw.ncpu)" -d0 toolset=clang variant=release link=static threading=multi \
cxxflags=-fPIC --layout=tagged -sNO_BZIP2=1 -sNO_ZSTD=1 -sNO_LZMA=1 \
cxxstd=17 cxxflags=-fPIC --layout=tagged -sNO_BZIP2=1 -sNO_ZSTD=1 -sNO_LZMA=1 \
install --prefix="${BOOST_PREFIX}"

- name: Save Boost
Expand Down
48 changes: 39 additions & 9 deletions .github/workflows/platform-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ on:
description: 'Id of a Release workflow run whose artifacts hold the Linux, Windows and macOS packages (empty = the latest successful one)'
default: ''
timeout_min:
description: 'Minutes each node has to reach the head'
default: '180'
description: 'Minutes each node has to reach the head (Linux needs about 5)'
default: '30'
min_common:
description: 'Blocks with a digest from all three systems needed to pass'
default: '5'
Expand Down Expand Up @@ -74,7 +74,7 @@ jobs:
package: '*-macos-arm64.tar.gz'
node: ./node
runs-on: ${{ matrix.runner }}
timeout-minutes: 300
timeout-minutes: 90
defaults:
run:
shell: bash
Expand Down Expand Up @@ -123,23 +123,45 @@ jobs:

- name: Restore the published snapshot
working-directory: csnode
run: ${{ matrix.node }} --config-file config.ini --download-snapshot </dev/null
# the import prints hundreds of thousands of debug lines to the console (the job log would be
# cut before the next steps): only the node's own report lines are kept in the log
run: |
${{ matrix.node }} --config-file config.ini --download-snapshot </dev/null > download.log 2>&1 || { tail -n 60 download.log | cut -c1-300; exit 1; }
grep -aE '^(download-snapshot|import-snapshot|snapshot)' download.log | cut -c1-250
tail -n 3 download.log | cut -c1-250

- name: Sync to the head
env:
TIMEOUT_MIN: ${{ inputs.timeout_min || '180' }}
TIMEOUT_MIN: ${{ inputs.timeout_min || '30' }}
run: |
cd csnode
${{ matrix.node }} --config-file config.ini </dev/null > stdout.log 2>&1 &
echo "NODE_PID=$!" >> "$GITHUB_ENV"
# how the node ended, if it ends before the head
# (not "cmd; echo $?": with -e the subshell would end at the failing node)
( rc=0; ${{ matrix.node }} --config-file config.ini </dev/null > stdout.log 2>&1 || rc=$?; echo "the node exited with status $rc" > node-exit.txt ) &
sleep 2
pid=
[ "${{ matrix.os }}" = windows ] || pid=$(pgrep -n -f -- '--config-file config.ini' || true)
echo "NODE_PID=$pid" >> "$GITHUB_ENV"
cd ..
python ci/snapshot/check_node.py wait-head --api 127.0.0.1:9090 --timeout-min "$TIMEOUT_MIN"
# Linux syncs this stretch at 30000+ blocks/min and reaches the head in about 5 minutes: a node
# that ends, has no API after 4 minutes or moves at less than a tenth of that speed is
# looked at at once, not after the timeout
python ci/snapshot/check_node.py wait-head --api 127.0.0.1:9090 --timeout-min "$TIMEOUT_MIN" \
--exit-file csnode/node-exit.txt --api-grace-min 4 --min-rate 3000 --window-min 2 2>&1 | tee csnode/waithead.txt || {
echo "---- the end of the node's output"; tail -n 60 csnode/stdout.log
exit 1
}

- name: Stop and collect the digests
if: always()
env:
OS: ${{ matrix.os }}
run: |
mkdir -p result
# a node that never reached the head: where its threads are
if [ "$OS" = macos ] && [ -n "${NODE_PID:-}" ] && kill -0 "$NODE_PID" 2>/dev/null; then
sample "$NODE_PID" 5 -file result/sample-macos.txt >/dev/null 2>&1 || true
fi
if [ "$OS" = windows ]; then
taskkill //F //IM node.exe >/dev/null 2>&1 || true
taskkill //F //IM java.exe >/dev/null 2>&1 || true
Expand All @@ -148,10 +170,18 @@ jobs:
for _ in $(seq 1 120); do [ -z "${NODE_PID:-}" ] || kill -0 "$NODE_PID" 2>/dev/null || break; sleep 1; done
pkill -f '[c]ontract-executor[.]jar' || true
fi
mkdir -p result
cat csnode/log/log_*.txt 2>/dev/null | grep -a "STATE DIGEST #" > "result/digests-$OS.txt" || true
cat csnode/log/log_*.txt 2>/dev/null | grep -aE "counted transactions is different|Couldn't save block|is not on this chain|but its caches already hold block|snapshot.verified|snapshot.mismatch" | head -50 > "result/problems-$OS.txt" || true
tail -c 2000000 csnode/stdout.log > "result/stdout-$OS.log" 2>/dev/null || true
cat csnode/node-exit.txt 2>/dev/null | tee result/exit-$OS.txt || true
cp csnode/waithead.txt "result/waithead-$OS.txt" 2>/dev/null || true
cp csnode/download.log "result/download-$OS.log" 2>/dev/null || true
if [ "$OS" = macos ]; then
sleep 15 # the crash report is written a moment after the process ends
cp ~/Library/Logs/DiagnosticReports/node*.ips result/ 2>/dev/null || true
ls ~/Library/Logs/DiagnosticReports/ 2>/dev/null | head || true
fi
for f in csnode/log/*; do [ -f "$f" ] && tail -c 3000000 "$f" > "result/$OS-$(basename "$f")"; done 2>/dev/null || true
echo "$(wc -l < "result/digests-$OS.txt") digest lines"; tail -3 "result/digests-$OS.txt"

- name: Upload the digests
Expand Down
45 changes: 39 additions & 6 deletions ci/snapshot/check_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,19 +59,47 @@ def monitor(path):


def wait_head(args):
node = None
deadline = time.time() + args.timeout_min * 60
started = time.time()
deadline = started + args.timeout_min * 60
near_since = None
api_up_since = None
window = [] # (time, block) of the last --window-min minutes
while time.time() < deadline:
# a node that ended is not waited for
if args.exit_file and os.path.exists(args.exit_file):
print(f"FAIL: {open(args.exit_file).read().strip()}", flush=True)
return 1
try:
# the node opens its API only once its database is loaded: connect (again) until it does
node = node or client(args.api)
s = node.SyncStateGet()
# a new connection for every poll: the node closes a connection idle for its
# server_receive_timeout (30 s), the interval of these polls, and a kept connection would
# race that close (lost on macOS every time, sometimes on Windows: "TSocket read 0 bytes").
# The node opens its API only once its database is loaded: connect until it does.
node = client(args.api)
try:
s = node.SyncStateGet()
finally:
node.close()
gap = s.currRound - s.lastBlock
api_up_since = api_up_since or time.time()
except Exception as e: # the node may still be starting
print(f"sync state unavailable: {e}", flush=True)
node = None
gap = None
if api_up_since is None and args.api_grace_min and time.time() - started > args.api_grace_min * 60:
print(f"FAIL: the node's API is not up {args.api_grace_min} min after the start", flush=True)
return 1
if gap is not None:
# the speed of the sync: a node far behind has to move, and at a speed in line with the
# others (Linux: 30000 blocks/min and more); near the head the chain sets the pace
now = time.time()
window.append((now, s.lastBlock))
while window and now - window[0][0] > args.window_min * 60 + 1:
window.pop(0)
if args.min_rate and gap > args.near_gap and now - window[0][0] >= args.window_min * 60:
rate = (s.lastBlock - window[0][1]) / ((now - window[0][0]) / 60)
print(f"speed over the last {args.window_min} min: {rate:.0f} blocks/min (at least {args.min_rate} wanted)", flush=True)
if rate < args.min_rate:
print(f"FAIL: the sync is too slow: {rate:.0f} blocks/min, {gap} blocks behind", flush=True)
return 1
if gap is not None and s.lastBlock > 0 and gap <= 3:
near_since = near_since or time.time()
if time.time() - near_since >= 180:
Expand Down Expand Up @@ -189,6 +217,11 @@ def main():
w = sub.add_parser("wait-head")
w.add_argument("--api", required=True)
w.add_argument("--timeout-min", type=int, default=90)
w.add_argument("--exit-file", help="a file the caller writes when the node process ends: the wait fails at once")
w.add_argument("--api-grace-min", type=int, default=0, help="minutes the node's API may take to come up (0 = no limit)")
w.add_argument("--min-rate", type=int, default=0, help="blocks per minute the sync must keep while far behind (0 = no check)")
w.add_argument("--window-min", type=int, default=2, help="minutes the speed is measured over")
w.add_argument("--near-gap", type=int, default=3000, help="no speed check this close to the head, in blocks")
b = sub.add_parser("balances")
b.add_argument("--api", required=True)
b.add_argument("--count", type=int, default=50)
Expand Down
191 changes: 191 additions & 0 deletions csnode/include/csnode/archive_stream.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
#ifndef ARCHIVE_STREAM_HPP
#define ARCHIVE_STREAM_HPP

#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <istream>
#include <streambuf>
#include <vector>

namespace cs {

// The input of a boost binary_iarchive that reads a checkpoint written on another system.
//
// A binary_oarchive header records the native sizes of int, long, float and double, and
// binary_iarchive refuses a header whose sizes differ from its own: long is 8 bytes on Linux and
// macOS, 4 on Windows, so a checkpoint (or a snapshot made of one) from one system did not load on
// the other. No checkpoint field is a long: 64-bit fields are int64_t/uint64_t, written as 8 bytes
// on every system, and boost writes its own counts as 8-byte size_t on 64-bit systems. The size of
// long in the header is the only difference. Archives are written with 8 there on every system
// (ArchiveOutput), so only Windows has anything to change on read: it takes an 8 as its own 4. On
// Linux and macOS the input is passed through untouched, the plain read, and boost keeps refusing
// an archive of a 32-bit system.
//
// Header of boost 1.83: size_t length (22) | "serialization::archive" | uint16 library version |
// sizeof(int) | sizeof(long) | sizeof(float) | sizeof(double) | int 1 (byte order). Anything else is
// passed through unchanged, for binary_iarchive to accept or refuse as before.
class ArchiveInputBuf : public std::streambuf {
public:
explicit ArchiveInputBuf(std::streambuf* source)
: source_(source) {
head_.resize(kHeaderSize);
head_.resize(static_cast<size_t>(source_->sgetn(head_.data(), static_cast<std::streamsize>(kHeaderSize))));
patchHeader();
setg(head_.data(), head_.data(), head_.data() + head_.size());
}

static constexpr size_t kSignatureSize = 22;
static constexpr size_t kLongSizeOffset = sizeof(uint64_t) + kSignatureSize + sizeof(uint16_t) + 1;
static constexpr size_t kHeaderSize = kLongSizeOffset + 3 + sizeof(int32_t);

protected:
int_type underflow() override {
if (gptr() < egptr()) {
return traits_type::to_int_type(*gptr());
}
buffer_.resize(kChunk);
const auto n = source_->sgetn(buffer_.data(), static_cast<std::streamsize>(buffer_.size()));
if (n <= 0) {
return traits_type::eof();
}
setg(buffer_.data(), buffer_.data(), buffer_.data() + n);
return traits_type::to_int_type(*gptr());
}

private:
static constexpr size_t kChunk = 1 << 16;

void patchHeader() {
if (head_.size() < kHeaderSize) {
return;
}
uint64_t length = 0;
for (size_t i = 0; i < sizeof(length); ++i) {
length |= static_cast<uint64_t>(static_cast<unsigned char>(head_[i])) << (8 * i);
}
static const char kSignature[] = "serialization::archive";
if (length != kSignatureSize || std::memcmp(head_.data() + sizeof(uint64_t), kSignature, kSignatureSize) != 0) {
return;
}
auto byteAt = [this](size_t i) { return static_cast<unsigned char>(head_[i]); };
const bool sizesAsExpected = byteAt(kLongSizeOffset - 1) == sizeof(int32_t) && byteAt(kLongSizeOffset + 1) == sizeof(float) &&
byteAt(kLongSizeOffset + 2) == sizeof(double);
if (sizesAsExpected && sizeof(long) == 4 && byteAt(kLongSizeOffset) == 8) {
head_[kLongSizeOffset] = static_cast<char>(sizeof(long));
}
}

std::streambuf* source_;
std::vector<char> head_;
std::vector<char> buffer_;
};

// usage: std::ifstream ifs(path, std::ios::binary); cs::ArchiveInput in(ifs); boost::archive::binary_iarchive ia(in);
class ArchiveInput : public std::istream {
public:
explicit ArchiveInput(std::istream& source)
: std::istream(nullptr)
, buf_(source.rdbuf()) {
rdbuf(&buf_);
if (!source) {
setstate(std::ios::badbit);
}
}

private:
ArchiveInputBuf buf_;
};

// The output of a boost binary_oarchive for a checkpoint: the header records long as 8 bytes on
// every system, so that a checkpoint (and a snapshot made of one) has the same bytes on Windows as
// on Linux and macOS, and nodes without ArchiveInput read it too. Only that header byte changes.
class ArchiveOutputBuf : public std::streambuf {
public:
explicit ArchiveOutputBuf(std::streambuf* target)
: target_(target) {
}

protected:
int_type overflow(int_type c) override {
if (traits_type::eq_int_type(c, traits_type::eof())) {
return traits_type::not_eof(c);
}
const char ch = traits_type::to_char_type(c);
return xsputn(&ch, 1) == 1 ? c : traits_type::eof();
}

std::streamsize xsputn(const char* s, std::streamsize n) override {
if (n <= 0) {
return 0;
}
const auto count = static_cast<size_t>(n);
if (written_ > ArchiveInputBuf::kLongSizeOffset) {
return put(s, n);
}
std::vector<char> copy(s, s + count);
for (size_t i = 0; i < count && written_ + i <= ArchiveInputBuf::kLongSizeOffset; ++i) {
const auto at = written_ + i;
if (at < ArchiveInputBuf::kLongSizeOffset) {
head_[at] = copy[i];
}
else if (signatureWritten() && (copy[i] == 4 || copy[i] == 8)) {
copy[i] = 8;
}
}
return put(copy.data(), n);
}

int sync() override {
return target_->pubsync();
}

private:
std::streamsize put(const char* s, std::streamsize n) {
const auto done = target_->sputn(s, n);
written_ += static_cast<size_t>(done > 0 ? done : 0);
return done;
}

bool signatureWritten() const {
static const char kSignature[] = "serialization::archive";
for (size_t i = 1; i < sizeof(uint64_t); ++i) {
if (head_[i] != 0) {
return false;
}
}
return static_cast<unsigned char>(head_[0]) == ArchiveInputBuf::kSignatureSize &&
std::memcmp(head_.data() + sizeof(uint64_t), kSignature, ArchiveInputBuf::kSignatureSize) == 0 &&
static_cast<unsigned char>(head_[ArchiveInputBuf::kLongSizeOffset - 1]) == sizeof(int32_t);
}

std::streambuf* target_;
size_t written_ = 0;
std::array<char, ArchiveInputBuf::kLongSizeOffset> head_{};
};

// usage: std::ofstream ofs(path, std::ios::binary); cs::ArchiveOutput out(ofs); boost::archive::binary_oarchive oa(out);
class ArchiveOutput : public std::ostream {
public:
explicit ArchiveOutput(std::ostream& target)
: std::ostream(nullptr)
, buf_(target.rdbuf()) {
rdbuf(&buf_);
if (!target) {
setstate(std::ios::badbit);
}
}

~ArchiveOutput() override {
flush();
}

private:
ArchiveOutputBuf buf_;
};

} // namespace cs

#endif // ARCHIVE_STREAM_HPP
2 changes: 1 addition & 1 deletion csnode/include/csnode/serializers_helper.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class SerializersHelper {
static const size_t kBufSize = 1 << 27;
std::vector<uint8_t> buf(::cscrypto::kHashSize + kBufSize, 0);
{
std::ifstream ifs(fileName);
std::ifstream ifs(fileName, std::ios::binary); // text mode on Windows would change the bytes hashed
while (!ifs.eof()) {
ifs.read(
reinterpret_cast<char*>(buf.data() + ::cscrypto::kHashSize),
Expand Down
Loading
Loading