Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ name: Robustness
# - a pruned node killed while it prunes;
# - every block database, full and pruned, in one network, also with crashes and a power cut;
# - packet loss, corruption, delay, reordering and a partition on some nodes, slow nodes, random
# datagrams, and all of them at once;
# datagrams, packet headers announcing huge sizes, and all of them at once;
# - smart contracts across crashes, a power cut, packet loss, a pruned node killed and a node joining,
# and a database conversion.
# All scenarios run funded transfers and the usual restart test.
Expand Down Expand Up @@ -140,14 +140,27 @@ jobs:
git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch"
git -C node-alt diff --stat
fi
# the submodules are copied from this run's checkout, which must point to the same commits
# the submodules are copied from this run's checkout, which must point to the same commits,
# except p2p: this repository runs its fork (EnzinoBB/p2p, which holds the upstream history),
# and the node under comparison gets the files of the p2p commit it points to (same submodules)
for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \
third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do
if [ "$(git ls-tree HEAD -- "$m")" != "$(git -C node-alt ls-tree HEAD -- "$m")" ]; then
alt=$(git -C node-alt ls-tree HEAD -- "$m" | awk '{print $3}')
if [ "$(git ls-tree HEAD -- "$m" | awk '{print $3}')" != "$alt" ] && [ "$m" != p2p ]; then
echo "$NODE_ALT_REF points $m to another commit than this run"; exit 1
fi
rm -rf "node-alt/$m"
cp -a "$m" "node-alt/$m"
if [ "$m" = p2p ] && [ "$(git -C p2p rev-parse HEAD)" != "$alt" ]; then
git -C p2p cat-file -e "$alt^{commit}" 2>/dev/null || git -C p2p fetch -q origin "$alt"
for s in $(git -C p2p ls-tree -r HEAD | awk '$2 == "commit" {print $4}'); do
[ "$(git -C p2p ls-tree HEAD -- "$s")" = "$(git -C p2p ls-tree "$alt" -- "$s")" ] \
|| { echo "p2p $alt points $s to another commit than this run"; exit 1; }
done
git -C p2p ls-files -s | awk '$1 != "160000" { sub(/^[^\t]*\t/, ""); print }' | (cd node-alt/p2p && xargs rm -f)
git -C p2p archive "$alt" | tar -x -C node-alt/p2p
echo "$NODE_ALT_REF: p2p at $alt (this run: $(git -C p2p rev-parse --short HEAD))"
fi
done
cmake -S node-alt -B build-alt -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
Expand Down Expand Up @@ -273,6 +286,12 @@ jobs:
# random datagrams to every node's p2p port
- name: garbage-datagrams
env: GARBAGE_AT=150 GARBAGE_RATE=300
# p2p packet headers above the limits of the p2p fork (64 bytes before registration, 256 MiB
# after) and just below them, to every node of this build (the odd ones with node_alt_ref):
# each closes the oversized ones at once, holds almost no memory for the others, keeps running
# and stays in consensus
- name: net-huge-frame
env: HUGE_FRAME_AT=150
# all at once, on a mixed-backend network
- name: chaos-mixed
env: DB_BACKEND=mixed NET_FAULT=corrupt NET_FAULT_PCT=3 NET_FAULT_NODES=4,5 SLOW_NODES=6 GARBAGE_AT=200
Expand Down Expand Up @@ -339,11 +358,9 @@ jobs:
- name: snapshot-p2p-fallback
env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb SNAPSHOT_SERVE_NODES=3 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_FULL=1 JOIN_MIN_SOURCES=2 JOIN_WAIT=60 JOIN_FALLBACK=sync JOIN_EXPECT=fallback TARGET_SEQ=3300
fixed: true
# two serving nodes, a third one (n7) starts serving later: the joiner waits (fetch_fallback=wait).
# Manual until plan P9: peers keep the capability bit they first saw for n7 (the p2p routing
# table is not refreshed), so the joiner finds n7 only through snapshot_sources.txt (-wait-file)
# two serving nodes, a third one (n7) starts serving later: the joiner waits (fetch_fallback=wait)
# and finds n7 by the capability bit its peers refresh from n7's datagrams (p2p fork, plan P9)
- name: snapshot-p2p-wait
manual: true
env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 SNAPSHOT_SERVE_NODES=3,5 SNAPSHOT_LATE_SERVE_NODES=7 JOIN_MIN_SOURCES=3 JOIN_WAIT=60 JOIN_FALLBACK=wait TARGET_SEQ=3300
fixed: true
# the same with the serving nodes listed in the joiner's snapshot_sources.txt
Expand Down
17 changes: 15 additions & 2 deletions .github/workflows/scale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,27 @@ jobs:
git worktree add -q node-alt FETCH_HEAD
git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch"
git -C node-alt diff --stat
# the submodules are copied from this run's checkout, which must point to the same commits
# the submodules are copied from this run's checkout, which must point to the same commits,
# except p2p: this repository runs its fork (EnzinoBB/p2p, which holds the upstream history),
# and the node under comparison gets the files of the p2p commit it points to (same submodules)
for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \
third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do
if [ "$(git ls-tree HEAD -- "$m")" != "$(git -C node-alt ls-tree HEAD -- "$m")" ]; then
alt=$(git -C node-alt ls-tree HEAD -- "$m" | awk '{print $3}')
if [ "$(git ls-tree HEAD -- "$m" | awk '{print $3}')" != "$alt" ] && [ "$m" != p2p ]; then
echo "the official node points $m to another commit than this run"; exit 1
fi
rm -rf "node-alt/$m"
cp -a "$m" "node-alt/$m"
if [ "$m" = p2p ] && [ "$(git -C p2p rev-parse HEAD)" != "$alt" ]; then
git -C p2p cat-file -e "$alt^{commit}" 2>/dev/null || git -C p2p fetch -q origin "$alt"
for s in $(git -C p2p ls-tree -r HEAD | awk '$2 == "commit" {print $4}'); do
[ "$(git -C p2p ls-tree HEAD -- "$s")" = "$(git -C p2p ls-tree "$alt" -- "$s")" ] \
|| { echo "p2p $alt points $s to another commit than this run"; exit 1; }
done
git -C p2p ls-files -s | awk '$1 != "160000" { sub(/^[^\t]*\t/, ""); print }' | (cd node-alt/p2p && xargs rm -f)
git -C p2p archive "$alt" | tar -x -C node-alt/p2p
echo "the official node: p2p at $alt (this run: $(git -C p2p rev-parse --short HEAD))"
fi
done
cmake -S node-alt -B build-alt -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
Expand Down
17 changes: 15 additions & 2 deletions .github/workflows/testnet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -184,14 +184,27 @@ jobs:
git -C node-alt apply --whitespace=nowarn "$PWD/ci/official/testnet-hooks.patch"
git -C node-alt diff --stat
fi
# the submodules are copied from this run's checkout, which must point to the same commits
# the submodules are copied from this run's checkout, which must point to the same commits,
# except p2p: this repository runs its fork (EnzinoBB/p2p, which holds the upstream history),
# and the node under comparison gets the files of the p2p commit it points to (same submodules)
for m in cscrypto p2p third-party/berkeleydb third-party/thrift third-party/thrift-interface-definitions \
third-party/googletest third-party/lmdb third-party/lmdbxx third-party/nameof third-party/rocksdb; do
if [ "$(git ls-tree HEAD -- "$m")" != "$(git -C node-alt ls-tree HEAD -- "$m")" ]; then
alt=$(git -C node-alt ls-tree HEAD -- "$m" | awk '{print $3}')
if [ "$(git ls-tree HEAD -- "$m" | awk '{print $3}')" != "$alt" ] && [ "$m" != p2p ]; then
echo "$NODE_ALT_REF points $m to another commit than this run"; exit 1
fi
rm -rf "node-alt/$m"
cp -a "$m" "node-alt/$m"
if [ "$m" = p2p ] && [ "$(git -C p2p rev-parse HEAD)" != "$alt" ]; then
git -C p2p cat-file -e "$alt^{commit}" 2>/dev/null || git -C p2p fetch -q origin "$alt"
for s in $(git -C p2p ls-tree -r HEAD | awk '$2 == "commit" {print $4}'); do
[ "$(git -C p2p ls-tree HEAD -- "$s")" = "$(git -C p2p ls-tree "$alt" -- "$s")" ] \
|| { echo "p2p $alt points $s to another commit than this run"; exit 1; }
done
git -C p2p ls-files -s | awk '$1 != "160000" { sub(/^[^\t]*\t/, ""); print }' | (cd node-alt/p2p && xargs rm -f)
git -C p2p archive "$alt" | tar -x -C node-alt/p2p
echo "$NODE_ALT_REF: p2p at $alt (this run: $(git -C p2p rev-parse --short HEAD))"
fi
done
cmake -S node-alt -B build-alt -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
Expand Down
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
url = https://github.com/Neargye/nameof.git
[submodule "p2p"]
path = p2p
url = https://github.com/SychevS/p2p.git
url = https://github.com/EnzinoBB/p2p.git
[submodule "third-party/rocksdb"]
path = third-party/rocksdb
url = https://github.com/facebook/rocksdb.git
153 changes: 153 additions & 0 deletions ci/testnet/huge_frame.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""Robustness test client (scenario net-huge-frame): opens connections to a node's p2p TCP port and
sends on each one a packet header announcing a large data size, then checks how the node treats them
and that its memory stays flat.

The p2p packet header is 77 bytes, little-endian: type (1 byte), data_size (8 bytes), sender id
(32 bytes), receiver id (32 bytes), reserved (4 bytes). The first packet of a connection must be a
registration (its data: address type, an IPv4 address and a port, 7 bytes).

Limits of the p2p fork: a registration may announce at most 64 bytes, any later packet at most
256 MiB; above that the node closes the connection at once. Below the limit it reads the data in
chunks of 1 MiB, so the memory it holds follows the bytes actually received, not the size announced.

Usage: huge_frame.py HOST PORT [options]
--size N announced data size, repeatable (default 2^40)
--registered send a valid registration first, then the header (a direct packet)
--expect close the node must close each connection within --timeout seconds (default)
--expect hold the node must keep each connection open for --timeout seconds (it waits for the
data), then close it on its idle timeout (10 s) within --idle seconds
--connections N connections per size, all held open together (default 4)
--pid PID the node's pid: its resident memory (Linux /proc) is sampled during the test and
may not grow by more than --max-growth MiB (default 64)
Exit status: 0 when the node behaved as expected (and its memory stayed flat), 1 otherwise.
"""
import argparse
import os
import select
import socket
import struct
import sys
import time

DIRECT = 0
REGISTRATION = 2


def header(packet_type, data_size, receiver=None):
return struct.pack("<BQ32s32sI", packet_type, data_size, os.urandom(32), receiver or os.urandom(32), 0)


def registration():
data = struct.pack("<B4sH", 0, bytes([127, 0, 0, 1]), 1) # AddrType::v4, 127.0.0.1, port 1
return header(REGISTRATION, len(data)) + data


def rss_kib(pid):
try:
with open(f"/proc/{pid}/status") as f:
for line in f:
if line.startswith("VmRSS:"):
return int(line.split()[1])
except OSError:
pass
return None


class Peak:
def __init__(self, pid):
self.pid = pid
self.value = rss_kib(pid) if pid else None

def sample(self):
if self.pid:
r = rss_kib(self.pid)
if r is not None and (self.value is None or r > self.value):
self.value = r


def wait_closed(socks, seconds, peak):
"""Reads from socks until each is closed by the node or `seconds` pass; returns the closed count."""
closed = 0
deadline = time.monotonic() + seconds
while socks and time.monotonic() < deadline:
readable, _, _ = select.select(socks, [], [], 0.1)
for s in readable:
try:
done = s.recv(65536) == b""
except (ConnectionResetError, BrokenPipeError):
done = True
except BlockingIOError:
done = False
if done:
closed += 1
socks.remove(s)
s.close()
peak.sample()
return closed


def run(a, size, peak):
socks, errors = [], 0
for _ in range(a.connections):
try:
s = socket.create_connection((a.host, a.port), timeout=5)
payload = header(REGISTRATION, size)
if a.registered:
payload = registration() + header(DIRECT, size)
s.sendall(payload)
s.setblocking(False)
socks.append(s)
except OSError as e:
print(f"huge-frame: connect/send to {a.host}:{a.port}: {e}")
errors += 1
what = f"{len(socks)} connection(s) announcing {size} bytes{' after a registration' if a.registered else ''}"
ok = errors == 0
if a.expect == "close":
closed = wait_closed(socks, a.timeout, peak)
print(f"huge-frame: {a.host}:{a.port}: {what}: {closed} closed by the node within {a.timeout:g} s")
ok = ok and closed == a.connections
else:
early = wait_closed(socks, a.timeout, peak)
late = wait_closed(socks, a.idle, peak)
print(f"huge-frame: {a.host}:{a.port}: {what}: {early} closed within {a.timeout:g} s (expected 0, "
f"the node waits for the data), {late} closed on the idle timeout within {a.idle:g} s more")
ok = ok and early == 0 and late == a.connections
for s in socks:
s.close()
return ok


def main():
ap = argparse.ArgumentParser()
ap.add_argument("host")
ap.add_argument("port", type=int)
ap.add_argument("--size", type=int, action="append")
ap.add_argument("--registered", action="store_true")
ap.add_argument("--expect", choices=("close", "hold"), default="close")
ap.add_argument("--connections", type=int, default=4)
ap.add_argument("--timeout", type=float, default=5.0)
ap.add_argument("--idle", type=float, default=15.0)
ap.add_argument("--pid", type=int)
ap.add_argument("--max-growth", type=int, default=64, help="MiB")
a = ap.parse_args()

peak = Peak(a.pid)
before = peak.value
ok = True
for size in a.size or [1 << 40]:
ok = run(a, size, peak) and ok
if a.pid:
if before is None or peak.value is None:
print(f"huge-frame: no memory reading for pid {a.pid}")
ok = False
else:
growth = (peak.value - before) / 1024
print(f"huge-frame: node memory {before / 1024:.0f} MiB before, peak {peak.value / 1024:.0f} MiB "
f"during the test (+{growth:.0f} MiB, limit {a.max_growth} MiB)")
ok = ok and growth <= a.max_growth
return 0 if ok else 1


if __name__ == "__main__":
sys.exit(main())
Loading
Loading