Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions .github/workflows/known-hosts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ name: Known hosts
# while from an empty database, and the peers it meets are checked: a node syncing from block 0
# asks its neighbours for the oldest blocks first, so a peer that answers keeps the chain's history.
# The report is published in the rolling release "known-hosts" (known_hosts.txt, peers.md,
# peers.json), and the list is proposed as a pull request updating packaging/mainnet/known_hosts.txt:
# peers.json, and from the survey of `node --survey-snapshots`: snapshots.md, snapshot_sources.txt),
# and the list is proposed as a pull request updating packaging/mainnet/known_hosts.txt (and
# snapshot_sources.txt, when the survey found serving nodes):
# it reaches the packages only after someone reviews it, since peers that misbehave on purpose could
# otherwise steer new nodes towards themselves.
#
Expand Down Expand Up @@ -109,6 +111,15 @@ jobs:
pkill -f '[c]ontract-executor[.]jar' 2>/dev/null || true # the brackets keep this shell's own command line from matching
grep -ah 'Start contract executor\|Node initialized\|Last written sequence' log/log_*.txt | tail -3 || true

# the snapshots the neighbours serve: the same folder, no chain opened; its output is data of
# untrusted peers, checked by the publishing job
- name: Survey snapshots
run: |
cd csnode
./node --db-path db --config-file config.ini --survey-snapshots 600 </dev/null > survey.txt 2> survey.err || echo "the survey ended with an error"
pkill -f '[c]ontract-executor[.]jar' 2>/dev/null || true
grep -a '^SURVEY' survey.txt | cut -c1-300 | tail -20 || true

- name: Collect
run: |
python3 ci/known-hosts/collect.py csnode out
Expand All @@ -129,6 +140,9 @@ jobs:
out/
csnode/log/
csnode/known_hosts.txt
csnode/known_peers.json
csnode/survey.txt
csnode/survey.err
csnode/stdout.log
csnode/debug.log
retention-days: 14
Expand Down Expand Up @@ -167,6 +181,8 @@ jobs:
&& { echo "unexpected line in known_hosts.txt"; exit 1; } || true
listed=$(wc -l < run/out/known_hosts.txt)
[ "$listed" -ge 5 ] || { echo "only $listed peers verified, not publishing"; exit 1; }
# the survey: every field is checked against its exact shape by the script
python3 ci/known-hosts/snapshot_sources.py run/csnode/survey.txt run/csnode/known_peers.json run/out

- name: Report
run: |
Expand All @@ -183,15 +199,19 @@ jobs:
gh release edit known-hosts --repo "$GITHUB_REPOSITORY" --notes-file notes.md
fi
gh release upload known-hosts --repo "$GITHUB_REPOSITORY" --clobber \
run/out/known_hosts.txt run/out/peers.md run/out/peers.json
run/out/known_hosts.txt run/out/peers.md run/out/peers.json run/out/snapshot_sources.txt run/out/snapshots.md

# the packages ship packaging/mainnet/known_hosts.txt: propose the new list for review
- name: Pull request
env:
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
cp run/out/known_hosts.txt packaging/mainnet/known_hosts.txt
if git diff --quiet; then echo "known_hosts.txt unchanged"; exit 0; fi
# the survey's sources only when it found some: no serving node must not empty the shipped file
if grep -qv '^#' run/out/snapshot_sources.txt; then
cp run/out/snapshot_sources.txt packaging/mainnet/snapshot_sources.txt
fi
if git diff --quiet; then echo "known_hosts.txt and snapshot_sources.txt unchanged"; exit 0; fi
branch=auto/known-hosts
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,11 @@ jobs:
- name: snapshot-p2p-join
env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 TARGET_SEQ=3300
fixed: true
# `node --survey-snapshots` from the joined node's folder, at the end of the same run: the three
# serving nodes in one group, and the sources file the weekly job makes from it
- name: snapshot-p2p-survey
env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 JOIN_SURVEY=1 TARGET_SEQ=3300
fixed: true
- name: snapshot-p2p-join-full
env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,rocksdb SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_FULL=1 JOIN_MIN_SOURCES=3 TARGET_SEQ=3300
fixed: true
Expand Down
19 changes: 16 additions & 3 deletions ci/known-hosts/run-weekly.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
# The pruned package of the latest community release joins mainnet for MINUTES minutes from an
# empty database with a throwaway key, in a CPU- and memory-limited container. collect.py lists the
# peers that served the oldest blocks; the report goes to the rolling release "known-hosts" and the
# list is proposed as a pull request updating packaging/mainnet/known_hosts.txt, for review.
# list is proposed as a pull request updating packaging/mainnet/known_hosts.txt, for review. Then
# `node --survey-snapshots` asks the neighbours for the snapshots they serve: snapshots.md and
# snapshot_sources.txt (the serving nodes that answered) go to the same release and pull request.
#
# Usage: run-weekly.sh
# Env: WORK (default ~/Projects/credits/known-hosts-run), MINUTES (25), REPO (EnzinoBB/node),
# CPUS (1.0), MEMORY (3g), PUBLISH (1)
# CPUS (1.0), MEMORY (3g), PUBLISH (1), SURVEY_SECONDS (600)
# Needs: docker, gh (authenticated), python3 with base58 and pynacl.
set -euo pipefail

Expand Down Expand Up @@ -67,6 +69,13 @@ grep -ah 'Last written sequence' csnode/log/log_*.txt | tail -1 || true

python3 "$root/ci/known-hosts/collect.py" csnode out
cat out/peers.md

# the snapshots the neighbours serve: the same folder, no chain opened (its key, its routing table)
docker run --rm --name "$container-survey" --cpus "$CPUS" --memory "$MEMORY" --cpu-shares 256 --user "$(id -u):$(id -g)" \
-v "$WORK/run/csnode:/node" -w /node ubuntu:24.04 \
./node --db-path db --config-file config.ini --survey-snapshots "${SURVEY_SECONDS:-600}" > survey.txt 2> survey.err || echo "the survey ended with an error"
python3 "$root/ci/known-hosts/snapshot_sources.py" survey.txt csnode/known_peers.json out
head -20 out/snapshots.md
listed=$(wc -l < out/known_hosts.txt)
# the database and the package are not needed any more; keep the logs of the last run
rm -rf csnode/db csnode/caches csnode/ojdkbuild
Expand All @@ -87,14 +96,18 @@ if ! gh release view known-hosts --repo "$REPO" >/dev/null 2>&1; then
else
gh release edit known-hosts --repo "$REPO" --notes-file notes.md
fi
gh release upload known-hosts --repo "$REPO" --clobber out/known_hosts.txt out/peers.md out/peers.json
gh release upload known-hosts --repo "$REPO" --clobber out/known_hosts.txt out/peers.md out/peers.json out/snapshot_sources.txt out/snapshots.md

# propose the list for review: a branch from the default branch with only this file changed
default=$(gh repo view "$REPO" --json defaultBranchRef --jq .defaultBranchRef.name)
branch=auto/known-hosts
clone="$WORK/run/repo"
git clone -q --depth 1 --branch "$default" "https://github.com/$REPO.git" "$clone"
cp out/known_hosts.txt "$clone/packaging/mainnet/known_hosts.txt"
# the survey's sources only when it found some: no serving node must not empty the shipped file
if grep -qv '^#' out/snapshot_sources.txt; then
cp out/snapshot_sources.txt "$clone/packaging/mainnet/snapshot_sources.txt"
fi
cd "$clone"
if git diff --quiet; then
echo "known_hosts.txt unchanged"
Expand Down
214 changes: 214 additions & 0 deletions ci/known-hosts/snapshot_sources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
#!/usr/bin/env python3
"""The snapshot sources of the weekly known-hosts job (docs/specs/2026-10-25-snapshot-p2p-design.md, 10).

The job runs `node --survey-snapshots 600` after collect.py in the same node folder. The node prints
one line per snapshot the neighbours declared and one per serving peer that gave no manifest:

SURVEY group seq=<n> head=<hex> state=<hex> settings=<hex> sources=<key>@<address>@<created>,...
SURVEY silent <key>@<address>
SURVEY end groups=<n> serving=<n> silent=<n>

From that output and the node's known_peers.json this script writes

snapshot_sources.txt "ip:port key" lines, the peers that answered a manifest that verified, at most
--max-per-subnet per /24 (/48 for IPv6) so that one party cannot fill the
file; the strongest snapshot's sources first. Shipped in the packages: it
only says which peers to bring into the neighbour table first, the node
finds every serving peer anyway.
snapshots.md what was seen: the snapshots with their sources, the serving peers that gave
no manifest, and the count of independent serving addresses that the
`min_sources` decision depends on.

Usage: snapshot_sources.py SURVEY_OUTPUT KNOWN_PEERS_JSON OUT_DIR [--max-per-subnet N]
snapshot_sources.py --self-test
"""
import argparse
import datetime
import ipaddress
import json
import os
import re
import sys
import tempfile

SNAPSHOT_PROTOCOL_V1 = 1 # net/hostdata.hpp: the capability byte of the peers that serve snapshots
# the output comes from a node that talked to untrusted peers: every field is checked against its
# exact shape before it reaches a file or a pull request
HEX = r"[0-9a-fA-F]{64}"
GROUP = re.compile(rf"^SURVEY group seq=(\d{{1,12}}) head=({HEX}) state=({HEX}) settings=({HEX}) sources=(\S*)$")
CREATED = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")
SILENT = re.compile(r"^SURVEY silent (\S+)$")
END = re.compile(r"^SURVEY end groups=(\d+) serving=(\d+) silent=(\d+)$")
KEY = re.compile(r"^[1-9A-HJ-NP-Za-km-z]{32,64}$")


def subnet(address):
"""The /24 (IPv4) or /48 (IPv6) of an address, None when it is not an IP."""
try:
ip = ipaddress.ip_address(address.strip("[]"))
except ValueError:
return None
return str(ipaddress.ip_network(f"{ip}/{24 if ip.version == 4 else 48}", strict=False))


def parse_survey(text):
"""-> (groups, silent, summary). Lines that are not survey lines (log output) are ignored."""
groups, silent, summary = [], [], None
for line in text.splitlines():
line = line.strip()
if m := GROUP.match(line):
sources = []
for item in filter(None, m.group(5).split(",")):
parts = item.split("@")
if len(parts) == 3 and KEY.match(parts[0]) and subnet(parts[1]) and CREATED.match(parts[2]):
sources.append({"key": parts[0], "address": parts[1], "created": parts[2]})
groups.append({"seq": int(m.group(1)), "head": m.group(2), "state": m.group(3),
"settings": m.group(4), "sources": sources})
elif m := SILENT.match(line):
parts = m.group(1).split("@")
if len(parts) == 2 and KEY.match(parts[0]) and subnet(parts[1]):
silent.append({"key": parts[0], "address": parts[1]})
elif m := END.match(line):
summary = {"groups": int(m.group(1)), "serving": int(m.group(2)), "silent": int(m.group(3))}
groups.sort(key=lambda g: (-len(g["sources"]), -g["seq"]))
return groups, silent, summary


def load_peers(path):
"""known_peers.json -> {key: peer}; a missing or damaged file is no peers."""
try:
peers = json.load(open(path))
except (OSError, ValueError):
return {}
return {p["id"]: p for p in peers if isinstance(p, dict) and isinstance(p.get("id"), str)}


def sources_lines(groups, peers, max_per_subnet):
"""The "ip:port key" lines: strongest group first, one line per key, a few per /24."""
lines, seen, per_subnet = [], set(), {}
for g in groups:
for s in g["sources"]:
peer = peers.get(s["key"])
net = subnet(s["address"])
if s["key"] in seen or peer is None or net is None or not isinstance(peer.get("port"), int):
continue
if per_subnet.get(net, 0) >= max_per_subnet:
continue
host = s["address"]
lines.append(f"{'[' + host + ']' if ':' in host else host}:{peer['port']} {s['key']}")
seen.add(s["key"])
per_subnet[net] = per_subnet.get(net, 0) + 1
return lines


def report(groups, silent, summary, peers, today):
serving_peers = [p for p in peers.values() if int(p.get("capabilities", 0)) & SNAPSHOT_PROTOCOL_V1]
serving_subnets = {subnet(p.get("ip", "")) for p in serving_peers} - {None}
out = [f"# Snapshots served on mainnet, {today}", "",
"A node asked its neighbours for snapshot manifests for several minutes (`node --survey-snapshots`).", "",
f"- peers known to the node: {len(peers)}",
f"- peers announcing that they serve snapshots: {len(serving_peers)}, in {len(serving_subnets)} different /24 (/48) networks",
f"- snapshots declared: {len(groups)}; serving peers that gave no manifest: {len(silent)}"]
if summary is None:
out += ["", "_The survey output had no summary line: the node may have stopped early._"]
out += ["", "`min_sources` (the sources that must agree) counts independent networks: "
f"**{len(serving_subnets)}** serving network(s) are known now.", ""]
if groups:
out += ["| block | state digest | head | created | sources | independent |", "|---|---|---|---|---|---|"]
for g in groups:
nets = {subnet(s["address"]) for s in g["sources"]} - {None}
created = max((s["created"] for s in g["sources"]), default="")
out.append(f"| {g['seq']} | `{g['state'][:16]}…` | `{g['head'][:16]}…` | {created} | {len(g['sources'])} | {len(nets)} |")
out.append("")
for g in groups:
out += [f"### Block {g['seq']}", "",
f"state digest `{g['state']}`, head `{g['head']}`, settings `{g['settings']}`", ""]
out += [f"- `{s['key']}` at {s['address']} (created {s['created']})" for s in g["sources"]]
out.append("")
else:
out += ["No snapshot was declared.", ""]
if silent:
out += ["### Serving peers without a manifest", ""]
out += [f"- `{s['key']}` at {s['address']}" for s in silent]
out.append("")
return "\n".join(out)


def run(survey_path, peers_path, out_dir, max_per_subnet):
groups, silent, summary = parse_survey(open(survey_path, errors="replace").read())
peers = load_peers(peers_path)
os.makedirs(out_dir, exist_ok=True)
lines = sources_lines(groups, peers, max_per_subnet)
with open(os.path.join(out_dir, "snapshot_sources.txt"), "w") as f:
f.write("# serving nodes that answered a snapshot manifest, from the weekly survey\n")
f.write("".join(line + "\n" for line in lines))
today = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
with open(os.path.join(out_dir, "snapshots.md"), "w") as f:
f.write(report(groups, silent, summary, peers, today) + "\n")
print(f"{len(groups)} snapshots, {len(silent)} silent serving peers, {len(lines)} sources listed")
return 0


def self_test():
k1, k2, k3, k4, k5 = ("1" * 40, "2" * 40, "3" * 40, "4" * 40, "5" * 40)
h = ["aa" * 32, "bb" * 32, "cc" * 32, "dd" * 32, "ee" * 32, "ff" * 32]
survey = "\n".join([
"[2026-10-27 10:00:00.1] [info] some log line",
f"SURVEY group seq=186600000 head={h[0]} state={h[1]} settings={h[2]} sources={k1}@10.0.1.5@2026-10-27T00:00:00Z,"
f"{k2}@10.0.1.6@2026-10-27T00:00:00Z,{k3}@10.0.1.7@2026-10-27T00:00:00Z,"
f"{k4}@2001:db8:1::1@2026-10-27T00:00:00Z",
f"SURVEY group seq=186500000 head={h[3]} state={h[4]} settings={h[5]} sources={k5}@10.0.9.9@2026-10-26T00:00:00Z",
f"SURVEY silent {k5}@10.0.9.10",
"SURVEY group seq=bad",
"SURVEY end groups=2 serving=6 silent=1", ""])
groups, silent, summary = parse_survey(survey)
assert [g["seq"] for g in groups] == [186600000, 186500000], groups
assert len(groups[0]["sources"]) == 4 and silent == [{"key": k5, "address": "10.0.9.10"}], (groups, silent)
assert summary == {"groups": 2, "serving": 6, "silent": 1}, summary
peers = {k: {"id": k, "ip": ip, "port": 9100 + i, "capabilities": 1} for i, (k, ip) in enumerate(
[(k1, "10.0.1.5"), (k2, "10.0.1.6"), (k3, "10.0.1.7"), (k4, "2001:db8:1::1"), (k5, "10.0.9.9")])}
peers["6" * 40] = {"id": "6" * 40, "ip": "10.0.7.7", "port": 1, "capabilities": 0}
lines = sources_lines(groups, peers, 2)
# two per /24: k3 is the third of 10.0.1.0/24; the IPv6 source has its own /48
assert lines == [f"10.0.1.5:9100 {k1}", f"10.0.1.6:9101 {k2}", f"[2001:db8:1::1]:9103 {k4}", f"10.0.9.9:9104 {k5}"], lines
assert sources_lines(groups, {}, 2) == [], "no port without known_peers.json"
# what is not of the exact shape does not parse: keys, addresses, hashes, dates, markup
g, s, _ = parse_survey(f"SURVEY group seq=1 head={h[0]} state={h[1]} settings={h[2]} sources=nokey@1.2.3.4@2026-10-27T00:00:00Z,"
f"{k1}@not-an-ip@2026-10-27T00:00:00Z,{k1}@10.0.0.1@<script>,{k1}@10.0.0.1@2026-10-27T00:00:00Z@x,@@\n"
"SURVEY silent bad\nSURVEY silent " + k1 + "@<b>\n"
"SURVEY group seq=1 head=a state=b settings=c sources=" + k1 + "@10.0.0.1@2026-10-27T00:00:00Z\n"
"SURVEY group seq=99999999999999 head=" + h[0] + " state=" + h[1] + " settings=" + h[2] + " sources=")
assert len(g) == 1 and g[0]["sources"] == [] and s == [], (g, s)
with tempfile.TemporaryDirectory() as tmp:
sp, pp, out = (os.path.join(tmp, n) for n in ("survey.txt", "known_peers.json", "out"))
open(sp, "w").write(survey)
json.dump(list(peers.values()), open(pp, "w"))
assert run(sp, pp, out, 3) == 0
text = open(os.path.join(out, "snapshots.md")).read()
assert "186600000" in text and "min_sources" in text and "Serving peers without a manifest" in text, text
assert open(os.path.join(out, "snapshot_sources.txt")).read().count("\n") == 6 # 1 comment + 5 sources
# no survey summary, no peers file
open(sp, "w").write("")
assert run(sp, os.path.join(tmp, "missing.json"), out, 3) == 0
assert "No snapshot was declared." in open(os.path.join(out, "snapshots.md")).read()
print("self-test ok")
return 0


def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("survey", nargs="?")
parser.add_argument("known_peers", nargs="?")
parser.add_argument("out_dir", nargs="?")
parser.add_argument("--max-per-subnet", type=int, default=3)
parser.add_argument("--self-test", action="store_true")
args = parser.parse_args()
if args.self_test:
return self_test()
if not (args.survey and args.known_peers and args.out_dir):
parser.error("SURVEY_OUTPUT KNOWN_PEERS_JSON OUT_DIR are required")
return run(args.survey, args.known_peers, args.out_dir, args.max_per_subnet)


if __name__ == "__main__":
sys.exit(main())
Loading
Loading