diff --git a/.github/workflows/known-hosts.yml b/.github/workflows/known-hosts.yml index cbb7d85d6..8beb8b86b 100644 --- a/.github/workflows/known-hosts.yml +++ b/.github/workflows/known-hosts.yml @@ -4,7 +4,9 @@ name: Known hosts # while from an empty database, and the peers it meets are checked: a node syncing from block 0 # asks its neighbours for the oldest blocks first, so a peer that answers keeps the chain's history. # The report is published in the rolling release "known-hosts" (known_hosts.txt, peers.md, -# peers.json), and the list is proposed as a pull request updating packaging/mainnet/known_hosts.txt: +# peers.json, and from the survey of `node --survey-snapshots`: snapshots.md, snapshot_sources.txt), +# and the list is proposed as a pull request updating packaging/mainnet/known_hosts.txt (and +# snapshot_sources.txt, when the survey found serving nodes): # it reaches the packages only after someone reviews it, since peers that misbehave on purpose could # otherwise steer new nodes towards themselves. # @@ -109,6 +111,15 @@ jobs: pkill -f '[c]ontract-executor[.]jar' 2>/dev/null || true # the brackets keep this shell's own command line from matching grep -ah 'Start contract executor\|Node initialized\|Last written sequence' log/log_*.txt | tail -3 || true + # the snapshots the neighbours serve: the same folder, no chain opened; its output is data of + # untrusted peers, checked by the publishing job + - name: Survey snapshots + run: | + cd csnode + ./node --db-path db --config-file config.ini --survey-snapshots 600 survey.txt 2> survey.err || echo "the survey ended with an error" + pkill -f '[c]ontract-executor[.]jar' 2>/dev/null || true + grep -a '^SURVEY' survey.txt | cut -c1-300 | tail -20 || true + - name: Collect run: | python3 ci/known-hosts/collect.py csnode out @@ -129,6 +140,9 @@ jobs: out/ csnode/log/ csnode/known_hosts.txt + csnode/known_peers.json + csnode/survey.txt + csnode/survey.err csnode/stdout.log csnode/debug.log retention-days: 14 @@ -167,6 +181,8 @@ jobs: && { echo "unexpected line in known_hosts.txt"; exit 1; } || true listed=$(wc -l < run/out/known_hosts.txt) [ "$listed" -ge 5 ] || { echo "only $listed peers verified, not publishing"; exit 1; } + # the survey: every field is checked against its exact shape by the script + python3 ci/known-hosts/snapshot_sources.py run/csnode/survey.txt run/csnode/known_peers.json run/out - name: Report run: | @@ -183,7 +199,7 @@ jobs: gh release edit known-hosts --repo "$GITHUB_REPOSITORY" --notes-file notes.md fi gh release upload known-hosts --repo "$GITHUB_REPOSITORY" --clobber \ - run/out/known_hosts.txt run/out/peers.md run/out/peers.json + run/out/known_hosts.txt run/out/peers.md run/out/peers.json run/out/snapshot_sources.txt run/out/snapshots.md # the packages ship packaging/mainnet/known_hosts.txt: propose the new list for review - name: Pull request @@ -191,7 +207,11 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | cp run/out/known_hosts.txt packaging/mainnet/known_hosts.txt - if git diff --quiet; then echo "known_hosts.txt unchanged"; exit 0; fi + # the survey's sources only when it found some: no serving node must not empty the shipped file + if grep -qv '^#' run/out/snapshot_sources.txt; then + cp run/out/snapshot_sources.txt packaging/mainnet/snapshot_sources.txt + fi + if git diff --quiet; then echo "known_hosts.txt and snapshot_sources.txt unchanged"; exit 0; fi branch=auto/known-hosts git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 6865da0f4..2ae54d67b 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -327,6 +327,11 @@ jobs: - name: snapshot-p2p-join env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 TARGET_SEQ=3300 fixed: true + # `node --survey-snapshots` from the joined node's folder, at the end of the same run: the three + # serving nodes in one group, and the sources file the weekly job makes from it + - name: snapshot-p2p-survey + env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 JOIN_SURVEY=1 TARGET_SEQ=3300 + fixed: true - name: snapshot-p2p-join-full env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,rocksdb SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_FULL=1 JOIN_MIN_SOURCES=3 TARGET_SEQ=3300 fixed: true diff --git a/ci/known-hosts/run-weekly.sh b/ci/known-hosts/run-weekly.sh index 81065aecc..f2bd1ccc3 100755 --- a/ci/known-hosts/run-weekly.sh +++ b/ci/known-hosts/run-weekly.sh @@ -5,11 +5,13 @@ # The pruned package of the latest community release joins mainnet for MINUTES minutes from an # empty database with a throwaway key, in a CPU- and memory-limited container. collect.py lists the # peers that served the oldest blocks; the report goes to the rolling release "known-hosts" and the -# list is proposed as a pull request updating packaging/mainnet/known_hosts.txt, for review. +# list is proposed as a pull request updating packaging/mainnet/known_hosts.txt, for review. Then +# `node --survey-snapshots` asks the neighbours for the snapshots they serve: snapshots.md and +# snapshot_sources.txt (the serving nodes that answered) go to the same release and pull request. # # Usage: run-weekly.sh # Env: WORK (default ~/Projects/credits/known-hosts-run), MINUTES (25), REPO (EnzinoBB/node), -# CPUS (1.0), MEMORY (3g), PUBLISH (1) +# CPUS (1.0), MEMORY (3g), PUBLISH (1), SURVEY_SECONDS (600) # Needs: docker, gh (authenticated), python3 with base58 and pynacl. set -euo pipefail @@ -67,6 +69,13 @@ grep -ah 'Last written sequence' csnode/log/log_*.txt | tail -1 || true python3 "$root/ci/known-hosts/collect.py" csnode out cat out/peers.md + +# the snapshots the neighbours serve: the same folder, no chain opened (its key, its routing table) +docker run --rm --name "$container-survey" --cpus "$CPUS" --memory "$MEMORY" --cpu-shares 256 --user "$(id -u):$(id -g)" \ + -v "$WORK/run/csnode:/node" -w /node ubuntu:24.04 \ + ./node --db-path db --config-file config.ini --survey-snapshots "${SURVEY_SECONDS:-600}" > survey.txt 2> survey.err || echo "the survey ended with an error" +python3 "$root/ci/known-hosts/snapshot_sources.py" survey.txt csnode/known_peers.json out +head -20 out/snapshots.md listed=$(wc -l < out/known_hosts.txt) # the database and the package are not needed any more; keep the logs of the last run rm -rf csnode/db csnode/caches csnode/ojdkbuild @@ -87,7 +96,7 @@ if ! gh release view known-hosts --repo "$REPO" >/dev/null 2>&1; then else gh release edit known-hosts --repo "$REPO" --notes-file notes.md fi -gh release upload known-hosts --repo "$REPO" --clobber out/known_hosts.txt out/peers.md out/peers.json +gh release upload known-hosts --repo "$REPO" --clobber out/known_hosts.txt out/peers.md out/peers.json out/snapshot_sources.txt out/snapshots.md # propose the list for review: a branch from the default branch with only this file changed default=$(gh repo view "$REPO" --json defaultBranchRef --jq .defaultBranchRef.name) @@ -95,6 +104,10 @@ branch=auto/known-hosts clone="$WORK/run/repo" git clone -q --depth 1 --branch "$default" "https://github.com/$REPO.git" "$clone" cp out/known_hosts.txt "$clone/packaging/mainnet/known_hosts.txt" +# the survey's sources only when it found some: no serving node must not empty the shipped file +if grep -qv '^#' out/snapshot_sources.txt; then + cp out/snapshot_sources.txt "$clone/packaging/mainnet/snapshot_sources.txt" +fi cd "$clone" if git diff --quiet; then echo "known_hosts.txt unchanged" diff --git a/ci/known-hosts/snapshot_sources.py b/ci/known-hosts/snapshot_sources.py new file mode 100755 index 000000000..d9e84074d --- /dev/null +++ b/ci/known-hosts/snapshot_sources.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""The snapshot sources of the weekly known-hosts job (docs/specs/2026-10-25-snapshot-p2p-design.md, 10). + +The job runs `node --survey-snapshots 600` after collect.py in the same node folder. The node prints +one line per snapshot the neighbours declared and one per serving peer that gave no manifest: + + SURVEY group seq= head= state= settings= sources=@
@,... + SURVEY silent @
+ SURVEY end groups= serving= silent= + +From that output and the node's known_peers.json this script writes + + snapshot_sources.txt "ip:port key" lines, the peers that answered a manifest that verified, at most + --max-per-subnet per /24 (/48 for IPv6) so that one party cannot fill the + file; the strongest snapshot's sources first. Shipped in the packages: it + only says which peers to bring into the neighbour table first, the node + finds every serving peer anyway. + snapshots.md what was seen: the snapshots with their sources, the serving peers that gave + no manifest, and the count of independent serving addresses that the + `min_sources` decision depends on. + +Usage: snapshot_sources.py SURVEY_OUTPUT KNOWN_PEERS_JSON OUT_DIR [--max-per-subnet N] + snapshot_sources.py --self-test +""" +import argparse +import datetime +import ipaddress +import json +import os +import re +import sys +import tempfile + +SNAPSHOT_PROTOCOL_V1 = 1 # net/hostdata.hpp: the capability byte of the peers that serve snapshots +# the output comes from a node that talked to untrusted peers: every field is checked against its +# exact shape before it reaches a file or a pull request +HEX = r"[0-9a-fA-F]{64}" +GROUP = re.compile(rf"^SURVEY group seq=(\d{{1,12}}) head=({HEX}) state=({HEX}) settings=({HEX}) sources=(\S*)$") +CREATED = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") +SILENT = re.compile(r"^SURVEY silent (\S+)$") +END = re.compile(r"^SURVEY end groups=(\d+) serving=(\d+) silent=(\d+)$") +KEY = re.compile(r"^[1-9A-HJ-NP-Za-km-z]{32,64}$") + + +def subnet(address): + """The /24 (IPv4) or /48 (IPv6) of an address, None when it is not an IP.""" + try: + ip = ipaddress.ip_address(address.strip("[]")) + except ValueError: + return None + return str(ipaddress.ip_network(f"{ip}/{24 if ip.version == 4 else 48}", strict=False)) + + +def parse_survey(text): + """-> (groups, silent, summary). Lines that are not survey lines (log output) are ignored.""" + groups, silent, summary = [], [], None + for line in text.splitlines(): + line = line.strip() + if m := GROUP.match(line): + sources = [] + for item in filter(None, m.group(5).split(",")): + parts = item.split("@") + if len(parts) == 3 and KEY.match(parts[0]) and subnet(parts[1]) and CREATED.match(parts[2]): + sources.append({"key": parts[0], "address": parts[1], "created": parts[2]}) + groups.append({"seq": int(m.group(1)), "head": m.group(2), "state": m.group(3), + "settings": m.group(4), "sources": sources}) + elif m := SILENT.match(line): + parts = m.group(1).split("@") + if len(parts) == 2 and KEY.match(parts[0]) and subnet(parts[1]): + silent.append({"key": parts[0], "address": parts[1]}) + elif m := END.match(line): + summary = {"groups": int(m.group(1)), "serving": int(m.group(2)), "silent": int(m.group(3))} + groups.sort(key=lambda g: (-len(g["sources"]), -g["seq"])) + return groups, silent, summary + + +def load_peers(path): + """known_peers.json -> {key: peer}; a missing or damaged file is no peers.""" + try: + peers = json.load(open(path)) + except (OSError, ValueError): + return {} + return {p["id"]: p for p in peers if isinstance(p, dict) and isinstance(p.get("id"), str)} + + +def sources_lines(groups, peers, max_per_subnet): + """The "ip:port key" lines: strongest group first, one line per key, a few per /24.""" + lines, seen, per_subnet = [], set(), {} + for g in groups: + for s in g["sources"]: + peer = peers.get(s["key"]) + net = subnet(s["address"]) + if s["key"] in seen or peer is None or net is None or not isinstance(peer.get("port"), int): + continue + if per_subnet.get(net, 0) >= max_per_subnet: + continue + host = s["address"] + lines.append(f"{'[' + host + ']' if ':' in host else host}:{peer['port']} {s['key']}") + seen.add(s["key"]) + per_subnet[net] = per_subnet.get(net, 0) + 1 + return lines + + +def report(groups, silent, summary, peers, today): + serving_peers = [p for p in peers.values() if int(p.get("capabilities", 0)) & SNAPSHOT_PROTOCOL_V1] + serving_subnets = {subnet(p.get("ip", "")) for p in serving_peers} - {None} + out = [f"# Snapshots served on mainnet, {today}", "", + "A node asked its neighbours for snapshot manifests for several minutes (`node --survey-snapshots`).", "", + f"- peers known to the node: {len(peers)}", + f"- peers announcing that they serve snapshots: {len(serving_peers)}, in {len(serving_subnets)} different /24 (/48) networks", + f"- snapshots declared: {len(groups)}; serving peers that gave no manifest: {len(silent)}"] + if summary is None: + out += ["", "_The survey output had no summary line: the node may have stopped early._"] + out += ["", "`min_sources` (the sources that must agree) counts independent networks: " + f"**{len(serving_subnets)}** serving network(s) are known now.", ""] + if groups: + out += ["| block | state digest | head | created | sources | independent |", "|---|---|---|---|---|---|"] + for g in groups: + nets = {subnet(s["address"]) for s in g["sources"]} - {None} + created = max((s["created"] for s in g["sources"]), default="") + out.append(f"| {g['seq']} | `{g['state'][:16]}…` | `{g['head'][:16]}…` | {created} | {len(g['sources'])} | {len(nets)} |") + out.append("") + for g in groups: + out += [f"### Block {g['seq']}", "", + f"state digest `{g['state']}`, head `{g['head']}`, settings `{g['settings']}`", ""] + out += [f"- `{s['key']}` at {s['address']} (created {s['created']})" for s in g["sources"]] + out.append("") + else: + out += ["No snapshot was declared.", ""] + if silent: + out += ["### Serving peers without a manifest", ""] + out += [f"- `{s['key']}` at {s['address']}" for s in silent] + out.append("") + return "\n".join(out) + + +def run(survey_path, peers_path, out_dir, max_per_subnet): + groups, silent, summary = parse_survey(open(survey_path, errors="replace").read()) + peers = load_peers(peers_path) + os.makedirs(out_dir, exist_ok=True) + lines = sources_lines(groups, peers, max_per_subnet) + with open(os.path.join(out_dir, "snapshot_sources.txt"), "w") as f: + f.write("# serving nodes that answered a snapshot manifest, from the weekly survey\n") + f.write("".join(line + "\n" for line in lines)) + today = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") + with open(os.path.join(out_dir, "snapshots.md"), "w") as f: + f.write(report(groups, silent, summary, peers, today) + "\n") + print(f"{len(groups)} snapshots, {len(silent)} silent serving peers, {len(lines)} sources listed") + return 0 + + +def self_test(): + k1, k2, k3, k4, k5 = ("1" * 40, "2" * 40, "3" * 40, "4" * 40, "5" * 40) + h = ["aa" * 32, "bb" * 32, "cc" * 32, "dd" * 32, "ee" * 32, "ff" * 32] + survey = "\n".join([ + "[2026-10-27 10:00:00.1] [info] some log line", + f"SURVEY group seq=186600000 head={h[0]} state={h[1]} settings={h[2]} sources={k1}@10.0.1.5@2026-10-27T00:00:00Z," + f"{k2}@10.0.1.6@2026-10-27T00:00:00Z,{k3}@10.0.1.7@2026-10-27T00:00:00Z," + f"{k4}@2001:db8:1::1@2026-10-27T00:00:00Z", + f"SURVEY group seq=186500000 head={h[3]} state={h[4]} settings={h[5]} sources={k5}@10.0.9.9@2026-10-26T00:00:00Z", + f"SURVEY silent {k5}@10.0.9.10", + "SURVEY group seq=bad", + "SURVEY end groups=2 serving=6 silent=1", ""]) + groups, silent, summary = parse_survey(survey) + assert [g["seq"] for g in groups] == [186600000, 186500000], groups + assert len(groups[0]["sources"]) == 4 and silent == [{"key": k5, "address": "10.0.9.10"}], (groups, silent) + assert summary == {"groups": 2, "serving": 6, "silent": 1}, summary + peers = {k: {"id": k, "ip": ip, "port": 9100 + i, "capabilities": 1} for i, (k, ip) in enumerate( + [(k1, "10.0.1.5"), (k2, "10.0.1.6"), (k3, "10.0.1.7"), (k4, "2001:db8:1::1"), (k5, "10.0.9.9")])} + peers["6" * 40] = {"id": "6" * 40, "ip": "10.0.7.7", "port": 1, "capabilities": 0} + lines = sources_lines(groups, peers, 2) + # two per /24: k3 is the third of 10.0.1.0/24; the IPv6 source has its own /48 + assert lines == [f"10.0.1.5:9100 {k1}", f"10.0.1.6:9101 {k2}", f"[2001:db8:1::1]:9103 {k4}", f"10.0.9.9:9104 {k5}"], lines + assert sources_lines(groups, {}, 2) == [], "no port without known_peers.json" + # what is not of the exact shape does not parse: keys, addresses, hashes, dates, markup + g, s, _ = parse_survey(f"SURVEY group seq=1 head={h[0]} state={h[1]} settings={h[2]} sources=nokey@1.2.3.4@2026-10-27T00:00:00Z," + f"{k1}@not-an-ip@2026-10-27T00:00:00Z,{k1}@10.0.0.1@