From 06f5a3adf3c30ea1fa1d75ed56988f8b7fb86582 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Tue, 27 Oct 2026 09:40:56 +0100 Subject: [PATCH 1/2] snapshot: node --survey-snapshots, and the weekly job's snapshots.md and snapshot_sources.txt The node asks its neighbours for snapshot manifests for N seconds without opening its chain, prints the snapshots they declared (one line per group with its sources) and the serving peers that gave no manifest, and exits 0. The weekly known-hosts job runs it after the collection and ci/known-hosts/snapshot_sources.py (strict parsing: the output comes from a node that talked to untrusted peers) writes snapshots.md and snapshot_sources.txt, published in the known-hosts release and proposed with the known hosts; the packages ship snapshot_sources.txt when the repository has it. - SnapshotFetcher::surveyText, Node::startSurvey/finishSurvey (the bootstrap's discovery and manifest phases with the fetcher in dry-run), the --survey-snapshots option - robustness row snapshot-p2p-survey (JOIN_SURVEY=1): at the end of a p2p join the joiner is stopped and surveyed; the three serving nodes must be in one group and in the sources file Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/known-hosts.yml | 26 ++- .github/workflows/robustness.yml | 5 + ci/known-hosts/run-weekly.sh | 19 +- ci/known-hosts/snapshot_sources.py | 214 +++++++++++++++++++++ ci/testnet/run.sh | 32 +++ client/src/main.cpp | 10 + csnode/include/csnode/node.hpp | 13 ++ csnode/include/csnode/snapshot_fetcher.hpp | 7 + csnode/src/node.cpp | 50 ++++- csnode/src/snapshot_fetcher.cpp | 62 ++++++ docs/snapshots.md | 21 ++ packaging/docker/entrypoint.sh | 2 + packaging/make-package.sh | 2 + tests/csnode/snapshot_fetcher_tests.cpp | 36 ++++ 14 files changed, 488 insertions(+), 11 deletions(-) create mode 100755 ci/known-hosts/snapshot_sources.py diff --git a/.github/workflows/known-hosts.yml b/.github/workflows/known-hosts.yml index cbb7d85d6..8beb8b86b 100644 --- a/.github/workflows/known-hosts.yml +++ b/.github/workflows/known-hosts.yml @@ -4,7 +4,9 @@ name: Known hosts # while from an empty database, and the peers it meets are checked: a node syncing from block 0 # asks its neighbours for the oldest blocks first, so a peer that answers keeps the chain's history. # The report is published in the rolling release "known-hosts" (known_hosts.txt, peers.md, -# peers.json), and the list is proposed as a pull request updating packaging/mainnet/known_hosts.txt: +# peers.json, and from the survey of `node --survey-snapshots`: snapshots.md, snapshot_sources.txt), +# and the list is proposed as a pull request updating packaging/mainnet/known_hosts.txt (and +# snapshot_sources.txt, when the survey found serving nodes): # it reaches the packages only after someone reviews it, since peers that misbehave on purpose could # otherwise steer new nodes towards themselves. # @@ -109,6 +111,15 @@ jobs: pkill -f '[c]ontract-executor[.]jar' 2>/dev/null || true # the brackets keep this shell's own command line from matching grep -ah 'Start contract executor\|Node initialized\|Last written sequence' log/log_*.txt | tail -3 || true + # the snapshots the neighbours serve: the same folder, no chain opened; its output is data of + # untrusted peers, checked by the publishing job + - name: Survey snapshots + run: | + cd csnode + ./node --db-path db --config-file config.ini --survey-snapshots 600 survey.txt 2> survey.err || echo "the survey ended with an error" + pkill -f '[c]ontract-executor[.]jar' 2>/dev/null || true + grep -a '^SURVEY' survey.txt | cut -c1-300 | tail -20 || true + - name: Collect run: | python3 ci/known-hosts/collect.py csnode out @@ -129,6 +140,9 @@ jobs: out/ csnode/log/ csnode/known_hosts.txt + csnode/known_peers.json + csnode/survey.txt + csnode/survey.err csnode/stdout.log csnode/debug.log retention-days: 14 @@ -167,6 +181,8 @@ jobs: && { echo "unexpected line in known_hosts.txt"; exit 1; } || true listed=$(wc -l < run/out/known_hosts.txt) [ "$listed" -ge 5 ] || { echo "only $listed peers verified, not publishing"; exit 1; } + # the survey: every field is checked against its exact shape by the script + python3 ci/known-hosts/snapshot_sources.py run/csnode/survey.txt run/csnode/known_peers.json run/out - name: Report run: | @@ -183,7 +199,7 @@ jobs: gh release edit known-hosts --repo "$GITHUB_REPOSITORY" --notes-file notes.md fi gh release upload known-hosts --repo "$GITHUB_REPOSITORY" --clobber \ - run/out/known_hosts.txt run/out/peers.md run/out/peers.json + run/out/known_hosts.txt run/out/peers.md run/out/peers.json run/out/snapshot_sources.txt run/out/snapshots.md # the packages ship packaging/mainnet/known_hosts.txt: propose the new list for review - name: Pull request @@ -191,7 +207,11 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | cp run/out/known_hosts.txt packaging/mainnet/known_hosts.txt - if git diff --quiet; then echo "known_hosts.txt unchanged"; exit 0; fi + # the survey's sources only when it found some: no serving node must not empty the shipped file + if grep -qv '^#' run/out/snapshot_sources.txt; then + cp run/out/snapshot_sources.txt packaging/mainnet/snapshot_sources.txt + fi + if git diff --quiet; then echo "known_hosts.txt and snapshot_sources.txt unchanged"; exit 0; fi branch=auto/known-hosts git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 6865da0f4..2ae54d67b 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -327,6 +327,11 @@ jobs: - name: snapshot-p2p-join env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 TARGET_SEQ=3300 fixed: true + # `node --survey-snapshots` from the joined node's folder, at the end of the same run: the three + # serving nodes in one group, and the sources file the weekly job makes from it + - name: snapshot-p2p-survey + env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,lmdb:pruned SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_MIN_SOURCES=3 JOIN_SURVEY=1 TARGET_SEQ=3300 + fixed: true - name: snapshot-p2p-join-full env: NODES=8 TRUSTED_NODES=7 NODE_STORAGE=lmdb,berkeleydb,lmdb:pruned,berkeleydb,rocksdb,berkeleydb,lmdb,berkeleydb,rocksdb SNAPSHOT_SERVE_NODES=3,5,7 JOIN_FETCH=p2p JOIN_PRUNED_AT=2300 JOIN_FULL=1 JOIN_MIN_SOURCES=3 TARGET_SEQ=3300 fixed: true diff --git a/ci/known-hosts/run-weekly.sh b/ci/known-hosts/run-weekly.sh index 81065aecc..f2bd1ccc3 100755 --- a/ci/known-hosts/run-weekly.sh +++ b/ci/known-hosts/run-weekly.sh @@ -5,11 +5,13 @@ # The pruned package of the latest community release joins mainnet for MINUTES minutes from an # empty database with a throwaway key, in a CPU- and memory-limited container. collect.py lists the # peers that served the oldest blocks; the report goes to the rolling release "known-hosts" and the -# list is proposed as a pull request updating packaging/mainnet/known_hosts.txt, for review. +# list is proposed as a pull request updating packaging/mainnet/known_hosts.txt, for review. Then +# `node --survey-snapshots` asks the neighbours for the snapshots they serve: snapshots.md and +# snapshot_sources.txt (the serving nodes that answered) go to the same release and pull request. # # Usage: run-weekly.sh # Env: WORK (default ~/Projects/credits/known-hosts-run), MINUTES (25), REPO (EnzinoBB/node), -# CPUS (1.0), MEMORY (3g), PUBLISH (1) +# CPUS (1.0), MEMORY (3g), PUBLISH (1), SURVEY_SECONDS (600) # Needs: docker, gh (authenticated), python3 with base58 and pynacl. set -euo pipefail @@ -67,6 +69,13 @@ grep -ah 'Last written sequence' csnode/log/log_*.txt | tail -1 || true python3 "$root/ci/known-hosts/collect.py" csnode out cat out/peers.md + +# the snapshots the neighbours serve: the same folder, no chain opened (its key, its routing table) +docker run --rm --name "$container-survey" --cpus "$CPUS" --memory "$MEMORY" --cpu-shares 256 --user "$(id -u):$(id -g)" \ + -v "$WORK/run/csnode:/node" -w /node ubuntu:24.04 \ + ./node --db-path db --config-file config.ini --survey-snapshots "${SURVEY_SECONDS:-600}" > survey.txt 2> survey.err || echo "the survey ended with an error" +python3 "$root/ci/known-hosts/snapshot_sources.py" survey.txt csnode/known_peers.json out +head -20 out/snapshots.md listed=$(wc -l < out/known_hosts.txt) # the database and the package are not needed any more; keep the logs of the last run rm -rf csnode/db csnode/caches csnode/ojdkbuild @@ -87,7 +96,7 @@ if ! gh release view known-hosts --repo "$REPO" >/dev/null 2>&1; then else gh release edit known-hosts --repo "$REPO" --notes-file notes.md fi -gh release upload known-hosts --repo "$REPO" --clobber out/known_hosts.txt out/peers.md out/peers.json +gh release upload known-hosts --repo "$REPO" --clobber out/known_hosts.txt out/peers.md out/peers.json out/snapshot_sources.txt out/snapshots.md # propose the list for review: a branch from the default branch with only this file changed default=$(gh repo view "$REPO" --json defaultBranchRef --jq .defaultBranchRef.name) @@ -95,6 +104,10 @@ branch=auto/known-hosts clone="$WORK/run/repo" git clone -q --depth 1 --branch "$default" "https://github.com/$REPO.git" "$clone" cp out/known_hosts.txt "$clone/packaging/mainnet/known_hosts.txt" +# the survey's sources only when it found some: no serving node must not empty the shipped file +if grep -qv '^#' out/snapshot_sources.txt; then + cp out/snapshot_sources.txt "$clone/packaging/mainnet/snapshot_sources.txt" +fi cd "$clone" if git diff --quiet; then echo "known_hosts.txt unchanged" diff --git a/ci/known-hosts/snapshot_sources.py b/ci/known-hosts/snapshot_sources.py new file mode 100755 index 000000000..d9e84074d --- /dev/null +++ b/ci/known-hosts/snapshot_sources.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""The snapshot sources of the weekly known-hosts job (docs/specs/2026-10-25-snapshot-p2p-design.md, 10). + +The job runs `node --survey-snapshots 600` after collect.py in the same node folder. The node prints +one line per snapshot the neighbours declared and one per serving peer that gave no manifest: + + SURVEY group seq= head= state= settings= sources=@
@,... + SURVEY silent @
+ SURVEY end groups= serving= silent= + +From that output and the node's known_peers.json this script writes + + snapshot_sources.txt "ip:port key" lines, the peers that answered a manifest that verified, at most + --max-per-subnet per /24 (/48 for IPv6) so that one party cannot fill the + file; the strongest snapshot's sources first. Shipped in the packages: it + only says which peers to bring into the neighbour table first, the node + finds every serving peer anyway. + snapshots.md what was seen: the snapshots with their sources, the serving peers that gave + no manifest, and the count of independent serving addresses that the + `min_sources` decision depends on. + +Usage: snapshot_sources.py SURVEY_OUTPUT KNOWN_PEERS_JSON OUT_DIR [--max-per-subnet N] + snapshot_sources.py --self-test +""" +import argparse +import datetime +import ipaddress +import json +import os +import re +import sys +import tempfile + +SNAPSHOT_PROTOCOL_V1 = 1 # net/hostdata.hpp: the capability byte of the peers that serve snapshots +# the output comes from a node that talked to untrusted peers: every field is checked against its +# exact shape before it reaches a file or a pull request +HEX = r"[0-9a-fA-F]{64}" +GROUP = re.compile(rf"^SURVEY group seq=(\d{{1,12}}) head=({HEX}) state=({HEX}) settings=({HEX}) sources=(\S*)$") +CREATED = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") +SILENT = re.compile(r"^SURVEY silent (\S+)$") +END = re.compile(r"^SURVEY end groups=(\d+) serving=(\d+) silent=(\d+)$") +KEY = re.compile(r"^[1-9A-HJ-NP-Za-km-z]{32,64}$") + + +def subnet(address): + """The /24 (IPv4) or /48 (IPv6) of an address, None when it is not an IP.""" + try: + ip = ipaddress.ip_address(address.strip("[]")) + except ValueError: + return None + return str(ipaddress.ip_network(f"{ip}/{24 if ip.version == 4 else 48}", strict=False)) + + +def parse_survey(text): + """-> (groups, silent, summary). Lines that are not survey lines (log output) are ignored.""" + groups, silent, summary = [], [], None + for line in text.splitlines(): + line = line.strip() + if m := GROUP.match(line): + sources = [] + for item in filter(None, m.group(5).split(",")): + parts = item.split("@") + if len(parts) == 3 and KEY.match(parts[0]) and subnet(parts[1]) and CREATED.match(parts[2]): + sources.append({"key": parts[0], "address": parts[1], "created": parts[2]}) + groups.append({"seq": int(m.group(1)), "head": m.group(2), "state": m.group(3), + "settings": m.group(4), "sources": sources}) + elif m := SILENT.match(line): + parts = m.group(1).split("@") + if len(parts) == 2 and KEY.match(parts[0]) and subnet(parts[1]): + silent.append({"key": parts[0], "address": parts[1]}) + elif m := END.match(line): + summary = {"groups": int(m.group(1)), "serving": int(m.group(2)), "silent": int(m.group(3))} + groups.sort(key=lambda g: (-len(g["sources"]), -g["seq"])) + return groups, silent, summary + + +def load_peers(path): + """known_peers.json -> {key: peer}; a missing or damaged file is no peers.""" + try: + peers = json.load(open(path)) + except (OSError, ValueError): + return {} + return {p["id"]: p for p in peers if isinstance(p, dict) and isinstance(p.get("id"), str)} + + +def sources_lines(groups, peers, max_per_subnet): + """The "ip:port key" lines: strongest group first, one line per key, a few per /24.""" + lines, seen, per_subnet = [], set(), {} + for g in groups: + for s in g["sources"]: + peer = peers.get(s["key"]) + net = subnet(s["address"]) + if s["key"] in seen or peer is None or net is None or not isinstance(peer.get("port"), int): + continue + if per_subnet.get(net, 0) >= max_per_subnet: + continue + host = s["address"] + lines.append(f"{'[' + host + ']' if ':' in host else host}:{peer['port']} {s['key']}") + seen.add(s["key"]) + per_subnet[net] = per_subnet.get(net, 0) + 1 + return lines + + +def report(groups, silent, summary, peers, today): + serving_peers = [p for p in peers.values() if int(p.get("capabilities", 0)) & SNAPSHOT_PROTOCOL_V1] + serving_subnets = {subnet(p.get("ip", "")) for p in serving_peers} - {None} + out = [f"# Snapshots served on mainnet, {today}", "", + "A node asked its neighbours for snapshot manifests for several minutes (`node --survey-snapshots`).", "", + f"- peers known to the node: {len(peers)}", + f"- peers announcing that they serve snapshots: {len(serving_peers)}, in {len(serving_subnets)} different /24 (/48) networks", + f"- snapshots declared: {len(groups)}; serving peers that gave no manifest: {len(silent)}"] + if summary is None: + out += ["", "_The survey output had no summary line: the node may have stopped early._"] + out += ["", "`min_sources` (the sources that must agree) counts independent networks: " + f"**{len(serving_subnets)}** serving network(s) are known now.", ""] + if groups: + out += ["| block | state digest | head | created | sources | independent |", "|---|---|---|---|---|---|"] + for g in groups: + nets = {subnet(s["address"]) for s in g["sources"]} - {None} + created = max((s["created"] for s in g["sources"]), default="") + out.append(f"| {g['seq']} | `{g['state'][:16]}…` | `{g['head'][:16]}…` | {created} | {len(g['sources'])} | {len(nets)} |") + out.append("") + for g in groups: + out += [f"### Block {g['seq']}", "", + f"state digest `{g['state']}`, head `{g['head']}`, settings `{g['settings']}`", ""] + out += [f"- `{s['key']}` at {s['address']} (created {s['created']})" for s in g["sources"]] + out.append("") + else: + out += ["No snapshot was declared.", ""] + if silent: + out += ["### Serving peers without a manifest", ""] + out += [f"- `{s['key']}` at {s['address']}" for s in silent] + out.append("") + return "\n".join(out) + + +def run(survey_path, peers_path, out_dir, max_per_subnet): + groups, silent, summary = parse_survey(open(survey_path, errors="replace").read()) + peers = load_peers(peers_path) + os.makedirs(out_dir, exist_ok=True) + lines = sources_lines(groups, peers, max_per_subnet) + with open(os.path.join(out_dir, "snapshot_sources.txt"), "w") as f: + f.write("# serving nodes that answered a snapshot manifest, from the weekly survey\n") + f.write("".join(line + "\n" for line in lines)) + today = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") + with open(os.path.join(out_dir, "snapshots.md"), "w") as f: + f.write(report(groups, silent, summary, peers, today) + "\n") + print(f"{len(groups)} snapshots, {len(silent)} silent serving peers, {len(lines)} sources listed") + return 0 + + +def self_test(): + k1, k2, k3, k4, k5 = ("1" * 40, "2" * 40, "3" * 40, "4" * 40, "5" * 40) + h = ["aa" * 32, "bb" * 32, "cc" * 32, "dd" * 32, "ee" * 32, "ff" * 32] + survey = "\n".join([ + "[2026-10-27 10:00:00.1] [info] some log line", + f"SURVEY group seq=186600000 head={h[0]} state={h[1]} settings={h[2]} sources={k1}@10.0.1.5@2026-10-27T00:00:00Z," + f"{k2}@10.0.1.6@2026-10-27T00:00:00Z,{k3}@10.0.1.7@2026-10-27T00:00:00Z," + f"{k4}@2001:db8:1::1@2026-10-27T00:00:00Z", + f"SURVEY group seq=186500000 head={h[3]} state={h[4]} settings={h[5]} sources={k5}@10.0.9.9@2026-10-26T00:00:00Z", + f"SURVEY silent {k5}@10.0.9.10", + "SURVEY group seq=bad", + "SURVEY end groups=2 serving=6 silent=1", ""]) + groups, silent, summary = parse_survey(survey) + assert [g["seq"] for g in groups] == [186600000, 186500000], groups + assert len(groups[0]["sources"]) == 4 and silent == [{"key": k5, "address": "10.0.9.10"}], (groups, silent) + assert summary == {"groups": 2, "serving": 6, "silent": 1}, summary + peers = {k: {"id": k, "ip": ip, "port": 9100 + i, "capabilities": 1} for i, (k, ip) in enumerate( + [(k1, "10.0.1.5"), (k2, "10.0.1.6"), (k3, "10.0.1.7"), (k4, "2001:db8:1::1"), (k5, "10.0.9.9")])} + peers["6" * 40] = {"id": "6" * 40, "ip": "10.0.7.7", "port": 1, "capabilities": 0} + lines = sources_lines(groups, peers, 2) + # two per /24: k3 is the third of 10.0.1.0/24; the IPv6 source has its own /48 + assert lines == [f"10.0.1.5:9100 {k1}", f"10.0.1.6:9101 {k2}", f"[2001:db8:1::1]:9103 {k4}", f"10.0.9.9:9104 {k5}"], lines + assert sources_lines(groups, {}, 2) == [], "no port without known_peers.json" + # what is not of the exact shape does not parse: keys, addresses, hashes, dates, markup + g, s, _ = parse_survey(f"SURVEY group seq=1 head={h[0]} state={h[1]} settings={h[2]} sources=nokey@1.2.3.4@2026-10-27T00:00:00Z," + f"{k1}@not-an-ip@2026-10-27T00:00:00Z,{k1}@10.0.0.1@