diff --git a/.github/actions/pin-socket-hosts/action.yml b/.github/actions/pin-socket-hosts/action.yml new file mode 100644 index 000000000..c7807042e --- /dev/null +++ b/.github/actions/pin-socket-hosts/action.yml @@ -0,0 +1,42 @@ +name: Pin Socket patch hosts +description: >- + On macOS runners, resolve the production patch hosts once (system resolver, + then DNS-over-HTTPS by IP literal), TLS-verify every address for its host, + and pin them in /etc/hosts for the rest of the job +inputs: + hosts: + description: Space-separated hostnames to pin + default: patch.socket.dev patches-api.socket.dev +runs: + using: composite + steps: + # GitHub's hosted macOS runners intermittently answer patch.socket.dev + # with EAI_NONAME ("[Errno 8] nodename nor servname provided", bun's + # `FailedToOpenSocket`) for minutes at a time — at job start or mid-job — + # while the service is up: the ubuntu / windows legs of the same run pass + # and the same macOS cells pass before and after the window. A pre-flight + # wait cannot cover a mid-job window and the failing processes are the + # real package managers, not the CLI, so the job takes the runner's + # resolver out of the path instead. Every request still goes to the + # production service over TLS verified for the hostname. + # scripts/pin-socket-hosts.py documents the resolution and verification. + - name: Pin hosts + if: runner.os == 'macOS' + shell: bash + env: + PIN_HOSTS: ${{ inputs.hosts }} + run: | + set -euo pipefail + # shellcheck disable=SC2086 # PIN_HOSTS is a space-separated list + lines=$(python3 "$GITHUB_WORKSPACE/scripts/pin-socket-hosts.py" $PIN_HOSTS) + printf '%s\n' "$lines" + printf '\n# pinned by .github/actions/pin-socket-hosts\n%s\n' "$lines" | sudo tee -a /etc/hosts >/dev/null + sudo dscacheutil -flushcache + sudo killall -HUP mDNSResponder || true + for host in $PIN_HOSTS; do + got=$(python3 -c 'import socket, sys; print(" ".join(sorted({i[4][0] for i in socket.getaddrinfo(sys.argv[1], 443)})))' "$host" || true) + echo "$host now resolves to: ${got:-nothing}" + if [ -z "$got" ] || ! grep -qE "^(${got// /|}) $host\$" <<<"$lines"; then + echo "::warning::$host does not resolve to its pinned address after pinning (got: ${got:-nothing})" + fi + done diff --git a/.github/workflows/bun-compatibility.yml b/.github/workflows/bun-compatibility.yml index 3f6b325f3..9894fe3b4 100644 --- a/.github/workflows/bun-compatibility.yml +++ b/.github/workflows/bun-compatibility.yml @@ -17,6 +17,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/bun-compatibility.yml' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' @@ -57,6 +59,8 @@ on: branches: [main] paths: - '.github/workflows/bun-compatibility.yml' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' - 'scripts/bun-historical-shas.json' @@ -187,6 +191,11 @@ jobs: with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts + - name: Download Bun ${{ matrix.bun }} id: bun # Pre-populate the exact directory layout the script's install_tool() diff --git a/.github/workflows/poetry-compatibility.yml b/.github/workflows/poetry-compatibility.yml index c4b07934d..585e83786 100644 --- a/.github/workflows/poetry-compatibility.yml +++ b/.github/workflows/poetry-compatibility.yml @@ -15,6 +15,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/poetry-compatibility.yml' - 'scripts/backtest-poetry.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' @@ -29,6 +31,8 @@ on: branches: [main] paths: - 'scripts/backtest-poetry.py' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' - 'crates/socket-patch-core/src/patch/redirect/**' - 'crates/socket-patch-core/src/vendor/pypi*.rs' @@ -91,6 +95,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts # uv bootstraps every pinned Poetry release (and its interpreter) # itself; pinning uv keeps the bootstrap reproducible. - run: python -m pip install uv==0.11.19 diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 738a2f548..a30976d9c 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -21,6 +21,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -60,6 +62,8 @@ on: branches: [main] paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -407,6 +411,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts - name: Backtest against production # Every hosted cell probes the artifact first; it records # blocked-by-server-encoding only when that probe saw a non-identity diff --git a/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs b/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs index 55a4708ac..3807f9e8f 100644 --- a/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs +++ b/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs @@ -29,6 +29,24 @@ fn hosted_leg(name: &'static str) -> Option { Leg::start(SUITE, name) } +/// v5 restores upstream pins without a saved lock fragment. The earliest +/// vlt releases record npmjs URLs even with the harness registry configured; +/// restore may omit that redundant slot or point it at the harness registry. +/// All other bytes, including bystanders and line endings, must still match. +fn assert_restored_lock(fx: &Fixture, before: &[u8]) { + let mut expected = String::from_utf8(before.to_vec()).unwrap(); + let mut actual = String::from_utf8(lock_bytes(&fx.proj)).unwrap(); + if fx.leg.version() <= VltVersion::zero(11) { + for target in &fx.svc.targets { + let bare = target.name.rsplit('/').next().unwrap(); + let path = Registry::tarball_path(&target.name, bare, &target.version); + expected = expected.replace(&format!(",\"https://registry.npmjs.org{path}\""), ""); + actual = actual.replace(&format!(",\"{}{path}\"", fx.reg.server.uri()), ""); + } + } + assert_eq!(actual, expected, "rollback restores the upstream lock"); +} + // ── drivers and fresh checkouts ─────────────────────────────────────────── /// `scan --mode hosted --vex`: the lock pins the artifact (one preflight @@ -167,7 +185,7 @@ async fn vlt_pinned_matrix_hosted_tamper_cold_eintegrity() { // ── rollback, rerun, heal ───────────────────────────────────────────────── -/// Rollback restores the lock byte-for-byte, heals the patched store copy +/// Rollback restores the upstream lock, heals the patched store copy /// (and nothing else), and the next `vlt install` is pristine. #[tokio::test(flavor = "multi_thread")] #[ignore = "real vlt: SOCKET_PATCH_VLT_E2E_JS"] @@ -184,11 +202,7 @@ async fn vlt_pinned_matrix_hosted_rollback_byte_exact() { let out = fx.rollback(&[]); assert_eq!(out.code, 0, "{out}"); let doc = out.json(); - assert_eq!( - String::from_utf8_lossy(&lock_bytes(&fx.proj)), - String::from_utf8_lossy(&fx.lock_before), - "rollback restores vlt-lock.json byte-for-byte" - ); + assert_restored_lock(&fx, &fx.lock_before); assert!( fx.ledger().is_none(), "no hosted ledger is ever written (v5)" @@ -762,11 +776,10 @@ async fn vlt_pinned_matrix_hosted_crlf_lock() { assert_eq!(state(&co, fx.t()), State::Patched); let out = fx.rollback(&[]); assert_eq!(out.code, 0, "{out}"); - assert_eq!( - lock_bytes(&fx.proj), - crlf, - "rollback restores the CRLF lock" - ); + assert_restored_lock(&fx, &crlf); + let co = fx.checkout("restored-crlf"); + fx.vlt_ok_profile(&co, &fx.leg.locked_install_args(), "restored-crlf"); + assert_eq!(state(&co, fx.t()), State::Pristine); fx.leg.ran(); } @@ -1185,11 +1198,7 @@ async fn vlt_pinned_matrix_hosted_idempotence() { assert!(fx.ledger().is_none()); let out = fx.rollback(&[]); assert_eq!(out.code, 0, "{out}"); - assert_eq!( - String::from_utf8_lossy(&lock_bytes(&fx.proj)), - String::from_utf8_lossy(&fx.lock_before), - "rollback restores the registry lock byte-for-byte: {out}" - ); + assert_restored_lock(&fx, &fx.lock_before); assert!(fx.ledger().is_none()); let files = package_files(&fx.proj); let out = fx.rollback(&[]); @@ -1199,6 +1208,9 @@ async fn vlt_pinned_matrix_hosted_idempotence() { "a second rollback finds no state: {out}" ); assert_eq!(package_files(&fx.proj), files, "and writes nothing"); + let co = fx.checkout("restored-idempotence"); + fx.vlt_ok_profile(&co, &fx.leg.locked_install_args(), "restored-idempotence"); + assert_eq!(state(&co, fx.t()), State::Pristine); fx.leg.ran(); } diff --git a/crates/socket-patch-cli/tests/mode_migration_npm.rs b/crates/socket-patch-cli/tests/mode_migration_npm.rs index f734f77af..7e7a3b718 100644 --- a/crates/socket-patch-cli/tests/mode_migration_npm.rs +++ b/crates/socket-patch-cli/tests/mode_migration_npm.rs @@ -388,11 +388,15 @@ async fn mount_hosted_mocks( /// Serve, from `server` (as `SOCKET_NPM_REGISTRY`), the npm registry version /// document the v5 upstream restore reads for DEP — mirrored from what the /// PRISTINE classic lock recorded (`resolved "#"`, -/// `integrity`). The restore of a hosted classic entry must reproduce the -/// registry entry yarn wrote from exactly that document; mirroring it keeps +/// `integrity`, or the SHA-1 fragment on pre-1.10 releases). The restore must +/// reproduce the registry entry yarn wrote from that document; mirroring it keeps /// the unwind hermetic (the binary's TLS stack need not reach the real -/// registry). Returns the registry base to hand the binary. -async fn mount_registry_from_classic_lock(server: &MockServer, lock: &str) -> String { +/// registry). Returns the registry base and expected upstream lock. Hosted +/// mode adds an integrity line even on pre-1.10 yarn, and v5 restores that +/// line's registry hash without a saved fragment to recover its absence. +async fn mount_registry_from_classic_lock(server: &MockServer, lock: &str) -> (String, String) { + use base64::Engine as _; + let block = lock .split("\n\n") .find(|b| { @@ -404,23 +408,43 @@ async fn mount_registry_from_classic_lock(server: &MockServer, lock: &str) -> St .lines() .find_map(|l| l.trim().strip_prefix(&format!("{name} "))) .map(|v| v.trim_matches('"').to_string()) - .unwrap_or_else(|| panic!("no `{name}` in {block}")) }; - let resolved = field("resolved"); + let resolved = field("resolved").unwrap_or_else(|| panic!("no `resolved` in {block}")); let (tarball, shasum) = resolved .split_once('#') .map(|(t, s)| (t.to_string(), Some(s.to_string()))) .unwrap_or((resolved.clone(), None)); + let integrity = field("integrity").unwrap_or_else(|| { + let sha1 = hex::decode( + shasum + .as_ref() + .expect("pre-1.10 yarn pins a SHA-1 fragment"), + ) + .expect("the resolved fragment is hex SHA-1"); + format!( + "sha1-{}", + base64::engine::general_purpose::STANDARD.encode(sha1) + ) + }); + let upstream_lock = if field("integrity").is_some() { + lock.to_string() + } else { + lock.replacen( + &format!(" resolved \"{resolved}\""), + &format!(" resolved \"{resolved}\"\n integrity {integrity}"), + 1, + ) + }; Mock::given(method("GET")) .and(path(format!("/registry/{DEP}/{DEP_VERSION}"))) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "name": DEP, "version": DEP_VERSION, - "dist": { "tarball": tarball, "integrity": field("integrity"), "shasum": shasum } + "dist": { "tarball": tarball, "integrity": integrity, "shasum": shasum } }))) .mount(server) .await; - format!("{}/registry", server.uri()) + (format!("{}/registry", server.uri()), upstream_lock) } fn run_hosted_scan(proj: &Path, server_uri: &str) -> (i32, String, String) { @@ -642,7 +666,7 @@ fn assert_pure_vendored_and_round_trip( "fresh vendored install must carry the PATCHED bytes ({tag})" ); - // (b) Round trip: `vendor --revert` restores the REGISTRY lock + // (b) Round trip: `vendor --revert` restores the expected REGISTRY lock // byte-identically (pre-fix it restored the hosted fragment, with no CLI // path back to registry state). let (code, stdout, stderr) = run_socket( @@ -659,8 +683,8 @@ fn assert_pure_vendored_and_round_trip( assert_eq!( std::fs::read(proj.join("yarn.lock")).unwrap(), lock_pristine, - "yarn.lock must be restored byte-identical to the pre-hosted \ - REGISTRY pristine ({tag}); got:\n{}", + "yarn.lock must be restored byte-identical to the expected \ + upstream REGISTRY lock ({tag}); got:\n{}", read(proj, "yarn.lock") ); assert_eq!( @@ -809,7 +833,7 @@ async fn classic_hosted_then_vendored_takeover_round_trips_to_registry() { // upstream restore re-resolves the registry entry (mirrored from the // pristine lock), and the mock origin is named hosted via // --patch-server-url. - let registry = + let (registry, lock_upstream) = mount_registry_from_classic_lock(&server, &String::from_utf8_lossy(&lock_pristine)).await; stage_patch(&proj, &fx.orig, &fx.patched); let (code, stdout, stderr) = run_socket_env( @@ -847,7 +871,7 @@ async fn classic_hosted_then_vendored_takeover_round_trips_to_registry() { "classic", false, &hosted_url, - &lock_pristine, + lock_upstream.as_bytes(), &pkg_json_pristine, &stdout, ); @@ -1055,10 +1079,10 @@ async fn classic_vendored_then_hosted_takeover_leaves_pure_hosted() { ) }); - // The originals chain across migrations: `rollback` restores the hosted - // pin's upstream registry entry, which is the pristine lock byte for - // byte (online: the entry is re-resolved from the registry document). - let registry = + // Rollback re-resolves the upstream registry entry. Hosted mode added an + // integrity line even on pre-1.10 yarn; v5 has no saved fragment to tell + // whether it was originally absent, so it restores the registry hash. + let (registry, lock_upstream) = mount_registry_from_classic_lock(&server, &String::from_utf8_lossy(&lock_pristine)).await; let (code, stdout, stderr) = run_socket_env( &proj, @@ -1076,8 +1100,27 @@ async fn classic_vendored_then_hosted_takeover_leaves_pure_hosted() { assert_eq!(code, 0, "rollback failed: {stdout}\n{stderr}"); assert_eq!( read(&proj, "yarn.lock"), - String::from_utf8_lossy(&lock_pristine), - "rollback lands on the pristine registry lock" + lock_upstream, + "rollback restores the registry lock, allowing the added upstream integrity" + ); + let fresh = fresh_checkout(&proj, fx.tmp.path(), "classic-rollback", false); + let fresh_cache = fx.tmp.path().join("fresh-cache-classic-rollback"); + let ci = corepack( + &fresh, + &yarn_classic_vex::yarn_classic(), + &["install", "--frozen-lockfile", "--no-progress"], + &[("YARN_CACHE_FOLDER", fresh_cache.to_str().unwrap())], + ); + assert!( + ci.status.success(), + "fresh-checkout rollback install must succeed.\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&ci.stdout), + String::from_utf8_lossy(&ci.stderr), + ); + assert_eq!( + std::fs::read(fresh.join("node_modules").join(DEP).join("index.js")).unwrap(), + fx.orig, + "rollback installs the pristine registry bytes" ); } diff --git a/docs/testing/bun-compatibility.md b/docs/testing/bun-compatibility.md index 90b30702e..d6782a2ef 100644 --- a/docs/testing/bun-compatibility.md +++ b/docs/testing/bun-compatibility.md @@ -345,6 +345,18 @@ matrix to six concurrent jobs, with three cells per job. Each cell has its own temporary directory so historical Bun processes cannot collide while extracting identically named packages. +On macOS the job first runs `.github/actions/pin-socket-hosts` +(`scripts/pin-socket-hosts.py`): the hosted macOS resolver intermittently +answers `patch.socket.dev` with EAI_NONAME for minutes at a time, at job start +or mid-job, while the service is up, which failed every hosted cell in the +window (`FailedToOpenSocket`, `[Errno 8] nodename nor servname provided`, or a +Bun 1.3.x workspace install that never exits). The action resolves the patch +hosts once (the system resolver, then DNS-over-HTTPS by IP literal), keeps only +addresses whose TLS handshake verifies the hostname, and pins them in +`/etc/hosts`, so cells still reach the production service over verified TLS +without depending on the runner's resolver. The vlt and Poetry workflows run +the same step. + **Pinned versions:** 0.8.1, 1.0.0, 1.0.36, 1.1.0, 1.1.38 (binary lock), 1.1.39 (first text lock, version 0), 1.1.43 (first `--lockfile-only`), 1.1.45 (last version-0 writer), 1.2.0, 1.2.23, 1.3.0 (version 1), 1.3.9 / 1.3.10 diff --git a/docs/testing/vlt-compatibility.md b/docs/testing/vlt-compatibility.md index b61b7aa20..4fd506694 100644 --- a/docs/testing/vlt-compatibility.md +++ b/docs/testing/vlt-compatibility.md @@ -75,7 +75,10 @@ asserted and each named test or row exists. collation golden, and the store linkers auto / hardlink / copy / unpack / a `/dev/shm` cache root; a cell `ci.yml`'s `e2e` rows run identically is left to them, see `scripts/ci-vlt-proof-suites.py`, except on dispatch); `native` (the backtest against production, artifacts - `vlt-results--` in depscan's capture `result.json` shape); + `vlt-results--` in depscan's capture `result.json` shape; on macOS + it first pins the TLS-verified patch hosts in `/etc/hosts` through + `.github/actions/pin-socket-hosts`, because the hosted macOS resolver + intermittently loses `patch.socket.dev` for minutes while the service is up); `lock-diff` (the same cell's `vlt-lock.json` must be byte-identical on Linux, macOS and Windows); `matrix-coverage` (every era × suite × OS). - **Nightly:** `canary` runs every capstone on `vlt@latest` on 3 OS (only the @@ -131,6 +134,13 @@ The capstones serve npmjs bytes from a local wiremock registry `R` and write | rc.30 … rc.32 | `{"config":{"registry": R}}` | yes | URL-segment DepIDs | | ≥ rc.33 | `{"config":{"registries":{"npm": R}}}` (+ `config.registry = R` for rc.33 … 1.0.4) | yes | | +The 0.0.0-1 and 0.0.0-11 writers can record an explicit npmjs tarball URL +despite the configured harness registry. v5 rollback reconstructs the upstream +pin without a saved lock fragment, so the rollback assertions allow that target +URL to be omitted or restored on the harness registry. They still compare every +other byte, including bystanders and CRLF line endings, and verify pristine +package contents after a real install from the restored lock. + `scripts/backtest-vlt.py`'s `write_vlt_json` follows the same table against public npm (a `registry` equal to vlt's npmjs default is left out: vlt strips it from the lock anyway). diff --git a/scripts/backtest-vlt.py b/scripts/backtest-vlt.py index ed8b54367..f9e565ae9 100644 --- a/scripts/backtest-vlt.py +++ b/scripts/backtest-vlt.py @@ -992,13 +992,20 @@ def __init__(self, ctx, version, mode, shape_name): self.record = ctx['record'] self.envelopes = [] self.fresh_patched = {} + self.cli_failures = [] # CLI ------------------------------------------------------------------- def cli(self, args, cwd=None): command = [self.ctx['cli'], *args, '--json', '--no-telemetry'] if self.ctx.get('patch_server_url'): command += ['--patch-server-url', self.ctx['patch_server_url']] - return run(command, cwd or self.project, self.ctx['cli_env'], self.log) + code, out, err = run(command, cwd or self.project, self.ctx['cli_env'], self.log) + if code: + # --json errors go to stdout, including on repeat/revert calls. + # Keep them on the result row so the transport retry sees them. + self.cli_failures.append(dict(command=str(args[0]), exitCode=code, + envelope=parse_envelope(out), stderr=tail(err, 3000))) + return code, out, err def patch_run(self, mode, cwd=None): cwd = cwd or self.project @@ -1195,6 +1202,8 @@ def blocked_refusal(self, row, envelopes, reference): return clean def finish(self, row, checks, started): + if self.cli_failures: + row['cliFailures'] = self.cli_failures row['failingChecks'] = [k for k, v in checks.items() if v is False] row['notEvaluated'] = [k for k, v in checks.items() if v is None] row.setdefault('codes', []) diff --git a/scripts/pin-socket-hosts.py b/scripts/pin-socket-hosts.py new file mode 100644 index 000000000..df36bb1ea --- /dev/null +++ b/scripts/pin-socket-hosts.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Pin the production Socket patch hosts in the hosts file of a CI runner. + +The compatibility workflows drive REAL package managers (bun, vlt, poetry, +...) against the production patch service. On GitHub's hosted macOS runners +the system resolver intermittently answers `patch.socket.dev` with +EAI_NONAME ("[Errno 8] nodename nor servname provided, or not known"; +bun: `FailedToOpenSocket`) for minutes at a time, starting at job start or +mid-job, while the service itself is up (the ubuntu and windows legs of the +same run pass, and the same macOS cells pass before and after the window). +The runner's resolver is not under test, so the workflow takes it out of the +path: this script resolves each host once, verifies every address, and +prints `hosts(5)` lines the workflow appends to /etc/hosts. + +Resolution tries the system resolver first, then DNS-over-HTTPS to IP-literal +endpoints (no DNS needed to reach them), retrying with backoff inside a +bounded window. An address is only pinned after a TLS handshake to it with +SNI = the host verifies the host's certificate, so a pinned address is one +that really serves that name; the package managers still verify TLS for the +hostname on every request. Both families are resolved; an IPv6 address is +pinned only when it verifies too, so a runner without an IPv6 route never gets +an unreachable entry. + +Exit status is non-zero, with nothing printed, when a host cannot be pinned +within the window: the job then fails at this step, naming the host, rather +than in a hundred cells downstream. +""" + +import argparse +import ipaddress +import json +import socket +import ssl +import sys +import time +import urllib.request + +DEFAULT_HOSTS = ['patch.socket.dev', 'patches-api.socket.dev'] +# DoH JSON endpoints reached by IP literal; both serve certificates with the +# IP in the subjectAltName, so they verify without any name resolution. +DOH_ENDPOINTS = [ + 'https://1.1.1.1/dns-query?name={host}&type={rrtype}', + 'https://8.8.8.8/resolve?name={host}&type={rrtype}', +] +RR_TYPES = {'A': 1, 'AAAA': 28} + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def system_resolve(host): + infos = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) + return [info[4][0] for info in infos] + + +def doh_resolve(host, template, timeout): + addresses = [] + for rrtype, code in RR_TYPES.items(): + request = urllib.request.Request(template.format(host=host, rrtype=rrtype), + headers={'accept': 'application/dns-json'}) + with urllib.request.urlopen(request, timeout=timeout) as response: + answer = json.loads(response.read()).get('Answer') or [] + # CNAME answers (type 5) precede the address records and are skipped. + addresses += [record['data'] for record in answer if record.get('type') == code] + return addresses + + +def verified(host, address, timeout): + """A TLS handshake to `address` with SNI `host` verifies `host`'s cert.""" + context = ssl.create_default_context() + try: + with socket.create_connection((address, 443), timeout=timeout) as raw: + with context.wrap_socket(raw, server_hostname=host): + return True + except (OSError, ssl.SSLError) as error: + log(f'{host}: {address} failed verification: {error}') + return False + + +def resolve(host, window, timeout): + """Verified addresses of `host` (IPv4 first), or [] once `window` seconds pass.""" + sources = [('system resolver', lambda: system_resolve(host))] + sources += [(template.split('/')[2], lambda t=template: doh_resolve(host, t, timeout)) + for template in DOH_ENDPOINTS] + deadline = time.monotonic() + window + attempt = 0 + fallback = [] + while True: + attempt += 1 + for name, source in sources: + try: + candidates = source() + except Exception as error: # noqa: BLE001 - every source is best effort + log(f'{host}: {name} attempt {attempt} failed: {error}') + continue + addresses = [] + for candidate in dict.fromkeys(candidates): + try: + ipaddress.ip_address(candidate) + except ValueError: + continue + if verified(host, candidate, timeout): + addresses.append(candidate) + addresses.sort(key=lambda a: ipaddress.ip_address(a).version) + # A source that only verified IPv6 is not enough on its own: try + # the next one for an IPv4 address before settling for it. + if any(ipaddress.ip_address(a).version == 4 for a in addresses): + log(f'{host}: pinned {" ".join(addresses)} (from {name}, attempt {attempt})') + return addresses + log(f'{host}: {name} attempt {attempt} gave no verified IPv4 address') + fallback = fallback or addresses + remaining = deadline - time.monotonic() + if remaining <= 0: + return fallback + time.sleep(min(5 * attempt, 30, remaining)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('hosts', nargs='*', default=DEFAULT_HOSTS) + parser.add_argument('--window', type=float, default=300, + help='seconds to keep retrying a host before failing (default 300)') + parser.add_argument('--timeout', type=float, default=10, + help='per-request timeout in seconds (default 10)') + args = parser.parse_args(argv) + lines = [] + for host in args.hosts: + addresses = resolve(host, args.window, args.timeout) + if not addresses: + log(f'::error::could not resolve and verify {host} within {args.window:.0f} s') + return 1 + lines += [f'{address} {host}' for address in addresses] + print('\n'.join(lines)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/tests/test_backtest_harnesses.py b/scripts/tests/test_backtest_harnesses.py index b5b7dc07a..5b393228d 100644 --- a/scripts/tests/test_backtest_harnesses.py +++ b/scripts/tests/test_backtest_harnesses.py @@ -608,6 +608,29 @@ def test_snapshot_never_follows_a_link(self): class VltRetryTests(unittest.TestCase): + def test_json_cli_failures_reach_the_retry_classifier(self): + for phase in ('initial', 'repeat', 'rollback'): + for status in (504, 404): + with self.subTest(phase=phase, status=status), tempfile.TemporaryDirectory() as temp: + cell = vlt.Cell({'out': Path(temp), 'record': {}, 'cli': 'socket-patch', + 'cli_env': {}}, '1.2.0', 'hosted', 'direct') + cell.project.mkdir(parents=True) + row = dict(cell=cell.name, expectedVerdict='patched', passed=False, checks={}) + envelope = {'status': 'error', + 'error': f'API request failed with status {status}: error code: {status}'} + with patch.object(vlt, 'run', return_value=(1, json.dumps(envelope), '')): + if phase == 'initial': + cell.patch_run('hosted') + elif phase == 'repeat': + cell.repeat(row, row['checks'], b'') + else: + cell.revert() + with patch('sys.stdout'): + cell.finish(row, row['checks'], vlt.time.time()) + captured = json.loads((cell.case / 'result.json').read_text()) + self.assertEqual(captured['cliFailures'][0]['envelope'], envelope) + self.assertEqual(vlt.transient(captured), status == 504) + def test_only_transport_failures_retry(self): self.assertFalse(vlt.transient({'matchesExpectation': True, 'serveProbe': {'curlExit': 7}})) self.assertTrue(vlt.transient({'serveProbe': {'curlExit': 7}})) diff --git a/scripts/tests/test_pin_socket_hosts.py b/scripts/tests/test_pin_socket_hosts.py new file mode 100644 index 000000000..56094000b --- /dev/null +++ b/scripts/tests/test_pin_socket_hosts.py @@ -0,0 +1,76 @@ +"""Hermetic coverage for scripts/pin-socket-hosts.py's resolution fallbacks.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import socket +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location( + 'pin_socket_hosts', Path(__file__).resolve().parents[1] / 'pin-socket-hosts.py') +pin = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pin) + +HOST = 'patch.socket.dev' +EAI_NONAME = socket.gaierror(8, 'nodename nor servname provided, or not known') + + +def run(fn): + with contextlib.redirect_stderr(io.StringIO()): + return fn() + + +class PinSocketHostsTests(unittest.TestCase): + def test_system_resolver_answer_is_pinned_when_it_verifies(self): + with patch.object(pin, 'system_resolve', return_value=['172.66.3.58', '172.66.3.58']), \ + patch.object(pin, 'doh_resolve') as doh, \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['172.66.3.58']) + doh.assert_not_called() + + def test_doh_takes_over_when_the_system_resolver_fails(self): + with patch.object(pin, 'system_resolve', side_effect=EAI_NONAME), \ + patch.object(pin, 'doh_resolve', return_value=['2606:4700:7::32d', '162.159.143.62']), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), + ['162.159.143.62', '2606:4700:7::32d']) + + def test_unverified_addresses_are_never_pinned(self): + with patch.object(pin, 'system_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'doh_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'verified', return_value=False): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), []) + + def test_ipv6_only_is_a_last_resort(self): + with patch.object(pin, 'system_resolve', return_value=['2606:4700:7::32d']), \ + patch.object(pin, 'doh_resolve', return_value=[]), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['2606:4700:7::32d']) + + def test_retries_until_the_resolver_recovers(self): + answers = [EAI_NONAME, ['172.66.3.58']] + with patch.object(pin, 'system_resolve', side_effect=answers), \ + patch.object(pin, 'doh_resolve', side_effect=OSError('no route')), \ + patch.object(pin, 'verified', return_value=True), \ + patch.object(pin.time, 'sleep') as sleep: + self.assertEqual(run(lambda: pin.resolve(HOST, 60, 1)), ['172.66.3.58']) + sleep.assert_called_once() + + def test_main_prints_hosts_lines_and_fails_closed(self): + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=['172.66.3.58']), \ + contextlib.redirect_stdout(out): + self.assertEqual(pin.main([HOST]), 0) + self.assertEqual(out.getvalue(), f'172.66.3.58 {HOST}\n') + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=[]), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(pin.main([HOST]), 1) + self.assertEqual(out.getvalue(), '') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/yarn-classic-vex-matrix.sh b/scripts/yarn-classic-vex-matrix.sh index 5b0ffc9d2..ea509ba05 100755 --- a/scripts/yarn-classic-vex-matrix.sh +++ b/scripts/yarn-classic-vex-matrix.sh @@ -41,11 +41,13 @@ for release in "${releases[@]}"; do for entry in "${suites[@]}"; do IFS=: read -r suite filter ignored <<<"$entry" echo "== yarn@$release $suite ${filter:-}" >&2 + start_line=$(wc -l < "$log") if ! (cd "$root" && SOCKET_PATCH_YARN_CLASSIC_E2E_VERSION="$release" \ SOCKET_PATCH_YARN_E2E_REQUIRED=1 \ cargo test -q -p socket-patch-cli --test "$suite" -- ${filter:+"$filter"} \ ${ignored:+"$ignored"} --nocapture --test-threads=1 >>"$log" 2>&1); then echo "FAIL yarn@$release $suite (log: $log)" >&2 + tail -n "+$((start_line + 1))" "$log" >&2 echo "VEXCELL leg=$suite yarn=$release mode=- cell=SUITE FAIL" >>"$log" status=1 fi