From 2091defa1d60a7421b5883b9b68b953a8f1bae9b Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:14:50 +0200 Subject: [PATCH] ws-head-latency: sub-ms histogram floor + keyed slots for base/solana (#1225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the leader-collapses-to-2.5ms artifact: histogram_quantile on a [0,5ms] first bucket interpolates every winner's p50 to ~2.5ms (chain-independent). Replace with a 0.5ms floor climbing to 10s so sub-ms wins actually resolve on Solana (~400ms slots) and Base (2s). Add keyed provider slots for Base (Alchemy/Infura/Chainstack/QuickNode) and Solana (Alchemy/Chainstack/Helius/QuickNode) so the cohort can extend beyond keyless once contributor keys are wired. Keyless tier is saturated on all three chains (audit failed: Ankr, Blast, LlamaRPC, BlockPI, 1RPC, NodeReal, Omniatech, base.org, extrnode, chainstack- public — none accepted a keyless WSS handshake as of 2026-07). Co-authored-by: Florent Tapponnier --- benchmarks/ws-head-latency-base.yml | 5 ++-- benchmarks/ws-head-latency-ethereum.yml | 7 +++-- benchmarks/ws-head-latency-solana.yml | 5 ++-- .../ws-head-latency/cmd/script/config.go | 28 +++++++++++++++---- .../ws-head-latency/cmd/script/metrics.go | 14 ++++++++-- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/benchmarks/ws-head-latency-base.yml b/benchmarks/ws-head-latency-base.yml index 44bc9fb9a..bf9f3643e 100644 --- a/benchmarks/ws-head-latency-base.yml +++ b/benchmarks/ws-head-latency-base.yml @@ -40,11 +40,12 @@ abstract: | benchmark; this page is the Base-scoped answer. methodology: - - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://base-rpc.publicnode.com), dRPC (wss://base.drpc.org). Tenderly does not expose a keyless Base WebSocket path and is scoped out until a public gateway is announced." + - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://base-rpc.publicnode.com), dRPC (wss://base.drpc.org). Tenderly does not expose a keyless Base WebSocket path and is scoped out until a public gateway is announced. Keyed providers (Alchemy, Infura, Chainstack, QuickNode) join the cohort when contributor keys are wired via WS_URL__BASE env vars." + - "Cohort saturation: Blast, LlamaRPC, BlockPI, 1RPC, mainnet.base.org and Ankr all reject a keyless WebSocket handshake at the time of ship (verified 2026-07), which is why the keyless tier stops at two." - "Chain scope: every query on this page is pinned to chain=\"base\". Base block cadence is 2 seconds, so a 24h window carries roughly 43000 samples per provider (versus ~7200 on Ethereum), well above the 7000 sample floor used by the parent bench." - "Race scoring: identical to the parent [ws-head-latency](https://openchainbench.com/benchmarks/ws-head-latency-ethereum) benchmark. First provider to deliver block N sets T0. Each provider's sample for block N is arrival(N) minus T0 in milliseconds. Cohort closes 5 seconds after the first arrival; late arrivals are scored as missed blocks via ws_block_gap_total rather than as huge latency samples." - "Relative, not absolute: subtraction of two arrivals over the same path removes vantage-point network latency. Consequence: the fastest provider reads near zero by construction. The honest readings are win rate, trailers' lag distribution, and gap counts." - - "Aggregation: per-provider lag histogram (5ms to 10s exponential buckets), p50/p90/p99 via histogram_quantile over the 24h rate. Coverage score = ws_head_blocks_seen_total divided by cohort maximum, clamped to 1. A provider that pushes fast but skips blocks loses on this column, not on the latency percentiles." + - "Aggregation: per-provider lag histogram with a sub-millisecond floor (0.5 ms bucket up through 10 s tail), p50/p90/p99 via histogram_quantile over the 24h rate. Coverage score = ws_head_blocks_seen_total divided by cohort maximum, clamped to 1. A provider that pushes fast but skips blocks loses on this column, not on the latency percentiles." - "Connection discipline: one persistent WebSocket per (provider, chain), client ping every 30s, read deadline 60s, 90s head watchdog forcing reconnect when a socket keeps the heartbeat alive but silently drops the subscription. Exponential backoff 2s to 60s on reconnect." - "Vantage point: a single eu-west host. Multi-region deployment is a follow-up on the parent bench roadmap; the same limitation and the reasoning behind it are documented there." diff --git a/benchmarks/ws-head-latency-ethereum.yml b/benchmarks/ws-head-latency-ethereum.yml index 2d7c9eb71..68f8f4a4d 100644 --- a/benchmarks/ws-head-latency-ethereum.yml +++ b/benchmarks/ws-head-latency-ethereum.yml @@ -46,14 +46,15 @@ abstract: | `slotSubscribe` slot notifications. methodology: - - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://ethereum-rpc.publicnode.com), dRPC (wss://eth.drpc.org), Tenderly (wss://mainnet.gateway.tenderly.co). Keyed providers (Alchemy, Infura, Chainstack) join the cohort when contributor keys are wired; the harness reads them from env vars and skips them cleanly when unset." + - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://ethereum-rpc.publicnode.com), dRPC (wss://eth.drpc.org), Tenderly (wss://mainnet.gateway.tenderly.co). Keyed providers (Alchemy, Infura, Chainstack, QuickNode) join the cohort when contributor keys are wired via WS_URL__ETHEREUM env vars; the harness skips them cleanly when unset." + - "Cohort saturation: Ankr, Blast, LlamaRPC, BlockPI, 1RPC, NodeReal and Omniatech all reject a keyless WebSocket handshake at the time of ship (verified 2026-07), which is why the keyless tier stops at three." - "Chains: Ethereum (headline, all three providers compete), Base via newHeads and Solana via slotSubscribe as secondary cohorts. Headline queries on this page are pinned to chain=ethereum so the ranking never mixes block cadences." - "Race scoring: the first provider to deliver block N sets T0. Each provider's sample for block N is arrival(N) minus T0 in milliseconds. The cohort closes 5 seconds after the first arrival; a provider arriving later than that is scored as a missed block (ws_block_gap_total), not as a huge latency sample." - "Relative, not absolute: a single vantage point cannot separate its own network path from provider pipeline time, so we subtract the two arrivals over the same path instead. Consequence: the fastest provider reads ~0 by construction, and the honest readings are win rate, the trailers' lag distribution, and gap counts, not the leader's absolute number." - "Blocks where only one provider delivered within the window emit no lag samples and no win: a one-horse race carries no relative information. Gap counters still increment for live providers that missed the block." - "Connection discipline: one persistent WebSocket per (provider, chain), client ping every 30s, read deadline 60s, plus a 90s head watchdog that forces a reconnect when a provider keeps the socket heartbeat alive but silently drops the subscription (a failure mode we have observed in production on other benches). Reconnects use exponential backoff 2s to 60s and are counted in ws_reconnects_total." - "Coverage score: ws_head_blocks_seen_total per provider divided by the cohort maximum over the same 24h window, clamped to 1. A provider that pushes fast but skips blocks loses on this column rather than hiding in the latency percentiles." - - "Aggregation: per-provider lag histogram (5ms to 10s exponential buckets), p50/p90/p99 via histogram_quantile over the 24h rate. Duplicate announcements of the same block on one connection are dropped; per-provider liveness is exported as ws_health (1 = connected and delivered a message in the last 120s)." + - "Aggregation: per-provider lag histogram with a sub-millisecond floor (0.5 ms bucket up through 10 s tail), p50/p90/p99 via histogram_quantile over the 24h rate. Duplicate announcements of the same block on one connection are dropped; per-provider liveness is exported as ws_health (1 = connected and delivered a message in the last 120s)." - "Cheat vector disclosure: a provider could theoretically pre-announce heads it has not fully validated to win the race. We keyed the cohort on block number as pushed; hash-level cross-checking against a reference node is on the v2 list. Reading win rate together with the gap counter already flags a provider that pushes early but wrong (retracted heads show up as cohort inconsistencies and reorg noise)." - "Vantage point: a single eu-west host. Geography moves absolute arrival times but affects all providers over the same path; still, a provider whose nearest edge is far from eu-west is disadvantaged, and a us-east probe is the planned second vantage before any cross-region claim is made." @@ -94,7 +95,7 @@ faq: - q: "Could a provider cheat by pushing heads before validating them?" a: "In principle yes: pre-announcing an unvalidated head wins the race at the cost of occasionally pushing a block that gets reorged or retracted. That is why the bench exports more than the lag histogram. A provider that pushes early but wrong shows up as cohort inconsistencies, elevated reorg noise and gaps, and the coverage column (blocks seen versus the cohort maximum) penalises skipped or retracted blocks. Hash-level cross-checking against an independent reference node is planned for v2 of the harness." - q: "How is WebSocket head lag measured on OpenChainBench?" - a: "One Go harness holds a persistent eth_subscribe('newHeads') WebSocket per provider from the same eu-west host, plus slotSubscribe for the Solana cohort. Every frame is timestamped on receipt with time.Now() precision. A block's cohort stays open for 5 seconds after the first arrival, then each provider's lag versus the earliest is observed into a Prometheus histogram (5 ms to 10 s exponential buckets); p50/p90/p99 come from histogram_quantile over the 24 h window. Connections carry a 90 s silent-subscription watchdog and exponential-backoff reconnects so a flaky endpoint degrades to health=0 instead of corrupting the race." + a: "One Go harness holds a persistent eth_subscribe('newHeads') WebSocket per provider from the same eu-west host, plus slotSubscribe for the Solana cohort. Every frame is timestamped on receipt with time.Now() precision. A block's cohort stays open for 5 seconds after the first arrival, then each provider's lag versus the earliest is observed into a Prometheus histogram (0.5 ms to 10 s buckets, sub-millisecond floor so the winner does not collapse to a single interpolation artifact); p50/p90/p99 come from histogram_quantile over the 24 h window. Connections carry a 90 s silent-subscription watchdog and exponential-backoff reconnects so a flaky endpoint degrades to health=0 instead of corrupting the race." - q: "Why are only PublicNode, dRPC and Tenderly listed?" a: "V1 measures the keyless tier: endpoints anyone can open a WebSocket to without an account, which is also the tier where claims are hardest to verify and marketing is loudest. Alchemy, Infura and Chainstack require API keys; the harness already accepts them via environment variables and they join the leaderboard once contributor keys are wired into the deployment. Keyed tiers usually run on better-provisioned infrastructure, so expect the keyless numbers here to be the floor, not the ceiling, of each vendor's performance." diff --git a/benchmarks/ws-head-latency-solana.yml b/benchmarks/ws-head-latency-solana.yml index 5e73184ec..284c63193 100644 --- a/benchmarks/ws-head-latency-solana.yml +++ b/benchmarks/ws-head-latency-solana.yml @@ -40,12 +40,13 @@ abstract: | benchmark; this page is the Solana-scoped answer. methodology: - - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://solana-rpc.publicnode.com) and Solana Foundation (wss://api.mainnet-beta.solana.com). Alchemy and Helius are keyed-only and join the cohort when contributor keys are wired." + - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://solana-rpc.publicnode.com) and Solana Foundation (wss://api.mainnet-beta.solana.com). Keyed providers (Alchemy, Chainstack, Helius, QuickNode) join the cohort when contributor keys are wired via WS_URL__SOLANA env vars." + - "Cohort saturation: dRPC's free Solana tier explicitly disallows slotSubscribe (\"method is not available on freetier\") and the other public Solana WSS gateways audited (Blast, BlockPI, Ankr, extrnode, Chainstack public) all reject the handshake (verified 2026-07), which is why the keyless tier stops at two." - "Chain scope: every query on this page is pinned to chain=\"solana\". Solana slot cadence is ~400ms, so a 24h window carries roughly 216000 samples per provider, an order of magnitude beyond the 7000 sample floor used across the head-lag cluster." - "RPC method: `slotSubscribe`, not `signatureSubscribe` or `logsSubscribe`. The notification payload includes the slot number and the parent slot; the harness dedupes on slot number and uses the first arrival per slot to define T0." - "Race scoring: identical to the parent [ws-head-latency](https://openchainbench.com/benchmarks/ws-head-latency-ethereum) benchmark, applied per slot instead of per block. First provider to deliver slot N sets T0. Each provider's sample for slot N is arrival(N) minus T0 in milliseconds. Cohort closes 5 seconds after the first arrival." - "Relative, not absolute: subtraction of two arrivals over the same path removes vantage-point network latency. Consequence: the fastest provider reads near zero by construction. The honest readings are win rate, trailers' lag distribution, and gap counts, not the leader's absolute number." - - "Aggregation: per-provider lag histogram (5ms to 10s exponential buckets), p50/p90/p99 via histogram_quantile over the 24h rate. Coverage score = ws_head_blocks_seen_total divided by cohort maximum, clamped to 1." + - "Aggregation: per-provider lag histogram with a sub-millisecond floor (0.5 ms bucket up through 10 s tail), p50/p90/p99 via histogram_quantile over the 24h rate. Coverage score = ws_head_blocks_seen_total divided by cohort maximum, clamped to 1." - "Connection discipline: one persistent WebSocket per (provider, chain), client ping every 30s, read deadline 60s, 90s slot watchdog forcing reconnect when a socket keeps the heartbeat alive but silently drops the subscription. Exponential backoff 2s to 60s on reconnect." - "Vantage point: a single eu-west host. Multi-region deployment is a follow-up on the parent bench roadmap; the same limitation and reasoning are documented there." diff --git a/harnesses/ws-head-latency/cmd/script/config.go b/harnesses/ws-head-latency/cmd/script/config.go index 1c30cfaa0..f3d5a7cb3 100644 --- a/harnesses/ws-head-latency/cmd/script/config.go +++ b/harnesses/ws-head-latency/cmd/script/config.go @@ -42,14 +42,30 @@ func endpoints() []Endpoint { } // Keyed providers, skipped when the env var is unset. URLs carry the // key inline (e.g. wss://eth-mainnet.g.alchemy.com/v2/) so they - // live in the deployment env, never in the repo. - for _, opt := range []struct{ env, provider, chain string }{ - {"WS_URL_ALCHEMY_ETHEREUM", "alchemy", "ethereum"}, - {"WS_URL_INFURA_ETHEREUM", "infura", "ethereum"}, - {"WS_URL_CHAINSTACK_ETHEREUM", "chainstack", "ethereum"}, + // live in the deployment env, never in the repo. The public keyless + // cohort is already saturated on every chain (Ankr, Blast, LlamaRPC, + // BlockPI, 1RPC, NodeReal, Omniatech all failed a live WSS handshake + // as of 2026-07 audit), so any new cohort member comes in via one of + // the slots below. + for _, opt := range []struct{ env, provider, chain, kind string }{ + // Ethereum keyed + {"WS_URL_ALCHEMY_ETHEREUM", "alchemy", "ethereum", "evm"}, + {"WS_URL_INFURA_ETHEREUM", "infura", "ethereum", "evm"}, + {"WS_URL_CHAINSTACK_ETHEREUM", "chainstack", "ethereum", "evm"}, + {"WS_URL_QUICKNODE_ETHEREUM", "quicknode", "ethereum", "evm"}, + // Base keyed + {"WS_URL_ALCHEMY_BASE", "alchemy", "base", "evm"}, + {"WS_URL_INFURA_BASE", "infura", "base", "evm"}, + {"WS_URL_CHAINSTACK_BASE", "chainstack", "base", "evm"}, + {"WS_URL_QUICKNODE_BASE", "quicknode", "base", "evm"}, + // Solana keyed + {"WS_URL_ALCHEMY_SOLANA", "alchemy", "solana", "solana"}, + {"WS_URL_CHAINSTACK_SOLANA", "chainstack", "solana", "solana"}, + {"WS_URL_HELIUS_SOLANA", "helius", "solana", "solana"}, + {"WS_URL_QUICKNODE_SOLANA", "quicknode", "solana", "solana"}, } { if url := strings.TrimSpace(os.Getenv(opt.env)); url != "" { - eps = append(eps, Endpoint{Provider: opt.provider, Chain: opt.chain, Kind: "evm", URL: url}) + eps = append(eps, Endpoint{Provider: opt.provider, Chain: opt.chain, Kind: opt.kind, URL: url}) } } return eps diff --git a/harnesses/ws-head-latency/cmd/script/metrics.go b/harnesses/ws-head-latency/cmd/script/metrics.go index af8f28427..9c589c1c8 100644 --- a/harnesses/ws-head-latency/cmd/script/metrics.go +++ b/harnesses/ws-head-latency/cmd/script/metrics.go @@ -41,9 +41,17 @@ var ( prometheus.HistogramOpts{ Name: "ws_head_lag_milliseconds", Help: "Per-block push lag vs the earliest provider in the cohort, in milliseconds. Relative measurement: the fastest provider on a block reads 0 by construction.", - // 5ms → 10.24s exponential; covers same-frame ties through - // multi-second stragglers. - Buckets: prometheus.ExponentialBuckets(5, 2, 12), + // Sub-ms floor because the winner reads near zero by + // construction; a 5ms first bucket used to collapse every + // leader's p50 to ~2.5ms (histogram_quantile interpolation + // artifact inside [0,5ms]). Fine buckets from 0.5ms up + // through 10s cover same-frame ties through multi-second + // stragglers on every chain in the cohort (Solana ~400ms + // slots, Base 2s blocks, Ethereum 12s blocks). + Buckets: []float64{ + 0.5, 1, 2, 3, 5, 8, 12, 20, 35, 60, + 100, 160, 260, 420, 680, 1100, 1800, 3000, 5000, 10000, + }, ConstLabels: regionLabels, }, []string{"provider", "chain"},