From eedf6a36935b219b11c5e1f1b9c0d688a2fcfad3 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Thu, 16 Jul 2026 13:09:19 +0200 Subject: [PATCH 1/2] ws-head-latency: region label via ConstLabels + Railway config Adds a static region ConstLabel to every ws-head-latency metric, bound at process startup from OCB_REGION (preferred) or RAILWAY_REPLICA_REGION (Railway auto-set), with eu-west fallback matching the historical single-vantage Paris VPS deployment. No observation site needs to change: the label is baked into every counter, gauge and histogram at registration, so all existing WithLabelValues(provider, chain) calls stay identical. Adds railway.toml so Railway can build the harness from Dockerfile with a /health check, matching the rpc-capabilities service layout. Follow-up: create the Railway ws-head-latency service with 3 replicas (us-east, eu-west, sgp) and OCB_REGION set on each, then decommission or leave the Paris VPS container as a 4th vantage. --- .../ws-head-latency/cmd/script/metrics.go | 57 ++++++++++++++----- harnesses/ws-head-latency/railway.toml | 7 +++ 2 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 harnesses/ws-head-latency/railway.toml diff --git a/harnesses/ws-head-latency/cmd/script/metrics.go b/harnesses/ws-head-latency/cmd/script/metrics.go index e2940ee6b..8df51f2f8 100644 --- a/harnesses/ws-head-latency/cmd/script/metrics.go +++ b/harnesses/ws-head-latency/cmd/script/metrics.go @@ -2,6 +2,7 @@ package main import ( "net/http" + "os" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -9,9 +10,31 @@ import ( ) // Prom metrics for the ws-head-latency harness. Names follow the -// `__` OCB convention. All series carry -// {provider, chain} so the spec can pin headline queries to one chain -// while keeping base/solana series available for future tabs. +// `__` OCB convention. Series carry +// {provider, chain} plus a static {region} ConstLabel bound at +// process startup, so a multi-region deploy (Railway replicas or +// VPS + Railway hybrid) never collides series across vantage points +// and no observation site needs to know about the region. +// +// Region resolution order at startup: OCB_REGION, then +// RAILWAY_REPLICA_REGION (Railway sets it automatically), then the +// eu-west default that matches the historical single-vantage Paris +// VPS deployment. Set OCB_REGION explicitly on each replica to +// guarantee a stable value across redeploys. +var region = resolveRegion() + +func resolveRegion() string { + if r := os.Getenv("OCB_REGION"); r != "" { + return r + } + if r := os.Getenv("RAILWAY_REPLICA_REGION"); r != "" { + return r + } + return "eu-west" +} + +var regionLabels = prometheus.Labels{"region": region} + var ( headLagHist = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -19,47 +42,53 @@ var ( Help: "Per-block push lag vs the earliest provider in the cohort, in milliseconds. Relative measurement: the fastest provider on a block reads 0 by construction.", // 5ms → 10.24s exponential; covers same-frame ties through // multi-second stragglers. - Buckets: prometheus.ExponentialBuckets(5, 2, 12), + Buckets: prometheus.ExponentialBuckets(5, 2, 12), + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) headWins = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_head_wins_total", - Help: "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.", + Name: "ws_head_wins_total", + Help: "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) headBlocksSeen = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_head_blocks_seen_total", - Help: "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.", + Name: "ws_head_blocks_seen_total", + Help: "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) wsReconnects = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_reconnects_total", - Help: "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.", + Name: "ws_reconnects_total", + Help: "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) wsHealth = promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ws_health", - Help: "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.", + Name: "ws_health", + Help: "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) headBlockGap = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_block_gap_total", - Help: "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).", + Name: "ws_block_gap_total", + Help: "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) diff --git a/harnesses/ws-head-latency/railway.toml b/harnesses/ws-head-latency/railway.toml new file mode 100644 index 000000000..2abbb1f00 --- /dev/null +++ b/harnesses/ws-head-latency/railway.toml @@ -0,0 +1,7 @@ +[build] +builder = "DOCKERFILE" +dockerfilePath = "Dockerfile" + +[deploy] +healthcheckPath = "/health" +restartPolicyType = "ON_FAILURE" From a2e7c3e0dea17d3c7247fb627c87e643cd1697c2 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Thu, 16 Jul 2026 13:11:26 +0200 Subject: [PATCH 2/2] ws-head-latency specs: add region dimension tabs (081 + 087 + 088) Adds `dimensions.region` block + `rank_matrix_query` on provider, region + per-provider `regions:` sub-arrays with region-scoped queries on the 3 ws-head-latency specs, following the polkadot-rpc pattern. Depends on the harness region label change (previous commit): every metric now carries a region ConstLabel so the region-scoped histogram queries return values as soon as multi-region replicas publish. Until Railway 3-replica deploy lands, only region=eu-west has data and the other tabs render empty; the All regions tab keeps working because the aggregate query is unscoped by region. --- benchmarks/ws-head-latency-base.yml | 29 ++++++++++++++++++ benchmarks/ws-head-latency-solana.yml | 29 ++++++++++++++++++ benchmarks/ws-head-latency.yml | 42 +++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/benchmarks/ws-head-latency-base.yml b/benchmarks/ws-head-latency-base.yml index 052d70a36..dbe4f8124 100644 --- a/benchmarks/ws-head-latency-base.yml +++ b/benchmarks/ws-head-latency-base.yml @@ -62,6 +62,15 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="base"}[24h]))) + +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "What is the fastest free Base RPC over WebSocket right now?" a: "{{best_name}} currently wins most per-block races on Base with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Base block, roughly every 2 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Base bot to any single provider." @@ -87,6 +96,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="base"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[1h]))) - slug: drpc name: dRPC @@ -100,3 +119,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="base"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[1h]))) diff --git a/benchmarks/ws-head-latency-solana.yml b/benchmarks/ws-head-latency-solana.yml index c906903be..cc9ee17c7 100644 --- a/benchmarks/ws-head-latency-solana.yml +++ b/benchmarks/ws-head-latency-solana.yml @@ -63,6 +63,15 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="solana"}[24h]))) + +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "What is the fastest free Solana RPC over WebSocket right now?" a: "{{best_name}} currently wins most per-slot races on Solana with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Solana slot, roughly every 400ms, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Solana keeper or MEV integration to any single provider." @@ -90,6 +99,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="solana"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[1h]))) - slug: solana-labs name: Solana Foundation @@ -103,3 +122,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="solana-labs", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="solana-labs", chain="solana"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[1h]))) diff --git a/benchmarks/ws-head-latency.yml b/benchmarks/ws-head-latency.yml index 49a42ebd2..7b66ea78f 100644 --- a/benchmarks/ws-head-latency.yml +++ b/benchmarks/ws-head-latency.yml @@ -72,6 +72,18 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +# Per-cell (region) ranking matrix for scoped badge claims. Chain is +# fixed for the whole bench, so cells key on provider and region. +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="ethereum"}[24h]))) + +# Region is the only dimension: chain is baked into every query. +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "Which RPC provider pushes new Ethereum blocks first over WebSocket?" a: "{{best_name}} currently wins most per-block races, with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Ethereum block, roughly every 12 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive bot to any single provider." @@ -99,6 +111,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="ethereum"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[1h]))) - slug: drpc name: dRPC @@ -112,6 +134,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="ethereum"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[1h]))) - slug: tenderly name: Tenderly @@ -125,3 +157,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="tenderly", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="tenderly", chain="ethereum"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[1h])))