diff --git a/benchmarks/ws-head-latency-base.yml b/benchmarks/ws-head-latency-base.yml index 052d70a36..dbe4f8124 100644 --- a/benchmarks/ws-head-latency-base.yml +++ b/benchmarks/ws-head-latency-base.yml @@ -62,6 +62,15 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="base"}[24h]))) + +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "What is the fastest free Base RPC over WebSocket right now?" a: "{{best_name}} currently wins most per-block races on Base with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Base block, roughly every 2 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Base bot to any single provider." @@ -87,6 +96,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="base"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[1h]))) - slug: drpc name: dRPC @@ -100,3 +119,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="base"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[1h]))) diff --git a/benchmarks/ws-head-latency-solana.yml b/benchmarks/ws-head-latency-solana.yml index c906903be..cc9ee17c7 100644 --- a/benchmarks/ws-head-latency-solana.yml +++ b/benchmarks/ws-head-latency-solana.yml @@ -63,6 +63,15 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="solana"}[24h]))) + +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "What is the fastest free Solana RPC over WebSocket right now?" a: "{{best_name}} currently wins most per-slot races on Solana with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Solana slot, roughly every 400ms, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Solana keeper or MEV integration to any single provider." @@ -90,6 +99,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="solana"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[1h]))) - slug: solana-labs name: Solana Foundation @@ -103,3 +122,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="solana-labs", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="solana-labs", chain="solana"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[1h]))) diff --git a/benchmarks/ws-head-latency.yml b/benchmarks/ws-head-latency.yml index 49a42ebd2..7b66ea78f 100644 --- a/benchmarks/ws-head-latency.yml +++ b/benchmarks/ws-head-latency.yml @@ -72,6 +72,18 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +# Per-cell (region) ranking matrix for scoped badge claims. Chain is +# fixed for the whole bench, so cells key on provider and region. +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="ethereum"}[24h]))) + +# Region is the only dimension: chain is baked into every query. +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "Which RPC provider pushes new Ethereum blocks first over WebSocket?" a: "{{best_name}} currently wins most per-block races, with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Ethereum block, roughly every 12 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive bot to any single provider." @@ -99,6 +111,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="ethereum"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[1h]))) - slug: drpc name: dRPC @@ -112,6 +134,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="ethereum"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[1h]))) - slug: tenderly name: Tenderly @@ -125,3 +157,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="tenderly", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="tenderly", chain="ethereum"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[1h]))) diff --git a/harnesses/ws-head-latency/cmd/script/metrics.go b/harnesses/ws-head-latency/cmd/script/metrics.go index e2940ee6b..8df51f2f8 100644 --- a/harnesses/ws-head-latency/cmd/script/metrics.go +++ b/harnesses/ws-head-latency/cmd/script/metrics.go @@ -2,6 +2,7 @@ package main import ( "net/http" + "os" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -9,9 +10,31 @@ import ( ) // Prom metrics for the ws-head-latency harness. Names follow the -// `__` OCB convention. All series carry -// {provider, chain} so the spec can pin headline queries to one chain -// while keeping base/solana series available for future tabs. +// `__` OCB convention. Series carry +// {provider, chain} plus a static {region} ConstLabel bound at +// process startup, so a multi-region deploy (Railway replicas or +// VPS + Railway hybrid) never collides series across vantage points +// and no observation site needs to know about the region. +// +// Region resolution order at startup: OCB_REGION, then +// RAILWAY_REPLICA_REGION (Railway sets it automatically), then the +// eu-west default that matches the historical single-vantage Paris +// VPS deployment. Set OCB_REGION explicitly on each replica to +// guarantee a stable value across redeploys. +var region = resolveRegion() + +func resolveRegion() string { + if r := os.Getenv("OCB_REGION"); r != "" { + return r + } + if r := os.Getenv("RAILWAY_REPLICA_REGION"); r != "" { + return r + } + return "eu-west" +} + +var regionLabels = prometheus.Labels{"region": region} + var ( headLagHist = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -19,47 +42,53 @@ var ( Help: "Per-block push lag vs the earliest provider in the cohort, in milliseconds. Relative measurement: the fastest provider on a block reads 0 by construction.", // 5ms → 10.24s exponential; covers same-frame ties through // multi-second stragglers. - Buckets: prometheus.ExponentialBuckets(5, 2, 12), + Buckets: prometheus.ExponentialBuckets(5, 2, 12), + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) headWins = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_head_wins_total", - Help: "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.", + Name: "ws_head_wins_total", + Help: "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) headBlocksSeen = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_head_blocks_seen_total", - Help: "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.", + Name: "ws_head_blocks_seen_total", + Help: "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) wsReconnects = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_reconnects_total", - Help: "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.", + Name: "ws_reconnects_total", + Help: "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) wsHealth = promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ws_health", - Help: "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.", + Name: "ws_health", + Help: "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) headBlockGap = promauto.NewCounterVec( prometheus.CounterOpts{ - Name: "ws_block_gap_total", - Help: "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).", + Name: "ws_block_gap_total", + Help: "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).", + ConstLabels: regionLabels, }, []string{"provider", "chain"}, ) diff --git a/harnesses/ws-head-latency/railway.toml b/harnesses/ws-head-latency/railway.toml new file mode 100644 index 000000000..2abbb1f00 --- /dev/null +++ b/harnesses/ws-head-latency/railway.toml @@ -0,0 +1,7 @@ +[build] +builder = "DOCKERFILE" +dockerfilePath = "Dockerfile" + +[deploy] +healthcheckPath = "/health" +restartPolicyType = "ON_FAILURE"