ws-head-latency: multi-region (harness region label + spec region tabs)

benchmarks/ws-head-latency-base.yml

@@ -62,6 +62,15 @@ prometheus:
   window: 24h
   freshness_metric: ws_head_blocks_seen_total
 
+rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="base"}[24h])))
+
+dimensions:
+  region:
+    - { value: all,     label: All regions }
+    - { value: us-east, label: US-East }
+    - { value: eu-west, label: EU-West }
+    - { value: sgp,     label: Singapore }
+
 faq:
   - q: "What is the fastest free Base RPC over WebSocket right now?"
     a: "{{best_name}} currently wins most per-block races on Base with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Base block, roughly every 2 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Base bot to any single provider."
@@ -87,6 +96,16 @@ providers:
       success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1)
       sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="base"}[24h]))
       series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base"}[1h])))
+      regions:
+        - region: us-east
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[1h])))
+        - region: eu-west
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[1h])))
+        - region: ap-southeast
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[1h])))
 
   - slug: drpc
     name: dRPC
@@ -100,3 +119,13 @@ providers:
       success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1)
       sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="base"}[24h]))
       series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base"}[1h])))
+      regions:
+        - region: us-east
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[1h])))
+        - region: eu-west
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[1h])))
+        - region: ap-southeast
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[1h])))

benchmarks/ws-head-latency-solana.yml

@@ -63,6 +63,15 @@ prometheus:
   window: 24h
   freshness_metric: ws_head_blocks_seen_total
 
+rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="solana"}[24h])))
+
+dimensions:
+  region:
+    - { value: all,     label: All regions }
+    - { value: us-east, label: US-East }
+    - { value: eu-west, label: EU-West }
+    - { value: sgp,     label: Singapore }
+
 faq:
   - q: "What is the fastest free Solana RPC over WebSocket right now?"
     a: "{{best_name}} currently wins most per-slot races on Solana with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Solana slot, roughly every 400ms, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Solana keeper or MEV integration to any single provider."
@@ -90,6 +99,16 @@ providers:
       success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1)
       sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="solana"}[24h]))
       series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana"}[1h])))
+      regions:
+        - region: us-east
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[1h])))
+        - region: eu-west
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[1h])))
+        - region: ap-southeast
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[1h])))
 
   - slug: solana-labs
     name: Solana Foundation
@@ -103,3 +122,13 @@ providers:
       success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="solana-labs", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1)
       sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="solana-labs", chain="solana"}[24h]))
       series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana"}[1h])))
+      regions:
+        - region: us-east
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[1h])))
+        - region: eu-west
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[1h])))
+        - region: ap-southeast
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[1h])))

benchmarks/ws-head-latency.yml

@@ -72,6 +72,18 @@ prometheus:
   window: 24h
   freshness_metric: ws_head_blocks_seen_total
 
+# Per-cell (region) ranking matrix for scoped badge claims. Chain is
+# fixed for the whole bench, so cells key on provider and region.
+rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="ethereum"}[24h])))
+
+# Region is the only dimension: chain is baked into every query.
+dimensions:
+  region:
+    - { value: all,     label: All regions }
+    - { value: us-east, label: US-East }
+    - { value: eu-west, label: EU-West }
+    - { value: sgp,     label: Singapore }
+
 faq:
   - q: "Which RPC provider pushes new Ethereum blocks first over WebSocket?"
     a: "{{best_name}} currently wins most per-block races, with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Ethereum block, roughly every 12 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive bot to any single provider."
@@ -99,6 +111,16 @@ providers:
       success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1)
       sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="ethereum"}[24h]))
       series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[1h])))
+      regions:
+        - region: us-east
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[1h])))
+        - region: eu-west
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[1h])))
+        - region: ap-southeast
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[1h])))
 
   - slug: drpc
     name: dRPC
@@ -112,6 +134,16 @@ providers:
       success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1)
       sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="ethereum"}[24h]))
       series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[1h])))
+      regions:
+        - region: us-east
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[1h])))
+        - region: eu-west
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[1h])))
+        - region: ap-southeast
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[1h])))
 
   - slug: tenderly
     name: Tenderly
@@ -125,3 +157,13 @@ providers:
       success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="tenderly", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1)
       sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="tenderly", chain="ethereum"}[24h]))
       series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[1h])))
+      regions:
+        - region: us-east
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[1h])))
+        - region: eu-west
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[1h])))
+        - region: ap-southeast
+          p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[24h])))
+          series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[1h])))

harnesses/ws-head-latency/cmd/script/metrics.go

@@ -2,64 +2,93 @@ package main
 
 import (
 	"net/http"
+	"os"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // Prom metrics for the ws-head-latency harness. Names follow the
-// `<bench>_<measurement>_<unit>` OCB convention. All series carry
-// {provider, chain} so the spec can pin headline queries to one chain
-// while keeping base/solana series available for future tabs.
+// `<bench>_<measurement>_<unit>` OCB convention. Series carry
+// {provider, chain} plus a static {region} ConstLabel bound at
+// process startup, so a multi-region deploy (Railway replicas or
+// VPS + Railway hybrid) never collides series across vantage points
+// and no observation site needs to know about the region.
+//
+// Region resolution order at startup: OCB_REGION, then
+// RAILWAY_REPLICA_REGION (Railway sets it automatically), then the
+// eu-west default that matches the historical single-vantage Paris
+// VPS deployment. Set OCB_REGION explicitly on each replica to
+// guarantee a stable value across redeploys.
+var region = resolveRegion()
+
+func resolveRegion() string {
+	if r := os.Getenv("OCB_REGION"); r != "" {
+		return r
+	}
+	if r := os.Getenv("RAILWAY_REPLICA_REGION"); r != "" {
+		return r
+	}
+	return "eu-west"
+}
+
+var regionLabels = prometheus.Labels{"region": region}
+
 var (
 	headLagHist = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name: "ws_head_lag_milliseconds",
 			Help: "Per-block push lag vs the earliest provider in the cohort, in milliseconds. Relative measurement: the fastest provider on a block reads 0 by construction.",
 			// 5ms → 10.24s exponential; covers same-frame ties through
 			// multi-second stragglers.
-			Buckets: prometheus.ExponentialBuckets(5, 2, 12),
+			Buckets:     prometheus.ExponentialBuckets(5, 2, 12),
+			ConstLabels: regionLabels,
 		},
 		[]string{"provider", "chain"},
 	)
 
 	headWins = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "ws_head_wins_total",
-			Help: "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.",
+			Name:        "ws_head_wins_total",
+			Help:        "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.",
+			ConstLabels: regionLabels,
 		},
 		[]string{"provider", "chain"},
 	)
 
 	headBlocksSeen = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "ws_head_blocks_seen_total",
-			Help: "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.",
+			Name:        "ws_head_blocks_seen_total",
+			Help:        "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.",
+			ConstLabels: regionLabels,
 		},
 		[]string{"provider", "chain"},
 	)
 
 	wsReconnects = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "ws_reconnects_total",
-			Help: "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.",
+			Name:        "ws_reconnects_total",
+			Help:        "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.",
+			ConstLabels: regionLabels,
 		},
 		[]string{"provider", "chain"},
 	)
 
 	wsHealth = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "ws_health",
-			Help: "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.",
+			Name:        "ws_health",
+			Help:        "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.",
+			ConstLabels: regionLabels,
 		},
 		[]string{"provider", "chain"},
 	)
 
 	headBlockGap = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "ws_block_gap_total",
-			Help: "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).",
+			Name:        "ws_block_gap_total",
+			Help:        "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).",
+			ConstLabels: regionLabels,
 		},
 		[]string{"provider", "chain"},
 	)

harnesses/ws-head-latency/railway.toml

@@ -0,0 +1,7 @@
+[build]
+builder = "DOCKERFILE"
+dockerfilePath = "Dockerfile"
+
+[deploy]
+healthcheckPath = "/health"
+restartPolicyType = "ON_FAILURE"