diff --git a/src/app/api/freshness/route.ts b/src/app/api/freshness/route.ts index f2584ced9..636133ea8 100644 --- a/src/app/api/freshness/route.ts +++ b/src/app/api/freshness/route.ts @@ -1,5 +1,5 @@ import { unstable_cache } from "next/cache"; -import { Prometheus } from "@/lib/prometheus"; +import { Prometheus, extractMetricName } from "@/lib/prometheus"; import { getSpecs } from "@/lib/spec"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; import { stripQueryRedirect } from "@/lib/canonical-query"; @@ -7,9 +7,10 @@ import { stripQueryRedirect } from "@/lib/canonical-query"; export const runtime = "nodejs"; /** - * Ultra-light freshness probe. One Prometheus instant query per spec - * (`scalar(time() - max(timestamp()))`), returning just the - * resolved data timestamp per slug. + * Ultra-light freshness probe. One Prometheus instant query for every + * spec's probe metric (`time() - max(timestamp())` per metric, + * OR-ed into one union), + * returning just the resolved data timestamp per slug. * * Separate from /api/citable so the LiveIndicator can poll cheaply * without re-running the heavy spec → rankings → sparkline → series @@ -42,32 +43,40 @@ const computeFreshness = unstable_cache( ); const fallback = process.env.PROMETHEUS_URL; - const entries = await Promise.all( - specs.map(async (spec) => { - const url = spec.prometheus?.url ?? fallback; - if (!url) return [spec.slug, null] as const; - // Use the first provider's p50 query as the freshness probe. Same - // logic as src/lib/spec.ts tryLoadLive - keeps the asOf reported - // here consistent with what /api/citable would compute. - const probe = spec.providers.find((p) => p.queries?.p50)?.queries?.p50; - if (!probe) return [spec.slug, null] as const; + // Use the first provider's p50 query as the freshness probe. Same + // logic as src/lib/spec.ts tryLoadLive - keeps the asOf reported + // here consistent with what /api/citable would compute. + // + // One query per Prom instance, not one per spec: the per-spec fan-out + // was ~98 queries per fill, 509k a day, 30% of every outbound call + // the project made (2026-10-10), and each one bills as an + // Observability Event. + const metricsByUrl = new Map>(); + for (const spec of specs) { + const url = spec.prometheus?.url ?? fallback; + const probe = spec.providers.find((p) => p.queries?.p50)?.queries?.p50; + const metric = probe ? extractMetricName(probe) : null; + if (!url || !metric) continue; + if (!metricsByUrl.has(url)) metricsByUrl.set(url, new Map()); + metricsByUrl.get(url)!.set(spec.slug, metric); + } + + const freshness: Record = {}; + await Promise.all( + [...metricsByUrl].map(async ([url, slugToMetric]) => { try { - const prom = new Prometheus(url); - const ageSec = await prom.dataAgeSec(probe); - if (ageSec == null || !Number.isFinite(ageSec) || ageSec < 0) { - return [spec.slug, null] as const; + const ages = await new Prometheus(url).dataAgesSec([...slugToMetric.values()]); + const now = Date.now(); + for (const [slug, metric] of slugToMetric) { + const ageSec = ages.get(metric); + if (ageSec == null || ageSec < 0) continue; + freshness[slug] = now - Math.floor(ageSec * 1000); } - return [spec.slug, Date.now() - Math.floor(ageSec * 1000)] as const; } catch { - return [spec.slug, null] as const; + // this instance's specs stay absent, as a failed probe did before } }), ); - - const freshness: Record = {}; - for (const [slug, asOf] of entries) { - if (asOf != null) freshness[slug] = asOf; - } // Store fallback: the Vercel site has no PROMETHEUS_URL (Prom lives // on a private VPS), so on prod every probe above fails and this // map came back EMPTY since launch, blanking the LiveIndicator. diff --git a/src/lib/prometheus.test.ts b/src/lib/prometheus.test.ts index 7bdc699ea..a03099c1d 100644 --- a/src/lib/prometheus.test.ts +++ b/src/lib/prometheus.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { extractMetricName, trimScalar } from "./prometheus"; +import { Prometheus, extractMetricName, trimScalar } from "./prometheus"; describe("extractMetricName", () => { test("plain metric with label selector", () => { @@ -144,3 +144,36 @@ describe("trimScalar", () => { expect(trimScalar(Number.POSITIVE_INFINITY)).toBeNull(); }); }); + +describe("dataAgesSec", () => { + test("one query for every metric, keyed back by name", async () => { + const seen: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request) => { + seen.push(new URL(String(input)).searchParams.get("query") ?? ""); + return Response.json({ + status: "success", + data: { + resultType: "vector", + result: [ + { metric: { m: "a_ms" }, value: [1, "12.5"] }, + { metric: { m: "b_ms" }, value: [1, "NaN"] }, + ], + }, + }); + }) as typeof fetch; + try { + const ages = await new Prometheus("http://localhost:9090").dataAgesSec([ + "a_ms", + "b_ms", + "a_ms", + 'bad"name', + ]); + expect(seen).toHaveLength(1); + expect(seen[0]).toBe('label_replace(time() - max(timestamp(a_ms)), "m", "a_ms", "", "") or label_replace(time() - max(timestamp(b_ms)), "m", "b_ms", "", "")'); + expect([...ages]).toEqual([["a_ms", 12.5]]); + } finally { + globalThis.fetch = realFetch; + } + }); +}); diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 1849e43d9..220a84032 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -112,6 +112,29 @@ export class Prometheus { return this.scalar(`scalar(time() - max(timestamp(${metric})))`); } + /** `dataAgeSec` for many metrics in ONE instant query: a union of + * per-metric terms, each labelled with its name. `timestamp()` has to + * wrap the bare selector: around anything else (a `label_replace`, a + * regex selector inside one) it returns the evaluation time, so every + * recording rule read as 0 s old. Metrics with no samples are absent + * from the map. Throws on a Prom error, like `query`. */ + async dataAgesSec(metrics: string[]): Promise> { + const ages = new Map(); + const names = [...new Set(metrics)].filter((m) => /^[a-zA-Z_:][a-zA-Z0-9_:]*$/.test(m)); + if (names.length === 0) return ages; + const res = await this.query( + names + .map((m) => `label_replace(time() - max(timestamp(${m})), "m", "${m}", "", "")`) + .join(" or "), + ); + if (res.resultType !== "vector") return ages; + for (const s of res.result) { + const age = Number(s.value[1]); + if (s.metric.m && Number.isFinite(age)) ages.set(s.metric.m, age); + } + return ages; + } + /** Convenience: scalar number from any instant query, or null if empty * / error. Errors are logged with the query that failed and the * underlying reason ("empty", "timeout", network error, parse error)