From ebdc6256977110f172d0c762b06b7ee27b45951c Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:57:07 +0200 Subject: [PATCH] freshness asked Prometheus 98 times per fill, now once /api/freshness ran one instant query per live spec on every 30 s cache fill: 509k Prom queries a day, 30% of all outbound calls the project made, each billed as an Observability Event. Prometheus.dataAgesSec() folds them into one union query. timestamp() must wrap the bare selector, otherwise it returns evaluation time and every recording rule reads 0 s old, so each metric gets its own term. Checked live: 246/246 specs agree with the per-spec probe within one scrape interval, 77 metrics in ~240 ms. --- src/app/api/freshness/route.ts | 57 ++++++++++++++++++++-------------- src/lib/prometheus.test.ts | 35 ++++++++++++++++++++- src/lib/prometheus.ts | 23 ++++++++++++++ 3 files changed, 90 insertions(+), 25 deletions(-) diff --git a/src/app/api/freshness/route.ts b/src/app/api/freshness/route.ts index f2584ced9..636133ea8 100644 --- a/src/app/api/freshness/route.ts +++ b/src/app/api/freshness/route.ts @@ -1,5 +1,5 @@ import { unstable_cache } from "next/cache"; -import { Prometheus } from "@/lib/prometheus"; +import { Prometheus, extractMetricName } from "@/lib/prometheus"; import { getSpecs } from "@/lib/spec"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; import { stripQueryRedirect } from "@/lib/canonical-query"; @@ -7,9 +7,10 @@ import { stripQueryRedirect } from "@/lib/canonical-query"; export const runtime = "nodejs"; /** - * Ultra-light freshness probe. One Prometheus instant query per spec - * (`scalar(time() - max(timestamp()))`), returning just the - * resolved data timestamp per slug. + * Ultra-light freshness probe. One Prometheus instant query for every + * spec's probe metric (`time() - max(timestamp())` per metric, + * OR-ed into one union), + * returning just the resolved data timestamp per slug. * * Separate from /api/citable so the LiveIndicator can poll cheaply * without re-running the heavy spec → rankings → sparkline → series @@ -42,32 +43,40 @@ const computeFreshness = unstable_cache( ); const fallback = process.env.PROMETHEUS_URL; - const entries = await Promise.all( - specs.map(async (spec) => { - const url = spec.prometheus?.url ?? fallback; - if (!url) return [spec.slug, null] as const; - // Use the first provider's p50 query as the freshness probe. Same - // logic as src/lib/spec.ts tryLoadLive - keeps the asOf reported - // here consistent with what /api/citable would compute. - const probe = spec.providers.find((p) => p.queries?.p50)?.queries?.p50; - if (!probe) return [spec.slug, null] as const; + // Use the first provider's p50 query as the freshness probe. Same + // logic as src/lib/spec.ts tryLoadLive - keeps the asOf reported + // here consistent with what /api/citable would compute. + // + // One query per Prom instance, not one per spec: the per-spec fan-out + // was ~98 queries per fill, 509k a day, 30% of every outbound call + // the project made (2026-10-10), and each one bills as an + // Observability Event. + const metricsByUrl = new Map>(); + for (const spec of specs) { + const url = spec.prometheus?.url ?? fallback; + const probe = spec.providers.find((p) => p.queries?.p50)?.queries?.p50; + const metric = probe ? extractMetricName(probe) : null; + if (!url || !metric) continue; + if (!metricsByUrl.has(url)) metricsByUrl.set(url, new Map()); + metricsByUrl.get(url)!.set(spec.slug, metric); + } + + const freshness: Record = {}; + await Promise.all( + [...metricsByUrl].map(async ([url, slugToMetric]) => { try { - const prom = new Prometheus(url); - const ageSec = await prom.dataAgeSec(probe); - if (ageSec == null || !Number.isFinite(ageSec) || ageSec < 0) { - return [spec.slug, null] as const; + const ages = await new Prometheus(url).dataAgesSec([...slugToMetric.values()]); + const now = Date.now(); + for (const [slug, metric] of slugToMetric) { + const ageSec = ages.get(metric); + if (ageSec == null || ageSec < 0) continue; + freshness[slug] = now - Math.floor(ageSec * 1000); } - return [spec.slug, Date.now() - Math.floor(ageSec * 1000)] as const; } catch { - return [spec.slug, null] as const; + // this instance's specs stay absent, as a failed probe did before } }), ); - - const freshness: Record = {}; - for (const [slug, asOf] of entries) { - if (asOf != null) freshness[slug] = asOf; - } // Store fallback: the Vercel site has no PROMETHEUS_URL (Prom lives // on a private VPS), so on prod every probe above fails and this // map came back EMPTY since launch, blanking the LiveIndicator. diff --git a/src/lib/prometheus.test.ts b/src/lib/prometheus.test.ts index 7bdc699ea..a03099c1d 100644 --- a/src/lib/prometheus.test.ts +++ b/src/lib/prometheus.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { extractMetricName, trimScalar } from "./prometheus"; +import { Prometheus, extractMetricName, trimScalar } from "./prometheus"; describe("extractMetricName", () => { test("plain metric with label selector", () => { @@ -144,3 +144,36 @@ describe("trimScalar", () => { expect(trimScalar(Number.POSITIVE_INFINITY)).toBeNull(); }); }); + +describe("dataAgesSec", () => { + test("one query for every metric, keyed back by name", async () => { + const seen: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request) => { + seen.push(new URL(String(input)).searchParams.get("query") ?? ""); + return Response.json({ + status: "success", + data: { + resultType: "vector", + result: [ + { metric: { m: "a_ms" }, value: [1, "12.5"] }, + { metric: { m: "b_ms" }, value: [1, "NaN"] }, + ], + }, + }); + }) as typeof fetch; + try { + const ages = await new Prometheus("http://localhost:9090").dataAgesSec([ + "a_ms", + "b_ms", + "a_ms", + 'bad"name', + ]); + expect(seen).toHaveLength(1); + expect(seen[0]).toBe('label_replace(time() - max(timestamp(a_ms)), "m", "a_ms", "", "") or label_replace(time() - max(timestamp(b_ms)), "m", "b_ms", "", "")'); + expect([...ages]).toEqual([["a_ms", 12.5]]); + } finally { + globalThis.fetch = realFetch; + } + }); +}); diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 1849e43d9..220a84032 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -112,6 +112,29 @@ export class Prometheus { return this.scalar(`scalar(time() - max(timestamp(${metric})))`); } + /** `dataAgeSec` for many metrics in ONE instant query: a union of + * per-metric terms, each labelled with its name. `timestamp()` has to + * wrap the bare selector: around anything else (a `label_replace`, a + * regex selector inside one) it returns the evaluation time, so every + * recording rule read as 0 s old. Metrics with no samples are absent + * from the map. Throws on a Prom error, like `query`. */ + async dataAgesSec(metrics: string[]): Promise> { + const ages = new Map(); + const names = [...new Set(metrics)].filter((m) => /^[a-zA-Z_:][a-zA-Z0-9_:]*$/.test(m)); + if (names.length === 0) return ages; + const res = await this.query( + names + .map((m) => `label_replace(time() - max(timestamp(${m})), "m", "${m}", "", "")`) + .join(" or "), + ); + if (res.resultType !== "vector") return ages; + for (const s of res.result) { + const age = Number(s.value[1]); + if (s.metric.m && Number.isFinite(age)) ages.set(s.metric.m, age); + } + return ages; + } + /** Convenience: scalar number from any instant query, or null if empty * / error. Errors are logged with the query that failed and the * underlying reason ("empty", "timeout", network error, parse error)