From b56c7f256e977b67c2902110988edafd451a6bf0 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:36:17 +0200 Subject: [PATCH] the public Prometheus host had no credential and an enabled admin API prom.openchainbench.com answers the internet. It is a bare reverse_proxy with no auth, and Prometheus runs behind it with --web.enable-admin-api, so a POST from anywhere could delete series. Measured from a laptop: /api/v1/status/config returned 200 and the delete_series route answered 405 to a GET, which is Prometheus saying "wrong verb", not "go away". The destructive routes are now blocked at the edge, which breaks nothing because the access log shows the only paths in use are /api/v1/query and /api/v1/query_range. This commit closes the rest: an Authorization header on the one fetch every Prometheus request passes through. It has to be a header. Node's fetch refuses a URL that embeds credentials ("Request cannot be constructed from a URL that includes credentials"), so user:pass@ in PROMETHEUS_URL would not weaken authentication, it would break every query. Tested before writing this. Absent PROMETHEUS_BASIC_AUTH the helper adds nothing, and that is the point: the rollout only works in one order, because Vercel binds env vars at deployment rather than at request time. 1. this commit ships, inert 2. PROMETHEUS_BASIC_AUTH set in production 3. redeploy, so the value is actually bound 4. basic_auth turned on in Caddy Reversed, every bench 401s until step 3 lands. A test pins step 1: an unset or empty variable must produce no header at all. Not sent over plain http, so the worker's in-network hop to ocb-prom:9090 never carries a credential it has no use for. Co-Authored-By: Claude Opus 5 (1M context) --- src/lib/prometheus-auth.test.ts | 65 +++++++++++++++++++++++++++++++++ src/lib/prometheus.ts | 35 ++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/lib/prometheus-auth.test.ts diff --git a/src/lib/prometheus-auth.test.ts b/src/lib/prometheus-auth.test.ts new file mode 100644 index 000000000..992d0735c --- /dev/null +++ b/src/lib/prometheus-auth.test.ts @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { promAuthHeaders } from "./prometheus"; + +const KEY = "PROMETHEUS_BASIC_AUTH"; +const original = process.env[KEY]; + +afterEach(() => { + if (original === undefined) delete process.env[KEY]; + else process.env[KEY] = original; +}); + +describe("promAuthHeaders", () => { + it("sends nothing when no credential is configured", () => { + delete process.env[KEY]; + expect(promAuthHeaders(new URL("https://prom.openchainbench.com/api/v1/query"))).toEqual({}); + }); + + // The order of rollout depends on this: the code ships first and stays inert, + // the variable is set second, and only then does Caddy start demanding auth. + // If an unset variable produced a header, step one would already 401. + it("stays inert so it can ship before the edge demands auth", () => { + process.env[KEY] = ""; + expect(promAuthHeaders(new URL("https://prom.openchainbench.com/api/v1/query"))).toEqual({}); + }); + + it("encodes user:password as a basic credential", () => { + process.env[KEY] = "florent:s3cret"; + const h = promAuthHeaders(new URL("https://prom.openchainbench.com/api/v1/query")); + expect(h.authorization).toBe(`Basic ${Buffer.from("florent:s3cret").toString("base64")}`); + // Decodes back to exactly what Caddy's basic_auth compares against. + const decoded = Buffer.from(h.authorization.slice("Basic ".length), "base64").toString("utf8"); + expect(decoded).toBe("florent:s3cret"); + }); + + it("trims surrounding whitespace, which a copied env value carries", () => { + process.env[KEY] = " florent:s3cret\n"; + const h = promAuthHeaders(new URL("https://prom.openchainbench.com/api/v1/query")); + const decoded = Buffer.from(h.authorization.slice("Basic ".length), "base64").toString("utf8"); + expect(decoded).toBe("florent:s3cret"); + }); + + // A value with no colon is a misconfiguration. Sending it would answer 401 + // on every bench with nothing to point at; refusing to send it leaves the + // benches working and the missing auth visible. + it("refuses a value that cannot be user:password", () => { + process.env[KEY] = "justatoken"; + expect(promAuthHeaders(new URL("https://prom.openchainbench.com/api/v1/query"))).toEqual({}); + }); + + // The worker reaches Prometheus in-network over plain http at ocb-prom:9090, + // which needs no credential. Forwarding one would put it somewhere it never + // has to be. + it("never sends the credential over plain http", () => { + process.env[KEY] = "florent:s3cret"; + expect(promAuthHeaders(new URL("http://ocb-prom:9090/api/v1/query"))).toEqual({}); + expect(promAuthHeaders(new URL("http://localhost:9090/api/v1/query"))).toEqual({}); + }); + + it("keeps a password containing colons intact", () => { + process.env[KEY] = "florent:pa:ss:word"; + const h = promAuthHeaders(new URL("https://prom.openchainbench.com/api/v1/query")); + const decoded = Buffer.from(h.authorization.slice("Basic ".length), "base64").toString("utf8"); + expect(decoded).toBe("florent:pa:ss:word"); + }); +}); diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 75602b89a..a04259c25 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -187,6 +187,8 @@ export class Prometheus { } private async fetchEnvelope(url: URL, signal?: AbortSignal): Promise { + const authHeaders = promAuthHeaders(url); + // Defense in depth against DNS-rebinding: the schema guards URL // literals, but a hostname can resolve to a private address at fetch // time. Refuse if any resolved IP is loopback / RFC1918 / link-local @@ -206,6 +208,7 @@ export class Prometheus { const merged = signal ? mergeSignals(signal, controller.signal) : controller.signal; const res = await fetch(url, { signal: merged, + headers: authHeaders, // Refuse to follow redirects. blocks 3xx into a private host. redirect: "manual", // Cache at the platform level. pages call us through ISR. @@ -442,6 +445,38 @@ const hostCheckCache = new Map(); // failure storms. First caller does the lookup, the rest await it. const hostCheckInFlight = new Map>(); +/** Basic-auth header for the public Prometheus host, when one is configured. + * + * prom.openchainbench.com answers the internet, because the Vercel site + * queries it directly (PROMETHEUS_URL is set in production). Until something + * authenticates, Prometheus runs there with --web.enable-admin-api and no + * credential, so the destructive routes are blocked at the edge instead. + * This closes the rest. + * + * It has to be a header: Node's fetch refuses a URL that embeds credentials + * outright ("Request cannot be constructed from a URL that includes + * credentials"), so putting user:pass@ in PROMETHEUS_URL would not weaken + * auth, it would break every query. + * + * Absent the variable this adds nothing, which is what makes the edge + * switchable in the only safe order: ship this, set the variable, redeploy + * (env binds at deployment, not at request time), then turn auth on in + * Caddy. Reversed, every bench 401s until the redeploy lands. + * + * Never sent over plain http. The worker's in-network hop to ocb-prom needs + * no credential, and forwarding one would put it somewhere it never has to + * be. */ +export function promAuthHeaders(url: URL): Record { + if (url.protocol !== "https:") return {}; + const raw = process.env.PROMETHEUS_BASIC_AUTH?.trim(); + // A value without a colon cannot be user:password; sending it would be a + // silent 401 on every bench rather than a visible misconfiguration. + if (!raw || !raw.includes(":")) return {}; + return { + authorization: `Basic ${Buffer.from(raw, "utf8").toString("base64")}`, + }; +} + async function assertPublicHost(url: URL): Promise { // The worker on the VPS reaches Prometheus over the docker bridge by // its container name (resolves to a private 172.18.0.0/16 address).