From 9741791e835a76c1830d238eda1b00a2ee6abe5f Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Wed, 16 Sep 2026 15:20:21 -0400
Subject: [PATCH 1/7] feat: upgrade clickhouse versions and use promql remote
endpoints
---
.changeset/promql-via-clickhouse-http-api.md | 7 +
docker-compose.ci.yml | 2 +-
docker-compose.dev.yml | 2 +-
docker-compose.yml | 2 +-
docker/clickhouse/local/config.xml | 13 ++
docker/hyperdx/Dockerfile | 2 +-
.../api/src/controllers/timeseriesEngine.ts | 8 +-
packages/api/src/fixtures.ts | 10 +-
.../api/__tests__/prometheus.int.test.ts | 84 +++++++-
.../routers/api/__tests__/prometheus.test.ts | 77 --------
packages/api/src/routers/api/prometheus.ts | 187 +++++-------------
packages/app/tests/e2e/docker-compose.yml | 2 +-
.../src/__tests__/metadata.int.test.ts | 4 -
13 files changed, 154 insertions(+), 246 deletions(-)
create mode 100644 .changeset/promql-via-clickhouse-http-api.md
diff --git a/.changeset/promql-via-clickhouse-http-api.md b/.changeset/promql-via-clickhouse-http-api.md
new file mode 100644
index 0000000000..a4995bc598
--- /dev/null
+++ b/.changeset/promql-via-clickhouse-http-api.md
@@ -0,0 +1,7 @@
+---
+'@hyperdx/api': minor
+---
+
+feat: run PromQL against ClickHouse through its Prometheus HTTP API
+
+ClickHouse-backed `/query` and `/query_range` now proxy to the connection's `/prometheus/api/v1/*` endpoint (the `prometheus_api_v1` HTTP handler, ClickHouse 26.8+) instead of calling the `prometheusQuery`/`prometheusQueryRange` table functions. The `database` and `table` query parameters are forwarded so one handler serves any TimeSeries table. ClickHouse holds the Prometheus API forward-compatible while the TimeSeries engine is in preview; the table functions and inner-table schema are not. Bundled ClickHouse images move to 26.8.
diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
index bc65a426e5..bbc07b9799 100644
--- a/docker-compose.ci.yml
+++ b/docker-compose.ci.yml
@@ -22,7 +22,7 @@ services:
depends_on:
- ch-server
ch-server:
- image: clickhouse/clickhouse-server:26.5-alpine
+ image: clickhouse/clickhouse-server:26.8-alpine
environment:
# default settings
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index deae228acf..54200ca08a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -105,7 +105,7 @@ services:
hdx.dev.service: clickhouse
hdx.dev.port: '${HDX_DEV_CH_HTTP_PORT:-8123}'
hdx.dev.url: 'http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}'
- image: clickhouse/clickhouse-server:26.5-alpine
+ image: clickhouse/clickhouse-server:26.8-alpine
ports:
- '${HDX_DEV_CH_HTTP_PORT:-8123}:8123' # http api
- '${HDX_DEV_CH_NATIVE_PORT:-9000}:9000' # native
diff --git a/docker-compose.yml b/docker-compose.yml
index feda9debd4..5a02572925 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -96,7 +96,7 @@ services:
- ch-server
- db
ch-server:
- image: clickhouse/clickhouse-server:26.5-alpine
+ image: clickhouse/clickhouse-server:26.8-alpine
# WARNING: Exposing the database port will make it accessible from outside the container,
# potentially allowing unauthorized access. If you uncomment the ports below,
# ensure to secure your database (e.g., with strong authentication, proper network rules, and firewalls).
diff --git a/docker/clickhouse/local/config.xml b/docker/clickhouse/local/config.xml
index cea179de32..3e70c297cb 100644
--- a/docker/clickhouse/local/config.xml
+++ b/docker/clickhouse/local/config.xml
@@ -73,6 +73,19 @@
+
+
+
+
+ /prometheus/api/v1
+
+ prometheus_api_v1
+
+
+
+
system
diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile
index 1568263449..d13edc5d21 100644
--- a/docker/hyperdx/Dockerfile
+++ b/docker/hyperdx/Dockerfile
@@ -8,7 +8,7 @@
# - Otel Collector (otelcol)
ARG NODE_VERSION=22.22
-ARG CLICKHOUSE_VERSION=26.5
+ARG CLICKHOUSE_VERSION=26.8
ARG OTEL_COLLECTOR_VERSION=0.155.0
ARG OTEL_COLLECTOR_CORE_VERSION=1.61.0
diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts
index 2ce263f340..471964d75a 100644
--- a/packages/api/src/controllers/timeseriesEngine.ts
+++ b/packages/api/src/controllers/timeseriesEngine.ts
@@ -11,8 +11,8 @@ import {
import { ClickhouseClient } from '@/clickhouse';
import logger from '@/utils/logger';
-export const PROMETHEUS_MAX_EXECUTION_SEC = 30;
-export const PROMETHEUS_MAX_RESULT_ROWS = 100000;
+const PROMETHEUS_MAX_EXECUTION_SEC = 30;
+const PROMETHEUS_MAX_RESULT_ROWS = 100000;
export type TimeSeriesTagsQueryArgs = {
client: ClickhouseClient;
@@ -241,8 +241,8 @@ export async function queryLabelNames({
limit,
...args
}: TimeSeriesTagsQueryArgs): Promise {
- // `all_tags` is EPHEMERAL by default, so the label names come from `tags`,
- // which the engine strips `__name__` out of. It is folded back in explicitly.
+ // The engine keeps `__name__` in `metric_name`, not `tags`, so it is folded
+ // back in explicitly.
const value = chSql`arrayJoin(arrayConcat([${{ String: '__name__' }}], mapKeys(${{ Identifier: 'tags' }})))`;
const conditions = await getSeriesFilterConditions(args);
diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts
index 407a1733cc..414ac9eebf 100644
--- a/packages/api/src/fixtures.ts
+++ b/packages/api/src/fixtures.ts
@@ -336,15 +336,11 @@ export const seedTimeSeriesTagsTable = async ({
.join(', ')})`;
const columns = storeTimeBounds
- ? '(metric_name, tags, all_tags, min_time, max_time)'
- : '(metric_name, tags, all_tags)';
+ ? '(metric_name, tags, min_time, max_time)'
+ : '(metric_name, tags)';
const values = series
.map(s => {
- const row = [
- quoted(s.metricName),
- mapLiteral(s.tags),
- mapLiteral({ __name__: s.metricName, ...s.tags }),
- ];
+ const row = [quoted(s.metricName), mapLiteral(s.tags)];
if (storeTimeBounds) {
row.push(
`toDateTime64(${s.startSec}, 3)`,
diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
index ac92c06efe..5a32b1326b 100644
--- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
+++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
@@ -685,20 +685,57 @@ describe('prometheus router', () => {
expect(requested.searchParams.get('step')).toBe('15s');
});
- it('does NOT proxy to Prometheus when connection is not isPrometheusEndpoint', async () => {
+ it('proxies a ClickHouse connection to its prometheus_api_v1 handler with db/table and credentials', async () => {
const { agent, team } = await getLoggedInAgent(server);
const conn = await seedClickHouseConnection(team._id);
+ const promResponse = {
+ status: 'success',
+ data: { resultType: 'matrix', result: [] },
+ };
+ mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse));
+
+ const res = await agent
+ .get('/v1/prometheus/query_range')
+ .query({
+ query: 'up',
+ start: '1700000000',
+ end: '1700000060',
+ step: '15s',
+ database: 'otel',
+ table: 'metrics_ts',
+ connectionId: conn._id.toString(),
+ })
+ .expect(200);
- // ClickHouse path: will likely fail with 400 because metrics_ts
- // is not seeded in the test CH, but the routing decision is what we
- // care about — fetch must not be called.
- await agent.get('/v1/prometheus/query_range').query({
- query: 'up',
- start: '1700000000',
- end: '1700000060',
- connectionId: conn._id.toString(),
+ expect(res.body).toEqual(promResponse);
+ const [calledUrl, init] = mockFetch.mock.calls[0];
+ expect(calledUrl).toContain(
+ `${config.CLICKHOUSE_HOST}/prometheus/api/v1/query_range`,
+ );
+ expect(calledUrl).toContain('database=otel');
+ expect(calledUrl).toContain('table=metrics_ts');
+ expect(calledUrl).not.toContain('connectionId');
+ expect(init?.headers).toMatchObject({
+ 'X-ClickHouse-User': config.CLICKHOUSE_USER,
+ 'X-ClickHouse-Key': config.CLICKHOUSE_PASSWORD,
});
+ });
+ it('returns 400 before proxying when a ClickHouse connection has no table', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+
+ const res = await agent
+ .get('/v1/prometheus/query_range')
+ .query({
+ query: 'up',
+ start: '1700000000',
+ end: '1700000060',
+ connectionId: conn._id.toString(),
+ })
+ .expect(400);
+
+ expect(res.body.error).toContain('table parameter required');
expect(mockFetch).not.toHaveBeenCalled();
});
@@ -775,6 +812,35 @@ describe('prometheus router', () => {
expect(calledUrl).toContain('/api/v1/query');
expect(calledUrl).not.toContain('/api/v1/query_range');
});
+
+ it('proxies a ClickHouse connection to its prometheus_api_v1 handler', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+ mockFetch.mockResolvedValueOnce(
+ fakeUpstreamResponse({
+ status: 'success',
+ data: { resultType: 'vector', result: [] },
+ }),
+ );
+
+ await agent
+ .get('/v1/prometheus/query')
+ .query({
+ query: 'up',
+ table: 'metrics_ts',
+ connectionId: conn._id.toString(),
+ })
+ .expect(200);
+
+ const [calledUrl, init] = mockFetch.mock.calls[0];
+ expect(calledUrl).toContain(
+ `${config.CLICKHOUSE_HOST}/prometheus/api/v1/query?`,
+ );
+ expect(calledUrl).toContain('table=metrics_ts');
+ expect(init?.headers).toMatchObject({
+ 'X-ClickHouse-User': config.CLICKHOUSE_USER,
+ });
+ });
});
describe('GET /v1/prometheus/label/:name/values', () => {
diff --git a/packages/api/src/routers/api/__tests__/prometheus.test.ts b/packages/api/src/routers/api/__tests__/prometheus.test.ts
index dbf11461bf..4fc42e1510 100644
--- a/packages/api/src/routers/api/__tests__/prometheus.test.ts
+++ b/packages/api/src/routers/api/__tests__/prometheus.test.ts
@@ -9,8 +9,6 @@ jest.mock('@/utils/instrumentation', () => {
});
import {
- formatMatrixResponse,
- formatVectorResponse,
isClientDisconnect,
joinPrometheusUpstreamUrl,
parseDuration,
@@ -73,81 +71,6 @@ describe('parseDuration', () => {
});
});
-describe('formatMatrixResponse', () => {
- it('converts ClickHouse rows into Prometheus matrix shape', () => {
- const rows = [
- {
- tags: [
- ['__name__', 'http_requests_total'],
- ['method', 'GET'],
- ] as [string, string][],
- time_series: [
- [1700000000, 5],
- [1700000060, 7],
- ] as [string | number, number][],
- },
- ];
- expect(formatMatrixResponse(rows as any)).toEqual([
- {
- metric: { __name__: 'http_requests_total', method: 'GET' },
- values: [
- [1700000000, '5'],
- [1700000060, '7'],
- ],
- },
- ]);
- });
-
- it('converts string timestamps to unix seconds', () => {
- const rows = [
- {
- tags: [] as [string, string][],
- time_series: [['2023-11-14T22:13:20.000Z', 1]] as [
- string | number,
- number,
- ][],
- },
- ];
- expect(formatMatrixResponse(rows as any)[0].values[0]).toEqual([
- 1700000000,
- '1',
- ]);
- });
-
- it('returns empty array for empty input', () => {
- expect(formatMatrixResponse([])).toEqual([]);
- });
-});
-
-describe('formatVectorResponse', () => {
- it('converts ClickHouse rows into Prometheus vector shape', () => {
- const rows = [
- {
- tags: [['service', 'api']] as [string, string][],
- timestamp: 1700000000 as unknown as string,
- value: 42,
- },
- ];
- expect(formatVectorResponse(rows as any)).toEqual([
- { metric: { service: 'api' }, value: [1700000000, '42'] },
- ]);
- });
-
- it('converts string timestamps to unix seconds', () => {
- const rows = [
- {
- tags: [] as [string, string][],
- timestamp: '2023-11-14T22:13:20.000Z',
- value: 3,
- },
- ];
- expect(formatVectorResponse(rows as any)[0].value).toEqual([
- 1700000000,
- '3',
- ]);
- });
-});
-
describe('resolveExemplarWindow', () => {
const DAY = 24 * 60 * 60;
const end = 1_700_000_000;
diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts
index 01a716d329..0f0beb33ad 100644
--- a/packages/api/src/routers/api/prometheus.ts
+++ b/packages/api/src/routers/api/prometheus.ts
@@ -7,8 +7,6 @@ import z from 'zod';
import { ClickhouseClient } from '@/clickhouse';
import { getConnectionById } from '@/controllers/connection';
import {
- PROMETHEUS_MAX_EXECUTION_SEC,
- PROMETHEUS_MAX_RESULT_ROWS,
queryLabelNames,
queryLabelValues,
TimeSeriesTagsQueryArgs,
@@ -100,61 +98,6 @@ function getParams(req: express.Request): Record {
};
}
-// --------------------------
-// Prometheus-compatible response types
-// --------------------------
-
-type PrometheusMetric = Record;
-type PrometheusMatrixResult = {
- metric: PrometheusMetric;
- values: [number, string][];
-};
-type PrometheusVectorResult = {
- metric: PrometheusMetric;
- value: [number, string];
-};
-
-// --------------------------
-// ClickHouse → Prometheus response formatters
-// --------------------------
-
-export function formatMatrixResponse(
- rows: { tags: [string, string][]; time_series: [string, number][] }[],
-): PrometheusMatrixResult[] {
- return rows.map(row => {
- const metric: PrometheusMetric = {};
- for (const [key, value] of row.tags) {
- metric[key] = value;
- }
- const values: [number, string][] = row.time_series.map(
- ([timestamp, value]) => {
- const ts =
- typeof timestamp === 'string'
- ? new Date(timestamp).getTime() / 1000
- : Number(timestamp);
- return [ts, String(value)];
- },
- );
- return { metric, values };
- });
-}
-
-export function formatVectorResponse(
- rows: { tags: [string, string][]; timestamp: string; value: number }[],
-): PrometheusVectorResult[] {
- return rows.map(row => {
- const metric: PrometheusMetric = {};
- for (const [key, value] of row.tags) {
- metric[key] = value;
- }
- const ts =
- typeof row.timestamp === 'string'
- ? new Date(row.timestamp).getTime() / 1000
- : Number(row.timestamp);
- return { metric, value: [ts, String(row.value)] };
- });
-}
-
// --------------------------
// Prometheus proxy (for real Prometheus backends)
// --------------------------
@@ -238,8 +181,26 @@ const CALLER_SETTABLE_PARAM_KEYS = new Set([
'limit',
'timeout',
'stats',
+ // ClickHouse's prometheus_api_v1 handler selects the TimeSeries table from
+ // these; a real Prometheus ignores them.
+ 'database',
+ 'table',
]);
+// ClickHouse serves the Prometheus HTTP API under this prefix on its main HTTP
+// port when the `prometheus_api_v1` handler is configured (26.8+).
+const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1';
+
+function clickhouseAuthHeaders(connection: {
+ username: string;
+ password?: string;
+}): Record {
+ return {
+ 'X-ClickHouse-User': connection.username,
+ 'X-ClickHouse-Key': connection.password ?? '',
+ };
+}
+
// Forwards the response straight from the upstream Prometheus to the
// HyperDX client. Returns the HTTP status it wrote, so callers can record an
// error metric: this helper handles its own failures by writing 400/502/504 and
@@ -255,6 +216,7 @@ async function proxyToPrometheus(
path: string,
params: Record,
res: express.Response,
+ headers?: Record,
): Promise {
let url: URL;
try {
@@ -314,6 +276,7 @@ async function proxyToPrometheus(
let upstreamResp: Response;
try {
upstreamResp = await fetch(target, {
+ headers,
signal: AbortSignal.timeout(PROMETHEUS_PROXY_TIMEOUT_MS),
});
} catch (err) {
@@ -465,21 +428,21 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => {
return;
}
- // Otherwise, use ClickHouse prometheusQuery()
+ // ClickHouse: the Prometheus HTTP API is the only surface ClickHouse
+ // holds forward-compatible while TimeSeries is in preview, so the request
+ // is proxied as-is (database/table included) rather than translated to a
+ // table function.
backend = 'clickhouse';
- const start = parseTimestamp(params.start);
- const end = parseTimestamp(params.end);
- const step = parseDuration(params.step ?? '60s');
- const database = params.database ?? 'default';
- const table = params.table;
- if (!table) {
+ if (!params.table) {
return res.status(400).json({
status: 'error',
errorType: 'bad_data',
error: `table parameter required for querying clickhouse via promql`,
});
}
-
+ const start = parseTimestamp(params.start);
+ const end = parseTimestamp(params.end);
+ const step = parseDuration(params.step ?? '60s');
if (step <= 0 || (end - start) / step > PROMETHEUS_MAX_RESOLUTION) {
return res.status(400).json({
status: 'error',
@@ -487,46 +450,15 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => {
error: `exceeded maximum resolution of ${PROMETHEUS_MAX_RESOLUTION.toLocaleString('en-US')} points per timeseries. Try decreasing the query resolution (?step=XX)`,
});
}
-
- const client = new ClickhouseClient({
- host: connection.host,
- username: connection.username,
- password: connection.password,
- requestTimeout: PROMETHEUS_CH_TIMEOUT_MS,
- });
-
- const startMs = Math.floor(start * 1000);
- const endMs = Math.floor(end * 1000);
- const stepSec = Math.max(Math.floor(step), 1);
-
- const resp = await client.query({
- query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`,
- query_params: {
- db: database,
- table,
- expr: query,
- startMs,
- endMs,
- stepSec,
- },
- format: 'JSON',
- clickhouse_settings: {
- allow_experimental_time_series_table: 1,
- max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC,
- max_result_rows: String(PROMETHEUS_MAX_RESULT_ROWS),
- },
- });
-
- const json = await resp.json();
- const result = formatMatrixResponse(json.data);
-
- return res.json({
- status: 'success',
- data: {
- resultType: 'matrix',
- result,
- },
- });
+ const status = await proxyToPrometheus(
+ connection.host,
+ `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`,
+ params,
+ res,
+ clickhouseAuthHeaders(connection),
+ );
+ recordProxyOutcome(status, 'query_range', backend);
+ return;
} catch (e) {
prometheusQueryErrors.add(1, { endpoint: 'query_range', backend });
logger.error(e, 'Prometheus query_range error');
@@ -600,47 +532,22 @@ const queryHandler: express.RequestHandler = async (req, res) => {
}
backend = 'clickhouse';
- const time = params.time ? parseTimestamp(params.time) : undefined;
- const database = params.database ?? 'default';
- const table = params.table;
- if (!table) {
+ if (!params.table) {
return res.status(400).json({
status: 'error',
errorType: 'bad_data',
error: `table parameter required for querying clickhouse via promql`,
});
}
-
- const client = new ClickhouseClient({
- host: connection.host,
- username: connection.username,
- password: connection.password,
- requestTimeout: PROMETHEUS_CH_TIMEOUT_MS,
- });
-
- const evalMs = time ? Math.floor(time * 1000) : Date.now();
-
- const resp = await client.query({
- query: `SELECT tags, timestamp, value FROM prometheusQuery({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({evalMs:Int64})) SETTINGS allow_experimental_time_series_table = 1`,
- query_params: { db: database, table, expr: query, evalMs },
- format: 'JSON',
- clickhouse_settings: {
- allow_experimental_time_series_table: 1,
- max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC,
- max_result_rows: String(PROMETHEUS_MAX_RESULT_ROWS),
- },
- });
-
- const json = await resp.json();
- const result = formatVectorResponse(json.data);
-
- return res.json({
- status: 'success',
- data: {
- resultType: 'vector',
- result,
- },
- });
+ const status = await proxyToPrometheus(
+ connection.host,
+ `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`,
+ params,
+ res,
+ clickhouseAuthHeaders(connection),
+ );
+ recordProxyOutcome(status, 'query', backend);
+ return;
} catch (e) {
prometheusQueryErrors.add(1, { endpoint: 'query', backend });
logger.error(e, 'Prometheus query error');
diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml
index fecb0bf937..b8152ccc6c 100644
--- a/packages/app/tests/e2e/docker-compose.yml
+++ b/packages/app/tests/e2e/docker-compose.yml
@@ -15,7 +15,7 @@ services:
memory: 512M
cpus: '0.5'
ch-server:
- image: clickhouse/clickhouse-server:26.5-alpine
+ image: clickhouse/clickhouse-server:26.8-alpine
ports:
- ${HDX_E2E_CH_PORT:-20500}:8123 # http api
- ${HDX_E2E_CH_NATIVE_PORT:-20600}:9000 # native
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index 478a9a84a1..1e7b25588e 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -1189,7 +1189,6 @@ describe('Metadata Integration Tests', () => {
const byName = new Map(columns.map(c => [c.name, c]));
expect([...byName.keys()].sort()).toEqual([
- 'all_tags',
'id',
'max_time',
'metric_name',
@@ -1199,9 +1198,6 @@ describe('Metadata Integration Tests', () => {
expect(byName.get('min_time')?.type).toBe(
'SimpleAggregateFunction(min, Nullable(DateTime64(3)))',
);
- // Ephemeral, so it cannot be selected — the reason a caller has to
- // inspect the columns rather than assume the documented shape.
- expect(byName.get('all_tags')?.default_type).toBe('EPHEMERAL');
});
// What `store_min_time_and_max_time = 0` costs: the time-bound columns are
From 6bca6cbe994a2037384f93e7acb3ceccd1a3a41b Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Wed, 16 Sep 2026 15:56:44 -0400
Subject: [PATCH 2/7] fix e2e tests
---
packages/app/tests/e2e/seed-clickhouse.ts | 81 +++++++++--------------
1 file changed, 31 insertions(+), 50 deletions(-)
diff --git a/packages/app/tests/e2e/seed-clickhouse.ts b/packages/app/tests/e2e/seed-clickhouse.ts
index e910e34fa4..98c537dc6f 100644
--- a/packages/app/tests/e2e/seed-clickhouse.ts
+++ b/packages/app/tests/e2e/seed-clickhouse.ts
@@ -577,7 +577,7 @@ function generateK8sGaugeMetrics(
: startMs;
for (let sample = 0; sample < samplesPerPod; sample++) {
const timestampMs = podSlotStart + sample * stepMs;
- const timestampNs = timestampMs * 1000000;
+ const timestampNs = Math.round(timestampMs * 1000000);
const _timeUnix = timestampMs / 1000;
// CPU metrics (percentage, 0-100)
@@ -594,39 +594,39 @@ function generateK8sGaugeMetrics(
// k8s.pod.phase
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.phase', 'Pod phase', '', {}, ${timestampNs}, ${timestampNs}, ${phase}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.phase', 'Pod phase', '', {}, '${timestampNs}', '${timestampNs}', ${phase}, 0, [], [], [], [], [])`,
);
// k8s.container.restarts
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.container.restarts', 'Container restarts', '', {}, ${timestampNs}, ${timestampNs}, ${restarts}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.container.restarts', 'Container restarts', '', {}, '${timestampNs}', '${timestampNs}', ${restarts}, 0, [], [], [], [], [])`,
);
// container.cpu.utilization (0-100%)
const containerCpuUtilization =
5 + (podIdx % 30) + Math.sin(sample / 3) * 3;
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'container.cpu.utilization', 'Container CPU utilization', '%', {}, ${timestampNs}, ${timestampNs}, ${containerCpuUtilization}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'container.cpu.utilization', 'Container CPU utilization', '%', {}, '${timestampNs}', '${timestampNs}', ${containerCpuUtilization}, 0, [], [], [], [], [])`,
);
// k8s.pod.cpu.utilization
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.cpu.utilization', 'Pod CPU utilization', '', {}, ${timestampNs}, ${timestampNs}, ${cpuUsage}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.cpu.utilization', 'Pod CPU utilization', '', {}, '${timestampNs}', '${timestampNs}', ${cpuUsage}, 0, [], [], [], [], [])`,
);
// k8s.pod.cpu_limit_utilization
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.cpu_limit_utilization', 'Pod CPU limit utilization', '%', {}, ${timestampNs}, ${timestampNs}, ${cpuLimitUtilization}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.cpu_limit_utilization', 'Pod CPU limit utilization', '%', {}, '${timestampNs}', '${timestampNs}', ${cpuLimitUtilization}, 0, [], [], [], [], [])`,
);
// k8s.pod.memory.usage
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.memory.usage', 'Pod memory usage', 'bytes', {}, ${timestampNs}, ${timestampNs}, ${memoryUsage}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.memory.usage', 'Pod memory usage', 'bytes', {}, '${timestampNs}', '${timestampNs}', ${memoryUsage}, 0, [], [], [], [], [])`,
);
// k8s.pod.memory_limit_utilization
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.memory_limit_utilization', 'Pod memory limit utilization', '%', {}, ${timestampNs}, ${timestampNs}, ${memoryLimitUtilization}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.memory_limit_utilization', 'Pod memory limit utilization', '%', {}, '${timestampNs}', '${timestampNs}', ${memoryLimitUtilization}, 0, [], [], [], [], [])`,
);
}
}
@@ -643,7 +643,7 @@ function generateK8sGaugeMetrics(
for (let sample = 0; sample < samplesPerPod; sample++) {
const timestampMs = nodeSlotStart + sample * stepMs;
- const timestampNs = timestampMs * 1000000;
+ const timestampNs = Math.round(timestampMs * 1000000);
const nodeCpuUsage = 30 + (nodeIdx % 30) + Math.sin(sample / 2) * 10;
const nodeMemoryUsage = (2 + nodeIdx) * 1024 * 1024 * 1024; // 2-4 GB
@@ -653,17 +653,17 @@ function generateK8sGaugeMetrics(
// k8s.node.cpu.utilization
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.node.cpu.utilization', 'Node CPU utilization', '', {}, ${timestampNs}, ${timestampNs}, ${nodeCpuUsage}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.node.cpu.utilization', 'Node CPU utilization', '', {}, '${timestampNs}', '${timestampNs}', ${nodeCpuUsage}, 0, [], [], [], [], [])`,
);
// k8s.node.memory.usage
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.node.memory.usage', 'Node memory usage', 'bytes', {}, ${timestampNs}, ${timestampNs}, ${nodeMemoryUsage}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.node.memory.usage', 'Node memory usage', 'bytes', {}, '${timestampNs}', '${timestampNs}', ${nodeMemoryUsage}, 0, [], [], [], [], [])`,
);
// k8s.node.condition_ready
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.node.condition_ready', 'Node condition ready', '', {}, ${timestampNs}, ${timestampNs}, ${nodeConditionReady}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.node.condition_ready', 'Node condition ready', '', {}, '${timestampNs}', '${timestampNs}', ${nodeConditionReady}, 0, [], [], [], [], [])`,
);
}
}
@@ -681,13 +681,13 @@ function generateK8sGaugeMetrics(
for (let sample = 0; sample < samplesPerPod; sample++) {
const timestampMs = nsSlotStart + sample * stepMs;
- const timestampNs = timestampMs * 1000000;
+ const timestampNs = Math.round(timestampMs * 1000000);
const resourceAttrs = `{'k8s.cluster.name':'${cluster}','k8s.namespace.name':'${namespace}'}`;
// k8s.namespace.phase
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.namespace.phase', 'Namespace phase', '', {}, ${timestampNs}, ${timestampNs}, ${namespacePhase}, 0, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.namespace.phase', 'Namespace phase', '', {}, '${timestampNs}', '${timestampNs}', ${namespacePhase}, 0, [], [], [], [], [])`,
);
}
}
@@ -712,7 +712,7 @@ function generateK8sSumMetrics(
const timestampMs =
podCount > 1 ? startMs + (podIdx / (podCount - 1)) * span : startMs;
- const timestampNs = timestampMs * 1000000;
+ const timestampNs = Math.round(timestampMs * 1000000);
// Pod uptime in seconds (1-10 hours)
const uptimeSeconds = (1 + (podIdx % 10)) * 3600;
@@ -722,7 +722,7 @@ function generateK8sSumMetrics(
// k8s.pod.uptime (Sum metric)
// AggregationTemporality: 1 = Delta, 2 = Cumulative
rows.push(
- `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.uptime', 'Pod uptime', 's', {}, ${timestampNs}, ${timestampNs}, ${uptimeSeconds}, 0, 2, true, [], [], [], [], [])`,
+ `(${resourceAttrs}, '', '', '', {}, 0, '', 'k8s-metrics', 'k8s.pod.uptime', 'Pod uptime', 's', {}, '${timestampNs}', '${timestampNs}', ${uptimeSeconds}, 0, 2, true, [], [], [], [], [])`,
);
}
@@ -1105,54 +1105,35 @@ export async function seedClickHouse(): Promise {
}
/**
- * Seed one `E2E_PROMQL_METRIC_NAME` series per service into the TimeSeries
- * table, through the `timeSeries*` table functions that expose its inner
- * tables (the engine names those after the table's UUID, so they can't be
- * addressed directly).
- *
- * A series' identity lives in the tags table while its samples live in the data
- * table, tied together by `id`. Rather than recompute the engine's
- * `reinterpretAsUUID(sipHash128(metric_name, all_tags))` here — where drifting
- * from the schema would silently orphan every sample — the samples select their
- * `id` back out of the tags table.
+ * Seed one `E2E_PROMQL_METRIC_NAME` series per service, one sample a minute
+ * across the window. Written through the TimeSeries table itself, not its inner
+ * tables: the engine derives each series' `id` and splits samples across inner
+ * tables in ways that change between ClickHouse versions, and rows written
+ * around it are invisible to the PromQL engine.
*/
async function seedPromqlSeries(
client: ReturnType,
startMs: number,
endMs: number,
) {
- const ts = (part: 'Tags' | 'Data' | 'Metrics') =>
- `timeSeries${part}('${E2E_CLICKHOUSE_DATABASE}', '${E2E_PROMQL_TABLE}')`;
+ const table = `${E2E_CLICKHOUSE_DATABASE}.${E2E_PROMQL_TABLE}`;
await client.query(`
- INSERT INTO FUNCTION ${ts('Metrics')} (metric_family_name, type, unit, help)
+ INSERT INTO ${table} (metric_family, type, unit, help)
VALUES ('${E2E_PROMQL_METRIC_NAME}', 'gauge', '', 'E2E service liveness')
`);
- const tagRows = SERVICES.map(
- service =>
- `('${E2E_PROMQL_METRIC_NAME}', map('service', '${service}'), ` +
- `map('__name__', '${E2E_PROMQL_METRIC_NAME}', 'service', '${service}'), ` +
- `fromUnixTimestamp64Milli(toInt64(${startMs})), fromUnixTimestamp64Milli(toInt64(${endMs})))`,
- ).join(',\n');
-
- await client.query(`
- INSERT INTO FUNCTION ${ts('Tags')} (metric_name, tags, all_tags, min_time, max_time)
- VALUES ${tagRows}
- `);
-
- // One sample a minute across the seeded window, for every series.
const stepMs = 60000;
const sampleCount = Math.max(1, Math.floor((endMs - startMs) / stepMs));
+ const rows = SERVICES.map(
+ service =>
+ `SELECT '${E2E_PROMQL_METRIC_NAME}', map('service', '${service}'), ` +
+ `arrayMap(n -> (fromUnixTimestamp64Milli(toInt64(${startMs} + n * ${stepMs})), 1.0), range(${sampleCount}))`,
+ ).join('\n UNION ALL\n ');
+
await client.query(`
- INSERT INTO FUNCTION ${ts('Data')} (id, timestamp, value)
- SELECT
- t.id,
- fromUnixTimestamp64Milli(toInt64(${startMs} + (n * ${stepMs}))),
- 1
- FROM ${ts('Tags')} AS t
- CROSS JOIN (SELECT number AS n FROM numbers(${sampleCount})) AS steps
- WHERE t.metric_name = '${E2E_PROMQL_METRIC_NAME}'
+ INSERT INTO ${table} (metric_name, tags, time_series)
+ ${rows}
`);
}
From 968d89e1874667f9b982c32dbed1b76768d7aba3 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Wed, 16 Sep 2026 17:07:19 -0400
Subject: [PATCH 3/7] fix suggestions
---
docker/clickhouse/local/config.xml | 2 +-
docker/hyperdx/clickhouseConfig.xml | 10 ++++++
.../api/src/controllers/timeseriesEngine.ts | 4 +--
.../api/__tests__/prometheus.int.test.ts | 31 +++++++++++++++++++
packages/api/src/routers/api/prometheus.ts | 25 +++++++++++++--
5 files changed, 67 insertions(+), 5 deletions(-)
diff --git a/docker/clickhouse/local/config.xml b/docker/clickhouse/local/config.xml
index 3e70c297cb..9257ef6b0c 100644
--- a/docker/clickhouse/local/config.xml
+++ b/docker/clickhouse/local/config.xml
@@ -77,13 +77,13 @@
port. database/table come from request query params so one handler
serves any TimeSeries table. -->
-
/prometheus/api/v1
prometheus_api_v1
+
diff --git a/docker/hyperdx/clickhouseConfig.xml b/docker/hyperdx/clickhouseConfig.xml
index 50cab2ce7f..db8bbc1dc5 100644
--- a/docker/hyperdx/clickhouseConfig.xml
+++ b/docker/hyperdx/clickhouseConfig.xml
@@ -47,6 +47,16 @@
+
+
+ /prometheus/api/v1
+
+ prometheus_api_v1
+
+
+
+
+
system
diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts
index 471964d75a..b1c4b2b95e 100644
--- a/packages/api/src/controllers/timeseriesEngine.ts
+++ b/packages/api/src/controllers/timeseriesEngine.ts
@@ -11,8 +11,8 @@ import {
import { ClickhouseClient } from '@/clickhouse';
import logger from '@/utils/logger';
-const PROMETHEUS_MAX_EXECUTION_SEC = 30;
-const PROMETHEUS_MAX_RESULT_ROWS = 100000;
+export const PROMETHEUS_MAX_EXECUTION_SEC = 30;
+export const PROMETHEUS_MAX_RESULT_ROWS = 100000;
export type TimeSeriesTagsQueryArgs = {
client: ClickhouseClient;
diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
index 5a32b1326b..133ee3374a 100644
--- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
+++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
@@ -714,6 +714,8 @@ describe('prometheus router', () => {
);
expect(calledUrl).toContain('database=otel');
expect(calledUrl).toContain('table=metrics_ts');
+ expect(calledUrl).toContain('max_execution_time=30');
+ expect(calledUrl).toContain('max_result_rows=100000');
expect(calledUrl).not.toContain('connectionId');
expect(init?.headers).toMatchObject({
'X-ClickHouse-User': config.CLICKHOUSE_USER,
@@ -721,6 +723,33 @@ describe('prometheus router', () => {
});
});
+ it('does not let the request loosen the pinned ClickHouse limits', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+
+ await agent
+ .get('/v1/prometheus/query_range')
+ .query({
+ query: 'up',
+ start: '1700000000',
+ end: '1700000060',
+ step: '15s',
+ table: 'metrics_ts',
+ max_execution_time: '9999',
+ max_result_rows: '0',
+ connectionId: conn._id.toString(),
+ })
+ .expect(200);
+
+ const calledUrl = new URL(String(mockFetch.mock.calls[0][0]));
+ expect(calledUrl.searchParams.getAll('max_execution_time')).toEqual([
+ '30',
+ ]);
+ expect(calledUrl.searchParams.getAll('max_result_rows')).toEqual([
+ '100000',
+ ]);
+ });
+
it('returns 400 before proxying when a ClickHouse connection has no table', async () => {
const { agent, team } = await getLoggedInAgent(server);
const conn = await seedClickHouseConnection(team._id);
@@ -837,6 +866,8 @@ describe('prometheus router', () => {
`${config.CLICKHOUSE_HOST}/prometheus/api/v1/query?`,
);
expect(calledUrl).toContain('table=metrics_ts');
+ expect(calledUrl).toContain('max_execution_time=30');
+ expect(calledUrl).toContain('max_result_rows=100000');
expect(init?.headers).toMatchObject({
'X-ClickHouse-User': config.CLICKHOUSE_USER,
});
diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts
index 0f0beb33ad..0e22886e1f 100644
--- a/packages/api/src/routers/api/prometheus.ts
+++ b/packages/api/src/routers/api/prometheus.ts
@@ -7,6 +7,8 @@ import z from 'zod';
import { ClickhouseClient } from '@/clickhouse';
import { getConnectionById } from '@/controllers/connection';
import {
+ PROMETHEUS_MAX_EXECUTION_SEC,
+ PROMETHEUS_MAX_RESULT_ROWS,
queryLabelNames,
queryLabelValues,
TimeSeriesTagsQueryArgs,
@@ -191,6 +193,25 @@ const CALLER_SETTABLE_PARAM_KEYS = new Set([
// port when the `prometheus_api_v1` handler is configured (26.8+).
const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1';
+/**
+ * The ClickHouse connection host with the query limits pinned as ClickHouse
+ * HTTP settings. The prometheus_api_v1 handler honours settings given in the
+ * URL and reports a breach as a Prometheus 400, so this is the only place to
+ * bound a PromQL evaluation — the request carries no SQL to attach SETTINGS to.
+ * Pinned on the host rather than passed as params so the merge in
+ * proxyToPrometheus (which lets the caller override any allowlisted key) can't
+ * loosen them.
+ */
+function clickhousePrometheusUpstream(host: string): string {
+ const url = new URL(host);
+ url.searchParams.set(
+ 'max_execution_time',
+ String(PROMETHEUS_MAX_EXECUTION_SEC),
+ );
+ url.searchParams.set('max_result_rows', String(PROMETHEUS_MAX_RESULT_ROWS));
+ return url.toString();
+}
+
function clickhouseAuthHeaders(connection: {
username: string;
password?: string;
@@ -451,7 +472,7 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => {
});
}
const status = await proxyToPrometheus(
- connection.host,
+ clickhousePrometheusUpstream(connection.host),
`${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`,
params,
res,
@@ -540,7 +561,7 @@ const queryHandler: express.RequestHandler = async (req, res) => {
});
}
const status = await proxyToPrometheus(
- connection.host,
+ clickhousePrometheusUpstream(connection.host),
`${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`,
params,
res,
From acf6568eb66ce4bca86e8c7d21c248647d7913f1 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 17 Sep 2026 14:51:55 -0400
Subject: [PATCH 4/7] feat: intelligent fallback for timeseries tables
---
.../__tests__/timeseriesEngine.test.ts | 47 +++++
.../api/src/controllers/timeseriesEngine.ts | 161 +++++++++++++++++
.../api/__tests__/prometheus.int.test.ts | 158 +++++++++++++++--
packages/api/src/routers/api/prometheus.ts | 165 ++++++++++++++----
.../src/__tests__/clickhouseVersion.test.ts | 24 +++
.../src/core/clickhouseVersion.ts | 8 +
packages/common-utils/src/core/metadata.ts | 26 +++
7 files changed, 549 insertions(+), 40 deletions(-)
create mode 100644 packages/api/src/controllers/__tests__/timeseriesEngine.test.ts
diff --git a/packages/api/src/controllers/__tests__/timeseriesEngine.test.ts b/packages/api/src/controllers/__tests__/timeseriesEngine.test.ts
new file mode 100644
index 0000000000..f1f0d2b36e
--- /dev/null
+++ b/packages/api/src/controllers/__tests__/timeseriesEngine.test.ts
@@ -0,0 +1,47 @@
+import {
+ formatMatrixResponse,
+ formatVectorResponse,
+} from '@/controllers/timeseriesEngine';
+
+describe('formatMatrixResponse', () => {
+ it('converts prometheusQueryRange rows into Prometheus matrix shape', () => {
+ const rows = [
+ {
+ tags: [
+ ['__name__', 'http_requests_total'],
+ ['method', 'GET'],
+ ] as [string, string][],
+ samples: [
+ [1700000000, 5],
+ ['2023-11-14T22:14:20.000Z', 7],
+ ] as [string | number, number][],
+ },
+ ];
+ expect(formatMatrixResponse(rows as any)).toEqual([
+ {
+ metric: { __name__: 'http_requests_total', method: 'GET' },
+ values: [
+ [1700000000, '5'],
+ [1700000060, '7'],
+ ],
+ },
+ ]);
+ });
+
+ it('returns empty array for empty input', () => {
+ expect(formatMatrixResponse([])).toEqual([]);
+ });
+});
+
+describe('formatVectorResponse', () => {
+ it('converts prometheusQuery rows into Prometheus vector shape', () => {
+ const rows = [
+ { tags: [['service', 'api']], timestamp: 1700000000, value: 42 },
+ { tags: [], timestamp: '2023-11-14T22:13:20.000Z', value: 3 },
+ ];
+ expect(formatVectorResponse(rows as any)).toEqual([
+ { metric: { service: 'api' }, value: [1700000000, '42'] },
+ { metric: {}, value: [1700000000, '3'] },
+ ]);
+ });
+});
diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts
index b1c4b2b95e..ad427a8806 100644
--- a/packages/api/src/controllers/timeseriesEngine.ts
+++ b/packages/api/src/controllers/timeseriesEngine.ts
@@ -3,7 +3,9 @@ import {
chSql,
concatChSql,
} from '@hyperdx/common-utils/dist/clickhouse';
+import { supportsPrometheusHttpApi } from '@hyperdx/common-utils/dist/core/clickhouseVersion';
import {
+ getMetadata,
Metadata,
MetadataCache,
} from '@hyperdx/common-utils/dist/core/metadata';
@@ -248,3 +250,162 @@ export async function queryLabelNames({
return queryDistinctTagsValues({ ...args, value, conditions, limit });
}
+
+// --------------------------
+// PromQL via table functions (ClickHouse < 26.6)
+// --------------------------
+//
+// Servers without the `prometheus_api_v1` HTTP handler can still evaluate
+// PromQL through `prometheusQuery`/`prometheusQueryRange`. ClickHouse only
+// holds the HTTP API forward-compatible while TimeSeries is in preview, so
+// this path is a fallback for old servers, not the primary route.
+
+type PrometheusMetric = Record;
+export type PrometheusMatrixResult = {
+ metric: PrometheusMetric;
+ values: [number, string][];
+};
+export type PrometheusVectorResult = {
+ metric: PrometheusMetric;
+ value: [number, string];
+};
+
+const toUnixSeconds = (timestamp: string | number) =>
+ typeof timestamp === 'string'
+ ? new Date(timestamp).getTime() / 1000
+ : Number(timestamp);
+
+export function formatMatrixResponse(
+ rows: { tags: [string, string][]; samples: [string, number][] }[],
+): PrometheusMatrixResult[] {
+ return rows.map(row => ({
+ metric: Object.fromEntries(row.tags),
+ values: row.samples.map(([ts, value]) => [
+ toUnixSeconds(ts),
+ String(value),
+ ]),
+ }));
+}
+
+/**
+ * The name of the per-series samples column, which the engine renamed from
+ * `time_series` to `samples` in TimeSeries schema version 3. The rename
+ * follows the table's pinned version, not the server's, so a v2 table on a new
+ * server still reads `time_series`. `prometheusQueryRange` returns whichever
+ * the table uses.
+ */
+async function timeSeriesSamplesColumn({
+ client,
+ connectionId,
+ databaseName,
+ tableName,
+}: {
+ client: ClickhouseClient;
+ connectionId: string;
+ databaseName: string;
+ tableName: string;
+}): Promise<'samples' | 'time_series'> {
+ const version = await getMetadata(client).getTimeSeriesTableVersion({
+ connectionId,
+ databaseName,
+ tableName,
+ });
+ return version >= 3 ? 'samples' : 'time_series';
+}
+
+export function formatVectorResponse(
+ rows: { tags: [string, string][]; timestamp: string; value: number }[],
+): PrometheusVectorResult[] {
+ return rows.map(row => ({
+ metric: Object.fromEntries(row.tags),
+ value: [toUnixSeconds(row.timestamp), String(row.value)],
+ }));
+}
+
+const PROMQL_TABLE_FUNCTION_SETTINGS = {
+ allow_experimental_time_series_table: 1,
+ max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC,
+ max_result_rows: String(PROMETHEUS_MAX_RESULT_ROWS),
+} as const;
+
+export async function queryRangeViaTableFunction({
+ client,
+ connectionId,
+ databaseName,
+ tableName,
+ expr,
+ startMs,
+ endMs,
+ stepSec,
+}: {
+ client: ClickhouseClient;
+ connectionId: string;
+ databaseName: string;
+ tableName: string;
+ expr: string;
+ startMs: number;
+ endMs: number;
+ stepSec: number;
+}): Promise {
+ const samplesColumn = await timeSeriesSamplesColumn({
+ client,
+ connectionId,
+ databaseName,
+ tableName,
+ });
+ const resp = await client.query({
+ query: `SELECT tags, ${samplesColumn} AS samples FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`,
+ query_params: {
+ db: databaseName,
+ table: tableName,
+ expr,
+ startMs,
+ endMs,
+ stepSec,
+ },
+ format: 'JSON',
+ clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS,
+ });
+ const json = await resp.json();
+ return formatMatrixResponse(json.data);
+}
+
+export async function queryInstantViaTableFunction({
+ client,
+ databaseName,
+ tableName,
+ expr,
+ evalMs,
+}: {
+ client: ClickhouseClient;
+ databaseName: string;
+ tableName: string;
+ expr: string;
+ evalMs: number;
+}): Promise {
+ const resp = await client.query({
+ query: `SELECT tags, timestamp, value FROM prometheusQuery({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({evalMs:Int64})) SETTINGS allow_experimental_time_series_table = 1`,
+ query_params: { db: databaseName, table: tableName, expr, evalMs },
+ format: 'JSON',
+ clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS,
+ });
+ const json = await resp.json();
+ return formatVectorResponse(json.data);
+}
+
+/**
+ * Whether the connection's ClickHouse serves the Prometheus HTTP API. The
+ * version is cached per connection in the process-wide metadata cache; an
+ * unknown version is treated as old, matching the other version-gated features.
+ */
+export async function connectionSupportsPrometheusHttpApi({
+ client,
+ connectionId,
+}: {
+ client: ClickhouseClient;
+ connectionId: string;
+}): Promise {
+ return supportsPrometheusHttpApi(
+ await getMetadata(client).getServerVersion({ connectionId }),
+ );
+}
diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
index 133ee3374a..f154aa91c7 100644
--- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
+++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
@@ -1,3 +1,4 @@
+import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
import { Types } from 'mongoose';
import * as config from '@/config';
@@ -43,6 +44,12 @@ function fakeUpstreamResponse(
} as unknown as Response;
}
+// What ClickHouse's prometheus_api_v1 handler answers the router's
+// `format_query` probe with. Queue this before the real upstream response in any
+// test that expects a ClickHouse connection to be proxied.
+const handlerProbeResponse = () =>
+ fakeUpstreamResponse({ status: 'success', data: 'up' });
+
describe('prometheus router', () => {
const server = getServer();
@@ -692,7 +699,9 @@ describe('prometheus router', () => {
status: 'success',
data: { resultType: 'matrix', result: [] },
};
- mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse));
+ mockFetch
+ .mockResolvedValueOnce(handlerProbeResponse())
+ .mockResolvedValueOnce(fakeUpstreamResponse(promResponse));
const res = await agent
.get('/v1/prometheus/query_range')
@@ -708,7 +717,15 @@ describe('prometheus router', () => {
.expect(200);
expect(res.body).toEqual(promResponse);
- const [calledUrl, init] = mockFetch.mock.calls[0];
+ expect(mockFetch).toHaveBeenCalledTimes(2);
+ const [probeUrl, probeInit] = mockFetch.mock.calls[0];
+ expect(String(probeUrl)).toContain(
+ `${config.CLICKHOUSE_HOST}/prometheus/api/v1/format_query?query=up`,
+ );
+ expect(probeInit?.headers).toMatchObject({
+ 'X-ClickHouse-User': config.CLICKHOUSE_USER,
+ });
+ const [calledUrl, init] = mockFetch.mock.calls[1];
expect(calledUrl).toContain(
`${config.CLICKHOUSE_HOST}/prometheus/api/v1/query_range`,
);
@@ -726,6 +743,7 @@ describe('prometheus router', () => {
it('does not let the request loosen the pinned ClickHouse limits', async () => {
const { agent, team } = await getLoggedInAgent(server);
const conn = await seedClickHouseConnection(team._id);
+ mockFetch.mockResolvedValueOnce(handlerProbeResponse());
await agent
.get('/v1/prometheus/query_range')
@@ -741,7 +759,7 @@ describe('prometheus router', () => {
})
.expect(200);
- const calledUrl = new URL(String(mockFetch.mock.calls[0][0]));
+ const calledUrl = new URL(String(mockFetch.mock.calls[1][0]));
expect(calledUrl.searchParams.getAll('max_execution_time')).toEqual([
'30',
]);
@@ -750,6 +768,124 @@ describe('prometheus router', () => {
]);
});
+ it('falls back to prometheusQueryRange() when the server has no prometheus_api_v1 handler', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+ // A stock ClickHouse answers the probe with a plain-text 404.
+ mockFetch.mockResolvedValueOnce({
+ status: 404,
+ text: jest
+ .fn()
+ .mockResolvedValue(
+ 'There is no handle /prometheus/api/v1/format_query',
+ ),
+ } as unknown as Response);
+
+ const res = await agent
+ .get('/v1/prometheus/query_range')
+ .query({
+ query: 'up',
+ start: '1700000000',
+ end: '1700000060',
+ step: '60s',
+ table: 'no_such_table_for_fallback',
+ connectionId: conn._id.toString(),
+ })
+ .expect(400);
+
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ // The table function ran (and failed on the missing table); the
+ // upstream was never asked to evaluate the query.
+ expect(res.body.error).toMatch(/no_such_table_for_fallback/);
+ });
+
+ describe('on ClickHouse without the Prometheus HTTP API (< 26.6)', () => {
+ const TABLE = 'prom_query_range_fallback';
+ const START = 1700000000;
+
+ beforeAll(async () => {
+ await seedTimeSeriesTagsTable({
+ table: TABLE,
+ series: [
+ {
+ metricName: 'fallback_up',
+ tags: { job: 'api' },
+ startSec: START,
+ endSec: START + 60,
+ },
+ ],
+ withSamples: true,
+ });
+ });
+
+ afterAll(async () => {
+ await dropTimeSeriesTable({ table: TABLE });
+ });
+
+ beforeEach(() => {
+ jest
+ .spyOn(Metadata.prototype, 'getServerVersion')
+ .mockResolvedValue([26, 5, 0, 0]);
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ it('evaluates through prometheusQueryRange() instead of proxying', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+
+ const res = await agent
+ .get('/v1/prometheus/query_range')
+ .query({
+ query: 'fallback_up',
+ start: String(START),
+ end: String(START + 60),
+ step: '60s',
+ database: DEFAULT_DATABASE,
+ table: TABLE,
+ connectionId: conn._id.toString(),
+ })
+ .expect(200);
+
+ expect(mockFetch).not.toHaveBeenCalled();
+ expect(res.body.status).toBe('success');
+ expect(res.body.data.resultType).toBe('matrix');
+ expect(res.body.data.result).toEqual([
+ {
+ metric: { __name__: 'fallback_up', job: 'api' },
+ values: expect.arrayContaining([[START, '1']]),
+ },
+ ]);
+ });
+
+ it('evaluates an instant query through prometheusQuery()', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+
+ const res = await agent
+ .get('/v1/prometheus/query')
+ .query({
+ query: 'fallback_up',
+ time: String(START + 60),
+ database: DEFAULT_DATABASE,
+ table: TABLE,
+ connectionId: conn._id.toString(),
+ })
+ .expect(200);
+
+ expect(mockFetch).not.toHaveBeenCalled();
+ expect(res.body.data.resultType).toBe('vector');
+ expect(res.body.data.result).toEqual([
+ {
+ metric: { __name__: 'fallback_up', job: 'api' },
+ value: [START + 60, '1'],
+ },
+ ]);
+ });
+ });
+
it('returns 400 before proxying when a ClickHouse connection has no table', async () => {
const { agent, team } = await getLoggedInAgent(server);
const conn = await seedClickHouseConnection(team._id);
@@ -845,12 +981,14 @@ describe('prometheus router', () => {
it('proxies a ClickHouse connection to its prometheus_api_v1 handler', async () => {
const { agent, team } = await getLoggedInAgent(server);
const conn = await seedClickHouseConnection(team._id);
- mockFetch.mockResolvedValueOnce(
- fakeUpstreamResponse({
- status: 'success',
- data: { resultType: 'vector', result: [] },
- }),
- );
+ mockFetch
+ .mockResolvedValueOnce(handlerProbeResponse())
+ .mockResolvedValueOnce(
+ fakeUpstreamResponse({
+ status: 'success',
+ data: { resultType: 'vector', result: [] },
+ }),
+ );
await agent
.get('/v1/prometheus/query')
@@ -861,7 +999,7 @@ describe('prometheus router', () => {
})
.expect(200);
- const [calledUrl, init] = mockFetch.mock.calls[0];
+ const [calledUrl, init] = mockFetch.mock.calls[1];
expect(calledUrl).toContain(
`${config.CLICKHOUSE_HOST}/prometheus/api/v1/query?`,
);
diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts
index 0e22886e1f..a2b52e9ab1 100644
--- a/packages/api/src/routers/api/prometheus.ts
+++ b/packages/api/src/routers/api/prometheus.ts
@@ -7,10 +7,13 @@ import z from 'zod';
import { ClickhouseClient } from '@/clickhouse';
import { getConnectionById } from '@/controllers/connection';
import {
+ connectionSupportsPrometheusHttpApi,
PROMETHEUS_MAX_EXECUTION_SEC,
PROMETHEUS_MAX_RESULT_ROWS,
+ queryInstantViaTableFunction,
queryLabelNames,
queryLabelValues,
+ queryRangeViaTableFunction,
TimeSeriesTagsQueryArgs,
} from '@/controllers/timeseriesEngine';
import { getNonNullUserWithTeam } from '@/middleware/auth';
@@ -212,6 +215,71 @@ function clickhousePrometheusUpstream(host: string): string {
return url.toString();
}
+function newClickhouseClient(connection: {
+ host: string;
+ username: string;
+ password?: string;
+}) {
+ return new ClickhouseClient({
+ host: connection.host,
+ username: connection.username,
+ password: connection.password,
+ requestTimeout: PROMETHEUS_CH_TIMEOUT_MS,
+ });
+}
+
+/**
+ * Whether a ClickHouse connection answers PromQL over HTTP. Two conditions,
+ * cheapest first: the server is new enough (26.6+), and the prometheus_api_v1
+ * handler is actually configured — a self-managed server needs an
+ * `` rule for that, which a stock install lacks. The handler
+ * answers every request, error or not, with Prometheus JSON (`{status, ...}`);
+ * a server without it answers a plain-text 404. `format_query` is used as the
+ * probe because it is the one endpoint that needs no table.
+ *
+ * ponytail: per-process TTL map, so a handler enabled later is picked up within
+ * PROBE_TTL_MS. Keyed by host too, so editing the connection re-probes.
+ */
+const promHttpApiProbe = new Map();
+const PROBE_TTL_MS = 5 * 60_000;
+const PROBE_TIMEOUT_MS = 5_000;
+
+async function clickhouseServesPrometheusHttpApi(
+ client: ClickhouseClient,
+ connection: { id: string; host: string; username: string; password?: string },
+): Promise {
+ if (
+ !(await connectionSupportsPrometheusHttpApi({
+ client,
+ connectionId: connection.id,
+ }))
+ ) {
+ return false;
+ }
+
+ const key = `${connection.id}:${connection.host}`;
+ const cached = promHttpApiProbe.get(key);
+ if (cached && cached.expires > Date.now()) return cached.ok;
+
+ let ok = false;
+ try {
+ const url = joinPrometheusUpstreamUrl(
+ connection.host,
+ `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/format_query`,
+ );
+ url.searchParams.set('query', 'up');
+ const resp = await fetch(url, {
+ headers: clickhouseAuthHeaders(connection),
+ signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+ });
+ ok = typeof JSON.parse(await resp.text())?.status === 'string';
+ } catch {
+ ok = false;
+ }
+ promHttpApiProbe.set(key, { ok, expires: Date.now() + PROBE_TTL_MS });
+ return ok;
+}
+
function clickhouseAuthHeaders(connection: {
username: string;
password?: string;
@@ -449,12 +517,9 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => {
return;
}
- // ClickHouse: the Prometheus HTTP API is the only surface ClickHouse
- // holds forward-compatible while TimeSeries is in preview, so the request
- // is proxied as-is (database/table included) rather than translated to a
- // table function.
backend = 'clickhouse';
- if (!params.table) {
+ const table = params.table;
+ if (!table) {
return res.status(400).json({
status: 'error',
errorType: 'bad_data',
@@ -471,15 +536,39 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => {
error: `exceeded maximum resolution of ${PROMETHEUS_MAX_RESOLUTION.toLocaleString('en-US')} points per timeseries. Try decreasing the query resolution (?step=XX)`,
});
}
- const status = await proxyToPrometheus(
- clickhousePrometheusUpstream(connection.host),
- `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`,
- params,
- res,
- clickhouseAuthHeaders(connection),
- );
- recordProxyOutcome(status, 'query_range', backend);
- return;
+
+ const client = newClickhouseClient(connection);
+
+ // The Prometheus HTTP API is the only surface ClickHouse holds
+ // forward-compatible while TimeSeries is in preview, so it is preferred
+ // wherever the server has it (26.6+); the table function is the fallback.
+ if (await clickhouseServesPrometheusHttpApi(client, connection)) {
+ const status = await proxyToPrometheus(
+ clickhousePrometheusUpstream(connection.host),
+ `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`,
+ params,
+ res,
+ clickhouseAuthHeaders(connection),
+ );
+ recordProxyOutcome(status, 'query_range', backend);
+ return;
+ }
+
+ // fallback for clickhouse version < 26.6
+ const result = await queryRangeViaTableFunction({
+ client,
+ connectionId,
+ databaseName: params.database ?? 'default',
+ tableName: table,
+ expr: query,
+ startMs: Math.floor(start * 1000),
+ endMs: Math.floor(end * 1000),
+ stepSec: Math.max(Math.floor(step), 1),
+ });
+ return res.json({
+ status: 'success',
+ data: { resultType: 'matrix', result },
+ });
} catch (e) {
prometheusQueryErrors.add(1, { endpoint: 'query_range', backend });
logger.error(e, 'Prometheus query_range error');
@@ -553,22 +642,43 @@ const queryHandler: express.RequestHandler = async (req, res) => {
}
backend = 'clickhouse';
- if (!params.table) {
+ const table = params.table;
+ if (!table) {
return res.status(400).json({
status: 'error',
errorType: 'bad_data',
error: `table parameter required for querying clickhouse via promql`,
});
}
- const status = await proxyToPrometheus(
- clickhousePrometheusUpstream(connection.host),
- `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`,
- params,
- res,
- clickhouseAuthHeaders(connection),
- );
- recordProxyOutcome(status, 'query', backend);
- return;
+
+ const client = newClickhouseClient(connection);
+
+ if (await clickhouseServesPrometheusHttpApi(client, connection)) {
+ const status = await proxyToPrometheus(
+ clickhousePrometheusUpstream(connection.host),
+ `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`,
+ params,
+ res,
+ clickhouseAuthHeaders(connection),
+ );
+ recordProxyOutcome(status, 'query', backend);
+ return;
+ }
+
+ // fallback for clickhouse version < 26.6
+ const result = await queryInstantViaTableFunction({
+ client,
+ databaseName: params.database ?? 'default',
+ tableName: table,
+ expr: query,
+ evalMs: params.time
+ ? Math.floor(parseTimestamp(params.time) * 1000)
+ : Date.now(),
+ });
+ return res.json({
+ status: 'success',
+ data: { resultType: 'vector', result },
+ });
} catch (e) {
prometheusQueryErrors.add(1, { endpoint: 'query', backend });
logger.error(e, 'Prometheus query error');
@@ -901,12 +1011,7 @@ async function handleLabelLookup(
});
}
- const client = new ClickhouseClient({
- host: connection.host,
- username: connection.username,
- password: connection.password,
- requestTimeout: PROMETHEUS_CH_TIMEOUT_MS,
- });
+ const client = newClickhouseClient(connection);
const startMs = start != null ? Math.floor(start * 1000) : undefined;
const endMs = end != null ? Math.ceil(end * 1000) : undefined;
diff --git a/packages/common-utils/src/__tests__/clickhouseVersion.test.ts b/packages/common-utils/src/__tests__/clickhouseVersion.test.ts
index 0df41e16d4..580bef9085 100644
--- a/packages/common-utils/src/__tests__/clickhouseVersion.test.ts
+++ b/packages/common-utils/src/__tests__/clickhouseVersion.test.ts
@@ -4,6 +4,7 @@ import {
parseClickHouseVersion,
supportsDirectReadMap,
supportsMergeTreeIndex,
+ supportsPrometheusHttpApi,
} from '@/core/clickhouseVersion';
type ClickHouseVersionTuple = readonly [number, number, number, number];
@@ -277,3 +278,26 @@ describe('supportsMergeTreeIndex', () => {
expect(supportsMergeTreeIndex(undefined)).toBe(false);
});
});
+
+describe('supportsPrometheusHttpApi', () => {
+ it.each([
+ [[26, 6, 0, 0]],
+ [[26, 6, 8, 7]],
+ [[26, 8, 6, 5]],
+ [[27, 0, 0, 0]],
+ ])('accepts %j', version => {
+ expect(supportsPrometheusHttpApi(version)).toBe(true);
+ });
+
+ it.each([
+ [[26, 5, 5, 8]],
+ [[26, 5, 99, 99]],
+ [[25, 12, 0, 0]],
+ ])('rejects %j', version => {
+ expect(supportsPrometheusHttpApi(version)).toBe(false);
+ });
+
+ it('returns false when the version is unknown', () => {
+ expect(supportsPrometheusHttpApi(undefined)).toBe(false);
+ });
+});
diff --git a/packages/common-utils/src/core/clickhouseVersion.ts b/packages/common-utils/src/core/clickhouseVersion.ts
index 3ad2657b8f..e4d1278075 100644
--- a/packages/common-utils/src/core/clickhouseVersion.ts
+++ b/packages/common-utils/src/core/clickhouseVersion.ts
@@ -173,3 +173,11 @@ export function supportsMergeTreeTextIndex(
): boolean {
return isClickHouseVersionAtLeast(version, MERGE_TREE_TEXT_INDEX_MIN);
}
+
+const PROMETHEUS_HTTP_API_MIN: ClickHouseVersion = [26, 6, 0, 0];
+
+export function supportsPrometheusHttpApi(
+ version: ClickHouseVersion | undefined,
+): boolean {
+ return isClickHouseVersionAtLeast(version, PROMETHEUS_HTTP_API_MIN);
+}
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 8cd8291ccc..80f5c3ebcd 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -615,6 +615,32 @@ export class Metadata {
);
}
+ async getTimeSeriesTableVersion({
+ connectionId,
+ databaseName,
+ tableName,
+ }: {
+ connectionId: string;
+ databaseName: string;
+ tableName: string;
+ }): Promise {
+ return this.cache.getOrFetch(
+ `${connectionId}.${databaseName}.${tableName}.getTimeSeriesTableVersion`,
+ async () => {
+ const sql = chSql`SELECT toUInt32OrZero(extract(engine_full, 'version = (\\d+)')) AS version FROM system.tables WHERE database = ${{ String: databaseName }} AND name = ${{ String: tableName }}`;
+ const json = await this.clickhouseClient
+ .query<'JSON'>({
+ query: sql.sql,
+ query_params: sql.params,
+ connectionId,
+ clickhouse_settings: this.getClickHouseSettings(),
+ })
+ .then(res => res.json<{ version: number }>());
+ return Number(json.data[0]?.version ?? 0);
+ },
+ );
+ }
+
async getMaterializedColumnsLookupTable({
databaseName,
tableName,
From 2fa8ca24e03c96437177d0fdde6741b377a2b9ad Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 17 Sep 2026 15:34:10 -0400
Subject: [PATCH 5/7] fix tests
---
packages/api/jest.setup.ts | 10 +-
.../api/__tests__/prometheus.int.test.ts | 115 ++++++++++++++++++
2 files changed, 124 insertions(+), 1 deletion(-)
diff --git a/packages/api/jest.setup.ts b/packages/api/jest.setup.ts
index 45d8305646..6861104f64 100644
--- a/packages/api/jest.setup.ts
+++ b/packages/api/jest.setup.ts
@@ -30,7 +30,15 @@ jest.mock('@/utils/slack', () => ({
postMessageToWebhook: jest.fn().mockResolvedValue(null),
}));
-// Mock global fetch for generic webhook calls
+// Mock global fetch for generic webhook calls. The real one is kept on
+// `globalThis.realFetch` so a test that needs to reach a live service (e.g.
+// ClickHouse's Prometheus API) can `mockFetch.mockImplementation(realFetch)`
+// for its duration.
+declare global {
+ // eslint-disable-next-line no-var
+ var realFetch: typeof fetch;
+}
+globalThis.realFetch = global.fetch;
global.fetch = jest.fn().mockResolvedValue({
ok: true,
text: jest.fn().mockResolvedValue(''),
diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
index f154aa91c7..2c3fced110 100644
--- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
+++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
@@ -799,6 +799,121 @@ describe('prometheus router', () => {
expect(res.body.error).toMatch(/no_such_table_for_fallback/);
});
+ // The one place the proxy meets a real prometheus_api_v1 handler: the CI
+ // ClickHouse (26.8, docker/clickhouse/local/config.xml) serves it, so the
+ // stubbed fetch is swapped for the real one and the answer has to be
+ // Prometheus-shaped data from a seeded table. Every other proxy test only
+ // pins the request shape.
+ describe('against the CI ClickHouse prometheus_api_v1 handler', () => {
+ const TABLE = 'prom_http_api_live';
+ const START = 1700000000;
+
+ beforeAll(async () => {
+ await seedTimeSeriesTagsTable({
+ table: TABLE,
+ series: [
+ {
+ metricName: 'live_up',
+ tags: { job: 'api' },
+ startSec: START,
+ endSec: START + 60,
+ },
+ {
+ metricName: 'live_up',
+ tags: { job: 'web' },
+ startSec: START,
+ endSec: START + 60,
+ },
+ ],
+ withSamples: true,
+ });
+ });
+
+ afterAll(async () => {
+ await dropTimeSeriesTable({ table: TABLE });
+ });
+
+ beforeEach(() => {
+ mockFetch.mockImplementation(globalThis.realFetch);
+ });
+
+ it('answers a range query with the series from the requested table', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+
+ const res = await agent
+ .get('/v1/prometheus/query_range')
+ .query({
+ query: 'sum by (job) (live_up)',
+ start: String(START),
+ end: String(START + 60),
+ step: '60s',
+ database: DEFAULT_DATABASE,
+ table: TABLE,
+ connectionId: conn._id.toString(),
+ })
+ .expect(200);
+
+ expect(res.headers['content-type']).toMatch(/application\/json/);
+ expect(res.body.status).toBe('success');
+ expect(res.body.data.resultType).toBe('matrix');
+ const byJob = Object.fromEntries(
+ res.body.data.result.map((r: any) => [r.metric.job, r.values]),
+ );
+ expect(Object.keys(byJob).sort()).toEqual(['api', 'web']);
+ expect(byJob.api).toEqual(
+ expect.arrayContaining([
+ [START, '1'],
+ [START + 60, '1'],
+ ]),
+ );
+ });
+
+ it('answers an instant query', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+
+ const res = await agent
+ .get('/v1/prometheus/query')
+ .query({
+ query: 'count(live_up)',
+ time: String(START + 60),
+ database: DEFAULT_DATABASE,
+ table: TABLE,
+ connectionId: conn._id.toString(),
+ })
+ .expect(200);
+
+ expect(res.body).toEqual({
+ status: 'success',
+ data: {
+ resultType: 'vector',
+ result: [{ metric: {}, value: [START + 60, '2'] }],
+ },
+ });
+ });
+
+ it('relays a PromQL parse error as a Prometheus 400', async () => {
+ const { agent, team } = await getLoggedInAgent(server);
+ const conn = await seedClickHouseConnection(team._id);
+
+ const res = await agent
+ .get('/v1/prometheus/query')
+ .query({
+ query: 'live_up(',
+ table: TABLE,
+ connectionId: conn._id.toString(),
+ })
+ .expect(400);
+
+ expect(res.body).toMatchObject({
+ status: 'error',
+ errorType: 'bad_data',
+ error: expect.stringContaining('PromQL'),
+ });
+ });
+ });
+
describe('on ClickHouse without the Prometheus HTTP API (< 26.6)', () => {
const TABLE = 'prom_query_range_fallback';
const START = 1700000000;
From d2b2b13667005be97c63b0aa3777bea65037b52e Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 17 Sep 2026 16:01:46 -0400
Subject: [PATCH 6/7] pr suggestions
---
.../api/src/controllers/timeseriesEngine.ts | 17 +++--
.../api/__tests__/prometheus.int.test.ts | 67 ++++++++++++++++++-
packages/api/src/routers/api/prometheus.ts | 46 ++++++-------
3 files changed, 96 insertions(+), 34 deletions(-)
diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts
index ad427a8806..2280012fb3 100644
--- a/packages/api/src/controllers/timeseriesEngine.ts
+++ b/packages/api/src/controllers/timeseriesEngine.ts
@@ -5,7 +5,6 @@ import {
} from '@hyperdx/common-utils/dist/clickhouse';
import { supportsPrometheusHttpApi } from '@hyperdx/common-utils/dist/core/clickhouseVersion';
import {
- getMetadata,
Metadata,
MetadataCache,
} from '@hyperdx/common-utils/dist/core/metadata';
@@ -292,7 +291,8 @@ export function formatMatrixResponse(
* `time_series` to `samples` in TimeSeries schema version 3. The rename
* follows the table's pinned version, not the server's, so a v2 table on a new
* server still reads `time_series`. `prometheusQueryRange` returns whichever
- * the table uses.
+ * the table uses. Looked up fresh each call: a throwaway cache, so a table
+ * dropped and recreated at a new version is seen at once.
*/
async function timeSeriesSamplesColumn({
client,
@@ -305,7 +305,8 @@ async function timeSeriesSamplesColumn({
databaseName: string;
tableName: string;
}): Promise<'samples' | 'time_series'> {
- const version = await getMetadata(client).getTimeSeriesTableVersion({
+ const metadata = new Metadata(client, new MetadataCache());
+ const version = await metadata.getTimeSeriesTableVersion({
connectionId,
databaseName,
tableName,
@@ -394,9 +395,10 @@ export async function queryInstantViaTableFunction({
}
/**
- * Whether the connection's ClickHouse serves the Prometheus HTTP API. The
- * version is cached per connection in the process-wide metadata cache; an
- * unknown version is treated as old, matching the other version-gated features.
+ * Whether the connection's ClickHouse is new enough for the Prometheus HTTP
+ * API. `SELECT version()` runs each call (throwaway cache) so an upgraded
+ * server is picked up without a restart; an unknown version is treated as
+ * old, matching the other version-gated features.
*/
export async function connectionSupportsPrometheusHttpApi({
client,
@@ -405,7 +407,8 @@ export async function connectionSupportsPrometheusHttpApi({
client: ClickhouseClient;
connectionId: string;
}): Promise {
+ const metadata = new Metadata(client, new MetadataCache());
return supportsPrometheusHttpApi(
- await getMetadata(client).getServerVersion({ connectionId }),
+ await metadata.getServerVersion({ connectionId }),
);
}
diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
index 2c3fced110..731a5d21ad 100644
--- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
+++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
@@ -11,7 +11,12 @@ import {
seedTimeSeriesTagsTable,
} from '@/fixtures';
import Connection from '@/models/connection';
-import { PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC } from '@/routers/api/prometheus';
+import {
+ CLICKHOUSE_PROMETHEUS_API_PREFIX,
+ clickhousePrometheusUpstream,
+ joinPrometheusUpstreamUrl,
+ PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC,
+} from '@/routers/api/prometheus';
const mockFetch = jest.mocked(global.fetch);
@@ -893,6 +898,66 @@ describe('prometheus router', () => {
});
});
+ // The router's limits are compile-time constants far above anything a
+ // test can exceed, so the same URL builder is driven at tiny values
+ // straight at the handler. This pins the property the router relies on:
+ // settings in the URL are applied to the PromQL evaluation and a breach
+ // comes back as a Prometheus 400, not as a truncated 200.
+ const liveRangeQuery = async (limits: {
+ maxExecutionSec?: number;
+ maxResultRows?: number;
+ }) => {
+ const url = joinPrometheusUpstreamUrl(
+ clickhousePrometheusUpstream(config.CLICKHOUSE_HOST, limits),
+ `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`,
+ );
+ for (const [k, v] of Object.entries({
+ query: 'live_up',
+ start: String(START),
+ end: String(START + 60),
+ step: '60',
+ database: DEFAULT_DATABASE,
+ table: TABLE,
+ })) {
+ url.searchParams.set(k, v);
+ }
+ const resp = await globalThis.realFetch(url, {
+ headers: {
+ 'X-ClickHouse-User': config.CLICKHOUSE_USER,
+ 'X-ClickHouse-Key': config.CLICKHOUSE_PASSWORD,
+ },
+ });
+ return { status: resp.status, body: await resp.json() };
+ };
+
+ it('enforces max_result_rows pinned on the upstream URL', async () => {
+ const { status, body } = await liveRangeQuery({ maxResultRows: 1 });
+ expect(status).toBe(400);
+ expect(body).toMatchObject({
+ status: 'error',
+ errorType: 'bad_data',
+ error: expect.stringMatching(/Limit for result exceeded/),
+ });
+ });
+
+ it('enforces max_execution_time pinned on the upstream URL', async () => {
+ const { status, body } = await liveRangeQuery({
+ maxExecutionSec: 0.0001,
+ });
+ expect(status).toBe(400);
+ expect(body).toMatchObject({
+ status: 'error',
+ errorType: 'bad_data',
+ error: expect.stringMatching(/Timeout exceeded/),
+ });
+ });
+
+ it('answers in full at the limits the router pins', async () => {
+ const { status, body } = await liveRangeQuery({});
+ expect(status).toBe(200);
+ expect(body.data.result).toHaveLength(2);
+ });
+
it('relays a PromQL parse error as a Prometheus 400', async () => {
const { agent, team } = await getLoggedInAgent(server);
const conn = await seedClickHouseConnection(team._id);
diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts
index a2b52e9ab1..2bd5492bea 100644
--- a/packages/api/src/routers/api/prometheus.ts
+++ b/packages/api/src/routers/api/prometheus.ts
@@ -193,25 +193,28 @@ const CALLER_SETTABLE_PARAM_KEYS = new Set([
]);
// ClickHouse serves the Prometheus HTTP API under this prefix on its main HTTP
-// port when the `prometheus_api_v1` handler is configured (26.8+).
-const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1';
+// port when the `prometheus_api_v1` handler is configured (26.6+).
+export const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1';
/**
* The ClickHouse connection host with the query limits pinned as ClickHouse
* HTTP settings. The prometheus_api_v1 handler honours settings given in the
- * URL and reports a breach as a Prometheus 400, so this is the only place to
- * bound a PromQL evaluation — the request carries no SQL to attach SETTINGS to.
- * Pinned on the host rather than passed as params so the merge in
- * proxyToPrometheus (which lets the caller override any allowlisted key) can't
- * loosen them.
+ * URL and reports a breach as a Prometheus 400 (pinned by the live int test),
+ * so this is the only place to bound a PromQL evaluation — the request carries
+ * no SQL to attach SETTINGS to. Pinned on the host rather than passed as params
+ * so the merge in proxyToPrometheus (which lets the caller override any
+ * allowlisted key) can't loosen them.
*/
-function clickhousePrometheusUpstream(host: string): string {
+export function clickhousePrometheusUpstream(
+ host: string,
+ {
+ maxExecutionSec = PROMETHEUS_MAX_EXECUTION_SEC,
+ maxResultRows = PROMETHEUS_MAX_RESULT_ROWS,
+ } = {},
+): string {
const url = new URL(host);
- url.searchParams.set(
- 'max_execution_time',
- String(PROMETHEUS_MAX_EXECUTION_SEC),
- );
- url.searchParams.set('max_result_rows', String(PROMETHEUS_MAX_RESULT_ROWS));
+ url.searchParams.set('max_execution_time', String(maxExecutionSec));
+ url.searchParams.set('max_result_rows', String(maxResultRows));
return url.toString();
}
@@ -237,11 +240,9 @@ function newClickhouseClient(connection: {
* a server without it answers a plain-text 404. `format_query` is used as the
* probe because it is the one endpoint that needs no table.
*
- * ponytail: per-process TTL map, so a handler enabled later is picked up within
- * PROBE_TTL_MS. Keyed by host too, so editing the connection re-probes.
+ * Probed on every request, like the version check, so a handler enabled or
+ * removed on the server is reflected immediately.
*/
-const promHttpApiProbe = new Map();
-const PROBE_TTL_MS = 5 * 60_000;
const PROBE_TIMEOUT_MS = 5_000;
async function clickhouseServesPrometheusHttpApi(
@@ -257,11 +258,6 @@ async function clickhouseServesPrometheusHttpApi(
return false;
}
- const key = `${connection.id}:${connection.host}`;
- const cached = promHttpApiProbe.get(key);
- if (cached && cached.expires > Date.now()) return cached.ok;
-
- let ok = false;
try {
const url = joinPrometheusUpstreamUrl(
connection.host,
@@ -272,12 +268,10 @@ async function clickhouseServesPrometheusHttpApi(
headers: clickhouseAuthHeaders(connection),
signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
});
- ok = typeof JSON.parse(await resp.text())?.status === 'string';
+ return typeof JSON.parse(await resp.text())?.status === 'string';
} catch {
- ok = false;
+ return false;
}
- promHttpApiProbe.set(key, { ok, expires: Date.now() + PROBE_TTL_MS });
- return ok;
}
function clickhouseAuthHeaders(connection: {
From a2925ebf822f59b7d661a842be3e471b5a4a714e Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 17 Sep 2026 16:22:38 -0400
Subject: [PATCH 7/7] fix lint
---
packages/api/jest.setup.ts | 6 +-----
.../api/src/routers/api/__tests__/prometheus.int.test.ts | 8 ++++++--
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/packages/api/jest.setup.ts b/packages/api/jest.setup.ts
index 6861104f64..3e270f01e9 100644
--- a/packages/api/jest.setup.ts
+++ b/packages/api/jest.setup.ts
@@ -34,11 +34,7 @@ jest.mock('@/utils/slack', () => ({
// `globalThis.realFetch` so a test that needs to reach a live service (e.g.
// ClickHouse's Prometheus API) can `mockFetch.mockImplementation(realFetch)`
// for its duration.
-declare global {
- // eslint-disable-next-line no-var
- var realFetch: typeof fetch;
-}
-globalThis.realFetch = global.fetch;
+Object.assign(globalThis, { realFetch: global.fetch });
global.fetch = jest.fn().mockResolvedValue({
ok: true,
text: jest.fn().mockResolvedValue(''),
diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
index 731a5d21ad..d7dbf94255 100644
--- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
+++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts
@@ -20,6 +20,10 @@ import {
const mockFetch = jest.mocked(global.fetch);
+// Set by jest.setup.ts before it stubs `fetch`, for tests that must reach a
+// live service.
+const { realFetch } = globalThis as unknown as { realFetch: typeof fetch };
+
const matchQueryString = (selectors: string[]) =>
selectors.map(s => `match[]=${encodeURIComponent(s)}`).join('&');
@@ -839,7 +843,7 @@ describe('prometheus router', () => {
});
beforeEach(() => {
- mockFetch.mockImplementation(globalThis.realFetch);
+ mockFetch.mockImplementation(realFetch);
});
it('answers a range query with the series from the requested table', async () => {
@@ -921,7 +925,7 @@ describe('prometheus router', () => {
})) {
url.searchParams.set(k, v);
}
- const resp = await globalThis.realFetch(url, {
+ const resp = await realFetch(url, {
headers: {
'X-ClickHouse-User': config.CLICKHOUSE_USER,
'X-ClickHouse-Key': config.CLICKHOUSE_PASSWORD,