feat: upgrade clickhouse versions and use promql remote endpoints

.changeset/promql-via-clickhouse-http-api.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/api': minor
+---
+
+feat: run PromQL against ClickHouse through its Prometheus HTTP API
+
+ClickHouse-backed `/query` and `/query_range` now proxy to the connection's `/prometheus/api/v1/*` endpoint (the `prometheus_api_v1` HTTP handler, ClickHouse 26.8+) instead of calling the `prometheusQuery`/`prometheusQueryRange` table functions. The `database` and `table` query parameters are forwarded so one handler serves any TimeSeries table. ClickHouse holds the Prometheus API forward-compatible while the TimeSeries engine is in preview; the table functions and inner-table schema are not. Bundled ClickHouse images move to 26.8.

docker-compose.ci.yml

@@ -22,7 +22,7 @@ services:
     depends_on:
       - ch-server
   ch-server:
-    image: clickhouse/clickhouse-server:26.5-alpine
+    image: clickhouse/clickhouse-server:26.8-alpine
     environment:
       # default settings
       CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1

docker-compose.dev.yml

@@ -105,7 +105,7 @@ services:
       hdx.dev.service: clickhouse
       hdx.dev.port: '${HDX_DEV_CH_HTTP_PORT:-8123}'
       hdx.dev.url: 'http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}'
-    image: clickhouse/clickhouse-server:26.5-alpine
+    image: clickhouse/clickhouse-server:26.8-alpine
     ports:
       - '${HDX_DEV_CH_HTTP_PORT:-8123}:8123' # http api
       - '${HDX_DEV_CH_NATIVE_PORT:-9000}:9000' # native

docker-compose.yml

@@ -96,7 +96,7 @@ services:
       - ch-server
       - db
   ch-server:
-    image: clickhouse/clickhouse-server:26.5-alpine
+    image: clickhouse/clickhouse-server:26.8-alpine
     # WARNING: Exposing the database port will make it accessible from outside the container,
     # potentially allowing unauthorized access. If you uncomment the ports below,
     # ensure to secure your database (e.g., with strong authentication, proper network rules, and firewalls).

docker/clickhouse/local/config.xml

@@ -73,6 +73,19 @@
         </handlers>
     </prometheus>
 
+    <!-- Prometheus HTTP API (/query, /query_range, ...) on the main HTTP
+         port. database/table come from request query params so one handler
+         serves any TimeSeries table. -->
+    <http_handlers>
+        <rule>
+            <url_prefix>/prometheus/api/v1</url_prefix>
+            <handler>
+                <type>prometheus_api_v1</type>
+            </handler>
+        </rule>
+        <defaults/>
+    </http_handlers>
+
     <!-- Query log. Used only for queries with setting log_queries = 1. -->
     <query_log>
         <database>system</database>

docker/hyperdx/Dockerfile

@@ -8,7 +8,7 @@
 # - Otel Collector (otelcol)
 
 ARG NODE_VERSION=22.22
-ARG CLICKHOUSE_VERSION=26.5
+ARG CLICKHOUSE_VERSION=26.8
 ARG OTEL_COLLECTOR_VERSION=0.155.0
 ARG OTEL_COLLECTOR_CORE_VERSION=1.61.0
 

docker/hyperdx/clickhouseConfig.xml

@@ -47,6 +47,16 @@
     <!--     <errors>true</errors> -->
     <!-- </prometheus> -->
 
+    <http_handlers>
+        <rule>
+            <url_prefix>/prometheus/api/v1</url_prefix>
+            <handler>
+                <type>prometheus_api_v1</type>
+            </handler>
+        </rule>
+        <defaults/>
+    </http_handlers>
+
     <!-- Query log. Used only for queries with setting log_queries = 1. -->
     <query_log>
         <database>system</database>

packages/api/jest.setup.ts

@@ -30,7 +30,11 @@ jest.mock('@/utils/slack', () => ({
   postMessageToWebhook: jest.fn().mockResolvedValue(null),
 }));
 
-// Mock global fetch for generic webhook calls
+// Mock global fetch for generic webhook calls. The real one is kept on
+// `globalThis.realFetch` so a test that needs to reach a live service (e.g.
+// ClickHouse's Prometheus API) can `mockFetch.mockImplementation(realFetch)`
+// for its duration.
+Object.assign(globalThis, { realFetch: global.fetch });
 global.fetch = jest.fn().mockResolvedValue({
   ok: true,
   text: jest.fn().mockResolvedValue(''),

packages/api/src/controllers/__tests__/timeseriesEngine.test.ts

@@ -0,0 +1,47 @@
+import {
+  formatMatrixResponse,
+  formatVectorResponse,
+} from '@/controllers/timeseriesEngine';
+
+describe('formatMatrixResponse', () => {
+  it('converts prometheusQueryRange rows into Prometheus matrix shape', () => {
+    const rows = [
+      {
+        tags: [
+          ['__name__', 'http_requests_total'],
+          ['method', 'GET'],
+        ] as [string, string][],
+        samples: [
+          [1700000000, 5],
+          ['2023-11-14T22:14:20.000Z', 7],
+        ] as [string | number, number][],
+      },
+    ];
+    expect(formatMatrixResponse(rows as any)).toEqual([
+      {
+        metric: { __name__: 'http_requests_total', method: 'GET' },
+        values: [
+          [1700000000, '5'],
+          [1700000060, '7'],
+        ],
+      },
+    ]);
+  });
+
+  it('returns empty array for empty input', () => {
+    expect(formatMatrixResponse([])).toEqual([]);
+  });
+});
+
+describe('formatVectorResponse', () => {
+  it('converts prometheusQuery rows into Prometheus vector shape', () => {
+    const rows = [
+      { tags: [['service', 'api']], timestamp: 1700000000, value: 42 },
+      { tags: [], timestamp: '2023-11-14T22:13:20.000Z', value: 3 },
+    ];
+    expect(formatVectorResponse(rows as any)).toEqual([
+      { metric: { service: 'api' }, value: [1700000000, '42'] },
+      { metric: {}, value: [1700000000, '3'] },
+    ]);
+  });
+});

packages/api/src/controllers/timeseriesEngine.ts

@@ -3,6 +3,7 @@ import {
   chSql,
   concatChSql,
 } from '@hyperdx/common-utils/dist/clickhouse';
+import { supportsPrometheusHttpApi } from '@hyperdx/common-utils/dist/core/clickhouseVersion';
 import {
   Metadata,
   MetadataCache,
@@ -241,10 +242,173 @@ export async function queryLabelNames({
   limit,
   ...args
 }: TimeSeriesTagsQueryArgs): Promise<string[]> {
-  // `all_tags` is EPHEMERAL by default, so the label names come from `tags`,
-  // which the engine strips `__name__` out of. It is folded back in explicitly.
+  // The engine keeps `__name__` in `metric_name`, not `tags`, so it is folded
+  // back in explicitly.
   const value = chSql`arrayJoin(arrayConcat([${{ String: '__name__' }}], mapKeys(${{ Identifier: 'tags' }})))`;
   const conditions = await getSeriesFilterConditions(args);
 
   return queryDistinctTagsValues({ ...args, value, conditions, limit });
 }
+
+// --------------------------
+// PromQL via table functions (ClickHouse < 26.6)
+// --------------------------
+//
+// Servers without the `prometheus_api_v1` HTTP handler can still evaluate
+// PromQL through `prometheusQuery`/`prometheusQueryRange`. ClickHouse only
+// holds the HTTP API forward-compatible while TimeSeries is in preview, so
+// this path is a fallback for old servers, not the primary route.
+
+type PrometheusMetric = Record<string, string>;
+export type PrometheusMatrixResult = {
+  metric: PrometheusMetric;
+  values: [number, string][];
+};
+export type PrometheusVectorResult = {
+  metric: PrometheusMetric;
+  value: [number, string];
+};
+
+const toUnixSeconds = (timestamp: string | number) =>
+  typeof timestamp === 'string'
+    ? new Date(timestamp).getTime() / 1000
+    : Number(timestamp);
+
+export function formatMatrixResponse(
+  rows: { tags: [string, string][]; samples: [string, number][] }[],
+): PrometheusMatrixResult[] {
+  return rows.map(row => ({
+    metric: Object.fromEntries(row.tags),
+    values: row.samples.map(([ts, value]) => [
+      toUnixSeconds(ts),
+      String(value),
+    ]),
+  }));
+}
+
+/**
+ * The name of the per-series samples column, which the engine renamed from
+ * `time_series` to `samples` in TimeSeries schema version 3. The rename
+ * follows the table's pinned version, not the server's, so a v2 table on a new
+ * server still reads `time_series`. `prometheusQueryRange` returns whichever
+ * the table uses. Looked up fresh each call: a throwaway cache, so a table
+ * dropped and recreated at a new version is seen at once.
+ */
+async function timeSeriesSamplesColumn({
+  client,
+  connectionId,
+  databaseName,
+  tableName,
+}: {
+  client: ClickhouseClient;
+  connectionId: string;
+  databaseName: string;
+  tableName: string;
+}): Promise<'samples' | 'time_series'> {
+  const metadata = new Metadata(client, new MetadataCache());
+  const version = await metadata.getTimeSeriesTableVersion({
+    connectionId,
+    databaseName,
+    tableName,
+  });
+  return version >= 3 ? 'samples' : 'time_series';
+}
+
+export function formatVectorResponse(
+  rows: { tags: [string, string][]; timestamp: string; value: number }[],
+): PrometheusVectorResult[] {
+  return rows.map(row => ({
+    metric: Object.fromEntries(row.tags),
+    value: [toUnixSeconds(row.timestamp), String(row.value)],
+  }));
+}
+
+const PROMQL_TABLE_FUNCTION_SETTINGS = {
+  allow_experimental_time_series_table: 1,
+  max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC,
+  max_result_rows: String(PROMETHEUS_MAX_RESULT_ROWS),
+} as const;
+
+export async function queryRangeViaTableFunction({
+  client,
+  connectionId,
+  databaseName,
+  tableName,
+  expr,
+  startMs,
+  endMs,
+  stepSec,
+}: {
+  client: ClickhouseClient;
+  connectionId: string;
+  databaseName: string;
+  tableName: string;
+  expr: string;
+  startMs: number;
+  endMs: number;
+  stepSec: number;
+}): Promise<PrometheusMatrixResult[]> {
+  const samplesColumn = await timeSeriesSamplesColumn({
+    client,
+    connectionId,
+    databaseName,
+    tableName,
+  });
+  const resp = await client.query({
+    query: `SELECT tags, ${samplesColumn} AS samples FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`,
+    query_params: {
+      db: databaseName,
+      table: tableName,
+      expr,
+      startMs,
+      endMs,
+      stepSec,
+    },
+    format: 'JSON',
+    clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS,
+  });
+  const json = await resp.json<any>();
+  return formatMatrixResponse(json.data);
+}
+
+export async function queryInstantViaTableFunction({
+  client,
+  databaseName,
+  tableName,
+  expr,
+  evalMs,
+}: {
+  client: ClickhouseClient;
+  databaseName: string;
+  tableName: string;
+  expr: string;
+  evalMs: number;
+}): Promise<PrometheusVectorResult[]> {
+  const resp = await client.query({
+    query: `SELECT tags, timestamp, value FROM prometheusQuery({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({evalMs:Int64})) SETTINGS allow_experimental_time_series_table = 1`,
+    query_params: { db: databaseName, table: tableName, expr, evalMs },
+    format: 'JSON',
+    clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS,
+  });
+  const json = await resp.json<any>();
+  return formatVectorResponse(json.data);
+}
+
+/**
+ * Whether the connection's ClickHouse is new enough for the Prometheus HTTP
+ * API. `SELECT version()` runs each call (throwaway cache) so an upgraded
+ * server is picked up without a restart; an unknown version is treated as
+ * old, matching the other version-gated features.
+ */
+export async function connectionSupportsPrometheusHttpApi({
+  client,
+  connectionId,
+}: {
+  client: ClickhouseClient;
+  connectionId: string;
+}): Promise<boolean> {
+  const metadata = new Metadata(client, new MetadataCache());
+  return supportsPrometheusHttpApi(
+    await metadata.getServerVersion({ connectionId }),
+  );
+}

packages/api/src/fixtures.ts

@@ -336,15 +336,11 @@ export const seedTimeSeriesTagsTable = async ({
       .join(', ')})`;
 
   const columns = storeTimeBounds
-    ? '(metric_name, tags, all_tags, min_time, max_time)'
-    : '(metric_name, tags, all_tags)';
+    ? '(metric_name, tags, min_time, max_time)'
+    : '(metric_name, tags)';
   const values = series
     .map(s => {
-      const row = [
-        quoted(s.metricName),
-        mapLiteral(s.tags),
-        mapLiteral({ __name__: s.metricName, ...s.tags }),
-      ];
+      const row = [quoted(s.metricName), mapLiteral(s.tags)];
       if (storeTimeBounds) {
         row.push(
           `toDateTime64(${s.startSec}, 3)`,