fix: route ClickHouse query debug logging through injectable loggers (pino on API, devtools in app)

.changeset/gate-clickhouse-query-debug-log.md

@@ -0,0 +1,25 @@
+---
+'@hyperdx/common-utils': patch
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+fix: route per-query SQL debug logging through an injectable logger (#2416)
+
+`BaseClickhouseClient` dumped raw SQL to the console on every ClickHouse query,
+unconditionally and outside the pino logger, flooding API logs with query spam.
+
+Query logging now goes through an optional per-client `customLogger` on
+`ClickhouseClientOptions`, logged at `debug`, and is silent when no logger is
+passed. The API injects a pino-backed logger, so query logging follows the
+existing `HYPERDX_LOG_LEVEL` setting instead of writing to `console.debug`. The
+browser client defaults to a console logger that pretty-prints the SQL as a
+single multi-line block, so query SQL stays visible and readable in devtools in
+all builds instead of wrapping into one long line.
+
+The API's log level now defaults to `info` (was `debug`), so SQL logging is
+silent in production unless `HYPERDX_LOG_LEVEL=debug` is set. Dev and CI env
+files already pin their levels explicitly and are unaffected. The default also
+now applies when `HYPERDX_LOG_LEVEL` is set but empty — which is what Compose
+passes when the variable is unset in the environment, and which previously made
+pino throw at startup.

packages/api/src/clickhouse.ts

@@ -0,0 +1,26 @@
+import type {
+  ClickhouseClientOptions,
+  Logger as ClickhouseLogger,
+} from '@hyperdx/common-utils/dist/clickhouse';
+import { ClickhouseClient as BaseClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
+
+import logger from '@/utils/logger';
+
+const customLogger: ClickhouseLogger = {
+  trace: ({ module, message, args }) =>
+    logger.trace({ ...args, module }, message),
+  debug: ({ module, message, args }) =>
+    logger.debug({ ...args, module }, message),
+  info: ({ module, message, args }) =>
+    logger.info({ ...args, module }, message),
+  warn: ({ module, message, args, err }) =>
+    logger.warn({ ...args, module, err }, message),
+  error: ({ module, message, args, err }) =>
+    logger.error({ ...args, module, err }, message),
+};
+
+export class ClickhouseClient extends BaseClickhouseClient {
+  constructor(options: ClickhouseClientOptions) {
+    super({ ...options, customLogger: options.customLogger ?? customLogger });
+  }
+}

packages/api/src/controllers/ai.ts

@@ -1,6 +1,5 @@
 import { createAnthropic } from '@ai-sdk/anthropic';
 import { createOpenAI } from '@ai-sdk/openai';
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import {
   getMetadata,
   TableMetadata,
@@ -16,6 +15,7 @@ import * as chrono from 'chrono-node';
 import ms from 'ms';
 import z from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import * as config from '@/config';
 import { ISource } from '@/models/source';
 import { parseJSON } from '@/utils/common';

packages/api/src/mcp/tools/query/denoise.ts

@@ -1,4 +1,3 @@
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils';
 import {
@@ -12,6 +11,7 @@ import {
 import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
 import { DisplayType } from '@hyperdx/common-utils/dist/types';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 

packages/api/src/mcp/tools/query/eventDeltas.ts

@@ -1,4 +1,3 @@
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import {
   getStableSampleExpression,
   rankProperties,
@@ -12,6 +11,7 @@ import {
 } from '@hyperdx/common-utils/dist/types';
 import { z } from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import type { ToolRegistrar } from '@/mcp/tools/types';

packages/api/src/mcp/tools/query/helpers.ts

@@ -1,5 +1,4 @@
 import { ClickHouseError } from '@clickhouse/client-common';
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import {
   convertToCategoricalChartConfig,
@@ -19,6 +18,7 @@ import {
 import { ObjectId } from 'mongodb';
 import ms from 'ms';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import type { McpErrorResult } from '@/mcp/utils/errors';

packages/api/src/mcp/tools/query/runEventPatterns.ts

@@ -1,4 +1,3 @@
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import {
   getFirstTimestampValueExpression,
@@ -8,6 +7,7 @@ import { minePatterns } from '@hyperdx/common-utils/dist/drain';
 import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
 import { DisplayType } from '@hyperdx/common-utils/dist/types';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import {

packages/api/src/mcp/tools/sources/describeMetric.ts

@@ -6,11 +6,11 @@ import {
   JSDataType,
   tableExpr,
 } from '@hyperdx/common-utils/dist/clickhouse';
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { SourceKind } from '@hyperdx/common-utils/dist/types';
 import { z } from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import { clickHouseErrorResult } from '@/mcp/tools/query/helpers';

packages/api/src/mcp/tools/sources/describeSource.ts

@@ -6,12 +6,12 @@ import {
   JSDataType,
   tableExpr,
 } from '@hyperdx/common-utils/dist/clickhouse';
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { type MetricTable, SourceKind } from '@hyperdx/common-utils/dist/types';
 import SqlString from 'sqlstring';
 import { z } from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import type { ToolRegistrar } from '@/mcp/tools/types';

packages/api/src/mcp/tools/sources/listMetrics.ts

@@ -3,11 +3,11 @@ import {
   concatChSql,
   tableExpr,
 } from '@hyperdx/common-utils/dist/clickhouse';
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { SourceKind } from '@hyperdx/common-utils/dist/types';
 import { z } from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import type { ToolRegistrar } from '@/mcp/tools/types';

packages/api/src/mcp/tools/trace/breakdown.ts

@@ -15,10 +15,10 @@
  * endpoint" without dropping into raw SQL with self-JOINs. The builder
  * tools (table / timeseries / search) can't express the TraceId subselect.
  */
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { SourceKind } from '@hyperdx/common-utils/dist/types';
 import { z } from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import {

packages/api/src/mcp/tools/trace/waterfall.ts

@@ -1,4 +1,3 @@
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import {
   type BuilderChartConfigWithDateRange,
@@ -8,6 +7,7 @@ import {
 } from '@hyperdx/common-utils/dist/types';
 import { z } from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import {

packages/api/src/routers/api/prometheus.ts

@@ -1,9 +1,9 @@
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import express from 'express';
 import { performance } from 'perf_hooks';
 import { Readable } from 'stream';
 import { pipeline } from 'stream/promises';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getNonNullUserWithTeam } from '@/middleware/auth';
 import { getCounter, getHistogram } from '@/utils/instrumentation';

packages/api/src/routers/external-api/v2/charts.ts

@@ -1,4 +1,3 @@
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { Granularity } from '@hyperdx/common-utils/dist/core/utils';
 import {
@@ -11,6 +10,7 @@ import express from 'express';
 import _ from 'lodash';
 import { z } from 'zod';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import { getTeam } from '@/controllers/team';

packages/api/src/routers/external-api/v2/utils/search.ts

@@ -1,4 +1,3 @@
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig';
 import {
@@ -15,6 +14,7 @@ import {
   isTraceSource,
 } from '@hyperdx/common-utils/dist/types';
 
+import { ClickhouseClient } from '@/clickhouse';
 import { getConnectionById } from '@/controllers/connection';
 import { getSource } from '@/controllers/sources';
 import type { ExternalDashboardSearchRequestConfig } from '@/utils/zod';

packages/api/src/tasks/checkAlerts/providers/default.ts

@@ -1,12 +1,12 @@
 import PQueue from '@esm2cjs/p-queue';
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { displayTypeSupportsRawSqlAlerts } from '@hyperdx/common-utils/dist/core/utils';
 import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
 import { Tile } from '@hyperdx/common-utils/dist/types';
 import mongoose from 'mongoose';
 import ms from 'ms';
 import { URLSearchParams } from 'url';
 
+import { ClickhouseClient } from '@/clickhouse';
 import * as config from '@/config';
 import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory';
 import { LOCAL_APP_TEAM } from '@/controllers/team';

packages/api/src/tasks/usageStats.ts

@@ -1,11 +1,11 @@
 import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse';
-import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types';
 import * as HyperDX from '@hyperdx/node-opentelemetry';
 import ms from 'ms';
 import os from 'os';
 import pino from 'pino';
 
+import { ClickhouseClient } from '@/clickhouse';
 import * as config from '@/config';
 import Connection from '@/models/connection';
 import { Source, SourceDocument } from '@/models/source';

packages/api/src/utils/logger.ts

@@ -8,7 +8,7 @@ import pinoHttp from 'pino-http';
 
 import * as config from '@/config';
 
-const MAX_LEVEL = config.HYPERDX_LOG_LEVEL ?? 'debug';
+const MAX_LEVEL = config.HYPERDX_LOG_LEVEL || 'info';
 
 const hyperdxTransport = config.HYPERDX_API_KEY
   ? getPinoTransport(MAX_LEVEL, {

packages/cli/src/api/client.ts

@@ -351,9 +351,6 @@ export class ProxyClickhouseClient extends BaseClickhouseClient {
     });
   }
 
-  // Silence the "Sending Query: ..." debug output from BaseClickhouseClient
-  protected override logDebugQuery(): void {}
-
   // This subclass always builds a node client, so narrow the base class's
   // platform-agnostic client type to the node-specific one.
   protected getClient(): NodeClickHouseClient {

packages/common-utils/src/clickhouse/__tests__/browser.test.ts

@@ -0,0 +1,65 @@
+import { consoleLogger } from '@/clickhouse/browser';
+
+describe('consoleLogger', () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it('pretty-prints SQL into a single multi-line message', () => {
+    const debugSpy = jest.spyOn(console, 'debug').mockImplementation(() => {});
+    const sql = 'SELECT a, b FROM default.otel_logs WHERE a = 1';
+
+    consoleLogger.debug({
+      module: 'clickhouse',
+      message: 'Sending query',
+      args: { sql },
+    });
+
+    // Exact layout is sqlFormatter's business; here we only pin the contract
+    // devtools needs: one call, one string, and SQL broken across lines.
+    expect(debugSpy).toHaveBeenCalledTimes(1);
+    expect(debugSpy.mock.calls[0]).toHaveLength(1);
+
+    const logged = String(debugSpy.mock.calls[0][0]);
+    expect(logged.startsWith('Sending query:\n')).toBe(true);
+    expect(logged.split('\n').length).toBeGreaterThan(2);
+  });
+
+  it('falls back to the raw SQL when it cannot be formatted', () => {
+    const debugSpy = jest.spyOn(console, 'debug').mockImplementation(() => {});
+    const sql = 'SELECT ((( FROM';
+
+    consoleLogger.debug({
+      module: 'clickhouse',
+      message: 'Sending query',
+      args: { sql },
+    });
+
+    expect(debugSpy).toHaveBeenCalledWith(`Sending query:\n${sql}`);
+  });
+
+  it('keeps the prefixed form for logs that carry no SQL', () => {
+    const debugSpy = jest.spyOn(console, 'debug').mockImplementation(() => {});
+
+    consoleLogger.debug({
+      module: 'clickhouse',
+      message: 'Response received',
+      args: { rows: 5 },
+    });
+
+    expect(debugSpy).toHaveBeenCalledWith('[clickhouse] Response received', 5);
+  });
+
+  it('appends the error to error logs', () => {
+    const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
+    const err = new Error('boom');
+
+    consoleLogger.error({
+      module: 'clickhouse',
+      message: 'Query failed',
+      err,
+    });
+
+    expect(errorSpy).toHaveBeenCalledWith('[clickhouse] Query failed', err);
+  });
+});