From 9e1b0d6d088dccc858c98ed0851a0160acaff96f Mon Sep 17 00:00:00 2001
From: Brandon Pereira <brandon.pereira@clickhouse.com>
Date: Fri, 7 Aug 2026 09:07:07 -0600
Subject: [PATCH 1/7] fix(mcp): steer agents toward builder query tools instead
 of raw SQL (HDX-4892)

---
 .changeset/mcp-steer-away-from-raw-sql.md     |  5 +++
 packages/api/src/mcp/mcpServer.ts             | 36 ++++++++++++++++---
 packages/api/src/mcp/tools/query/search.ts    |  3 ++
 packages/api/src/mcp/tools/query/sql.ts       | 20 +++++++----
 packages/api/src/mcp/tools/query/table.ts     |  2 ++
 .../api/src/mcp/tools/query/timeseries.ts     |  3 ++
 6 files changed, 58 insertions(+), 11 deletions(-)
 create mode 100644 .changeset/mcp-steer-away-from-raw-sql.md

diff --git a/.changeset/mcp-steer-away-from-raw-sql.md b/.changeset/mcp-steer-away-from-raw-sql.md
new file mode 100644
index 0000000000..a56c503ab5
--- /dev/null
+++ b/.changeset/mcp-steer-away-from-raw-sql.md
@@ -0,0 +1,5 @@
+---
+'@hyperdx/api': patch
+---
+
+fix(mcp): steer agents toward builder query tools instead of raw SQL (HDX-4892). Telemetry showed agents (notebook investigations) using `clickstack_sql` for ~73% of data queries — usually for single-source aggregations, top-N, and time-series that the builder tools express more reliably (raw SQL also had ~2x the error rate). Reworded the `clickstack_sql` description to mark it a last resort, added a reciprocal "prefer me over SQL" nudge to `clickstack_table`, `clickstack_timeseries`, and `clickstack_search`, and added a server-level `instructions` tool-selection policy so the guidance is surfaced on `initialize` rather than only via the opt-in `query_guide` prompt.
diff --git a/packages/api/src/mcp/mcpServer.ts b/packages/api/src/mcp/mcpServer.ts
index 3f65438259..8174162ecc 100644
--- a/packages/api/src/mcp/mcpServer.ts
+++ b/packages/api/src/mcp/mcpServer.ts
@@ -12,11 +12,39 @@ import traceTools from './tools/trace/index';
 import { McpContext } from './tools/types';
 import { createRegisterTool } from './utils/registerTool';
 
+const SERVER_INSTRUCTIONS = [
+  'ClickStack observability MCP server for querying logs, metrics, traces, and',
+  'sessions, and for managing dashboards, alerts, sources, and saved searches.',
+  '',
+  'TOOL SELECTION POLICY — read before querying:',
+  'Default to the builder query tools. They are more reliable and produce',
+  'structured, chart-ready results:',
+  '  • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns',
+  '  • clickstack_timeseries — trends / metrics over time',
+  '  • clickstack_search — browsing individual log/trace rows',
+  '  • clickstack_event_patterns / clickstack_event_deltas — pattern & cohort analysis',
+  '  • clickstack_trace_waterfall / clickstack_trace_top_time_consuming_operations — trace drill-down',
+  '',
+  'Do NOT reach for clickstack_sql first. Raw SQL is a last resort, reserved for',
+  'queries the builder tools genuinely cannot express — JOINs, sub-queries, CTEs,',
+  'window functions, or tables not registered as sources. A single-source',
+  'aggregation, top-N, time-series, or row browse must always use a builder tool.',
+  '',
+  'Discovery flow: clickstack_list_sources → clickstack_describe_source → query.',
+  'Fetch the query_guide prompt for detailed syntax, and create_dashboard for',
+  'dashboard authoring.',
+].join('\n');
+
 export function createServer(context: McpContext) {
-  const server = new McpServer({
-    name: 'clickstack',
-    version: `${CODE_VERSION}-beta`,
-  });
+  const server = new McpServer(
+    {
+      name: 'clickstack',
+      version: `${CODE_VERSION}-beta`,
+    },
+    {
+      instructions: SERVER_INSTRUCTIONS,
+    },
+  );
 
   const registerTool = createRegisterTool(server, context);
   const registrar = { server, context, registerTool };
diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts
index e97ae33b64..6777ab11c0 100644
--- a/packages/api/src/mcp/tools/query/search.ts
+++ b/packages/api/src/mcp/tools/query/search.ts
@@ -66,6 +66,9 @@ export function registerSearch({ context, registerTool }: ToolRegistrar) {
         'Browse individual log/event/trace rows. ' +
         'Use this when you need to see raw events, investigate specific log lines, ' +
         'or drill into individual records matching a filter.\n\n' +
+        'PREFER THIS over clickstack_sql for browsing/filtering rows from a single source. ' +
+        'Only drop to raw SQL when the query needs JOINs, sub-queries, or CTEs this tool ' +
+        'cannot express.\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
         'For aggregated metrics, use clickstack_table instead. ' +
         'For pattern discovery, use clickstack_event_patterns instead.\n\n' +
diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts
index fa2ee74f32..eb94d00309 100644
--- a/packages/api/src/mcp/tools/query/sql.ts
+++ b/packages/api/src/mcp/tools/query/sql.ts
@@ -60,16 +60,22 @@ export function registerSql({ context, registerTool }: ToolRegistrar) {
     {
       title: 'Raw SQL Query',
       description:
-        'Execute raw ClickHouse SQL. ' +
-        'ADVANCED: only use this when you need capabilities the builder tools cannot express — ' +
-        'JOINs, sub-queries, CTEs, or querying tables not registered as sources.\n\n' +
+        'Execute raw ClickHouse SQL. LAST-RESORT TOOL — do NOT reach for this first.\n\n' +
+        'Default to the builder tools for querying; they are more reliable and produce ' +
+        'richer, structured results:\n' +
+        '  • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns\n' +
+        '  • clickstack_timeseries — trends and metrics over time\n' +
+        '  • clickstack_search — browsing individual rows\n' +
+        '  • clickstack_event_patterns / clickstack_event_deltas — pattern & cohort analysis\n\n' +
+        'ONLY use raw SQL when the query genuinely cannot be expressed by a builder tool — ' +
+        'i.e. it requires JOINs, sub-queries, CTEs, window functions, or tables not registered ' +
+        'as sources. A single-table aggregation, top-N, time-series, or row browse is ALWAYS a ' +
+        'builder-tool job, never raw SQL. If you are unsure, try the builder tool first and only ' +
+        'fall back to SQL if it cannot express what you need.\n\n' +
         'Requires connectionId (not sourceId) — call clickstack_list_sources to find connections. ' +
         'Call clickstack_describe_source to discover column names before writing SQL.\n\n' +
         'Results are always returned as table rows — for time-series semantics, ' +
-        'include a time column and ORDER BY it in your SQL.\n\n' +
-        'For standard aggregations use clickstack_table. ' +
-        'For time-series charts use clickstack_timeseries. ' +
-        'For browsing rows use clickstack_search.',
+        'include a time column and ORDER BY it in your SQL.',
       inputSchema: sqlSchema,
     },
     async input => {
diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts
index 4ccc2b8532..2de1402c18 100644
--- a/packages/api/src/mcp/tools/query/table.ts
+++ b/packages/api/src/mcp/tools/query/table.ts
@@ -208,6 +208,8 @@ export function registerTable({ context, registerTool }: ToolRegistrar) {
         'Compute aggregated metrics as a table, single number, pie chart, or bar chart. ' +
         'Use this for grouped aggregations, top-N queries, single-value KPIs, ' +
         'or proportional breakdowns.\n\n' +
+        'PREFER THIS over clickstack_sql for any single-source aggregation. Only drop to raw ' +
+        'SQL when the query needs JOINs, sub-queries, or CTEs this tool cannot express.\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
         'Use the top-level "where" to scope the entire query (e.g. filter by service). ' +
         'Each select item can also have its own "where" for per-metric cohort ' +
diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts
index 78666bdc29..f91c630ee5 100644
--- a/packages/api/src/mcp/tools/query/timeseries.ts
+++ b/packages/api/src/mcp/tools/query/timeseries.ts
@@ -82,6 +82,9 @@ export function registerTimeseries({ context, registerTool }: ToolRegistrar) {
         'Plot metrics over time as a line or stacked bar chart. ' +
         'Use this when you need to visualize trends, compare time-series, ' +
         'or monitor metric changes over a time window.\n\n' +
+        'PREFER THIS over clickstack_sql for any time-bucketed query — do not hand-roll ' +
+        'toStartOfInterval SQL. Only drop to raw SQL when the query needs JOINs, sub-queries, ' +
+        'or CTEs this tool cannot express.\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first. ' +
         'Each select item defines one plotted series.\n\n' +
         'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' +

From 5f1ea5a74e14e5329e4f6af3f902b385bc56828f Mon Sep 17 00:00:00 2001
From: Brandon Pereira <brandon.pereira@clickhouse.com>
Date: Fri, 7 Aug 2026 15:05:40 -0600
Subject: [PATCH 2/7] feat(hdx-eval): preflight MCP reachability before running
 a batch

Probe each MCP server (initialize + tools/list) before spawning any
agents and abort the batch if a server is unreachable or serves zero
tools. Previously a dead API server produced an entire suite of silent
zero-tool-call runs that graded as real (bad) scores, masking the
failure. Add --no-preflight to bypass.
---
 .../hdx-eval/src/__tests__/preflight.test.ts  | 156 ++++++++++++++
 packages/hdx-eval/src/cli.ts                  |  39 ++++
 packages/hdx-eval/src/harness/preflight.ts    | 199 ++++++++++++++++++
 3 files changed, 394 insertions(+)
 create mode 100644 packages/hdx-eval/src/__tests__/preflight.test.ts
 create mode 100644 packages/hdx-eval/src/harness/preflight.ts

diff --git a/packages/hdx-eval/src/__tests__/preflight.test.ts b/packages/hdx-eval/src/__tests__/preflight.test.ts
new file mode 100644
index 0000000000..d527f36435
--- /dev/null
+++ b/packages/hdx-eval/src/__tests__/preflight.test.ts
@@ -0,0 +1,156 @@
+import { preflightMcps, probeMcp } from '@/harness/preflight';
+import type { McpDefinition } from '@/harness/types';
+
+const httpDef = (url = 'http://localhost:30199/mcp'): McpDefinition => ({
+  type: 'http',
+  url,
+  headers: { Authorization: 'Bearer test' },
+  toolPattern: 'mcp__hyperdx__*',
+  label: 'HyperDX',
+});
+
+/** Build a Response-like object for the mocked fetch. */
+function sseResponse(status: number, jsonRpc: unknown): Response {
+  const body = `event: message\ndata: ${JSON.stringify(jsonRpc)}\n\n`;
+  return {
+    status,
+    headers: { get: () => null },
+    text: async () => body,
+  } as unknown as Response;
+}
+
+function jsonResponse(status: number, jsonRpc: unknown): Response {
+  return {
+    status,
+    headers: { get: () => null },
+    text: async () => JSON.stringify(jsonRpc),
+  } as unknown as Response;
+}
+
+const originalFetch = global.fetch;
+afterEach(() => {
+  global.fetch = originalFetch;
+});
+
+describe('probeMcp', () => {
+  it('returns ok with a tool count when the server initializes and lists tools (SSE framing)', async () => {
+    const fetchMock = jest
+      .fn()
+      .mockResolvedValueOnce(sseResponse(200, { result: { serverInfo: {} } }))
+      .mockResolvedValueOnce(
+        sseResponse(200, {
+          result: { tools: [{ name: 'a' }, { name: 'b' }, { name: 'c' }] },
+        }),
+      );
+    global.fetch = fetchMock as unknown as typeof fetch;
+
+    const res = await probeMcp('hyperdx', httpDef());
+    expect(res).toEqual({ mcp: 'hyperdx', ok: true, toolCount: 3 });
+  });
+
+  it('handles plain JSON (non-SSE) response bodies', async () => {
+    global.fetch = jest
+      .fn()
+      .mockResolvedValueOnce(jsonResponse(200, { result: {} }))
+      .mockResolvedValueOnce(
+        jsonResponse(200, { result: { tools: [{ name: 'a' }] } }),
+      ) as unknown as typeof fetch;
+
+    const res = await probeMcp('hyperdx', httpDef());
+    expect(res.ok).toBe(true);
+    expect(res.toolCount).toBe(1);
+  });
+
+  it('fails when the server is reachable but serves zero tools', async () => {
+    global.fetch = jest
+      .fn()
+      .mockResolvedValueOnce(sseResponse(200, { result: {} }))
+      .mockResolvedValueOnce(
+        sseResponse(200, { result: { tools: [] } }),
+      ) as unknown as typeof fetch;
+
+    const res = await probeMcp('hyperdx', httpDef());
+    expect(res.ok).toBe(false);
+    expect(res.toolCount).toBe(0);
+    expect(res.error).toMatch(/0 tools/);
+  });
+
+  it('fails when initialize returns a non-2xx status', async () => {
+    global.fetch = jest
+      .fn()
+      .mockResolvedValueOnce(sseResponse(404, {})) as unknown as typeof fetch;
+
+    const res = await probeMcp('hyperdx', httpDef());
+    expect(res.ok).toBe(false);
+    expect(res.error).toMatch(/HTTP 404/);
+  });
+
+  it('fails with an actionable message when the connection is refused', async () => {
+    global.fetch = jest
+      .fn()
+      .mockRejectedValueOnce(
+        new Error('fetch failed'),
+      ) as unknown as typeof fetch;
+
+    const res = await probeMcp('hyperdx', httpDef());
+    expect(res.ok).toBe(false);
+    expect(res.error).toMatch(/could not connect/);
+    expect(res.error).toMatch(/API server/);
+  });
+
+  it('surfaces a JSON-RPC error object from initialize', async () => {
+    global.fetch = jest
+      .fn()
+      .mockResolvedValueOnce(
+        sseResponse(200, { error: { code: -32000, message: 'nope' } }),
+      ) as unknown as typeof fetch;
+
+    const res = await probeMcp('hyperdx', httpDef());
+    expect(res.ok).toBe(false);
+    expect(res.error).toMatch(/JSON-RPC error/);
+  });
+
+  it('treats stdio MCPs as ok without probing', async () => {
+    global.fetch = jest.fn(() => {
+      throw new Error('should not be called for stdio');
+    }) as unknown as typeof fetch;
+
+    const stdioDef: McpDefinition = {
+      type: 'stdio',
+      command: 'uvx',
+      args: ['mcp-clickhouse'],
+      toolPattern: 'mcp__clickhouse__*',
+      label: 'ClickHouse',
+    };
+    const res = await probeMcp('clickhouse', stdioDef);
+    expect(res).toEqual({ mcp: 'clickhouse', ok: true });
+  });
+});
+
+describe('preflightMcps', () => {
+  it('probes every entry and preserves order', async () => {
+    // Probes run concurrently (Promise.all), so route by URL rather than by
+    // call order. Slot 99 (branch) is up and serves a tool; slot 98 (main)
+    // refuses the connection.
+    global.fetch = jest.fn((url: string, init: { body: string }) => {
+      if (url.includes('30198')) {
+        return Promise.reject(new Error('fetch failed'));
+      }
+      const method = JSON.parse(init.body).method;
+      return Promise.resolve(
+        method === 'tools/list'
+          ? sseResponse(200, { result: { tools: [{ name: 'x' }] } })
+          : sseResponse(200, { result: {} }),
+      );
+    }) as unknown as typeof fetch;
+
+    const results = await preflightMcps([
+      { mcp: 'branch', def: httpDef('http://localhost:30199/mcp') },
+      { mcp: 'main', def: httpDef('http://localhost:30198/mcp') },
+    ]);
+
+    expect(results.map(r => r.mcp)).toEqual(['branch', 'main']);
+    expect(results[0].ok).toBe(true);
+    expect(results[1].ok).toBe(false);
+  });
+});
diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts
index 9d12467d88..6eca011b84 100644
--- a/packages/hdx-eval/src/cli.ts
+++ b/packages/hdx-eval/src/cli.ts
@@ -17,6 +17,7 @@ import {
   resolveBatchDir,
 } from './grading/grade';
 import { DEFAULT_JUDGE_SPEC, parseJudgeSpec } from './grading/judgeModel';
+import { preflightMcps } from './harness/preflight';
 import { runCell } from './harness/runRun';
 import { type McpKind, PLUGIN_NONE, type PromptVariant } from './harness/types';
 import {
@@ -464,6 +465,14 @@ program
     '--no-judge',
     'Run programmatic checks only during auto-grading (skip LLM judge)',
   )
+  .option(
+    '--no-preflight',
+    'Skip the MCP reachability preflight check. By default the runner probes ' +
+      'each MCP server (initialize + tools/list) before spawning any agents ' +
+      'and aborts if a server is unreachable or serves no tools — this ' +
+      'prevents an entire batch of silent zero-tool-call runs when an API ' +
+      'server is down.',
+  )
   .option(
     '--email <email>',
     `HyperDX account email for post-run inspection (default: ${DEFAULT_EVAL_EMAIL})`,
@@ -495,6 +504,7 @@ program
         report: boolean;
         judgeModel?: string;
         judge: boolean;
+        preflight: boolean;
         email: string;
         password: string;
       },
@@ -662,6 +672,35 @@ program
         }
       }
 
+      // Preflight: probe every MCP server before spawning any agents. A dead
+      // API server (or one serving zero tools) otherwise yields an entire
+      // batch of silent zero-tool-call runs that look like real (bad) scores
+      // but actually reflect an unreachable server — wasting a full suite.
+      if (cmdOpts.preflight) {
+        const probes = await preflightMcps(
+          mcpKinds.map(k => ({ mcp: k, def: getMcpDefinition(config, k) })),
+        );
+        for (const p of probes) {
+          if (p.ok) {
+            const tools =
+              p.toolCount !== undefined ? ` (${p.toolCount} tools)` : '';
+            console.log(`Preflight ${p.mcp}: OK${tools}`);
+          } else {
+            console.error(`Preflight ${p.mcp}: FAILED — ${p.error}`);
+          }
+        }
+        const failed = probes.filter(p => !p.ok);
+        if (failed.length > 0) {
+          throw new Error(
+            `MCP preflight failed for: ${failed
+              .map(p => p.mcp)
+              .join(', ')}. Fix the server(s) above (usually: the dev stack ` +
+              `for that slot is not running) and re-run, or pass ` +
+              `--no-preflight to bypass this check.`,
+          );
+        }
+      }
+
       const batchDir = batchDirName();
       console.log(`\nBatch: runs/${batchDir}`);
       console.log(`Scenario: ${scenario.name}`);
diff --git a/packages/hdx-eval/src/harness/preflight.ts b/packages/hdx-eval/src/harness/preflight.ts
new file mode 100644
index 0000000000..cf6ed648de
--- /dev/null
+++ b/packages/hdx-eval/src/harness/preflight.ts
@@ -0,0 +1,199 @@
+import type { McpDefinition, McpKind } from './types';
+
+/**
+ * Result of probing a single MCP server for reachability.
+ */
+export type PreflightResult = {
+  mcp: McpKind;
+  ok: boolean;
+  /** Number of tools the server advertised via `tools/list` (HTTP only). */
+  toolCount?: number;
+  /** Human-readable failure reason when `ok` is false. */
+  error?: string;
+};
+
+const MCP_PROTOCOL_VERSION = '2024-11-05';
+
+/**
+ * Parse a JSON-RPC response body that may be either a plain JSON object
+ * (Content-Type: application/json) or a Server-Sent Events stream
+ * (Content-Type: text/event-stream, one `data: {...}` line per event).
+ * The stateless Streamable HTTP transport used by the HyperDX MCP server
+ * replies with SSE, so we extract the first `data:` payload.
+ */
+function parseJsonRpcBody(body: string): unknown {
+  const trimmed = body.trim();
+  if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+    return JSON.parse(trimmed);
+  }
+  // SSE framing: find the first `data:` line and parse it.
+  for (const line of trimmed.split('\n')) {
+    const m = line.match(/^data:\s*(.*)$/);
+    if (m && m[1]) return JSON.parse(m[1]);
+  }
+  throw new Error('no JSON-RPC payload found in response body');
+}
+
+async function postJsonRpc(
+  url: string,
+  headers: Record<string, string>,
+  payload: unknown,
+  timeoutMs: number,
+): Promise<{ status: number; body: string; mcpSessionId: string | null }> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        // The Streamable HTTP transport requires the client to accept both.
+        Accept: 'application/json, text/event-stream',
+        ...headers,
+      },
+      body: JSON.stringify(payload),
+      signal: controller.signal,
+    });
+    const body = await res.text();
+    return {
+      status: res.status,
+      body,
+      mcpSessionId: res.headers.get('mcp-session-id'),
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Probe a single HTTP MCP server: run the `initialize` handshake and then
+ * `tools/list`, confirming the server is reachable and actually serving
+ * tools. stdio MCPs are reported as `ok` without probing (they are spawned
+ * per-run by Claude Code and cannot be probed out-of-band here).
+ */
+export async function probeMcp(
+  mcp: McpKind,
+  def: McpDefinition,
+  timeoutMs = 10_000,
+): Promise<PreflightResult> {
+  if (def.type !== 'http') {
+    // stdio servers are launched by Claude Code itself; nothing to probe.
+    return { mcp, ok: true };
+  }
+
+  const headers = def.headers ?? {};
+  try {
+    const init = await postJsonRpc(
+      def.url,
+      headers,
+      {
+        jsonrpc: '2.0',
+        id: 1,
+        method: 'initialize',
+        params: {
+          protocolVersion: MCP_PROTOCOL_VERSION,
+          capabilities: {},
+          clientInfo: { name: 'hdx-eval-preflight', version: '1.0' },
+        },
+      },
+      timeoutMs,
+    );
+
+    if (init.status < 200 || init.status >= 300) {
+      return {
+        mcp,
+        ok: false,
+        error: `initialize returned HTTP ${init.status} (is the API server on this slot running?)`,
+      };
+    }
+
+    let initResult: unknown;
+    try {
+      initResult = parseJsonRpcBody(init.body);
+    } catch (e) {
+      return {
+        mcp,
+        ok: false,
+        error: `initialize response was not valid JSON-RPC: ${(e as Error).message}`,
+      };
+    }
+    if ((initResult as { error?: unknown })?.error) {
+      return {
+        mcp,
+        ok: false,
+        error: `initialize returned a JSON-RPC error: ${JSON.stringify(
+          (initResult as { error: unknown }).error,
+        )}`,
+      };
+    }
+
+    // Some transports require the session id from initialize on later calls.
+    const sessionHeaders = init.mcpSessionId
+      ? { ...headers, 'mcp-session-id': init.mcpSessionId }
+      : headers;
+
+    const list = await postJsonRpc(
+      def.url,
+      sessionHeaders,
+      { jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} },
+      timeoutMs,
+    );
+    if (list.status < 200 || list.status >= 300) {
+      return {
+        mcp,
+        ok: false,
+        error: `tools/list returned HTTP ${list.status}`,
+      };
+    }
+
+    let toolCount: number | undefined;
+    try {
+      const parsed = parseJsonRpcBody(list.body) as {
+        result?: { tools?: unknown[] };
+      };
+      toolCount = parsed.result?.tools?.length;
+    } catch {
+      // tools/list body unparseable — treat as zero tools below.
+      toolCount = undefined;
+    }
+
+    if (!toolCount || toolCount === 0) {
+      return {
+        mcp,
+        ok: false,
+        toolCount: toolCount ?? 0,
+        error:
+          'server is reachable but advertised 0 tools via tools/list ' +
+          '(the agent would have nothing to call)',
+      };
+    }
+
+    return { mcp, ok: true, toolCount };
+  } catch (e) {
+    const err = e as Error;
+    const reason =
+      err.name === 'AbortError'
+        ? `no response within ${timeoutMs}ms`
+        : err.message;
+    return {
+      mcp,
+      ok: false,
+      error: `could not connect to ${def.url}: ${reason} (is the API server on this slot running?)`,
+    };
+  }
+}
+
+/**
+ * Probe every MCP that a run batch will use. Returns the per-MCP results so
+ * the caller can abort the batch before spawning any agents when a server is
+ * unreachable — preventing an entire suite of silent zero-tool-call runs that
+ * look like (bad) scores but actually reflect a dead server.
+ */
+export async function preflightMcps(
+  entries: Array<{ mcp: McpKind; def: McpDefinition }>,
+  timeoutMs = 10_000,
+): Promise<PreflightResult[]> {
+  return Promise.all(
+    entries.map(({ mcp, def }) => probeMcp(mcp, def, timeoutMs)),
+  );
+}

From 42a5b08eb34fb21726e53411eeb76e3e716ab28b Mon Sep 17 00:00:00 2001
From: Brandon Pereira <brandon.pereira@clickhouse.com>
Date: Mon, 10 Aug 2026 09:02:21 -0600
Subject: [PATCH 3/7] fix(hdx-eval): run MCP preflight before seed
 detection/reseed

Move the preflight probe ahead of the re-seed block so an unavailable
MCP server fails fast instead of after truncating and repopulating
scenario tables (--reseed/--live). The preflight is independent of
seeding, so running it first avoids the avoidable destructive work.

Addresses Greptile review feedback on #2840.
---
 packages/hdx-eval/src/cli.ts | 61 +++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts
index 6eca011b84..f0c83093d5 100644
--- a/packages/hdx-eval/src/cli.ts
+++ b/packages/hdx-eval/src/cli.ts
@@ -626,6 +626,38 @@ program
         anchorMs = anchor.anchorMs;
       }
 
+      // Preflight: probe every MCP server before doing any expensive or
+      // destructive setup (seed detection / reseeding) and before spawning
+      // any agents. A dead API server (or one serving zero tools) otherwise
+      // yields an entire batch of silent zero-tool-call runs that look like
+      // real (bad) scores but actually reflect an unreachable server —
+      // wasting a full suite. Running it first also avoids truncating and
+      // repopulating scenario tables (via --reseed/--live) only to abort.
+      if (cmdOpts.preflight) {
+        const probes = await preflightMcps(
+          mcpKinds.map(k => ({ mcp: k, def: getMcpDefinition(config, k) })),
+        );
+        for (const p of probes) {
+          if (p.ok) {
+            const tools =
+              p.toolCount !== undefined ? ` (${p.toolCount} tools)` : '';
+            console.log(`Preflight ${p.mcp}: OK${tools}`);
+          } else {
+            console.error(`Preflight ${p.mcp}: FAILED — ${p.error}`);
+          }
+        }
+        const failed = probes.filter(p => !p.ok);
+        if (failed.length > 0) {
+          throw new Error(
+            `MCP preflight failed for: ${failed
+              .map(p => p.mcp)
+              .join(', ')}. Fix the server(s) above (usually: the dev stack ` +
+              `for that slot is not running) and re-run, or pass ` +
+              `--no-preflight to bypass this check.`,
+          );
+        }
+      }
+
       // ── Re-seed ───────────────────────────────────────────────────
       // Default: skip reseed IF data already exists. Auto-seeds on first
       // run when scenario tables are empty or missing.
@@ -672,35 +704,6 @@ program
         }
       }
 
-      // Preflight: probe every MCP server before spawning any agents. A dead
-      // API server (or one serving zero tools) otherwise yields an entire
-      // batch of silent zero-tool-call runs that look like real (bad) scores
-      // but actually reflect an unreachable server — wasting a full suite.
-      if (cmdOpts.preflight) {
-        const probes = await preflightMcps(
-          mcpKinds.map(k => ({ mcp: k, def: getMcpDefinition(config, k) })),
-        );
-        for (const p of probes) {
-          if (p.ok) {
-            const tools =
-              p.toolCount !== undefined ? ` (${p.toolCount} tools)` : '';
-            console.log(`Preflight ${p.mcp}: OK${tools}`);
-          } else {
-            console.error(`Preflight ${p.mcp}: FAILED — ${p.error}`);
-          }
-        }
-        const failed = probes.filter(p => !p.ok);
-        if (failed.length > 0) {
-          throw new Error(
-            `MCP preflight failed for: ${failed
-              .map(p => p.mcp)
-              .join(', ')}. Fix the server(s) above (usually: the dev stack ` +
-              `for that slot is not running) and re-run, or pass ` +
-              `--no-preflight to bypass this check.`,
-          );
-        }
-      }
-
       const batchDir = batchDirName();
       console.log(`\nBatch: runs/${batchDir}`);
       console.log(`Scenario: ${scenario.name}`);

From 29cbc624436992d64388133acb8fb6253701ba20 Mon Sep 17 00:00:00 2001
From: Brandon Pereira <brandon.pereira@clickhouse.com>
Date: Mon, 10 Aug 2026 09:24:25 -0600
Subject: [PATCH 4/7] refactor(mcp): centralize builder-tool steering catalog

The 'prefer builder tools over raw SQL' list was hand-duplicated across
SERVER_INSTRUCTIONS, the clickstack_sql description, and each builder
tool's description, and had already drifted (sql.ts omitted the trace
tools; event_patterns/event_deltas lacked the reciprocal nudge).

Define the builder-tool catalog once in builderCatalog.ts and compose
the server instructions, the sql.ts alternatives list, and every builder
tool's 'prefer me over SQL' nudge from it. Adds event_patterns and
event_deltas nudges that were missing, and a drift-guard test that fails
if the catalog and the registered query/trace tools diverge.

Addresses Greptile review feedback on #2840.
---
 .../src/mcp/__tests__/builderCatalog.test.ts  |  84 +++++++++++++
 packages/api/src/mcp/mcpServer.ts             |  16 +--
 .../api/src/mcp/tools/query/builderCatalog.ts | 113 ++++++++++++++++++
 .../api/src/mcp/tools/query/eventDeltas.ts    |   3 +
 .../api/src/mcp/tools/query/eventPatterns.ts  |   3 +
 packages/api/src/mcp/tools/query/search.ts    |   6 +-
 packages/api/src/mcp/tools/query/sql.ts       |  11 +-
 packages/api/src/mcp/tools/query/table.ts     |   5 +-
 .../api/src/mcp/tools/query/timeseries.ts     |   6 +-
 9 files changed, 225 insertions(+), 22 deletions(-)
 create mode 100644 packages/api/src/mcp/__tests__/builderCatalog.test.ts
 create mode 100644 packages/api/src/mcp/tools/query/builderCatalog.ts

diff --git a/packages/api/src/mcp/__tests__/builderCatalog.test.ts b/packages/api/src/mcp/__tests__/builderCatalog.test.ts
new file mode 100644
index 0000000000..4b01deca9b
--- /dev/null
+++ b/packages/api/src/mcp/__tests__/builderCatalog.test.ts
@@ -0,0 +1,84 @@
+import { readdirSync, readFileSync } from 'fs';
+import { join } from 'path';
+
+import {
+  BUILDER_TOOLS,
+  builderToolBulletList,
+  preferOverSqlNudge,
+  SQL_FALLBACK_CRITERIA,
+} from '@/mcp/tools/query/builderCatalog';
+
+describe('builderCatalog', () => {
+  it('lists every builder tool as a bullet, one per line', () => {
+    const list = builderToolBulletList();
+    const lines = list.split('\n');
+    expect(lines).toHaveLength(BUILDER_TOOLS.length);
+    for (const tool of BUILDER_TOOLS) {
+      expect(list).toContain(`${tool.name} — ${tool.blurb}`);
+    }
+  });
+
+  it('never lists clickstack_sql among the builder tools', () => {
+    expect(BUILDER_TOOLS.some(t => t.name === 'clickstack_sql')).toBe(false);
+    expect(builderToolBulletList()).not.toContain('clickstack_sql —');
+  });
+
+  it('honors bullet/indent options', () => {
+    const list = builderToolBulletList({ indent: '', bullet: '-' });
+    expect(list.split('\n')[0]).toBe(
+      `- ${BUILDER_TOOLS[0].name} — ${BUILDER_TOOLS[0].blurb}`,
+    );
+  });
+
+  it('produces a consistent reciprocal nudge for every tool that has a preferHint', () => {
+    for (const tool of BUILDER_TOOLS) {
+      if (!tool.preferHint) continue;
+      const nudge = preferOverSqlNudge(tool.name);
+      expect(nudge).toContain('PREFER THIS over clickstack_sql');
+      expect(nudge).toContain(tool.preferHint);
+      // The SQL fallback criteria must be shared verbatim (no drift).
+      expect(nudge).toContain(SQL_FALLBACK_CRITERIA);
+    }
+  });
+
+  it('throws for an unknown tool or one without a preferHint', () => {
+    expect(() => preferOverSqlNudge('clickstack_does_not_exist')).toThrow();
+    const hintless = BUILDER_TOOLS.find(t => !t.preferHint);
+    if (hintless) {
+      expect(() => preferOverSqlNudge(hintless.name)).toThrow();
+    }
+  });
+
+  // Drift guard: the catalog must stay in sync with the tools actually
+  // registered under tools/query and tools/trace. If a new builder query
+  // tool is added (or one is renamed) without updating the catalog, this
+  // fails — which is the whole point of centralizing the list.
+  it('matches the set of registered non-SQL query/trace tools', () => {
+    const dirs = [
+      join(__dirname, '..', 'tools', 'query'),
+      join(__dirname, '..', 'tools', 'trace'),
+    ];
+    const registered = new Set<string>();
+    const files = dirs.flatMap(dir =>
+      // Paths are derived from __dirname (not user input); safe to read.
+      // eslint-disable-next-line security/detect-non-literal-fs-filename
+      readdirSync(dir)
+        .filter(f => f.endsWith('.ts') && !f.endsWith('.test.ts'))
+        .map(f => join(dir, f)),
+    );
+    for (const file of files) {
+      // eslint-disable-next-line security/detect-non-literal-fs-filename
+      const src = readFileSync(file, 'utf8');
+      for (const m of src.matchAll(
+        /registerTool\(\s*'(clickstack_[a-z_]+)'/g,
+      )) {
+        registered.add(m[1]);
+      }
+    }
+    // clickstack_sql is intentionally excluded from the builder catalog.
+    registered.delete('clickstack_sql');
+
+    const catalog = new Set(BUILDER_TOOLS.map(t => t.name));
+    expect([...catalog].sort()).toEqual([...registered].sort());
+  });
+});
diff --git a/packages/api/src/mcp/mcpServer.ts b/packages/api/src/mcp/mcpServer.ts
index 8174162ecc..9c0275ce12 100644
--- a/packages/api/src/mcp/mcpServer.ts
+++ b/packages/api/src/mcp/mcpServer.ts
@@ -5,6 +5,10 @@ import { CODE_VERSION } from '@/config';
 import dashboardPrompts from './prompts/dashboards/index';
 import alertsTools from './tools/alerts/index';
 import dashboardsTools from './tools/dashboards/index';
+import {
+  builderToolBulletList,
+  SQL_FALLBACK_CRITERIA,
+} from './tools/query/builderCatalog';
 import queryTools from './tools/query/index';
 import savedSearchesTools from './tools/savedSearches/index';
 import sourcesTools from './tools/sources/index';
@@ -19,16 +23,12 @@ const SERVER_INSTRUCTIONS = [
   'TOOL SELECTION POLICY — read before querying:',
   'Default to the builder query tools. They are more reliable and produce',
   'structured, chart-ready results:',
-  '  • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns',
-  '  • clickstack_timeseries — trends / metrics over time',
-  '  • clickstack_search — browsing individual log/trace rows',
-  '  • clickstack_event_patterns / clickstack_event_deltas — pattern & cohort analysis',
-  '  • clickstack_trace_waterfall / clickstack_trace_top_time_consuming_operations — trace drill-down',
+  builderToolBulletList(),
   '',
   'Do NOT reach for clickstack_sql first. Raw SQL is a last resort, reserved for',
-  'queries the builder tools genuinely cannot express — JOINs, sub-queries, CTEs,',
-  'window functions, or tables not registered as sources. A single-source',
-  'aggregation, top-N, time-series, or row browse must always use a builder tool.',
+  `queries the builder tools genuinely cannot express — ${SQL_FALLBACK_CRITERIA}.`,
+  'A single-source aggregation, top-N, time-series, or row browse must always',
+  'use a builder tool.',
   '',
   'Discovery flow: clickstack_list_sources → clickstack_describe_source → query.',
   'Fetch the query_guide prompt for detailed syntax, and create_dashboard for',
diff --git a/packages/api/src/mcp/tools/query/builderCatalog.ts b/packages/api/src/mcp/tools/query/builderCatalog.ts
new file mode 100644
index 0000000000..46cb5620b2
--- /dev/null
+++ b/packages/api/src/mcp/tools/query/builderCatalog.ts
@@ -0,0 +1,113 @@
+/**
+ * Single source of truth for the "prefer the builder tools over raw SQL"
+ * steering that appears in three places:
+ *   - the server-level instructions (mcpServer.ts),
+ *   - the clickstack_sql description (which lists the alternatives), and
+ *   - each builder tool's own description (the reciprocal "prefer me" nudge).
+ *
+ * Defining the catalog once keeps those lists from silently drifting apart
+ * (e.g. a new builder tool being added to one place but not the others).
+ */
+
+/** The canonical reason SQL is a last resort — shared by every nudge. */
+export const SQL_FALLBACK_CRITERIA =
+  'JOINs, sub-queries, CTEs, window functions, or tables not registered as sources';
+
+/**
+ * One entry per builder (non-SQL) query tool. `blurb` is the one-liner shown
+ * in the steering lists. `preferHint`, when present, is the tool-specific
+ * clause used in that tool's own "prefer me over SQL" nudge.
+ */
+export type BuilderTool = {
+  name: string;
+  /** One-liner for the bulleted steering lists. */
+  blurb: string;
+  /**
+   * Tool-specific phrasing for the reciprocal nudge in the tool's own
+   * description, e.g. "for any single-source aggregation". Omit for tools
+   * that only need to appear in the steering lists (not every tool has a
+   * hand-written description edit).
+   */
+  preferHint?: string;
+};
+
+/**
+ * The builder-tool catalog. Order is the order they appear in steering lists.
+ * Keep this in sync with the registered tools in tools/query and tools/trace.
+ */
+export const BUILDER_TOOLS: readonly BuilderTool[] = [
+  {
+    name: 'clickstack_table',
+    blurb: 'aggregations, top-N, single-value KPIs, breakdowns',
+    preferHint: 'for any single-source aggregation',
+  },
+  {
+    name: 'clickstack_timeseries',
+    blurb: 'trends / metrics over time',
+    preferHint:
+      'for any time-bucketed query — do not hand-roll toStartOfInterval SQL',
+  },
+  {
+    name: 'clickstack_search',
+    blurb: 'browsing individual log/trace rows',
+    preferHint: 'for browsing/filtering rows from a single source',
+  },
+  {
+    name: 'clickstack_event_patterns',
+    blurb: 'discover recurring log/event patterns (clustering)',
+    preferHint: 'for pattern / recurring-shape discovery',
+  },
+  {
+    name: 'clickstack_event_deltas',
+    blurb: 'rank the attributes that differ between two row groups',
+    preferHint: 'for target-vs-baseline attribute-shift analysis',
+  },
+  {
+    name: 'clickstack_emerging_signals',
+    blurb:
+      'detect log/event patterns that are new or gone vs a baseline window',
+  },
+  {
+    name: 'clickstack_trace_waterfall',
+    blurb: 'inspect one trace as a parent/child span tree',
+  },
+  {
+    name: 'clickstack_trace_top_time_consuming_operations',
+    blurb: 'rank the child operations contributing the most time in a trace',
+  },
+] as const;
+
+/**
+ * Bulleted steering list, e.g.
+ *   • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns
+ * `bullet` and `indent` let callers match the surrounding style (the server
+ * instructions and the sql.ts description differ slightly).
+ */
+export function builderToolBulletList(
+  opts: { indent?: string; bullet?: string } = {},
+): string {
+  const indent = opts.indent ?? '  ';
+  const bullet = opts.bullet ?? '•';
+  return BUILDER_TOOLS.map(
+    t => `${indent}${bullet} ${t.name} — ${t.blurb}`,
+  ).join('\n');
+}
+
+/**
+ * The reciprocal "prefer this over raw SQL" nudge for a single builder tool's
+ * own description. Throws if the tool has no `preferHint` (a wiring mistake we
+ * want to fail loudly at module load, not silently omit).
+ */
+export function preferOverSqlNudge(name: string): string {
+  const tool = BUILDER_TOOLS.find(t => t.name === name);
+  if (!tool || !tool.preferHint) {
+    throw new Error(
+      `preferOverSqlNudge: no builder tool with a preferHint named "${name}"`,
+    );
+  }
+  return (
+    `PREFER THIS over clickstack_sql ${tool.preferHint}. ` +
+    `Only drop to raw SQL for things this tool cannot express — ` +
+    `${SQL_FALLBACK_CRITERIA}.`
+  );
+}
diff --git a/packages/api/src/mcp/tools/query/eventDeltas.ts b/packages/api/src/mcp/tools/query/eventDeltas.ts
index 216a862f62..4b76873dce 100644
--- a/packages/api/src/mcp/tools/query/eventDeltas.ts
+++ b/packages/api/src/mcp/tools/query/eventDeltas.ts
@@ -18,6 +18,7 @@ import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 import { trimToolResponse } from '@/utils/trimToolResponse';
 
+import { preferOverSqlNudge } from './builderCatalog';
 import { clickHouseErrorResult, parseTimeRange } from './helpers';
 
 // ─── Schema ──────────────────────────────────────────────────────────────────
@@ -180,6 +181,8 @@ export function registerEventDeltas({ context, registerTool }: ToolRegistrar) {
         'specific attribute (use clickstack_table groupBy), when you want raw ' +
         'rows (use clickstack_search), or when you need a time-series shape ' +
         '(use clickstack_timeseries).\n\n' +
+        preferOverSqlNudge('clickstack_event_deltas') +
+        '\n\n' +
         'OUTPUT SHAPE: an array of properties, each with rank, key, score, ' +
         'semanticBoost (true for well-known OTel attrs like service.name / ' +
         'http.method / error.type / status), targetCount and baselineCount ' +
diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts
index 92e2a95bad..bea7609776 100644
--- a/packages/api/src/mcp/tools/query/eventPatterns.ts
+++ b/packages/api/src/mcp/tools/query/eventPatterns.ts
@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
+import { preferOverSqlNudge } from './builderCatalog';
 import { parseTimeRange } from './helpers';
 import { runEventPatterns } from './runEventPatterns';
 import {
@@ -86,6 +87,8 @@ export function registerEventPatterns({
         'Each pattern includes a "whereSnippet" — use it as the "where" parameter in ' +
         'a follow-up clickstack_search call to browse matching raw events.\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
+        preferOverSqlNudge('clickstack_event_patterns') +
+        '\n\n' +
         'When to use which tool:\n' +
         '  - clickstack_event_patterns: clustering / recurring shapes / noise analysis\n' +
         '  - clickstack_search: raw individual rows\n' +
diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts
index 640999ee1b..bb9cd1e70e 100644
--- a/packages/api/src/mcp/tools/query/search.ts
+++ b/packages/api/src/mcp/tools/query/search.ts
@@ -5,6 +5,7 @@ import { mcpUserError } from '@/mcp/utils/errors';
 import logger from '@/utils/logger';
 import { trimToolResponse } from '@/utils/trimToolResponse';
 
+import { preferOverSqlNudge } from './builderCatalog';
 import { denoiseSearchResults } from './denoise';
 import { buildTile, parseTimeRange, runConfigTile } from './helpers';
 import {
@@ -67,9 +68,8 @@ export function registerSearch({ context, registerTool }: ToolRegistrar) {
         'Browse individual log/event/trace rows. ' +
         'Use this when you need to see raw events, investigate specific log lines, ' +
         'or drill into individual records matching a filter.\n\n' +
-        'PREFER THIS over clickstack_sql for browsing/filtering rows from a single source. ' +
-        'Only drop to raw SQL when the query needs JOINs, sub-queries, or CTEs this tool ' +
-        'cannot express.\n\n' +
+        preferOverSqlNudge('clickstack_search') +
+        '\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
         'For aggregated metrics, use clickstack_table instead. ' +
         'For pattern discovery, use clickstack_event_patterns instead.\n\n' +
diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts
index 8208051941..62eb08f8d7 100644
--- a/packages/api/src/mcp/tools/query/sql.ts
+++ b/packages/api/src/mcp/tools/query/sql.ts
@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
+import { builderToolBulletList, SQL_FALLBACK_CRITERIA } from './builderCatalog';
 import { buildTile, parseTimeRange, runConfigTile } from './helpers';
 import { endTimeSchema, startTimeSchema } from './schemas';
 
@@ -65,13 +66,11 @@ export function registerSql({ context, registerTool }: ToolRegistrar) {
         'Execute raw ClickHouse SQL. LAST-RESORT TOOL — do NOT reach for this first.\n\n' +
         'Default to the builder tools for querying; they are more reliable and produce ' +
         'richer, structured results:\n' +
-        '  • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns\n' +
-        '  • clickstack_timeseries — trends and metrics over time\n' +
-        '  • clickstack_search — browsing individual rows\n' +
-        '  • clickstack_event_patterns / clickstack_event_deltas — pattern & cohort analysis\n\n' +
+        builderToolBulletList() +
+        '\n\n' +
         'ONLY use raw SQL when the query genuinely cannot be expressed by a builder tool — ' +
-        'i.e. it requires JOINs, sub-queries, CTEs, window functions, or tables not registered ' +
-        'as sources. A single-table aggregation, top-N, time-series, or row browse is ALWAYS a ' +
+        `i.e. it requires ${SQL_FALLBACK_CRITERIA}. ` +
+        'A single-table aggregation, top-N, time-series, or row browse is ALWAYS a ' +
         'builder-tool job, never raw SQL. If you are unsure, try the builder tool first and only ' +
         'fall back to SQL if it cannot express what you need.\n\n' +
         'Requires connectionId (not sourceId) — call clickstack_list_sources to find connections. ' +
diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts
index 3422f18721..2f6afbe204 100644
--- a/packages/api/src/mcp/tools/query/table.ts
+++ b/packages/api/src/mcp/tools/query/table.ts
@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
+import { preferOverSqlNudge } from './builderCatalog';
 import {
   annotateIncreaseTopNHint,
   buildTile,
@@ -209,8 +210,8 @@ export function registerTable({ context, registerTool }: ToolRegistrar) {
         'Compute aggregated metrics as a table, single number, pie chart, or bar chart. ' +
         'Use this for grouped aggregations, top-N queries, single-value KPIs, ' +
         'or proportional breakdowns.\n\n' +
-        'PREFER THIS over clickstack_sql for any single-source aggregation. Only drop to raw ' +
-        'SQL when the query needs JOINs, sub-queries, or CTEs this tool cannot express.\n\n' +
+        preferOverSqlNudge('clickstack_table') +
+        '\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
         'Use the top-level "where" to scope the entire query (e.g. filter by service). ' +
         'Each select item can also have its own "where" for per-metric cohort ' +
diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts
index 4bfd0244a4..56b9bf0044 100644
--- a/packages/api/src/mcp/tools/query/timeseries.ts
+++ b/packages/api/src/mcp/tools/query/timeseries.ts
@@ -4,6 +4,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
+import { preferOverSqlNudge } from './builderCatalog';
 import {
   annotateIncreaseTopNHint,
   appendHint,
@@ -83,9 +84,8 @@ export function registerTimeseries({ context, registerTool }: ToolRegistrar) {
         'Plot metrics over time as a line or stacked bar chart. ' +
         'Use this when you need to visualize trends, compare time-series, ' +
         'or monitor metric changes over a time window.\n\n' +
-        'PREFER THIS over clickstack_sql for any time-bucketed query — do not hand-roll ' +
-        'toStartOfInterval SQL. Only drop to raw SQL when the query needs JOINs, sub-queries, ' +
-        'or CTEs this tool cannot express.\n\n' +
+        preferOverSqlNudge('clickstack_timeseries') +
+        '\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first. ' +
         'Each select item defines one plotted series.\n\n' +
         'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' +

From 5c1af4c56da31859c7ab5eb7ab234128c5be6ecd Mon Sep 17 00:00:00 2001
From: Brandon Pereira <brandon.pereira@clickhouse.com>
Date: Mon, 10 Aug 2026 09:34:45 -0600
Subject: [PATCH 5/7] refactor(mcp): simplify builder-tool steering to shared
 string constants

Replace the catalog module's types/functions/lookup (and the drift-guard
test) with three plain constants: BUILDER_TOOLS_LIST, SQL_FALLBACK_CRITERIA,
and a single generic PREFER_BUILDER_OVER_SQL_NUDGE reused by every builder
tool. Same anti-drift guarantee, far less machinery.
---
 .../src/mcp/__tests__/builderCatalog.test.ts  |  84 ------------
 packages/api/src/mcp/mcpServer.ts             |   4 +-
 .../api/src/mcp/tools/query/builderCatalog.ts | 126 +++---------------
 .../api/src/mcp/tools/query/eventDeltas.ts    |   4 +-
 .../api/src/mcp/tools/query/eventPatterns.ts  |   4 +-
 packages/api/src/mcp/tools/query/search.ts    |   4 +-
 packages/api/src/mcp/tools/query/sql.ts       |   4 +-
 packages/api/src/mcp/tools/query/table.ts     |   4 +-
 .../api/src/mcp/tools/query/timeseries.ts     |   4 +-
 9 files changed, 35 insertions(+), 203 deletions(-)
 delete mode 100644 packages/api/src/mcp/__tests__/builderCatalog.test.ts

diff --git a/packages/api/src/mcp/__tests__/builderCatalog.test.ts b/packages/api/src/mcp/__tests__/builderCatalog.test.ts
deleted file mode 100644
index 4b01deca9b..0000000000
--- a/packages/api/src/mcp/__tests__/builderCatalog.test.ts
+++ /dev/null
@@ -1,84 +0,0 @@
-import { readdirSync, readFileSync } from 'fs';
-import { join } from 'path';
-
-import {
-  BUILDER_TOOLS,
-  builderToolBulletList,
-  preferOverSqlNudge,
-  SQL_FALLBACK_CRITERIA,
-} from '@/mcp/tools/query/builderCatalog';
-
-describe('builderCatalog', () => {
-  it('lists every builder tool as a bullet, one per line', () => {
-    const list = builderToolBulletList();
-    const lines = list.split('\n');
-    expect(lines).toHaveLength(BUILDER_TOOLS.length);
-    for (const tool of BUILDER_TOOLS) {
-      expect(list).toContain(`${tool.name} — ${tool.blurb}`);
-    }
-  });
-
-  it('never lists clickstack_sql among the builder tools', () => {
-    expect(BUILDER_TOOLS.some(t => t.name === 'clickstack_sql')).toBe(false);
-    expect(builderToolBulletList()).not.toContain('clickstack_sql —');
-  });
-
-  it('honors bullet/indent options', () => {
-    const list = builderToolBulletList({ indent: '', bullet: '-' });
-    expect(list.split('\n')[0]).toBe(
-      `- ${BUILDER_TOOLS[0].name} — ${BUILDER_TOOLS[0].blurb}`,
-    );
-  });
-
-  it('produces a consistent reciprocal nudge for every tool that has a preferHint', () => {
-    for (const tool of BUILDER_TOOLS) {
-      if (!tool.preferHint) continue;
-      const nudge = preferOverSqlNudge(tool.name);
-      expect(nudge).toContain('PREFER THIS over clickstack_sql');
-      expect(nudge).toContain(tool.preferHint);
-      // The SQL fallback criteria must be shared verbatim (no drift).
-      expect(nudge).toContain(SQL_FALLBACK_CRITERIA);
-    }
-  });
-
-  it('throws for an unknown tool or one without a preferHint', () => {
-    expect(() => preferOverSqlNudge('clickstack_does_not_exist')).toThrow();
-    const hintless = BUILDER_TOOLS.find(t => !t.preferHint);
-    if (hintless) {
-      expect(() => preferOverSqlNudge(hintless.name)).toThrow();
-    }
-  });
-
-  // Drift guard: the catalog must stay in sync with the tools actually
-  // registered under tools/query and tools/trace. If a new builder query
-  // tool is added (or one is renamed) without updating the catalog, this
-  // fails — which is the whole point of centralizing the list.
-  it('matches the set of registered non-SQL query/trace tools', () => {
-    const dirs = [
-      join(__dirname, '..', 'tools', 'query'),
-      join(__dirname, '..', 'tools', 'trace'),
-    ];
-    const registered = new Set<string>();
-    const files = dirs.flatMap(dir =>
-      // Paths are derived from __dirname (not user input); safe to read.
-      // eslint-disable-next-line security/detect-non-literal-fs-filename
-      readdirSync(dir)
-        .filter(f => f.endsWith('.ts') && !f.endsWith('.test.ts'))
-        .map(f => join(dir, f)),
-    );
-    for (const file of files) {
-      // eslint-disable-next-line security/detect-non-literal-fs-filename
-      const src = readFileSync(file, 'utf8');
-      for (const m of src.matchAll(
-        /registerTool\(\s*'(clickstack_[a-z_]+)'/g,
-      )) {
-        registered.add(m[1]);
-      }
-    }
-    // clickstack_sql is intentionally excluded from the builder catalog.
-    registered.delete('clickstack_sql');
-
-    const catalog = new Set(BUILDER_TOOLS.map(t => t.name));
-    expect([...catalog].sort()).toEqual([...registered].sort());
-  });
-});
diff --git a/packages/api/src/mcp/mcpServer.ts b/packages/api/src/mcp/mcpServer.ts
index 9c0275ce12..33365a8662 100644
--- a/packages/api/src/mcp/mcpServer.ts
+++ b/packages/api/src/mcp/mcpServer.ts
@@ -6,7 +6,7 @@ import dashboardPrompts from './prompts/dashboards/index';
 import alertsTools from './tools/alerts/index';
 import dashboardsTools from './tools/dashboards/index';
 import {
-  builderToolBulletList,
+  BUILDER_TOOLS_LIST,
   SQL_FALLBACK_CRITERIA,
 } from './tools/query/builderCatalog';
 import queryTools from './tools/query/index';
@@ -23,7 +23,7 @@ const SERVER_INSTRUCTIONS = [
   'TOOL SELECTION POLICY — read before querying:',
   'Default to the builder query tools. They are more reliable and produce',
   'structured, chart-ready results:',
-  builderToolBulletList(),
+  BUILDER_TOOLS_LIST,
   '',
   'Do NOT reach for clickstack_sql first. Raw SQL is a last resort, reserved for',
   `queries the builder tools genuinely cannot express — ${SQL_FALLBACK_CRITERIA}.`,
diff --git a/packages/api/src/mcp/tools/query/builderCatalog.ts b/packages/api/src/mcp/tools/query/builderCatalog.ts
index 46cb5620b2..ee3070b7bf 100644
--- a/packages/api/src/mcp/tools/query/builderCatalog.ts
+++ b/packages/api/src/mcp/tools/query/builderCatalog.ts
@@ -1,113 +1,29 @@
 /**
- * Single source of truth for the "prefer the builder tools over raw SQL"
- * steering that appears in three places:
- *   - the server-level instructions (mcpServer.ts),
- *   - the clickstack_sql description (which lists the alternatives), and
- *   - each builder tool's own description (the reciprocal "prefer me" nudge).
- *
- * Defining the catalog once keeps those lists from silently drifting apart
- * (e.g. a new builder tool being added to one place but not the others).
+ * Shared "prefer the builder tools over raw SQL" steering, defined once so the
+ * server instructions, the clickstack_sql description, and the builder tools'
+ * descriptions can't drift apart.
  */
 
-/** The canonical reason SQL is a last resort — shared by every nudge. */
+/** Bulleted list of the builder (non-SQL) query tools and what they're for. */
+export const BUILDER_TOOLS_LIST = [
+  '  • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns',
+  '  • clickstack_timeseries — trends / metrics over time',
+  '  • clickstack_search — browsing individual log/trace rows',
+  '  • clickstack_event_patterns — recurring log/event pattern discovery',
+  '  • clickstack_event_deltas — attributes that differ between two row groups',
+  '  • clickstack_emerging_signals — patterns new or gone vs a baseline window',
+  '  • clickstack_trace_waterfall — one trace as a parent/child span tree',
+  '  • clickstack_trace_top_time_consuming_operations — slowest child operations in a trace',
+].join('\n');
+
+/** When raw SQL is actually warranted. */
 export const SQL_FALLBACK_CRITERIA =
   'JOINs, sub-queries, CTEs, window functions, or tables not registered as sources';
 
 /**
- * One entry per builder (non-SQL) query tool. `blurb` is the one-liner shown
- * in the steering lists. `preferHint`, when present, is the tool-specific
- * clause used in that tool's own "prefer me over SQL" nudge.
- */
-export type BuilderTool = {
-  name: string;
-  /** One-liner for the bulleted steering lists. */
-  blurb: string;
-  /**
-   * Tool-specific phrasing for the reciprocal nudge in the tool's own
-   * description, e.g. "for any single-source aggregation". Omit for tools
-   * that only need to appear in the steering lists (not every tool has a
-   * hand-written description edit).
-   */
-  preferHint?: string;
-};
-
-/**
- * The builder-tool catalog. Order is the order they appear in steering lists.
- * Keep this in sync with the registered tools in tools/query and tools/trace.
- */
-export const BUILDER_TOOLS: readonly BuilderTool[] = [
-  {
-    name: 'clickstack_table',
-    blurb: 'aggregations, top-N, single-value KPIs, breakdowns',
-    preferHint: 'for any single-source aggregation',
-  },
-  {
-    name: 'clickstack_timeseries',
-    blurb: 'trends / metrics over time',
-    preferHint:
-      'for any time-bucketed query — do not hand-roll toStartOfInterval SQL',
-  },
-  {
-    name: 'clickstack_search',
-    blurb: 'browsing individual log/trace rows',
-    preferHint: 'for browsing/filtering rows from a single source',
-  },
-  {
-    name: 'clickstack_event_patterns',
-    blurb: 'discover recurring log/event patterns (clustering)',
-    preferHint: 'for pattern / recurring-shape discovery',
-  },
-  {
-    name: 'clickstack_event_deltas',
-    blurb: 'rank the attributes that differ between two row groups',
-    preferHint: 'for target-vs-baseline attribute-shift analysis',
-  },
-  {
-    name: 'clickstack_emerging_signals',
-    blurb:
-      'detect log/event patterns that are new or gone vs a baseline window',
-  },
-  {
-    name: 'clickstack_trace_waterfall',
-    blurb: 'inspect one trace as a parent/child span tree',
-  },
-  {
-    name: 'clickstack_trace_top_time_consuming_operations',
-    blurb: 'rank the child operations contributing the most time in a trace',
-  },
-] as const;
-
-/**
- * Bulleted steering list, e.g.
- *   • clickstack_table — aggregations, top-N, single-value KPIs, breakdowns
- * `bullet` and `indent` let callers match the surrounding style (the server
- * instructions and the sql.ts description differ slightly).
- */
-export function builderToolBulletList(
-  opts: { indent?: string; bullet?: string } = {},
-): string {
-  const indent = opts.indent ?? '  ';
-  const bullet = opts.bullet ?? '•';
-  return BUILDER_TOOLS.map(
-    t => `${indent}${bullet} ${t.name} — ${t.blurb}`,
-  ).join('\n');
-}
-
-/**
- * The reciprocal "prefer this over raw SQL" nudge for a single builder tool's
- * own description. Throws if the tool has no `preferHint` (a wiring mistake we
- * want to fail loudly at module load, not silently omit).
+ * Reciprocal nudge dropped into each builder tool's description. Keep it
+ * generic so a single string works for every builder tool.
  */
-export function preferOverSqlNudge(name: string): string {
-  const tool = BUILDER_TOOLS.find(t => t.name === name);
-  if (!tool || !tool.preferHint) {
-    throw new Error(
-      `preferOverSqlNudge: no builder tool with a preferHint named "${name}"`,
-    );
-  }
-  return (
-    `PREFER THIS over clickstack_sql ${tool.preferHint}. ` +
-    `Only drop to raw SQL for things this tool cannot express — ` +
-    `${SQL_FALLBACK_CRITERIA}.`
-  );
-}
+export const PREFER_BUILDER_OVER_SQL_NUDGE =
+  'PREFER THIS over clickstack_sql. Only drop to raw SQL for things the ' +
+  `builder tools cannot express — ${SQL_FALLBACK_CRITERIA}.`;
diff --git a/packages/api/src/mcp/tools/query/eventDeltas.ts b/packages/api/src/mcp/tools/query/eventDeltas.ts
index 4b76873dce..8134c9ea94 100644
--- a/packages/api/src/mcp/tools/query/eventDeltas.ts
+++ b/packages/api/src/mcp/tools/query/eventDeltas.ts
@@ -18,7 +18,7 @@ import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 import { trimToolResponse } from '@/utils/trimToolResponse';
 
-import { preferOverSqlNudge } from './builderCatalog';
+import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog';
 import { clickHouseErrorResult, parseTimeRange } from './helpers';
 
 // ─── Schema ──────────────────────────────────────────────────────────────────
@@ -181,7 +181,7 @@ export function registerEventDeltas({ context, registerTool }: ToolRegistrar) {
         'specific attribute (use clickstack_table groupBy), when you want raw ' +
         'rows (use clickstack_search), or when you need a time-series shape ' +
         '(use clickstack_timeseries).\n\n' +
-        preferOverSqlNudge('clickstack_event_deltas') +
+        PREFER_BUILDER_OVER_SQL_NUDGE +
         '\n\n' +
         'OUTPUT SHAPE: an array of properties, each with rank, key, score, ' +
         'semanticBoost (true for well-known OTel attrs like service.name / ' +
diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts
index bea7609776..5be52c6dd6 100644
--- a/packages/api/src/mcp/tools/query/eventPatterns.ts
+++ b/packages/api/src/mcp/tools/query/eventPatterns.ts
@@ -3,7 +3,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
-import { preferOverSqlNudge } from './builderCatalog';
+import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog';
 import { parseTimeRange } from './helpers';
 import { runEventPatterns } from './runEventPatterns';
 import {
@@ -87,7 +87,7 @@ export function registerEventPatterns({
         'Each pattern includes a "whereSnippet" — use it as the "where" parameter in ' +
         'a follow-up clickstack_search call to browse matching raw events.\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
-        preferOverSqlNudge('clickstack_event_patterns') +
+        PREFER_BUILDER_OVER_SQL_NUDGE +
         '\n\n' +
         'When to use which tool:\n' +
         '  - clickstack_event_patterns: clustering / recurring shapes / noise analysis\n' +
diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts
index bb9cd1e70e..13ea2269f0 100644
--- a/packages/api/src/mcp/tools/query/search.ts
+++ b/packages/api/src/mcp/tools/query/search.ts
@@ -5,7 +5,7 @@ import { mcpUserError } from '@/mcp/utils/errors';
 import logger from '@/utils/logger';
 import { trimToolResponse } from '@/utils/trimToolResponse';
 
-import { preferOverSqlNudge } from './builderCatalog';
+import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog';
 import { denoiseSearchResults } from './denoise';
 import { buildTile, parseTimeRange, runConfigTile } from './helpers';
 import {
@@ -68,7 +68,7 @@ export function registerSearch({ context, registerTool }: ToolRegistrar) {
         'Browse individual log/event/trace rows. ' +
         'Use this when you need to see raw events, investigate specific log lines, ' +
         'or drill into individual records matching a filter.\n\n' +
-        preferOverSqlNudge('clickstack_search') +
+        PREFER_BUILDER_OVER_SQL_NUDGE +
         '\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
         'For aggregated metrics, use clickstack_table instead. ' +
diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts
index 62eb08f8d7..94eb52f79c 100644
--- a/packages/api/src/mcp/tools/query/sql.ts
+++ b/packages/api/src/mcp/tools/query/sql.ts
@@ -3,7 +3,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
-import { builderToolBulletList, SQL_FALLBACK_CRITERIA } from './builderCatalog';
+import { BUILDER_TOOLS_LIST, SQL_FALLBACK_CRITERIA } from './builderCatalog';
 import { buildTile, parseTimeRange, runConfigTile } from './helpers';
 import { endTimeSchema, startTimeSchema } from './schemas';
 
@@ -66,7 +66,7 @@ export function registerSql({ context, registerTool }: ToolRegistrar) {
         'Execute raw ClickHouse SQL. LAST-RESORT TOOL — do NOT reach for this first.\n\n' +
         'Default to the builder tools for querying; they are more reliable and produce ' +
         'richer, structured results:\n' +
-        builderToolBulletList() +
+        BUILDER_TOOLS_LIST +
         '\n\n' +
         'ONLY use raw SQL when the query genuinely cannot be expressed by a builder tool — ' +
         `i.e. it requires ${SQL_FALLBACK_CRITERIA}. ` +
diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts
index 2f6afbe204..64e66e953e 100644
--- a/packages/api/src/mcp/tools/query/table.ts
+++ b/packages/api/src/mcp/tools/query/table.ts
@@ -3,7 +3,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
-import { preferOverSqlNudge } from './builderCatalog';
+import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog';
 import {
   annotateIncreaseTopNHint,
   buildTile,
@@ -210,7 +210,7 @@ export function registerTable({ context, registerTool }: ToolRegistrar) {
         'Compute aggregated metrics as a table, single number, pie chart, or bar chart. ' +
         'Use this for grouped aggregations, top-N queries, single-value KPIs, ' +
         'or proportional breakdowns.\n\n' +
-        preferOverSqlNudge('clickstack_table') +
+        PREFER_BUILDER_OVER_SQL_NUDGE +
         '\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
         'Use the top-level "where" to scope the entire query (e.g. filter by service). ' +
diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts
index 56b9bf0044..8a3aa7b698 100644
--- a/packages/api/src/mcp/tools/query/timeseries.ts
+++ b/packages/api/src/mcp/tools/query/timeseries.ts
@@ -4,7 +4,7 @@ import { z } from 'zod';
 import type { ToolRegistrar } from '@/mcp/tools/types';
 import { mcpUserError } from '@/mcp/utils/errors';
 
-import { preferOverSqlNudge } from './builderCatalog';
+import { PREFER_BUILDER_OVER_SQL_NUDGE } from './builderCatalog';
 import {
   annotateIncreaseTopNHint,
   appendHint,
@@ -84,7 +84,7 @@ export function registerTimeseries({ context, registerTool }: ToolRegistrar) {
         'Plot metrics over time as a line or stacked bar chart. ' +
         'Use this when you need to visualize trends, compare time-series, ' +
         'or monitor metric changes over a time window.\n\n' +
-        preferOverSqlNudge('clickstack_timeseries') +
+        PREFER_BUILDER_OVER_SQL_NUDGE +
         '\n\n' +
         'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first. ' +
         'Each select item defines one plotted series.\n\n' +

From ce47c11e26b9aed61a2fea91a4b1ac8dbbea8c75 Mon Sep 17 00:00:00 2001
From: Brandon Pereira <brandon.pereira@clickhouse.com>
Date: Mon, 10 Aug 2026 10:28:51 -0600
Subject: [PATCH 6/7] test(hdx-eval): trim preflight test to essential cases

Drop redundant coverage (plain-JSON body, non-2xx, JSON-RPC error, and the
preflightMcps ordering wrapper) and use one compact fetch mock helper. Keeps
the four behaviors that matter: happy path w/ tool count, zero-tools failure,
connection-refused actionable message, and stdio passthrough.
---
 .../hdx-eval/src/__tests__/preflight.test.ts  | 151 ++++--------------
 1 file changed, 34 insertions(+), 117 deletions(-)

diff --git a/packages/hdx-eval/src/__tests__/preflight.test.ts b/packages/hdx-eval/src/__tests__/preflight.test.ts
index d527f36435..8f86ca1070 100644
--- a/packages/hdx-eval/src/__tests__/preflight.test.ts
+++ b/packages/hdx-eval/src/__tests__/preflight.test.ts
@@ -1,30 +1,25 @@
-import { preflightMcps, probeMcp } from '@/harness/preflight';
+import { probeMcp } from '@/harness/preflight';
 import type { McpDefinition } from '@/harness/types';
 
-const httpDef = (url = 'http://localhost:30199/mcp'): McpDefinition => ({
+const httpDef: McpDefinition = {
   type: 'http',
-  url,
+  url: 'http://localhost:30199/mcp',
   headers: { Authorization: 'Bearer test' },
   toolPattern: 'mcp__hyperdx__*',
   label: 'HyperDX',
-});
-
-/** Build a Response-like object for the mocked fetch. */
-function sseResponse(status: number, jsonRpc: unknown): Response {
-  const body = `event: message\ndata: ${JSON.stringify(jsonRpc)}\n\n`;
-  return {
-    status,
-    headers: { get: () => null },
-    text: async () => body,
-  } as unknown as Response;
-}
-
-function jsonResponse(status: number, jsonRpc: unknown): Response {
-  return {
-    status,
-    headers: { get: () => null },
-    text: async () => JSON.stringify(jsonRpc),
-  } as unknown as Response;
+};
+
+/** Mock fetch so each queued value is returned as an SSE JSON-RPC response. */
+function mockFetch(...jsonRpc: unknown[]) {
+  const fn = jest.fn();
+  for (const payload of jsonRpc) {
+    fn.mockResolvedValueOnce({
+      status: 200,
+      headers: { get: () => null },
+      text: async () => `event: message\ndata: ${JSON.stringify(payload)}\n\n`,
+    });
+  }
+  global.fetch = fn as unknown as typeof fetch;
 }
 
 const originalFetch = global.fetch;
@@ -33,124 +28,46 @@ afterEach(() => {
 });
 
 describe('probeMcp', () => {
-  it('returns ok with a tool count when the server initializes and lists tools (SSE framing)', async () => {
-    const fetchMock = jest
-      .fn()
-      .mockResolvedValueOnce(sseResponse(200, { result: { serverInfo: {} } }))
-      .mockResolvedValueOnce(
-        sseResponse(200, {
-          result: { tools: [{ name: 'a' }, { name: 'b' }, { name: 'c' }] },
-        }),
-      );
-    global.fetch = fetchMock as unknown as typeof fetch;
-
-    const res = await probeMcp('hyperdx', httpDef());
-    expect(res).toEqual({ mcp: 'hyperdx', ok: true, toolCount: 3 });
-  });
-
-  it('handles plain JSON (non-SSE) response bodies', async () => {
-    global.fetch = jest
-      .fn()
-      .mockResolvedValueOnce(jsonResponse(200, { result: {} }))
-      .mockResolvedValueOnce(
-        jsonResponse(200, { result: { tools: [{ name: 'a' }] } }),
-      ) as unknown as typeof fetch;
-
-    const res = await probeMcp('hyperdx', httpDef());
-    expect(res.ok).toBe(true);
-    expect(res.toolCount).toBe(1);
+  it('returns ok with a tool count when initialize + tools/list succeed', async () => {
+    mockFetch(
+      { result: {} },
+      { result: { tools: [{ name: 'a' }, { name: 'b' }] } },
+    );
+    expect(await probeMcp('hyperdx', httpDef)).toEqual({
+      mcp: 'hyperdx',
+      ok: true,
+      toolCount: 2,
+    });
   });
 
   it('fails when the server is reachable but serves zero tools', async () => {
-    global.fetch = jest
-      .fn()
-      .mockResolvedValueOnce(sseResponse(200, { result: {} }))
-      .mockResolvedValueOnce(
-        sseResponse(200, { result: { tools: [] } }),
-      ) as unknown as typeof fetch;
-
-    const res = await probeMcp('hyperdx', httpDef());
+    mockFetch({ result: {} }, { result: { tools: [] } });
+    const res = await probeMcp('hyperdx', httpDef);
     expect(res.ok).toBe(false);
-    expect(res.toolCount).toBe(0);
     expect(res.error).toMatch(/0 tools/);
   });
 
-  it('fails when initialize returns a non-2xx status', async () => {
-    global.fetch = jest
-      .fn()
-      .mockResolvedValueOnce(sseResponse(404, {})) as unknown as typeof fetch;
-
-    const res = await probeMcp('hyperdx', httpDef());
-    expect(res.ok).toBe(false);
-    expect(res.error).toMatch(/HTTP 404/);
-  });
-
   it('fails with an actionable message when the connection is refused', async () => {
     global.fetch = jest
       .fn()
       .mockRejectedValueOnce(
         new Error('fetch failed'),
       ) as unknown as typeof fetch;
-
-    const res = await probeMcp('hyperdx', httpDef());
-    expect(res.ok).toBe(false);
-    expect(res.error).toMatch(/could not connect/);
-    expect(res.error).toMatch(/API server/);
-  });
-
-  it('surfaces a JSON-RPC error object from initialize', async () => {
-    global.fetch = jest
-      .fn()
-      .mockResolvedValueOnce(
-        sseResponse(200, { error: { code: -32000, message: 'nope' } }),
-      ) as unknown as typeof fetch;
-
-    const res = await probeMcp('hyperdx', httpDef());
+    const res = await probeMcp('hyperdx', httpDef);
     expect(res.ok).toBe(false);
-    expect(res.error).toMatch(/JSON-RPC error/);
+    expect(res.error).toMatch(/could not connect.*API server/s);
   });
 
   it('treats stdio MCPs as ok without probing', async () => {
-    global.fetch = jest.fn(() => {
-      throw new Error('should not be called for stdio');
-    }) as unknown as typeof fetch;
-
     const stdioDef: McpDefinition = {
       type: 'stdio',
       command: 'uvx',
-      args: ['mcp-clickhouse'],
       toolPattern: 'mcp__clickhouse__*',
       label: 'ClickHouse',
     };
-    const res = await probeMcp('clickhouse', stdioDef);
-    expect(res).toEqual({ mcp: 'clickhouse', ok: true });
-  });
-});
-
-describe('preflightMcps', () => {
-  it('probes every entry and preserves order', async () => {
-    // Probes run concurrently (Promise.all), so route by URL rather than by
-    // call order. Slot 99 (branch) is up and serves a tool; slot 98 (main)
-    // refuses the connection.
-    global.fetch = jest.fn((url: string, init: { body: string }) => {
-      if (url.includes('30198')) {
-        return Promise.reject(new Error('fetch failed'));
-      }
-      const method = JSON.parse(init.body).method;
-      return Promise.resolve(
-        method === 'tools/list'
-          ? sseResponse(200, { result: { tools: [{ name: 'x' }] } })
-          : sseResponse(200, { result: {} }),
-      );
-    }) as unknown as typeof fetch;
-
-    const results = await preflightMcps([
-      { mcp: 'branch', def: httpDef('http://localhost:30199/mcp') },
-      { mcp: 'main', def: httpDef('http://localhost:30198/mcp') },
-    ]);
-
-    expect(results.map(r => r.mcp)).toEqual(['branch', 'main']);
-    expect(results[0].ok).toBe(true);
-    expect(results[1].ok).toBe(false);
+    expect(await probeMcp('clickhouse', stdioDef)).toEqual({
+      mcp: 'clickhouse',
+      ok: true,
+    });
   });
 });

From 7534841c588bdfd889d058483fc4efc07e01eec4 Mon Sep 17 00:00:00 2001
From: Brandon Pereira <brandon.pereira@clickhouse.com>
Date: Wed, 12 Aug 2026 10:24:35 -0600
Subject: [PATCH 7/7] docs(mcp): note BUILDER_TOOLS_LIST must be updated
 alongside tool table

---
 MCP.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/MCP.md b/MCP.md
index cbb8881ee9..8d0640cbe4 100644
--- a/MCP.md
+++ b/MCP.md
@@ -147,6 +147,11 @@ with:
 | `clickstack_save_webhook`        | Create (omit id) or update (provide id) a webhook notification destination (slack, generic, or incidentio) |
 | `clickstack_delete_webhook`      | Permanently delete a webhook by ID (blocked while alerts still reference it)                   |
 
+> **Maintainers:** when adding/removing/renaming a query tool, update this table
+> and — for builder (non-SQL) tools — the hand-curated `BUILDER_TOOLS_LIST` in
+> `packages/api/src/mcp/tools/query/builderCatalog.ts`. It isn't auto-derived, so
+> it drifts silently.
+
 ### Metric Sources
 
 `clickstack_timeseries`, `clickstack_table`, and the dashboard builder tile