From 15d5b00691b3ee8796d0928b6d3bff961b6b50db Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 15:45:45 -0600 Subject: [PATCH 1/9] refactor(mcp): split hyperdx_query into 5 display-type-specific tools Replace the monolithic hyperdx_query tool (one schema carrying fields for all display types) with five narrow tools, each with only its relevant fields: - hyperdx_timeseries (line + stacked_bar) - hyperdx_table (table + number + pie, with shape auto-upgrade) - hyperdx_search (raw event browsing) - hyperdx_event_patterns (Drain pattern mining) - hyperdx_sql (raw ClickHouse SQL) hyperdx_query is removed from the tool surface. HDX-4252 --- .../api/src/mcp/__tests__/queryTool.test.ts | 371 +++++++++------ .../api/src/mcp/prompts/dashboards/content.ts | 16 +- .../src/mcp/tools/dashboards/listSources.ts | 9 +- .../src/mcp/tools/dashboards/saveDashboard.ts | 4 +- .../api/src/mcp/tools/query/eventPatterns.ts | 428 ++++-------------- packages/api/src/mcp/tools/query/index.ts | 166 +------ .../src/mcp/tools/query/runEventPatterns.ts | 346 ++++++++++++++ packages/api/src/mcp/tools/query/schemas.ts | 325 +++---------- packages/api/src/mcp/tools/query/search.ts | 99 ++++ packages/api/src/mcp/tools/query/sql.ts | 105 +++++ packages/api/src/mcp/tools/query/table.ts | 119 +++++ .../api/src/mcp/tools/query/timeseries.ts | 114 +++++ 12 files changed, 1208 insertions(+), 894 deletions(-) create mode 100644 packages/api/src/mcp/tools/query/runEventPatterns.ts create mode 100644 packages/api/src/mcp/tools/query/search.ts create mode 100644 packages/api/src/mcp/tools/query/sql.ts create mode 100644 packages/api/src/mcp/tools/query/table.ts create mode 100644 packages/api/src/mcp/tools/query/timeseries.ts diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index fbb5cb5967..4b3a2a0520 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -16,11 +16,12 @@ import { Source, type SourceDocument } from '@/models/source'; import { McpContext } from '../tools/types'; import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; -describe('MCP Query Tool', () => { +describe('MCP Query Tools', () => { const server = getServer(); let team: any; let user: any; let traceSource: SourceDocument; + let logSource: SourceDocument; let connection: any; let client: Client; @@ -53,6 +54,20 @@ describe('MCP Query Tool', () => { name: 'Traces', }); + logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_LOGS_TABLE, + }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + bodyExpression: 'Body', + severityTextExpression: 'SeverityText', + }); + const context: McpContext = { teamId: team._id.toString(), userId: user._id.toString(), @@ -69,37 +84,84 @@ describe('MCP Query Tool', () => { await server.stop(); }); + // ─── Schema serialization ──────────────────────────────────────────────────── + describe('schema serialization', () => { - it('should expose inputSchema with all expected properties via tools/list', async () => { + it('should expose hyperdx_timeseries with expected properties', async () => { const { tools } = await client.listTools(); - const queryTool = tools.find(t => t.name === 'hyperdx_query'); - expect(queryTool).toBeDefined(); + const tool = tools.find(t => t.name === 'hyperdx_timeseries'); + expect(tool).toBeDefined(); + + const props = Object.keys(tool!.inputSchema.properties ?? {}); + expect(props).toContain('sourceId'); + expect(props).toContain('select'); + expect(props).toContain('shape'); + expect(props).toContain('granularity'); + expect(props).toContain('groupBy'); + expect(tool!.inputSchema.required).toContain('sourceId'); + expect(tool!.inputSchema.required).toContain('select'); + }); - const schema = queryTool!.inputSchema; - expect(schema.type).toBe('object'); - expect(schema.properties).toBeDefined(); + it('should expose hyperdx_table with expected properties', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === 'hyperdx_table'); + expect(tool).toBeDefined(); - // Verify key properties are present (not silently stripped by the SDK) - const props = Object.keys(schema.properties ?? {}); - expect(props).toContain('displayType'); + const props = Object.keys(tool!.inputSchema.properties ?? {}); expect(props).toContain('sourceId'); expect(props).toContain('select'); + expect(props).toContain('shape'); + expect(props).toContain('groupBy'); + expect(tool!.inputSchema.required).toContain('sourceId'); + expect(tool!.inputSchema.required).toContain('select'); + }); + + it('should expose hyperdx_search with expected properties', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === 'hyperdx_search'); + expect(tool).toBeDefined(); + + const props = Object.keys(tool!.inputSchema.properties ?? {}); + expect(props).toContain('sourceId'); expect(props).toContain('where'); - expect(props).toContain('sql'); + expect(props).toContain('columns'); + expect(props).toContain('maxResults'); + expect(tool!.inputSchema.required).toContain('sourceId'); + }); + + it('should expose hyperdx_event_patterns with expected properties', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === 'hyperdx_event_patterns'); + expect(tool).toBeDefined(); + + const props = Object.keys(tool!.inputSchema.properties ?? {}); + expect(props).toContain('sourceId'); + expect(props).toContain('where'); + expect(props).toContain('sampleSize'); + expect(props).toContain('bodyExpression'); + expect(tool!.inputSchema.required).toContain('sourceId'); + }); + + it('should expose hyperdx_sql with expected properties', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === 'hyperdx_sql'); + expect(tool).toBeDefined(); + + const props = Object.keys(tool!.inputSchema.properties ?? {}); expect(props).toContain('connectionId'); + expect(props).toContain('sql'); expect(props).toContain('startTime'); expect(props).toContain('endTime'); - expect(props).toContain('groupBy'); - - // displayType should be required - expect(schema.required).toContain('displayType'); + expect(tool!.inputSchema.required).toContain('connectionId'); + expect(tool!.inputSchema.required).toContain('sql'); }); }); - describe('builder queries', () => { - it('should execute a number query', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'number', + // ─── hyperdx_timeseries ───────────────────────────────────────────────────── + + describe('hyperdx_timeseries', () => { + it('should execute a line chart query', async () => { + const result = await callTool(client, 'hyperdx_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -112,11 +174,11 @@ describe('MCP Query Tool', () => { expect(output).toHaveProperty('result'); }); - it('should execute a line chart query', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'line', + it('should execute a stacked bar chart query', async () => { + const result = await callTool(client, 'hyperdx_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], + shape: 'stacked_bar', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); @@ -125,12 +187,21 @@ describe('MCP Query Tool', () => { expect(result.content).toHaveLength(1); }); - it('should execute a table query', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'table', + it('should default to line shape when shape is omitted', async () => { + const result = await callTool(client, 'hyperdx_timeseries', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], - groupBy: 'SpanName', + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should accept granularity in correct format', async () => { + const result = await callTool(client, 'hyperdx_timeseries', { + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + granularity: '1 minute', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); @@ -139,9 +210,23 @@ describe('MCP Query Tool', () => { expect(result.content).toHaveLength(1); }); - it('should execute a pie query', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'pie', + it('should return error for invalid time range', async () => { + const result = await callTool(client, 'hyperdx_timeseries', { + sourceId: traceSource._id.toString(), + select: [{ aggFn: 'count' }], + startTime: 'invalid-date', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Invalid'); + }); + }); + + // ─── hyperdx_table ────────────────────────────────────────────────────────── + + describe('hyperdx_table', () => { + it('should execute a table query', async () => { + const result = await callTool(client, 'hyperdx_table', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], groupBy: 'SpanName', @@ -151,13 +236,15 @@ describe('MCP Query Tool', () => { expect(result.isError).toBeFalsy(); expect(result.content).toHaveLength(1); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveProperty('result'); }); - it('should execute a stacked_bar query', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'stacked_bar', + it('should execute a number query', async () => { + const result = await callTool(client, 'hyperdx_table', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], + shape: 'number', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); @@ -166,37 +253,46 @@ describe('MCP Query Tool', () => { expect(result.content).toHaveLength(1); }); - it('should use default time range when not provided', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'number', + it('should execute a pie query', async () => { + const result = await callTool(client, 'hyperdx_table', { sourceId: traceSource._id.toString(), select: [{ aggFn: 'count' }], + shape: 'pie', + groupBy: 'SpanName', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), }); expect(result.isError).toBeFalsy(); expect(result.content).toHaveLength(1); }); - it('should return result for query with no matching data', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'number', + it('should auto-upgrade shape:"number" to "table" when select has multiple items', async () => { + // This should NOT error — it should silently upgrade to table + const result = await callTool(client, 'hyperdx_table', { sourceId: traceSource._id.toString(), - select: [{ aggFn: 'count', where: 'SpanName:z_impossible_value_xyz' }], - startTime: new Date(Date.now() - 60 * 1000).toISOString(), + select: [ + { aggFn: 'count' }, + { aggFn: 'avg', valueExpression: 'Duration' }, + ], + shape: 'number', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); expect(result.isError).toBeFalsy(); expect(result.content).toHaveLength(1); }); - }); - describe('search queries', () => { - it('should execute a search query', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'search', + it('should auto-upgrade shape:"pie" to "table" when select has multiple items', async () => { + const result = await callTool(client, 'hyperdx_table', { sourceId: traceSource._id.toString(), - where: '', + select: [ + { aggFn: 'count' }, + { aggFn: 'sum', valueExpression: 'Duration' }, + ], + shape: 'pie', + groupBy: 'SpanName', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); @@ -205,25 +301,24 @@ describe('MCP Query Tool', () => { expect(result.content).toHaveLength(1); }); - it('should respect maxResults parameter', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'search', + it('should use default time range when not provided', async () => { + const result = await callTool(client, 'hyperdx_table', { sourceId: traceSource._id.toString(), - maxResults: 10, - startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), - endTime: new Date().toISOString(), + select: [{ aggFn: 'count' }], }); expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); }); }); - describe('SQL queries', () => { - it('should execute a raw SQL query', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'sql', - connectionId: connection._id.toString(), - sql: 'SELECT 1 AS value', + // ─── hyperdx_search ───────────────────────────────────────────────────────── + + describe('hyperdx_search', () => { + it('should execute a search query', async () => { + const result = await callTool(client, 'hyperdx_search', { + sourceId: traceSource._id.toString(), + where: '', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); @@ -232,42 +327,40 @@ describe('MCP Query Tool', () => { expect(result.content).toHaveLength(1); }); - it('should execute SQL with time macros', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'sql', - connectionId: connection._id.toString(), - sql: `SELECT count() AS cnt FROM ${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE} WHERE $__timeFilter(Timestamp) LIMIT 10`, + it('should respect maxResults parameter', async () => { + const result = await callTool(client, 'hyperdx_search', { + sourceId: traceSource._id.toString(), + maxResults: 10, startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); + expect(result.isError).toBeFalsy(); + }); + + it('should use default time range when not provided', async () => { + const result = await callTool(client, 'hyperdx_search', { + sourceId: traceSource._id.toString(), + }); + expect(result.isError).toBeFalsy(); expect(result.content).toHaveLength(1); }); - }); - describe('event_patterns queries', () => { - let logSource: SourceDocument; - - beforeEach(async () => { - logSource = await Source.create({ - kind: SourceKind.Log, - team: team._id, - from: { - databaseName: DEFAULT_DATABASE, - tableName: DEFAULT_LOGS_TABLE, - }, - timestampValueExpression: 'Timestamp', - connection: connection._id, - name: 'Logs', - bodyExpression: 'Body', - severityTextExpression: 'SeverityText', + it('should reject calls missing sourceId', async () => { + const result = await callTool(client, 'hyperdx_search', { + where: 'level:error', }); + + expect(result.isError).toBe(true); }); + }); + + // ─── hyperdx_event_patterns ───────────────────────────────────────────────── + describe('hyperdx_event_patterns', () => { it('should execute an event_patterns query on a log source', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + const result = await callTool(client, 'hyperdx_event_patterns', { sourceId: logSource._id.toString(), startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), @@ -286,9 +379,8 @@ describe('MCP Query Tool', () => { expect(Array.isArray(output.result.patterns)).toBe(true); }); - it('should execute an event_patterns query on a trace source with explicit bodyExpression', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + it('should execute with explicit bodyExpression on trace source', async () => { + const result = await callTool(client, 'hyperdx_event_patterns', { sourceId: traceSource._id.toString(), bodyExpression: 'SpanName', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), @@ -300,27 +392,27 @@ describe('MCP Query Tool', () => { const output = JSON.parse(getFirstText(result)); expect(output).toHaveProperty('result'); expect(Array.isArray(output.result.patterns)).toBe(true); + expect(output.result).toHaveProperty('bodyColumn', 'SpanName'); }); - it('should respect sampleSize parameter', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + it('should accept custom bodyExpression on log source', async () => { + const result = await callTool(client, 'hyperdx_event_patterns', { sourceId: logSource._id.toString(), - sampleSize: 100, + bodyExpression: 'SeverityText', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); expect(result.isError).toBeFalsy(); expect(result.content).toHaveLength(1); + const output = JSON.parse(getFirstText(result)); + expect(output.result).toHaveProperty('bodyColumn', 'SeverityText'); }); - it('should respect where filter', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + it('should respect sampleSize parameter', async () => { + const result = await callTool(client, 'hyperdx_event_patterns', { sourceId: logSource._id.toString(), - where: "SeverityText = 'ERROR'", - whereLanguage: 'sql', + sampleSize: 100, startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); @@ -329,35 +421,30 @@ describe('MCP Query Tool', () => { expect(result.content).toHaveLength(1); }); - it('should accept custom bodyExpression', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + it('should respect where filter', async () => { + const result = await callTool(client, 'hyperdx_event_patterns', { sourceId: logSource._id.toString(), - bodyExpression: 'SeverityText', + where: "SeverityText = 'ERROR'", + whereLanguage: 'sql', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); expect(result.isError).toBeFalsy(); expect(result.content).toHaveLength(1); - const output = JSON.parse(getFirstText(result)); - expect(output.result).toHaveProperty('bodyColumn', 'SeverityText'); }); - it('should require sourceId', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + it('should reject calls missing sourceId', async () => { + const result = await callTool(client, 'hyperdx_event_patterns', { startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), }); expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('sourceId is required'); }); it('should return error for non-existent source', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + const result = await callTool(client, 'hyperdx_event_patterns', { sourceId: '000000000000000000000000', startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), endTime: new Date().toISOString(), @@ -367,27 +454,11 @@ describe('MCP Query Tool', () => { expect(getFirstText(result)).toContain('Source not found'); }); - it('should include event_patterns in the schema displayType enum', async () => { - const { tools } = await client.listTools(); - const queryTool = tools.find(t => t.name === 'hyperdx_query'); - expect(queryTool).toBeDefined(); - - const schema = queryTool!.inputSchema; - const displayTypeSchema = (schema.properties as any)?.displayType; - expect(displayTypeSchema?.enum).toContain('event_patterns'); - - // Verify new fields are present - const props = Object.keys(schema.properties ?? {}); - expect(props).toContain('sampleSize'); - expect(props).toContain('bodyExpression'); - }); - describe('with seeded data', () => { const now = new Date(); const fiveMinAgo = new Date(now.getTime() - 5 * 60 * 1000); beforeEach(async () => { - // Seed several distinct log templates so Drain produces multiple patterns const logs: Parameters[0] = []; // Template A: "User logged in from " — 20 rows @@ -422,8 +493,7 @@ describe('MCP Query Tool', () => { }); it('should mine patterns from seeded data and return non-empty results', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'event_patterns', + const result = await callTool(client, 'hyperdx_event_patterns', { sourceId: logSource._id.toString(), startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), endTime: new Date(now.getTime() + 60 * 1000).toISOString(), @@ -433,18 +503,15 @@ describe('MCP Query Tool', () => { const output = JSON.parse(getFirstText(result)); const { patterns, totalCount, sampledRows } = output.result; - // Should have found data expect(totalCount).toBeGreaterThanOrEqual(31); expect(sampledRows).toBeGreaterThanOrEqual(31); - - // Should have mined at least one pattern expect(patterns.length).toBeGreaterThanOrEqual(1); // The most common pattern should contain <*> placeholders const topPattern = patterns[0]; expect(topPattern.pattern).toContain('<*>'); - // Patterns should be sorted by estimatedCount descending (monotonic) + // Patterns should be sorted by estimatedCount descending for (let i = 1; i < patterns.length; i++) { expect(patterns[i - 1].estimatedCount).toBeGreaterThanOrEqual( patterns[i].estimatedCount, @@ -456,7 +523,6 @@ describe('MCP Query Tool', () => { expect(p.trend.length).toBeGreaterThan(0); } - // At least one pattern should have a non-zero trend bucket count const hasNonZeroTrend = patterns.some((p: any) => p.trend.some((t: any) => t.count > 0), ); @@ -465,13 +531,48 @@ describe('MCP Query Tool', () => { }); }); - describe('error handling', () => { + // ─── hyperdx_sql ──────────────────────────────────────────────────────────── + + describe('hyperdx_sql', () => { + it('should execute a raw SQL query', async () => { + const result = await callTool(client, 'hyperdx_sql', { + connectionId: connection._id.toString(), + sql: 'SELECT 1 AS value', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should execute SQL with time macros', async () => { + const result = await callTool(client, 'hyperdx_sql', { + connectionId: connection._id.toString(), + sql: `SELECT count() AS cnt FROM ${DEFAULT_DATABASE}.${DEFAULT_TRACES_TABLE} WHERE $__timeFilter(Timestamp) LIMIT 10`, + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should use default time range when not provided', async () => { + const result = await callTool(client, 'hyperdx_sql', { + connectionId: connection._id.toString(), + sql: 'SELECT 1 AS value', + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + it('should return error for invalid time range', async () => { - const result = await callTool(client, 'hyperdx_query', { - displayType: 'number', - sourceId: traceSource._id.toString(), - select: [{ aggFn: 'count' }], - startTime: 'invalid-date', + const result = await callTool(client, 'hyperdx_sql', { + connectionId: connection._id.toString(), + sql: 'SELECT 1', + startTime: 'not-a-date', }); expect(result.isError).toBe(true); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 90d1aa98b1..ee45e3c4a3 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -760,7 +760,7 @@ SQL TEMPLATE REFERENCE: } export function buildQueryGuidePrompt(): string { - return `Reference guide for writing queries with HyperDX MCP tools (hyperdx_query and hyperdx_save_dashboard). + return `Reference guide for writing queries with HyperDX MCP tools (hyperdx_timeseries, hyperdx_table, hyperdx_search, hyperdx_event_patterns, hyperdx_sql, and hyperdx_save_dashboard). == AGGREGATION FUNCTIONS (aggFn) == @@ -1106,10 +1106,10 @@ containers[i].tabs[j].id). Read the path to know which input to fix. == EVENT PATTERN MINING == -Use displayType "event_patterns" on hyperdx_query when asked to find common log -messages, recurring patterns, noisy services, or top event types. This is the -right choice whenever the question is about "most common logs", "top patterns", -"what is generating the most log volume", or similar pattern-discovery questions. +Use hyperdx_event_patterns when asked to find common log messages, recurring +patterns, noisy services, or top event types. This is the right choice whenever +the question is about "most common logs", "top patterns", "what is generating +the most log volume", or similar pattern-discovery questions. It samples random events and clusters them with the Drain algorithm, returning: - pattern templates (with <*> wildcards for variable parts) @@ -1127,8 +1127,7 @@ Optional parameters: - bodyExpression — column to mine (auto-detected: Body for logs, SpanName for traces) Example: find top patterns for production services over the last 4 hours: - hyperdx_query({ - displayType: "event_patterns", + hyperdx_event_patterns({ sourceId: "", startTime: "<4 hours ago ISO>", endTime: "", @@ -1150,9 +1149,10 @@ Example: find top patterns for production services over the last 4 hours: Correct: groupBy: "SpanAttributes['http.method']" NOTE: JSON-type columns DO use dot notation. Check jsType from hyperdx_list_sources. -4. Multiple select items on number/pie tiles +4. Multiple select items on number/pie tiles (dashboard tiles) Wrong: displayType: "number", select: [{ aggFn: "count" }, { aggFn: "avg", ... }] Correct: displayType: "number", select: [{ aggFn: "count" }] + Note: hyperdx_table auto-upgrades shape:"number" to "table" when select has >1 item. 5. Missing level for quantile Wrong: { aggFn: "quantile", valueExpression: "Duration" } diff --git a/packages/api/src/mcp/tools/dashboards/listSources.ts b/packages/api/src/mcp/tools/dashboards/listSources.ts index 18f0a0fa47..97b491fb20 100644 --- a/packages/api/src/mcp/tools/dashboards/listSources.ts +++ b/packages/api/src/mcp/tools/dashboards/listSources.ts @@ -31,8 +31,9 @@ export function registerListSources( title: 'List Sources & Connections', description: 'List all data sources (logs, metrics, traces) and database connections available to this team. ' + - 'Returns source IDs (use as sourceId in hyperdx_query and dashboard tiles) and ' + - 'connection IDs (use as connectionId for advanced raw SQL queries). ' + + 'Returns source IDs (use as sourceId in hyperdx_timeseries, hyperdx_table, hyperdx_search, ' + + 'hyperdx_event_patterns, and dashboard tiles) and ' + + 'connection IDs (use as connectionId for hyperdx_sql raw SQL queries). ' + 'Each source includes its full column schema and sampled attribute keys from map columns ' + '(e.g. SpanAttributes, ResourceAttributes). ' + 'Column names are PascalCase (e.g. Duration, not duration). ' + @@ -167,9 +168,9 @@ export function registerListSources( mapAttributes: "Use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']", sourceIds: - 'Use sourceId with builder display types (line, stacked_bar, table, number, pie, search) for standard queries', + 'Use sourceId with hyperdx_timeseries, hyperdx_table, hyperdx_search, and hyperdx_event_patterns', connectionIds: - 'ADVANCED: Use connectionId only with raw SQL queries (displayType "sql" or configType "sql"). ' + + 'ADVANCED: Use connectionId with hyperdx_sql for raw ClickHouse SQL. ' + 'Raw SQL is for advanced use cases like JOINs, sub-queries, or querying tables not registered as sources.', }, }; diff --git a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts index 871af7996d..4784be3b85 100644 --- a/packages/api/src/mcp/tools/dashboards/saveDashboard.ts +++ b/packages/api/src/mcp/tools/dashboards/saveDashboard.ts @@ -272,7 +272,7 @@ async function createDashboard({ ...(frontendUrl ? { url: `${frontendUrl}/dashboards/${newDashboard._id}` } : {}), - hint: 'Use hyperdx_query to test individual tile queries before viewing the dashboard.', + hint: 'Use hyperdx_query_tile to test individual tile queries before viewing the dashboard.', }, null, 2, @@ -529,7 +529,7 @@ async function updateDashboard({ ...(frontendUrl ? { url: `${frontendUrl}/dashboards/${updatedDashboard._id}` } : {}), - hint: 'Use hyperdx_query to test individual tile queries before viewing the dashboard.', + hint: 'Use hyperdx_query_tile to test individual tile queries before viewing the dashboard.', }, null, 2, diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts index ab380ce8d4..9bb9cfd812 100644 --- a/packages/api/src/mcp/tools/query/eventPatterns.ts +++ b/packages/api/src/mcp/tools/query/eventPatterns.ts @@ -1,346 +1,92 @@ -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; -import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { - getFirstTimestampValueExpression, - splitAndTrimWithBracket, -} from '@hyperdx/common-utils/dist/core/utils'; -import { minePatterns } from '@hyperdx/common-utils/dist/drain'; -import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; -import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; - -import { getConnectionById } from '@/controllers/connection'; -import { getSource } from '@/controllers/sources'; -import { trimToolResponse } from '@/utils/trimToolResponse'; - -// ─── Source helpers ────────────────────────────────────────────────────────── - -interface SourceBodyFields { - kind: string; - spanNameExpression?: string; - bodyExpression?: string; - implicitColumnExpression?: string; -} - -/** - * Resolve the body column expression for pattern mining from a source. - * Mirrors the web app's getEventBody() logic (packages/app/src/source.ts). - */ -function resolveBodyExpression(source: SourceBodyFields): string | undefined { - let expression: string | undefined; - if (source.kind === SourceKind.Trace) { - expression = source.spanNameExpression; - } else if (source.kind === SourceKind.Log) { - expression = source.bodyExpression ?? source.implicitColumnExpression; - } - if (!expression) return undefined; - const multiExpr = splitAndTrimWithBracket(expression); - return multiExpr.length === 1 ? expression : multiExpr[0]; -} - -/** Reject bodyExpression values containing SQL-unsafe characters. */ -// eslint-disable-next-line no-useless-escape -const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; - -// ─── Event pattern mining ──────────────────────────────────────────────────── - -export async function runEventPatterns( - teamId: string, - sourceId: string, - startDate: Date, - endDate: Date, - options?: { - where?: string; - whereLanguage?: 'lucene' | 'sql'; - bodyExpression?: string; - sampleSize?: number; - }, -) { - const sampleSize = options?.sampleSize ?? 10_000; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; - // ── Resolve source & connection ── - const source = await getSource(teamId, sourceId); - if (!source) { - return { - isError: true as const, - content: [ - { type: 'text' as const, text: `Source not found: ${sourceId}` }, - ], - }; - } - - const connection = await getConnectionById( - teamId, - source.connection.toString(), - true, - ); - if (!connection) { - return { - isError: true as const, - content: [ - { - type: 'text' as const, - text: `Connection not found for source: ${sourceId}`, - }, - ], - }; - } - - // ── Determine body column ── - // Sanitize caller-supplied bodyExpression: must be a single column reference - // matching the documented format (e.g. "Body", "SpanAttributes['http.url']"). - // The allowlist rejects injection attempts like "Body) OR (1=1". - let bodyColumn: string | undefined; - if (options?.bodyExpression) { - const parts = splitAndTrimWithBracket(options.bodyExpression); - if (parts.length !== 1 || !SAFE_BODY_EXPR_CHARS.test(parts[0])) { - return { - isError: true as const, - content: [ - { - type: 'text' as const, - text: - 'bodyExpression must be a single column expression ' + - '(e.g. "Body", "SpanName", "SpanAttributes[\'http.url\']"). ' + - 'Multiple expressions, function calls, or sub-queries are not allowed.', - }, - ], - }; - } - bodyColumn = parts[0]; - } else { - bodyColumn = resolveBodyExpression(source); - } - if (!bodyColumn) { - return { - isError: true as const, - content: [ - { - type: 'text' as const, - text: - 'Could not determine body column for pattern mining. ' + - 'This source may not have a body/spanName expression configured. ' + - 'Try specifying bodyExpression explicitly.', - }, - ], - }; - } - - const clickhouseClient = new ClickhouseClient({ - host: connection.host, - username: connection.username, - password: connection.password, - }); - const metadata = getMetadata(clickhouseClient); - - const tsExpr = getFirstTimestampValueExpression( - source.timestampValueExpression, - ); - const implicitColumn = - 'implicitColumnExpression' in source - ? source.implicitColumnExpression - : undefined; - - // ── Query 1: Random sample of events ── - const sampleConfig = { - displayType: DisplayType.Search, - source: source._id.toString(), - select: `${bodyColumn} as __hdx_pattern_body, ${tsExpr} as __hdx_pattern_ts`, - from: { - databaseName: source.from.databaseName, - tableName: source.from.tableName, - }, - where: options?.where ?? '', - whereLanguage: options?.whereLanguage ?? ('lucene' as const), - connection: source.connection.toString(), - timestampValueExpression: source.timestampValueExpression, - implicitColumnExpression: implicitColumn, - orderBy: [{ ordering: 'DESC' as const, valueExpression: 'rand()' }], - limit: { limit: sampleSize, offset: 0 }, - dateRange: [startDate, endDate] as [Date, Date], - } satisfies ChartConfigWithDateRange; - - // ── Query 2: Total count ── - const countConfig = { - displayType: DisplayType.Table, - source: source._id.toString(), - select: 'count() as total', - from: { - databaseName: source.from.databaseName, - tableName: source.from.tableName, +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { parseTimeRange } from './helpers'; +import { runEventPatterns } from './runEventPatterns'; +import { + endTimeSchema, + sourceIdSchema, + startTimeSchema, + whereLanguageSchema, + whereSchema, +} from './schemas'; + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const eventPatternsSchema = z.object({ + sourceId: sourceIdSchema, + where: whereSchema, + whereLanguage: whereLanguageSchema, + // TODO: explore whether we can safely increase the max beyond 25_000 + sampleSize: z + .number() + .min(1) + .max(25_000) + .optional() + .describe( + 'Number of random rows to sample for pattern mining. ' + + 'Default: 10000. Higher values produce more accurate patterns but take longer.', + ), + bodyExpression: z + .string() + .optional() + .describe( + 'Column expression to mine patterns from. ' + + 'Auto-detected from the source if omitted (Body for logs, SpanName for traces). ' + + 'Example: "Body", "SpanName", "SpanAttributes[\'http.url\']"', + ), + startTime: startTimeSchema, + endTime: endTimeSchema, +}); + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerEventPatterns(server: McpServer, context: McpContext) { + const { teamId } = context; + + server.registerTool( + 'hyperdx_event_patterns', + { + title: 'Event Pattern Mining', + description: + 'Discover the most common log messages and event patterns. ' + + 'Samples random events, clusters them using the Drain algorithm, and returns ' + + 'patterns sorted by frequency with estimated counts and time trends.\n\n' + + 'Use this when asked about "top patterns", "common logs", "noisy services", ' + + '"recurring messages", or log noise analysis.\n\n' + + 'Each pattern includes a "whereSnippet" — use it as the "where" parameter in ' + + 'a follow-up hyperdx_search call to browse matching raw events.\n\n' + + 'Requires sourceId — call hyperdx_list_sources first.\n\n' + + 'When to use which tool:\n' + + ' - hyperdx_event_patterns: clustering / recurring shapes / noise analysis\n' + + ' - hyperdx_search: raw individual rows\n' + + ' - hyperdx_table: aggregated metrics / counts / top-N', + inputSchema: eventPatternsSchema, }, - where: options?.where ?? '', - whereLanguage: options?.whereLanguage ?? ('lucene' as const), - connection: source.connection.toString(), - timestampValueExpression: source.timestampValueExpression, - implicitColumnExpression: implicitColumn, - limit: { limit: 1, offset: 0 }, - dateRange: [startDate, endDate] as [Date, Date], - } satisfies ChartConfigWithDateRange; - - // Fire both queries in parallel - let sampleResult: Awaited< - ReturnType - >; - let countResult: Awaited< - ReturnType - >; - try { - [sampleResult, countResult] = await Promise.all([ - clickhouseClient.queryChartConfig({ - config: sampleConfig, - metadata, - querySettings: source.querySettings, - opts: { clickhouse_settings: { max_execution_time: 30 } }, - }), - clickhouseClient.queryChartConfig({ - config: countConfig, - metadata, - querySettings: source.querySettings, - opts: { clickhouse_settings: { max_execution_time: 30 } }, - }), - ]); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return { - isError: true as const, - content: [ + withToolTracing('hyperdx_event_patterns', context, async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + return runEventPatterns( + teamId.toString(), + input.sourceId, + startDate, + endDate, { - type: 'text' as const, - text: `ClickHouse query failed: ${message}`, + where: input.where, + whereLanguage: input.whereLanguage, + bodyExpression: input.bodyExpression, + sampleSize: input.sampleSize, }, - ], - }; - } - - const sampleRows = sampleResult.data; - const totalCount = Number(countResult.data?.[0]?.total ?? 0); - - if (!sampleRows || sampleRows.length === 0) { - return { - content: [ - { - type: 'text' as const, - text: JSON.stringify( - { - result: { - patterns: [], - totalCount, - sampledRows: 0, - sampleMultiplier: 1, - bodyColumn, - timeRange: { - start: startDate.toISOString(), - end: endDate.toISOString(), - }, - }, - hint: 'No data found in the queried time range. Try setting startTime to a wider window (e.g. 24 hours ago) or check that filters match existing data.', - }, - null, - 2, - ), - }, - ], - }; - } - - // ── Mine patterns using the shared Drain pipeline ── - let rawPatterns: ReturnType['patterns']; - let sampleMultiplier: number; - try { - ({ patterns: rawPatterns, sampleMultiplier } = minePatterns(sampleRows, { - totalCount, - startDate, - endDate, - maxSamples: 5, - getBody: row => { - const raw = row.__hdx_pattern_body; - return raw != null ? String(raw) : ''; - }, - getTimestamp: row => { - const tsRaw = row.__hdx_pattern_ts; - return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; - }, - })); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return { - isError: true as const, - content: [ - { - type: 'text' as const, - text: `Pattern mining failed: ${message}`, - }, - ], - }; - } - - // ── Format response ── - // Convert trend timestamps to ISO strings, extract sample body texts, - // and build a whereSnippet per pattern so the agent can drill into - // matching events via a follow-up displayType:"search" query. - const patterns = rawPatterns.map(p => { - // Build a Lucene-compatible where clause from the pattern's literal - // (non-<*>) tokens. This lets agents chain: pattern → search. - // Escape \ and " (the phrase-query metachars) in each token. - const literalTokens = p.pattern - .split(/\s+/) - .filter(t => t !== '<*>' && t.length > 0) - .map(t => t.replace(/[\\"]/g, '\\$&')); - const whereSnippet = - literalTokens.length > 0 - ? `${bodyColumn}:"${literalTokens.join(' ')}"` - : ''; - - return { - id: p.id, - pattern: p.pattern, - estimatedCount: p.estimatedCount, - sampleCount: p.sampleCount, - whereSnippet, - trend: p.trend.map(t => ({ - ts: new Date(t.ts).toISOString(), - count: t.count, - })), - samples: p.samples.map(row => { - const raw = row.__hdx_pattern_body; - return raw != null ? String(raw) : ''; - }), - }; - }); - - const output = { - patterns, - totalCount, - sampledRows: sampleRows.length, - sampleMultiplier: Math.round(sampleMultiplier * 100) / 100, - bodyColumn, - timeRange: { - start: startDate.toISOString(), - end: endDate.toISOString(), - }, - }; - - const trimmedOutput = trimToolResponse(output); - const isTrimmed = - JSON.stringify(trimmedOutput).length < JSON.stringify(output).length; - - return { - content: [ - { - type: 'text' as const, - text: JSON.stringify( - { - result: trimmedOutput, - ...(isTrimmed - ? { - note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', - } - : {}), - }, - null, - 2, - ), - }, - ], - }; + ); + }), + ); } diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index 052339e41c..d49c86f45c 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -1,152 +1,20 @@ -import { ObjectId } from 'mongodb'; - -import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import { externalDashboardTileSchemaWithId } from '@/utils/zod'; - -import { withToolTracing } from '../../utils/tracing'; -import type { ToolDefinition } from '../types'; -import { runEventPatterns } from './eventPatterns'; -import { parseTimeRange, runConfigTile } from './helpers'; -import { hyperdxQuerySchema, validateQueryInput } from './schemas'; - -// ─── Tool definition ───────────────────────────────────────────────────────── - -const queryTools: ToolDefinition = (server, context) => { - const { teamId } = context; - - server.registerTool( - 'hyperdx_query', - { - title: 'Query Data', - description: - 'Query observability data (logs, metrics, traces) from HyperDX. ' + - 'Use hyperdx_list_sources first to find sourceId/connectionId values. ' + - 'Set displayType to control the query shape.\n\n' + - 'PREFERRED: Use the builder display types (line, stacked_bar, table, number, pie) ' + - 'for aggregated metrics, or "search" for browsing individual log/event rows. ' + - 'These are safer, easier to construct, and cover most use cases.\n\n' + - 'PATTERN MINING: Use "event_patterns" whenever asked about the most common log messages, ' + - 'top log/event patterns, noisy or high-volume services, recurring messages, or log noise analysis. ' + - 'It samples random events, clusters them using the Drain algorithm, and returns patterns sorted ' + - 'by frequency with estimated counts and time trends. ' + - 'Each pattern includes a "whereSnippet" field — use it as the "where" parameter in a follow-up ' + - '"search" query on the same source to browse the matching raw events.\n\n' + - 'ADVANCED: Use displayType "sql" only when you need capabilities the builder cannot express, ' + - 'such as JOINs, sub-queries, CTEs, or querying tables not registered as sources. ' + - 'Raw SQL requires a connectionId (not sourceId) and a hand-written ClickHouse SQL query.\n\n' + - 'Column naming: Top-level columns are PascalCase (Duration, StatusCode, SpanName). ' + - "Map attributes use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']. " + - 'Call hyperdx_list_sources to discover available columns and attribute keys for each source.', - inputSchema: hyperdxQuerySchema, - }, - withToolTracing('hyperdx_query', context, async input => { - // Cross-field validation (kept out of the Zod schema to avoid - // .superRefine() wrapping in ZodEffects, which the MCP SDK's - // normalizeObjectSchema() cannot serialize to JSON Schema). - const validationError = validateQueryInput(input); - if (validationError) { - return { - isError: true, - content: [{ type: 'text' as const, text: validationError }], - }; - } - - const timeRange = parseTimeRange(input.startTime, input.endTime); - if ('error' in timeRange) { - return { - isError: true, - content: [{ type: 'text' as const, text: timeRange.error }], - }; - } - const { startDate, endDate } = timeRange; - - // Event patterns has its own execution path (sample + mine), - // so it returns early without going through runConfigTile. - if (input.displayType === 'event_patterns') { - return runEventPatterns( - teamId.toString(), - input.sourceId!, - startDate, - endDate, - { - where: input.where, - whereLanguage: input.whereLanguage, - bodyExpression: input.bodyExpression, - sampleSize: input.sampleSize, - }, - ); - } - - let tile: ExternalDashboardTileWithId; - - if (input.displayType === 'sql') { - tile = externalDashboardTileSchemaWithId.parse({ - id: new ObjectId().toString(), - name: 'MCP SQL', - x: 0, - y: 0, - w: 24, - h: 6, - config: { - configType: 'sql' as const, - displayType: 'table' as const, - connectionId: input.connectionId, - sqlTemplate: input.sql, - }, - }); - } else if (input.displayType === 'search') { - tile = externalDashboardTileSchemaWithId.parse({ - id: new ObjectId().toString(), - name: 'MCP Search', - x: 0, - y: 0, - w: 24, - h: 6, - config: { - displayType: 'search' as const, - sourceId: input.sourceId, - select: input.columns ?? '', - where: input.where ?? '', - whereLanguage: input.whereLanguage ?? 'lucene', - }, - }); - } else { - tile = externalDashboardTileSchemaWithId.parse({ - id: new ObjectId().toString(), - name: 'MCP Query', - x: 0, - y: 0, - w: 12, - h: 4, - config: { - displayType: input.displayType, - sourceId: input.sourceId, - select: input.select!.map(s => ({ - aggFn: s.aggFn, - where: s.where ?? '', - whereLanguage: s.whereLanguage ?? 'lucene', - valueExpression: s.valueExpression, - alias: s.alias, - level: s.level, - })), - groupBy: input.groupBy ?? undefined, - orderBy: input.orderBy ?? undefined, - ...(input.granularity ? { granularity: input.granularity } : {}), - }, - }); - } - - return runConfigTile( - teamId.toString(), - tile, - startDate, - endDate, - input.displayType === 'search' - ? { maxResults: input.maxResults } - : undefined, - ); - }), - ); +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + +import type { McpContext, ToolDefinition } from '../types'; +import { registerEventPatterns } from './eventPatterns'; +import { registerSearch } from './search'; +import { registerSql } from './sql'; +import { registerTable } from './table'; +import { registerTimeseries } from './timeseries'; + +export * from './schemas'; + +const queryTools: ToolDefinition = (server: McpServer, context: McpContext) => { + registerTimeseries(server, context); + registerTable(server, context); + registerSearch(server, context); + registerEventPatterns(server, context); + registerSql(server, context); }; export default queryTools; diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts new file mode 100644 index 0000000000..19ca8e0baa --- /dev/null +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -0,0 +1,346 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + getFirstTimestampValueExpression, + splitAndTrimWithBracket, +} from '@hyperdx/common-utils/dist/core/utils'; +import { minePatterns } from '@hyperdx/common-utils/dist/drain'; +import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + +// ─── Source helpers ────────────────────────────────────────────────────────── + +interface SourceBodyFields { + kind: string; + spanNameExpression?: string; + bodyExpression?: string; + implicitColumnExpression?: string; +} + +/** + * Resolve the body column expression for pattern mining from a source. + * Mirrors the web app's getEventBody() logic (packages/app/src/source.ts). + */ +function resolveBodyExpression(source: SourceBodyFields): string | undefined { + let expression: string | undefined; + if (source.kind === SourceKind.Trace) { + expression = source.spanNameExpression; + } else if (source.kind === SourceKind.Log) { + expression = source.bodyExpression ?? source.implicitColumnExpression; + } + if (!expression) return undefined; + const multiExpr = splitAndTrimWithBracket(expression); + return multiExpr.length === 1 ? expression : multiExpr[0]; +} + +/** Reject bodyExpression values containing SQL-unsafe characters. */ +// eslint-disable-next-line no-useless-escape +const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; + +// ─── Event pattern mining ──────────────────────────────────────────────────── + +export async function runEventPatterns( + teamId: string, + sourceId: string, + startDate: Date, + endDate: Date, + options?: { + where?: string; + whereLanguage?: 'lucene' | 'sql'; + bodyExpression?: string; + sampleSize?: number; + }, +) { + const sampleSize = options?.sampleSize ?? 10_000; + + // ── Resolve source & connection ── + const source = await getSource(teamId, sourceId); + if (!source) { + return { + isError: true as const, + content: [ + { type: 'text' as const, text: `Source not found: ${sourceId}` }, + ], + }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `Connection not found for source: ${sourceId}`, + }, + ], + }; + } + + // ── Determine body column ── + // Sanitize caller-supplied bodyExpression: must be a single column reference + // matching the documented format (e.g. "Body", "SpanAttributes['http.url']"). + // The allowlist rejects injection attempts like "Body) OR (1=1". + let bodyColumn: string | undefined; + if (options?.bodyExpression) { + const parts = splitAndTrimWithBracket(options.bodyExpression); + if (parts.length !== 1 || !SAFE_BODY_EXPR_CHARS.test(parts[0])) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + 'bodyExpression must be a single column expression ' + + '(e.g. "Body", "SpanName", "SpanAttributes[\'http.url\']"). ' + + 'Multiple expressions, function calls, or sub-queries are not allowed.', + }, + ], + }; + } + bodyColumn = parts[0]; + } else { + bodyColumn = resolveBodyExpression(source); + } + if (!bodyColumn) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + 'Could not determine body column for pattern mining. ' + + 'This source may not have a body/spanName expression configured. ' + + 'Try specifying bodyExpression explicitly.', + }, + ], + }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + + const tsExpr = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + const implicitColumn = + 'implicitColumnExpression' in source + ? source.implicitColumnExpression + : undefined; + + // ── Query 1: Random sample of events ── + const sampleConfig = { + displayType: DisplayType.Search, + source: source._id.toString(), + select: `${bodyColumn} as __hdx_pattern_body, ${tsExpr} as __hdx_pattern_ts`, + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + orderBy: [{ ordering: 'DESC' as const, valueExpression: 'rand()' }], + limit: { limit: sampleSize, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + // ── Query 2: Total count ── + const countConfig = { + displayType: DisplayType.Table, + source: source._id.toString(), + select: 'count() as total', + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + limit: { limit: 1, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + // Fire both queries in parallel + let sampleResult: Awaited< + ReturnType + >; + let countResult: Awaited< + ReturnType + >; + try { + [sampleResult, countResult] = await Promise.all([ + clickhouseClient.queryChartConfig({ + config: sampleConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + clickhouseClient.queryChartConfig({ + config: countConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + ]); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `ClickHouse query failed: ${message}`, + }, + ], + }; + } + + const sampleRows = sampleResult.data; + const totalCount = Number(countResult.data?.[0]?.total ?? 0); + + if (!sampleRows || sampleRows.length === 0) { + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: { + patterns: [], + totalCount, + sampledRows: 0, + sampleMultiplier: 1, + bodyColumn, + timeRange: { + start: startDate.toISOString(), + end: endDate.toISOString(), + }, + }, + hint: 'No data found in the queried time range. Try setting startTime to a wider window (e.g. 24 hours ago) or check that filters match existing data.', + }, + null, + 2, + ), + }, + ], + }; + } + + // ── Mine patterns using the shared Drain pipeline ── + let rawPatterns: ReturnType['patterns']; + let sampleMultiplier: number; + try { + ({ patterns: rawPatterns, sampleMultiplier } = minePatterns(sampleRows, { + totalCount, + startDate, + endDate, + maxSamples: 5, + getBody: row => { + const raw = row.__hdx_pattern_body; + return raw != null ? String(raw) : ''; + }, + getTimestamp: row => { + const tsRaw = row.__hdx_pattern_ts; + return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; + }, + })); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `Pattern mining failed: ${message}`, + }, + ], + }; + } + + // ── Format response ── + // Convert trend timestamps to ISO strings, extract sample body texts, + // and build a whereSnippet per pattern so the agent can drill into + // matching events via a follow-up hyperdx_search query. + const patterns = rawPatterns.map(p => { + // Build a Lucene-compatible where clause from the pattern's literal + // (non-<*>) tokens. This lets agents chain: pattern → search. + // Escape \ and " (the phrase-query metachars) in each token. + const literalTokens = p.pattern + .split(/\s+/) + .filter(t => t !== '<*>' && t.length > 0) + .map(t => t.replace(/[\\"]/g, '\\$&')); + const whereSnippet = + literalTokens.length > 0 + ? `${bodyColumn}:"${literalTokens.join(' ')}"` + : ''; + + return { + id: p.id, + pattern: p.pattern, + estimatedCount: p.estimatedCount, + sampleCount: p.sampleCount, + whereSnippet, + trend: p.trend.map(t => ({ + ts: new Date(t.ts).toISOString(), + count: t.count, + })), + samples: p.samples.map(row => { + const raw = row.__hdx_pattern_body; + return raw != null ? String(raw) : ''; + }), + }; + }); + + const output = { + patterns, + totalCount, + sampledRows: sampleRows.length, + sampleMultiplier: Math.round(sampleMultiplier * 100) / 100, + bodyColumn, + timeRange: { + start: startDate.toISOString(), + end: endDate.toISOString(), + }, + }; + + const trimmedOutput = trimToolResponse(output); + const isTrimmed = + JSON.stringify(trimmedOutput).length < JSON.stringify(output).length; + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: trimmedOutput, + ...(isTrimmed + ? { + note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', + } + : {}), + }, + null, + 2, + ), + }, + ], + }; +} diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 554fa2ce0e..85e05dfbe1 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -1,8 +1,27 @@ import { z } from 'zod'; -// ─── Shared schemas ────────────────────────────────────────────────────────── - -const mcpAggFnSchema = z +// ─── Shared description fragments ──────────────────────────────────────────── + +export const WHERE_DESCRIPTION = + 'Row filter.\n\n' + + 'FIRST: pick a language (whereLanguage):\n' + + ' Lucene (default): Column:value e.g. level:error\n' + + " SQL: Column = 'value' e.g. StatusCode = 500\n" + + " SQL map attrs: SpanAttributes['key'] = 'value'\n" + + " SQL DateTime64: Timestamp >= parseDateTime64BestEffortOrNull('2024-01-01T00:00:00Z')\n\n" + + "WRONG: SpanAttributes['key']:value (Lucene cannot parse bracket syntax)\n" + + 'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")\n\n' + + 'MAP ATTRIBUTE ACCESS requires whereLanguage:"sql" — the Lucene parser ' + + "does not accept SpanAttributes['key'] bracket syntax."; + +export const WHERE_LANGUAGE_DESCRIPTION = + 'Query language for the "where" filter. Default: lucene.\n' + + 'Use "sql" when filtering on map attributes (SpanAttributes[\'key\']) or ' + + 'when you need SQL operators like >=, LIKE, IN, etc.'; + +// ─── Shared Zod schemas ────────────────────────────────────────────────────── + +export const mcpAggFnSchema = z .enum([ 'avg', 'count', @@ -24,7 +43,7 @@ const mcpAggFnSchema = z ' none – pass a raw expression through unchanged', ); -const mcpSelectItemSchema = z.object({ +export const mcpSelectItemSchema = z.object({ aggFn: mcpAggFnSchema, valueExpression: z .string() @@ -40,14 +59,17 @@ const mcpSelectItemSchema = z.object({ .optional() .default('') .describe( - 'Row filter in Lucene syntax. ' + - 'Examples: "level:error", "service.name:api AND http.status_code:>=500"', + 'Per-metric row filter. Compiles to If(...) for multi-cohort ' + + 'aggregates in one call. ' + + 'Example: "level:error" (Lucene) or "StatusCode >= 500" (SQL)', ), whereLanguage: z .enum(['lucene', 'sql']) .optional() .default('lucene') - .describe('Query language for the where filter. Default: lucene'), + .describe( + "Query language for this select item's where filter. Default: lucene", + ), alias: z .string() .optional() @@ -61,255 +83,48 @@ const mcpSelectItemSchema = z.object({ ), }); -// ─── Display type groups (used for validation) ────────────────────────────── - -const BUILDER_DISPLAY_TYPES = [ - 'line', - 'stacked_bar', - 'table', - 'number', - 'pie', -] as const; - -type BuilderDisplayType = (typeof BUILDER_DISPLAY_TYPES)[number]; - -function isBuilderDisplayType(dt: string): dt is BuilderDisplayType { - return (BUILDER_DISPLAY_TYPES as readonly string[]).includes(dt); -} - -// ─── Flat object schema for hyperdx_query ─────────────────────────────────── -// Uses a single z.object() instead of z.discriminatedUnion() so the MCP SDK -// can serialize it to JSON Schema correctly. The SDK's normalizeObjectSchema() -// only recognizes z.object() schemas — discriminated unions and ZodEffects -// (from .superRefine/.refine/.transform) are silently replaced with an empty -// schema in the tools/list response. -// -// To work around this, the schema is split into two parts: -// 1. hyperdxQuerySchema — plain z.object() used as inputSchema for the SDK -// 2. validateQueryInput — cross-field validation applied at runtime - -export const hyperdxQuerySchema = z.object({ - // ── Shared fields (all display types) ── - displayType: z - .enum([ - 'line', - 'stacked_bar', - 'table', - 'number', - 'pie', - 'search', - 'event_patterns', - 'sql', - ]) - .describe( - 'How to query and visualize the data:\n' + - ' line – time-series line chart (builder)\n' + - ' stacked_bar – time-series stacked bar chart (builder)\n' + - ' table – grouped aggregation as rows (builder)\n' + - ' number – single aggregate scalar (builder)\n' + - ' pie – pie chart, one metric grouped (builder)\n' + - ' search – browse individual log/event rows\n' + - ' event_patterns – discover the most common log messages / event patterns. ' + - 'Use this when asked about "top patterns", "common logs", "noisy services", or "recurring messages". ' + - 'Returns message templates with estimated counts and time trends.\n' + - ' sql – ADVANCED: raw ClickHouse SQL query', - ), - startTime: z - .string() - .optional() - .describe( - 'Start of the query window as ISO 8601. Default: 15 minutes ago. ' + - 'If results are empty, try a wider range (e.g. 24 hours).', - ), - endTime: z - .string() - .optional() - .describe('End of the query window as ISO 8601. Default: now.'), - - // ── Builder + search fields ── - sourceId: z - .string() - .optional() - .describe( - 'Source ID — required for builder display types (line, stacked_bar, table, number, pie), ' + - '"search", and "event_patterns". Call hyperdx_list_sources to find available sources.', - ), - select: z - .array(mcpSelectItemSchema) - .min(1) - .max(10) - .optional() - .describe( - 'Metrics to compute — required for builder display types (line, stacked_bar, table, number, pie). ' + - 'Each item defines an aggregation. ' + - 'For "number" display, provide exactly 1 item. ' + - 'Example: [{ aggFn: "count" }, { aggFn: "avg", valueExpression: "Duration" }]', - ), - groupBy: z - .string() - .optional() - .describe( - 'Column to group/split by (builder display types only). ' + - 'Top-level columns use PascalCase (e.g. "SpanName", "StatusCode"). ' + - "Span attributes: SpanAttributes['key'] (e.g. SpanAttributes['http.method']). " + - "Resource attributes: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).", - ), - orderBy: z - .string() - .optional() - .describe( - 'Column to sort results by (builder display types only, mainly "table").', - ), - granularity: z - .string() - .optional() - .describe( - 'Time bucket size for time-series charts (line, stacked_bar). ' + - 'Format: " " where unit is second, minute, hour, or day. ' + - 'Examples: "1 minute", "5 minute", "1 hour", "1 day". ' + - 'Omit to let HyperDX pick automatically based on the time range.', - ), - - // ── Search-only fields ── - where: z - .string() - .optional() - .default('') - .describe( - 'Row filter for "search" and "event_patterns" display types. ' + - 'Examples: "level:error", "service.name:api AND duration:>500"', - ), - whereLanguage: z - .enum(['lucene', 'sql']) - .optional() - .default('lucene') - .describe( - 'Query language for the "where" filter ("search" and "event_patterns" display types). Default: lucene', - ), - columns: z - .string() - .optional() - .default('') - .describe( - 'Comma-separated columns to include in search results ("search" display type only). ' + - 'Leave empty for defaults. Example: "body,service.name,duration"', - ), - maxResults: z - .number() - .min(1) - .max(200) - .optional() - .default(50) - .describe( - 'Maximum number of rows to return for "search" display type (1–200). Default: 50. ' + - 'Use smaller values to reduce response size.', - ), - - // ── Event patterns fields ── - // TODO: explore whether we can safely increase the max beyond 25_000 - sampleSize: z - .number() - .min(1) - .max(25_000) - .optional() - .describe( - 'Number of random rows to sample for pattern mining ("event_patterns" only). ' + - 'Default: 10000. Higher values produce more accurate patterns but take longer.', - ), - bodyExpression: z - .string() - .optional() - .describe( - 'Column expression to mine patterns from ("event_patterns" only). ' + - 'Auto-detected from the source if omitted (Body for logs, SpanName for traces). ' + - 'Example: "Body", "SpanName", "SpanAttributes[\'http.url\']"', - ), - - // ── SQL-only fields ── - connectionId: z - .string() - .optional() - .describe( - 'Connection ID — required for "sql" display type (not sourceId). ' + - 'Call hyperdx_list_sources to find available connections.', - ), - sql: z - .string() - .optional() - .describe( - 'Raw ClickHouse SQL query — required for "sql" display type. ' + - 'Always include a LIMIT clause to avoid returning excessive data.\n\n' + - 'QUERY PARAMETERS (ClickHouse native parameterized syntax):\n' + - ' {startDateMilliseconds:Int64} — start of date range in ms since epoch\n' + - ' {endDateMilliseconds:Int64} — end of date range in ms since epoch\n' + - ' {intervalSeconds:Int64} — time bucket size in seconds (time-series only)\n' + - ' {intervalMilliseconds:Int64} — time bucket size in milliseconds (time-series only)\n\n' + - 'MACROS (expanded before execution):\n' + - ' $__timeFilter(column) — expands to: column >= AND column <= (DateTime precision)\n' + - ' $__timeFilter_ms(column) — same but with DateTime64 millisecond precision\n' + - ' $__dateFilter(column) — same but with Date precision\n' + - ' $__dateTimeFilter(dateCol, timeCol) — filters on both a Date and DateTime column\n' + - ' $__dt(dateCol, timeCol) — alias for $__dateTimeFilter\n' + - ' $__fromTime / $__toTime — start/end as DateTime values\n' + - ' $__fromTime_ms / $__toTime_ms — start/end as DateTime64 values\n' + - ' $__timeInterval(column) — time bucket expression: toStartOfInterval(toDateTime(column), INTERVAL ...)\n' + - ' $__timeInterval_ms(column) — same with millisecond precision\n' + - ' $__interval_s — raw interval in seconds\n' + - ' $__filters — placeholder for dashboard filter conditions (resolves to 1=1 when no filters)\n\n' + - 'Example (time-series): "SELECT $__timeInterval(TimestampTime) AS ts, ServiceName, count() ' + - 'FROM otel_logs WHERE $__timeFilter(TimestampTime) GROUP BY ServiceName, ts ORDER BY ts"\n\n' + - 'Example (table): "SELECT ServiceName, count() AS n FROM otel_logs ' + - 'WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) ' + - 'AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) ' + - 'GROUP BY ServiceName ORDER BY n DESC LIMIT 20"', - ), -}); +export const startTimeSchema = z + .string() + .optional() + .describe( + 'Start of the query window as ISO 8601. Default: 15 minutes ago. ' + + 'If results are empty, try a wider range (e.g. 24 hours).', + ); -/** - * Cross-field validation for the query schema. Applied at runtime in the - * handler rather than via .superRefine() so the base z.object() stays intact - * for the MCP SDK's schema serialization. - * - * Returns a user-facing error string if validation fails, or null if valid. - */ -export function validateQueryInput( - data: z.infer, -): string | null { - const { displayType } = data; +export const endTimeSchema = z + .string() + .optional() + .describe('End of the query window as ISO 8601. Default: now.'); - if (isBuilderDisplayType(displayType)) { - if (!data.sourceId) { - return `sourceId is required when displayType is "${displayType}"`; - } - if (!data.select || data.select.length === 0) { - return `select is required when displayType is "${displayType}"`; - } - } else if (displayType === 'search') { - if (!data.sourceId) { - return 'sourceId is required when displayType is "search"'; - } - } else if (displayType === 'event_patterns') { - if (!data.sourceId) { - return 'sourceId is required when displayType is "event_patterns"'; - } - } else if (displayType === 'sql') { - if (!data.connectionId) { - return 'connectionId is required when displayType is "sql"'; - } - if (!data.sql) { - return 'sql is required when displayType is "sql"'; - } - } +export const sourceIdSchema = z + .string() + .describe( + 'Source ID (required). Call hyperdx_list_sources to find available sources.', + ); - // Reject event_patterns-only fields on other display types - if (displayType !== 'event_patterns') { - if (data.sampleSize != null) { - return `sampleSize is only valid when displayType is "event_patterns"`; - } - if (data.bodyExpression != null) { - return `bodyExpression is only valid when displayType is "event_patterns"`; - } - } +export const whereSchema = z + .string() + .optional() + .default('') + .describe(WHERE_DESCRIPTION); + +export const whereLanguageSchema = z + .enum(['lucene', 'sql']) + .optional() + .default('lucene') + .describe(WHERE_LANGUAGE_DESCRIPTION); + +export const groupBySchema = z + .string() + .optional() + .describe( + 'Column to group/split by. ' + + 'Top-level columns use PascalCase (e.g. "SpanName", "StatusCode"). ' + + "Span attributes: SpanAttributes['key'] (e.g. SpanAttributes['http.method']). " + + "Resource attributes: ResourceAttributes['key'] (e.g. ResourceAttributes['service.name']).", + ); - return null; -} +export const orderBySchema = z + .string() + .optional() + .describe('Column to sort results by (mainly useful for table shape).'); diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts new file mode 100644 index 0000000000..bf216c881b --- /dev/null +++ b/packages/api/src/mcp/tools/query/search.ts @@ -0,0 +1,99 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { ObjectId } from 'mongodb'; +import { z } from 'zod'; + +import type { ExternalDashboardTileWithId } from '@/utils/zod'; +import { externalDashboardTileSchemaWithId } from '@/utils/zod'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { parseTimeRange, runConfigTile } from './helpers'; +import { + endTimeSchema, + sourceIdSchema, + startTimeSchema, + whereLanguageSchema, + whereSchema, +} from './schemas'; + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const searchSchema = z.object({ + sourceId: sourceIdSchema, + where: whereSchema, + whereLanguage: whereLanguageSchema, + columns: z + .string() + .optional() + .default('') + .describe( + 'Comma-separated columns to include in search results. ' + + 'Leave empty for defaults. Example: "body,service.name,duration"', + ), + maxResults: z + .number() + .min(1) + .max(200) + .optional() + .default(50) + .describe( + 'Maximum number of rows to return (1-200). Default: 50. ' + + 'Use smaller values to reduce response size.', + ), + startTime: startTimeSchema, + endTime: endTimeSchema, +}); + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerSearch(server: McpServer, context: McpContext) { + const { teamId } = context; + + server.registerTool( + 'hyperdx_search', + { + title: 'Search Events', + description: + 'Browse individual log/event/trace rows. ' + + 'Use this when you need to see raw events, investigate specific log lines, ' + + 'or drill into individual records matching a filter.\n\n' + + 'Requires sourceId — call hyperdx_list_sources first.\n\n' + + 'For aggregated metrics, use hyperdx_table instead. ' + + 'For pattern discovery, use hyperdx_event_patterns instead.\n\n' + + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + + "Map attributes use bracket syntax: SpanAttributes['http.method'].", + inputSchema: searchSchema, + }, + withToolTracing('hyperdx_search', context, async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + const tile: ExternalDashboardTileWithId = + externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name: 'MCP Search', + x: 0, + y: 0, + w: 24, + h: 6, + config: { + displayType: 'search' as const, + sourceId: input.sourceId, + select: input.columns ?? '', + where: input.where ?? '', + whereLanguage: input.whereLanguage ?? 'lucene', + }, + }); + + return runConfigTile(teamId.toString(), tile, startDate, endDate, { + maxResults: input.maxResults, + }); + }), + ); +} diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts new file mode 100644 index 0000000000..c7e275632c --- /dev/null +++ b/packages/api/src/mcp/tools/query/sql.ts @@ -0,0 +1,105 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { ObjectId } from 'mongodb'; +import { z } from 'zod'; + +import type { ExternalDashboardTileWithId } from '@/utils/zod'; +import { externalDashboardTileSchemaWithId } from '@/utils/zod'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { parseTimeRange, runConfigTile } from './helpers'; +import { endTimeSchema, startTimeSchema } from './schemas'; + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const sqlSchema = z.object({ + connectionId: z + .string() + .describe( + 'Connection ID (required). This is the only split tool that needs a connectionId ' + + 'instead of sourceId. Call hyperdx_list_sources to find available connections.', + ), + sql: z + .string() + .describe( + 'Raw ClickHouse SQL query. Always include a LIMIT clause to avoid excessive data.\n\n' + + 'TIME FILTERING (preferred path):\n' + + ' $__timeFilter(column) — column >= AND column <= (DateTime precision)\n' + + ' $__timeFilter_ms(column) — same with DateTime64 millisecond precision\n' + + ' {startDateMilliseconds:Int64} / {endDateMilliseconds:Int64} — epoch ms parameters\n\n' + + 'LITERAL-CAST FALLBACKS:\n' + + " parseDateTime64BestEffortOrNull('2024-01-01T00:00:00Z')\n" + + " toDateTime64('2024-01-01 00:00:00', 9)\n\n" + + 'OTHER MACROS:\n' + + ' $__dateFilter(column) — Date precision\n' + + ' $__dateTimeFilter(dateCol, timeCol) — filters on Date + DateTime columns\n' + + ' $__dt(dateCol, timeCol) — alias for $__dateTimeFilter\n' + + ' $__fromTime / $__toTime — start/end as DateTime values\n' + + ' $__fromTime_ms / $__toTime_ms — start/end as DateTime64 values\n' + + ' $__timeInterval(column) — time bucket expression: toStartOfInterval(toDateTime(column), INTERVAL ...)\n' + + ' $__timeInterval_ms(column) — same with millisecond precision\n' + + ' $__interval_s — raw interval in seconds\n' + + ' $__filters — dashboard filter conditions (resolves to 1=1 when no filters)\n' + + ' {intervalSeconds:Int64} / {intervalMilliseconds:Int64} — bucket size parameters\n\n' + + 'Example (time-series):\n' + + ' SELECT $__timeInterval(TimestampTime) AS ts, ServiceName, count() ' + + 'FROM otel_logs WHERE $__timeFilter(TimestampTime) GROUP BY ServiceName, ts ORDER BY ts\n\n' + + 'Example (table):\n' + + ' SELECT ServiceName, count() AS n FROM otel_logs ' + + 'WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) ' + + 'AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) ' + + 'GROUP BY ServiceName ORDER BY n DESC LIMIT 20', + ), + startTime: startTimeSchema, + endTime: endTimeSchema, +}); + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerSql(server: McpServer, context: McpContext) { + const { teamId } = context; + + server.registerTool( + 'hyperdx_sql', + { + title: 'Raw SQL Query', + description: + 'Execute raw ClickHouse SQL. ' + + 'ADVANCED: only use this when you need capabilities the builder tools cannot express — ' + + 'JOINs, sub-queries, CTEs, or querying tables not registered as sources.\n\n' + + 'Requires connectionId (not sourceId) — call hyperdx_list_sources to find connections.\n\n' + + 'For standard aggregations use hyperdx_table. ' + + 'For time-series charts use hyperdx_timeseries. ' + + 'For browsing rows use hyperdx_search.', + inputSchema: sqlSchema, + }, + withToolTracing('hyperdx_sql', context, async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + const tile: ExternalDashboardTileWithId = + externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name: 'MCP SQL', + x: 0, + y: 0, + w: 24, + h: 6, + config: { + configType: 'sql' as const, + displayType: 'table' as const, + connectionId: input.connectionId, + sqlTemplate: input.sql, + }, + }); + + return runConfigTile(teamId.toString(), tile, startDate, endDate); + }), + ); +} diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts new file mode 100644 index 0000000000..91c88e7849 --- /dev/null +++ b/packages/api/src/mcp/tools/query/table.ts @@ -0,0 +1,119 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { ObjectId } from 'mongodb'; +import { z } from 'zod'; + +import type { ExternalDashboardTileWithId } from '@/utils/zod'; +import { externalDashboardTileSchemaWithId } from '@/utils/zod'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { parseTimeRange, runConfigTile } from './helpers'; +import { + endTimeSchema, + groupBySchema, + mcpSelectItemSchema, + orderBySchema, + sourceIdSchema, + startTimeSchema, +} from './schemas'; + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const tableSchema = z.object({ + sourceId: sourceIdSchema, + select: z + .array(mcpSelectItemSchema) + .min(1) + .max(10) + .describe( + 'Metrics to compute. Each item defines one aggregation column. ' + + 'For "number" shape, provide exactly 1 item. ' + + 'Example: [{ aggFn: "count" }, { aggFn: "avg", valueExpression: "Duration" }]', + ), + shape: z + .enum(['table', 'number', 'pie']) + .optional() + .default('table') + .describe( + 'Output shape: "table" (grouped rows, default), "number" (single scalar), or "pie" (pie chart). ' + + 'If "number" or "pie" is set with select.length > 1, it is auto-upgraded to "table".', + ), + groupBy: groupBySchema, + orderBy: orderBySchema, + startTime: startTimeSchema, + endTime: endTimeSchema, +}); + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerTable(server: McpServer, context: McpContext) { + const { teamId } = context; + + server.registerTool( + 'hyperdx_table', + { + title: 'Aggregation Table', + description: + 'Compute aggregated metrics as a table, single number, or pie chart. ' + + 'Use this for grouped aggregations, top-N queries, single-value KPIs, ' + + 'or proportional breakdowns.\n\n' + + 'Requires sourceId — call hyperdx_list_sources first.\n\n' + + 'Each select item can have its own "where" filter, which compiles to ' + + 'If(...) for multi-cohort comparison in one call ' + + '(e.g. p99_before + p99_after side-by-side).\n\n' + + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + + "Map attributes use bracket syntax: SpanAttributes['http.method']. " + + 'Map attributes work in groupBy and valueExpression, including ' + + "toFloat64OrZero(SpanAttributes['key']).\n\n" + + 'Shape auto-upgrade: if shape is "number" or "pie" but select has >1 item, ' + + 'it is transparently upgraded to "table".', + inputSchema: tableSchema, + }, + withToolTracing('hyperdx_table', context, async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + // Auto-upgrade shape when select has multiple items but shape is + // single-value (number/pie). This is the #1 Zod error class from agents. + let displayType: string = input.shape ?? 'table'; + if ( + (displayType === 'number' || displayType === 'pie') && + input.select.length > 1 + ) { + displayType = 'table'; + } + + const tile: ExternalDashboardTileWithId = + externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name: 'MCP Table', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType, + sourceId: input.sourceId, + select: input.select.map(s => ({ + aggFn: s.aggFn, + where: s.where ?? '', + whereLanguage: s.whereLanguage ?? 'lucene', + valueExpression: s.valueExpression, + alias: s.alias, + level: s.level, + })), + groupBy: input.groupBy ?? undefined, + orderBy: input.orderBy ?? undefined, + }, + }); + + return runConfigTile(teamId.toString(), tile, startDate, endDate); + }), + ); +} diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts new file mode 100644 index 0000000000..b3de81d91f --- /dev/null +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -0,0 +1,114 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { ObjectId } from 'mongodb'; +import { z } from 'zod'; + +import type { ExternalDashboardTileWithId } from '@/utils/zod'; +import { externalDashboardTileSchemaWithId } from '@/utils/zod'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; +import { parseTimeRange, runConfigTile } from './helpers'; +import { + endTimeSchema, + groupBySchema, + mcpSelectItemSchema, + orderBySchema, + sourceIdSchema, + startTimeSchema, +} from './schemas'; + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const timeseriesSchema = z.object({ + sourceId: sourceIdSchema, + select: z + .array(mcpSelectItemSchema) + .min(1) + .max(10) + .describe( + 'Metrics to plot on the chart. Each item defines one series. ' + + 'Example: [{ aggFn: "count" }, { aggFn: "avg", valueExpression: "Duration" }]', + ), + shape: z + .enum(['line', 'stacked_bar']) + .optional() + .default('line') + .describe( + 'Chart shape. "line" for line chart (default), "stacked_bar" for stacked bar chart.', + ), + groupBy: groupBySchema, + orderBy: orderBySchema, + granularity: z + .string() + .optional() + .describe( + 'Time bucket size for the chart. ' + + 'Format: " " where unit is second, minute, hour, or day.\n' + + 'CORRECT: "1 minute", "5 minute", "1 hour", "1 day"\n' + + 'WRONG: "1m", "5min", "1h" (abbreviations are not supported)\n\n' + + 'Omit to let HyperDX pick automatically based on the time range.', + ), + startTime: startTimeSchema, + endTime: endTimeSchema, +}); + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerTimeseries(server: McpServer, context: McpContext) { + const { teamId } = context; + + server.registerTool( + 'hyperdx_timeseries', + { + title: 'Time-Series Chart', + description: + 'Plot metrics over time as a line or stacked bar chart. ' + + 'Use this when you need to visualize trends, compare time-series, ' + + 'or monitor metric changes over a time window.\n\n' + + 'Requires sourceId — call hyperdx_list_sources first. ' + + 'Each select item defines one plotted series.\n\n' + + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + + "Map attributes use bracket syntax: SpanAttributes['http.method'].", + inputSchema: timeseriesSchema, + }, + withToolTracing('hyperdx_timeseries', context, async input => { + const timeRange = parseTimeRange(input.startTime, input.endTime); + if ('error' in timeRange) { + return { + isError: true, + content: [{ type: 'text' as const, text: timeRange.error }], + }; + } + const { startDate, endDate } = timeRange; + + const displayType = input.shape ?? 'line'; + + const tile: ExternalDashboardTileWithId = + externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name: 'MCP Timeseries', + x: 0, + y: 0, + w: 12, + h: 4, + config: { + displayType, + sourceId: input.sourceId, + select: input.select.map(s => ({ + aggFn: s.aggFn, + where: s.where ?? '', + whereLanguage: s.whereLanguage ?? 'lucene', + valueExpression: s.valueExpression, + alias: s.alias, + level: s.level, + })), + groupBy: input.groupBy ?? undefined, + orderBy: input.orderBy ?? undefined, + ...(input.granularity ? { granularity: input.granularity } : {}), + }, + }); + + return runConfigTile(teamId.toString(), tile, startDate, endDate); + }), + ); +} From 9509098217353f2d6cbef943fdbfa26b12463adb Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 15:50:15 -0600 Subject: [PATCH 2/9] chore: add changeset and fix knip unused exports --- .changeset/split-query-tools.md | 16 ++++++++++++++++ packages/api/src/mcp/tools/query/index.ts | 2 -- packages/api/src/mcp/tools/query/schemas.ts | 6 +++--- 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 .changeset/split-query-tools.md diff --git a/.changeset/split-query-tools.md b/.changeset/split-query-tools.md new file mode 100644 index 0000000000..32aa27adc9 --- /dev/null +++ b/.changeset/split-query-tools.md @@ -0,0 +1,16 @@ +--- +'@hyperdx/api': patch +--- + +refactor(mcp): split hyperdx_query into 5 display-type-specific tools + +Replace the monolithic `hyperdx_query` tool with five narrow tools: +- `hyperdx_timeseries` (line + stacked_bar) +- `hyperdx_table` (table + number + pie, with shape auto-upgrade) +- `hyperdx_search` (raw event browsing) +- `hyperdx_event_patterns` (Drain pattern mining) +- `hyperdx_sql` (raw ClickHouse SQL) + +Each tool's schema contains only its relevant parameters — no displayType +discriminator, no fields from other modes, no conditional required fields. +`hyperdx_query` is removed from the tool surface. diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index d49c86f45c..a9c8a892fe 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -7,8 +7,6 @@ import { registerSql } from './sql'; import { registerTable } from './table'; import { registerTimeseries } from './timeseries'; -export * from './schemas'; - const queryTools: ToolDefinition = (server: McpServer, context: McpContext) => { registerTimeseries(server, context); registerTable(server, context); diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 85e05dfbe1..f7a464dd62 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; // ─── Shared description fragments ──────────────────────────────────────────── -export const WHERE_DESCRIPTION = +const WHERE_DESCRIPTION = 'Row filter.\n\n' + 'FIRST: pick a language (whereLanguage):\n' + ' Lucene (default): Column:value e.g. level:error\n' + @@ -14,14 +14,14 @@ export const WHERE_DESCRIPTION = 'MAP ATTRIBUTE ACCESS requires whereLanguage:"sql" — the Lucene parser ' + "does not accept SpanAttributes['key'] bracket syntax."; -export const WHERE_LANGUAGE_DESCRIPTION = +const WHERE_LANGUAGE_DESCRIPTION = 'Query language for the "where" filter. Default: lucene.\n' + 'Use "sql" when filtering on map attributes (SpanAttributes[\'key\']) or ' + 'when you need SQL operators like >=, LIKE, IN, etc.'; // ─── Shared Zod schemas ────────────────────────────────────────────────────── -export const mcpAggFnSchema = z +const mcpAggFnSchema = z .enum([ 'avg', 'count', From edf87562ec0ed3ec08119bdb1b6ab3b9131e9c35 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 15:51:33 -0600 Subject: [PATCH 3/9] chore(mcp): minor type/description cleanups from review - table.ts: narrow displayType to union type instead of string - search.ts: remove redundant ?? '' after zod .default('') - table.ts: note in shape description that groupBy is ignored for number - sql.ts: note in description that results are always table rows --- packages/api/src/mcp/tools/query/search.ts | 2 +- packages/api/src/mcp/tools/query/sql.ts | 2 ++ packages/api/src/mcp/tools/query/table.ts | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index bf216c881b..3bbb1c8c8c 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -85,7 +85,7 @@ export function registerSearch(server: McpServer, context: McpContext) { config: { displayType: 'search' as const, sourceId: input.sourceId, - select: input.columns ?? '', + select: input.columns, where: input.where ?? '', whereLanguage: input.whereLanguage ?? 'lucene', }, diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts index c7e275632c..c3e00b3106 100644 --- a/packages/api/src/mcp/tools/query/sql.ts +++ b/packages/api/src/mcp/tools/query/sql.ts @@ -68,6 +68,8 @@ export function registerSql(server: McpServer, context: McpContext) { 'ADVANCED: only use this when you need capabilities the builder tools cannot express — ' + 'JOINs, sub-queries, CTEs, or querying tables not registered as sources.\n\n' + 'Requires connectionId (not sourceId) — call hyperdx_list_sources to find connections.\n\n' + + 'Results are always returned as table rows — for time-series semantics, ' + + 'include a time column and ORDER BY it in your SQL.\n\n' + 'For standard aggregations use hyperdx_table. ' + 'For time-series charts use hyperdx_timeseries. ' + 'For browsing rows use hyperdx_search.', diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index 91c88e7849..d9501b9f5b 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -36,7 +36,8 @@ const tableSchema = z.object({ .default('table') .describe( 'Output shape: "table" (grouped rows, default), "number" (single scalar), or "pie" (pie chart). ' + - 'If "number" or "pie" is set with select.length > 1, it is auto-upgraded to "table".', + 'If "number" or "pie" is set with select.length > 1, it is auto-upgraded to "table". ' + + 'groupBy is ignored when shape is "number".', ), groupBy: groupBySchema, orderBy: orderBySchema, @@ -81,7 +82,7 @@ export function registerTable(server: McpServer, context: McpContext) { // Auto-upgrade shape when select has multiple items but shape is // single-value (number/pie). This is the #1 Zod error class from agents. - let displayType: string = input.shape ?? 'table'; + let displayType: 'table' | 'number' | 'pie' = input.shape ?? 'table'; if ( (displayType === 'number' || displayType === 'pie') && input.select.length > 1 From d1ccabd4cac6a902d77713307b4af4a1eb60ec3a Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 15:54:10 -0600 Subject: [PATCH 4/9] docs: update MCP.md tool table for split query tools Also assert on sourceId in missing-field rejection tests to catch schema-required-field regressions. --- MCP.md | 6 +++++- packages/api/src/mcp/__tests__/queryTool.test.ts | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/MCP.md b/MCP.md index fc3c62151f..9f702d43a6 100644 --- a/MCP.md +++ b/MCP.md @@ -109,7 +109,11 @@ with: | Tool | Description | | ----------------------------- | -------------------------------------------------------------------------------------------- | | `hyperdx_list_sources` | List all data sources and database connections, including column schemas and attribute keys | -| `hyperdx_query` | Query observability data (logs, metrics, traces) using builder mode, search mode, or raw SQL | +| `hyperdx_timeseries` | Plot metrics over time as a line or stacked bar chart | +| `hyperdx_table` | Compute aggregated metrics as a table, single number, or pie chart | +| `hyperdx_search` | Browse individual log, event, or trace rows | +| `hyperdx_event_patterns` | Discover the most common log messages and event patterns using Drain clustering | +| `hyperdx_sql` | Execute raw ClickHouse SQL for advanced queries (JOINs, CTEs, sub-queries) | | `hyperdx_get_dashboard` | List all dashboards or get full detail for a specific dashboard | | `hyperdx_save_dashboard` | Create or update a dashboard with tiles (charts, tables, numbers, search, markdown) | | `hyperdx_delete_dashboard` | Permanently delete a dashboard and its attached alerts | diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index 4b3a2a0520..0ecabba06d 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -353,6 +353,7 @@ describe('MCP Query Tools', () => { }); expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/sourceId/i); }); }); @@ -441,6 +442,7 @@ describe('MCP Query Tools', () => { }); expect(result.isError).toBe(true); + expect(getFirstText(result)).toMatch(/sourceId/i); }); it('should return error for non-existent source', async () => { From d4bc0aaa676c086ae999235460625617d74de7d9 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 15:57:19 -0600 Subject: [PATCH 5/9] refactor(mcp): extract buildTile helper to deduplicate tile envelope construction --- packages/api/src/mcp/tools/query/helpers.ts | 25 ++++++++++ packages/api/src/mcp/tools/query/search.ts | 29 ++++-------- packages/api/src/mcp/tools/query/sql.ts | 27 +++-------- packages/api/src/mcp/tools/query/table.ts | 43 ++++++----------- .../api/src/mcp/tools/query/timeseries.ts | 47 +++++++------------ 5 files changed, 71 insertions(+), 100 deletions(-) diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 79d34eaa79..64f948ae84 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -7,6 +7,7 @@ import type { MetricTable, } from '@hyperdx/common-utils/dist/types'; import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; +import { ObjectId } from 'mongodb'; import ms from 'ms'; import { getConnectionById } from '@/controllers/connection'; @@ -17,6 +18,30 @@ import { } from '@/routers/external-api/v2/utils/dashboards'; import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; +import { externalDashboardTileSchemaWithId } from '@/utils/zod'; + +// ─── Tile construction ─────────────────────────────────────────────────────── + +/** + * Build a validated tile envelope for MCP tool execution. + * Eliminates the repeated id/name/x/y/w/h boilerplate across tool handlers. + */ +export function buildTile( + name: string, + w: number, + h: number, + config: Record, +): ExternalDashboardTileWithId { + return externalDashboardTileSchemaWithId.parse({ + id: new ObjectId().toString(), + name, + x: 0, + y: 0, + w, + h, + config, + }); +} // ─── Time range ────────────────────────────────────────────────────────────── diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 3bbb1c8c8c..7018621cf0 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -1,13 +1,9 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ObjectId } from 'mongodb'; import { z } from 'zod'; -import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import { externalDashboardTileSchemaWithId } from '@/utils/zod'; - import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; -import { parseTimeRange, runConfigTile } from './helpers'; +import { buildTile, parseTimeRange, runConfigTile } from './helpers'; import { endTimeSchema, sourceIdSchema, @@ -74,22 +70,13 @@ export function registerSearch(server: McpServer, context: McpContext) { } const { startDate, endDate } = timeRange; - const tile: ExternalDashboardTileWithId = - externalDashboardTileSchemaWithId.parse({ - id: new ObjectId().toString(), - name: 'MCP Search', - x: 0, - y: 0, - w: 24, - h: 6, - config: { - displayType: 'search' as const, - sourceId: input.sourceId, - select: input.columns, - where: input.where ?? '', - whereLanguage: input.whereLanguage ?? 'lucene', - }, - }); + const tile = buildTile('MCP Search', 24, 6, { + displayType: 'search' as const, + sourceId: input.sourceId, + select: input.columns, + where: input.where ?? '', + whereLanguage: input.whereLanguage ?? 'lucene', + }); return runConfigTile(teamId.toString(), tile, startDate, endDate, { maxResults: input.maxResults, diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts index c3e00b3106..b5cf396970 100644 --- a/packages/api/src/mcp/tools/query/sql.ts +++ b/packages/api/src/mcp/tools/query/sql.ts @@ -1,13 +1,9 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ObjectId } from 'mongodb'; import { z } from 'zod'; -import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import { externalDashboardTileSchemaWithId } from '@/utils/zod'; - import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; -import { parseTimeRange, runConfigTile } from './helpers'; +import { buildTile, parseTimeRange, runConfigTile } from './helpers'; import { endTimeSchema, startTimeSchema } from './schemas'; // ─── Schema ────────────────────────────────────────────────────────────────── @@ -85,21 +81,12 @@ export function registerSql(server: McpServer, context: McpContext) { } const { startDate, endDate } = timeRange; - const tile: ExternalDashboardTileWithId = - externalDashboardTileSchemaWithId.parse({ - id: new ObjectId().toString(), - name: 'MCP SQL', - x: 0, - y: 0, - w: 24, - h: 6, - config: { - configType: 'sql' as const, - displayType: 'table' as const, - connectionId: input.connectionId, - sqlTemplate: input.sql, - }, - }); + const tile = buildTile('MCP SQL', 24, 6, { + configType: 'sql' as const, + displayType: 'table' as const, + connectionId: input.connectionId, + sqlTemplate: input.sql, + }); return runConfigTile(teamId.toString(), tile, startDate, endDate); }), diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index d9501b9f5b..08a1a17317 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -1,13 +1,9 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ObjectId } from 'mongodb'; import { z } from 'zod'; -import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import { externalDashboardTileSchemaWithId } from '@/utils/zod'; - import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; -import { parseTimeRange, runConfigTile } from './helpers'; +import { buildTile, parseTimeRange, runConfigTile } from './helpers'; import { endTimeSchema, groupBySchema, @@ -90,29 +86,20 @@ export function registerTable(server: McpServer, context: McpContext) { displayType = 'table'; } - const tile: ExternalDashboardTileWithId = - externalDashboardTileSchemaWithId.parse({ - id: new ObjectId().toString(), - name: 'MCP Table', - x: 0, - y: 0, - w: 12, - h: 4, - config: { - displayType, - sourceId: input.sourceId, - select: input.select.map(s => ({ - aggFn: s.aggFn, - where: s.where ?? '', - whereLanguage: s.whereLanguage ?? 'lucene', - valueExpression: s.valueExpression, - alias: s.alias, - level: s.level, - })), - groupBy: input.groupBy ?? undefined, - orderBy: input.orderBy ?? undefined, - }, - }); + const tile = buildTile('MCP Table', 12, 4, { + displayType, + sourceId: input.sourceId, + select: input.select.map(s => ({ + aggFn: s.aggFn, + where: s.where ?? '', + whereLanguage: s.whereLanguage ?? 'lucene', + valueExpression: s.valueExpression, + alias: s.alias, + level: s.level, + })), + groupBy: input.groupBy ?? undefined, + orderBy: input.orderBy ?? undefined, + }); return runConfigTile(teamId.toString(), tile, startDate, endDate); }), diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index b3de81d91f..5bb11e1749 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -1,13 +1,9 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { ObjectId } from 'mongodb'; import { z } from 'zod'; -import type { ExternalDashboardTileWithId } from '@/utils/zod'; -import { externalDashboardTileSchemaWithId } from '@/utils/zod'; - import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; -import { parseTimeRange, runConfigTile } from './helpers'; +import { buildTile, parseTimeRange, runConfigTile } from './helpers'; import { endTimeSchema, groupBySchema, @@ -81,32 +77,21 @@ export function registerTimeseries(server: McpServer, context: McpContext) { } const { startDate, endDate } = timeRange; - const displayType = input.shape ?? 'line'; - - const tile: ExternalDashboardTileWithId = - externalDashboardTileSchemaWithId.parse({ - id: new ObjectId().toString(), - name: 'MCP Timeseries', - x: 0, - y: 0, - w: 12, - h: 4, - config: { - displayType, - sourceId: input.sourceId, - select: input.select.map(s => ({ - aggFn: s.aggFn, - where: s.where ?? '', - whereLanguage: s.whereLanguage ?? 'lucene', - valueExpression: s.valueExpression, - alias: s.alias, - level: s.level, - })), - groupBy: input.groupBy ?? undefined, - orderBy: input.orderBy ?? undefined, - ...(input.granularity ? { granularity: input.granularity } : {}), - }, - }); + const tile = buildTile('MCP Timeseries', 12, 4, { + displayType: input.shape ?? 'line', + sourceId: input.sourceId, + select: input.select.map(s => ({ + aggFn: s.aggFn, + where: s.where ?? '', + whereLanguage: s.whereLanguage ?? 'lucene', + valueExpression: s.valueExpression, + alias: s.alias, + level: s.level, + })), + groupBy: input.groupBy ?? undefined, + orderBy: input.orderBy ?? undefined, + ...(input.granularity ? { granularity: input.granularity } : {}), + }); return runConfigTile(teamId.toString(), tile, startDate, endDate); }), From 52246eaa8e04c736abf89e035c612c964700b6e3 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 16:02:54 -0600 Subject: [PATCH 6/9] chore(mcp): remove redundant fallbacks after zod .default(), strip groupBy for number shape --- packages/api/src/mcp/tools/query/search.ts | 4 ++-- packages/api/src/mcp/tools/query/table.ts | 8 ++++---- packages/api/src/mcp/tools/query/timeseries.ts | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 7018621cf0..ef23704964 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -74,8 +74,8 @@ export function registerSearch(server: McpServer, context: McpContext) { displayType: 'search' as const, sourceId: input.sourceId, select: input.columns, - where: input.where ?? '', - whereLanguage: input.whereLanguage ?? 'lucene', + where: input.where, + whereLanguage: input.whereLanguage, }); return runConfigTile(teamId.toString(), tile, startDate, endDate, { diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index 08a1a17317..f0352d3c9f 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -78,7 +78,7 @@ export function registerTable(server: McpServer, context: McpContext) { // Auto-upgrade shape when select has multiple items but shape is // single-value (number/pie). This is the #1 Zod error class from agents. - let displayType: 'table' | 'number' | 'pie' = input.shape ?? 'table'; + let displayType: 'table' | 'number' | 'pie' = input.shape; if ( (displayType === 'number' || displayType === 'pie') && input.select.length > 1 @@ -91,13 +91,13 @@ export function registerTable(server: McpServer, context: McpContext) { sourceId: input.sourceId, select: input.select.map(s => ({ aggFn: s.aggFn, - where: s.where ?? '', - whereLanguage: s.whereLanguage ?? 'lucene', + where: s.where, + whereLanguage: s.whereLanguage, valueExpression: s.valueExpression, alias: s.alias, level: s.level, })), - groupBy: input.groupBy ?? undefined, + groupBy: displayType === 'number' ? undefined : input.groupBy, orderBy: input.orderBy ?? undefined, }); diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index 5bb11e1749..d712c2dbee 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -78,12 +78,12 @@ export function registerTimeseries(server: McpServer, context: McpContext) { const { startDate, endDate } = timeRange; const tile = buildTile('MCP Timeseries', 12, 4, { - displayType: input.shape ?? 'line', + displayType: input.shape, sourceId: input.sourceId, select: input.select.map(s => ({ aggFn: s.aggFn, - where: s.where ?? '', - whereLanguage: s.whereLanguage ?? 'lucene', + where: s.where, + whereLanguage: s.whereLanguage, valueExpression: s.valueExpression, alias: s.alias, level: s.level, From ba1a5fb987c5d82ea79830f3b24d96e36b2cff78 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 16:13:57 -0600 Subject: [PATCH 7/9] chore(mcp): align columns example to PascalCase, remove no-op fallbacks, add sampleSize .default() --- packages/api/src/mcp/tools/query/eventPatterns.ts | 1 + packages/api/src/mcp/tools/query/search.ts | 2 +- packages/api/src/mcp/tools/query/timeseries.ts | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts index 9bb9cfd812..8224190f90 100644 --- a/packages/api/src/mcp/tools/query/eventPatterns.ts +++ b/packages/api/src/mcp/tools/query/eventPatterns.ts @@ -25,6 +25,7 @@ const eventPatternsSchema = z.object({ .min(1) .max(25_000) .optional() + .default(10_000) .describe( 'Number of random rows to sample for pattern mining. ' + 'Default: 10000. Higher values produce more accurate patterns but take longer.', diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index ef23704964..de319ecf30 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -24,7 +24,7 @@ const searchSchema = z.object({ .default('') .describe( 'Comma-separated columns to include in search results. ' + - 'Leave empty for defaults. Example: "body,service.name,duration"', + 'Leave empty for defaults. Example: "Body,ServiceName,Duration"', ), maxResults: z .number() diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index d712c2dbee..3bb5bcafd6 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -88,8 +88,8 @@ export function registerTimeseries(server: McpServer, context: McpContext) { alias: s.alias, level: s.level, })), - groupBy: input.groupBy ?? undefined, - orderBy: input.orderBy ?? undefined, + groupBy: input.groupBy, + orderBy: input.orderBy, ...(input.granularity ? { granularity: input.granularity } : {}), }); From 386984da45df91c4b1ab7851e32f33a263daddf0 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 15 May 2026 16:22:10 -0600 Subject: [PATCH 8/9] fix(mcp): catch ClickHouse errors in runConfigTile, fix stale listSources description, simplify select passthrough --- .../src/mcp/tools/dashboards/listSources.ts | 5 +- packages/api/src/mcp/tools/query/helpers.ts | 48 +++++++++++++++---- packages/api/src/mcp/tools/query/table.ts | 11 +---- .../api/src/mcp/tools/query/timeseries.ts | 9 +--- 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/packages/api/src/mcp/tools/dashboards/listSources.ts b/packages/api/src/mcp/tools/dashboards/listSources.ts index 97b491fb20..8b12cc6a8d 100644 --- a/packages/api/src/mcp/tools/dashboards/listSources.ts +++ b/packages/api/src/mcp/tools/dashboards/listSources.ts @@ -38,8 +38,9 @@ export function registerListSources( '(e.g. SpanAttributes, ResourceAttributes). ' + 'Column names are PascalCase (e.g. Duration, not duration). ' + "Map attributes must be accessed via bracket syntax: SpanAttributes['key'].\n\n" + - 'NOTE: For most queries, use source IDs with the builder display types. ' + - 'Connection IDs are only needed for advanced raw SQL queries (displayType "sql").', + 'NOTE: For most queries, use source IDs with hyperdx_timeseries, hyperdx_table, ' + + 'hyperdx_search, or hyperdx_event_patterns. ' + + 'Connection IDs are only needed for hyperdx_sql (raw ClickHouse SQL).', inputSchema: z.object({}), }, withToolTracing('hyperdx_list_sources', context, async () => { diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 64f948ae84..b287f155b1 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -219,11 +219,25 @@ export async function runConfigTile( } satisfies ChartConfigWithDateRange; const metadata = getMetadata(clickhouseClient); - const result = await clickhouseClient.queryChartConfig({ - config: chartConfig, - metadata, - querySettings: source.querySettings, - }); + let result; + try { + result = await clickhouseClient.queryChartConfig({ + config: chartConfig, + metadata, + querySettings: source.querySettings, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `ClickHouse query failed: ${message}`, + }, + ], + }; + } return formatQueryResult(result); } @@ -279,11 +293,25 @@ export async function runConfigTile( } satisfies ChartConfigWithDateRange; const metadata = getMetadata(clickhouseClient); - const result = await clickhouseClient.queryChartConfig({ - config: chartConfig, - metadata, - querySettings: undefined, - }); + let result; + try { + result = await clickhouseClient.queryChartConfig({ + config: chartConfig, + metadata, + querySettings: undefined, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `ClickHouse query failed: ${message}`, + }, + ], + }; + } return formatQueryResult(result); } diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index f0352d3c9f..1d826f179b 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -89,16 +89,9 @@ export function registerTable(server: McpServer, context: McpContext) { const tile = buildTile('MCP Table', 12, 4, { displayType, sourceId: input.sourceId, - select: input.select.map(s => ({ - aggFn: s.aggFn, - where: s.where, - whereLanguage: s.whereLanguage, - valueExpression: s.valueExpression, - alias: s.alias, - level: s.level, - })), + select: input.select, groupBy: displayType === 'number' ? undefined : input.groupBy, - orderBy: input.orderBy ?? undefined, + orderBy: input.orderBy, }); return runConfigTile(teamId.toString(), tile, startDate, endDate); diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index 3bb5bcafd6..919f39a5ec 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -80,14 +80,7 @@ export function registerTimeseries(server: McpServer, context: McpContext) { const tile = buildTile('MCP Timeseries', 12, 4, { displayType: input.shape, sourceId: input.sourceId, - select: input.select.map(s => ({ - aggFn: s.aggFn, - where: s.where, - whereLanguage: s.whereLanguage, - valueExpression: s.valueExpression, - alias: s.alias, - level: s.level, - })), + select: input.select, groupBy: input.groupBy, orderBy: input.orderBy, ...(input.granularity ? { granularity: input.granularity } : {}), From b4f33ea90d70204fb1397f2a34d41234f9efcebd Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Tue, 19 May 2026 11:52:37 -0600 Subject: [PATCH 9/9] fix(mcp): document Lucene dot-notation for map attributes and accurate operator support Address Drew's PR review comment: the WHERE_DESCRIPTION incorrectly stated that map attribute access requires whereLanguage:"sql". In fact, Lucene fully supports map attributes via dot notation (e.g. SpanAttributes.http.method:GET). Changes: - Add Lucene dot-notation examples to WHERE_DESCRIPTION - Add MAP ATTRIBUTES section showing syntax for each language - Remove incorrect 'MAP ATTRIBUTE ACCESS requires sql' claim - Update WHERE_LANGUAGE_DESCRIPTION to accurately reflect that Lucene supports comparisons, wildcards, ranges, and map attributes natively - Only recommend SQL for IN(...) lists, complex expressions, or function calls --- packages/api/src/mcp/tools/query/schemas.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index f7a464dd62..9da11a64b6 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -6,18 +6,20 @@ const WHERE_DESCRIPTION = 'Row filter.\n\n' + 'FIRST: pick a language (whereLanguage):\n' + ' Lucene (default): Column:value e.g. level:error\n' + + ' Lucene map attrs: Column.key:value e.g. SpanAttributes.http.method:GET\n' + " SQL: Column = 'value' e.g. StatusCode = 500\n" + - " SQL map attrs: SpanAttributes['key'] = 'value'\n" + - " SQL DateTime64: Timestamp >= parseDateTime64BestEffortOrNull('2024-01-01T00:00:00Z')\n\n" + + " SQL map attrs: SpanAttributes['key'] = 'value'\n\n" + + 'MAP ATTRIBUTES:\n' + + ' Lucene uses DOT notation: SpanAttributes.http.method:GET\n' + + " SQL uses BRACKET notation: SpanAttributes['http.method'] = 'GET'\n\n" + "WRONG: SpanAttributes['key']:value (Lucene cannot parse bracket syntax)\n" + - 'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")\n\n' + - 'MAP ATTRIBUTE ACCESS requires whereLanguage:"sql" — the Lucene parser ' + - "does not accept SpanAttributes['key'] bracket syntax."; + 'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")'; const WHERE_LANGUAGE_DESCRIPTION = 'Query language for the "where" filter. Default: lucene.\n' + - 'Use "sql" when filtering on map attributes (SpanAttributes[\'key\']) or ' + - 'when you need SQL operators like >=, LIKE, IN, etc.'; + 'Lucene supports comparisons (>= > < <=), wildcards (field:val*), ranges ([1 TO 5]), ' + + 'and map attributes via dot notation. ' + + 'Use "sql" for IN(...) lists, complex expressions, or function calls.'; // ─── Shared Zod schemas ──────────────────────────────────────────────────────