diff --git a/.changeset/trace-breakdown-timeout.md b/.changeset/trace-breakdown-timeout.md new file mode 100644 index 0000000000..26c435b3f9 --- /dev/null +++ b/.changeset/trace-breakdown-timeout.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/api': patch +--- + +fix(mcp): time-limit the `clickstack_trace_top_time_consuming_operations` tool. +Its queries now use the MCP request timeout and `max_execution_time` ceiling, +which a source's `querySettings` can no longer override. A timeout returns +guidance to shorten the window or narrow `parentFilter` instead of an "invalid +SQL" hint, and the client-side `Timeout error.` now gets the timeout hint in +every MCP tool. Sources with a multi-column `timestampValueExpression` +(including a leading Date column such as `EventDate, EventTime`) now return +correct breakdowns. diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts index 4c6f0b5ba7..585ea7551f 100644 --- a/packages/api/src/mcp/__tests__/query.test.ts +++ b/packages/api/src/mcp/__tests__/query.test.ts @@ -309,6 +309,17 @@ describe('errorHint', () => { expect(hint).toContain('Narrow the time range'); }); + it('should match the client-side request timeout', () => { + const error = new Error('Timeout error.'); + expect(errorHint(error.message, error)).toContain('execution-time limit'); + }); + + it('should not give query-tuning advice for a socket timeout', () => { + const error: NodeJS.ErrnoException = new Error('connect ETIMEDOUT'); + error.code = 'ETIMEDOUT'; + expect(errorHint(error.message, error)).toBeNull(); + }); + it('should match SETTING_CONSTRAINT_VIOLATION errors', () => { const hint = errorHint( "Setting max_result_rows shouldn't be greater than 1000. (SETTING_CONSTRAINT_VIOLATION)", diff --git a/packages/api/src/mcp/__tests__/trace.int.test.ts b/packages/api/src/mcp/__tests__/trace.int.test.ts index ef5876a1d0..722af8b2bd 100644 --- a/packages/api/src/mcp/__tests__/trace.int.test.ts +++ b/packages/api/src/mcp/__tests__/trace.int.test.ts @@ -1,3 +1,4 @@ +import { ClickHouseError } from '@clickhouse/client-common'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; @@ -11,6 +12,7 @@ import { getLoggedInAgent, getServer, } from '@/fixtures'; +import { MCP_REQUEST_TIMEOUT } from '@/mcp/tools/query/helpers'; import { McpContext } from '@/mcp/tools/types'; import Connection from '@/models/connection'; import { Source, type SourceDocument } from '@/models/source'; @@ -19,6 +21,15 @@ import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; // ─── Helpers ────────────────────────────────────────────────────────────────── +type BreakdownOperation = { + service: string; + operation: string; + totalTimeMs: number; + calls: number; + inParents: number; + shareOfTotalTime: number; +}; + /** Insert trace spans into the default otel_traces table. */ async function bulkInsertTraces( spans: { @@ -1212,10 +1223,10 @@ describe('MCP Trace Tools', () => { // DB operations should be ranked first (higher total time) const dbOp = output.operations.find( - (op: any) => op.operation === CHILD_DB_OP, + (op: BreakdownOperation) => op.operation === CHILD_DB_OP, ); const cacheOp = output.operations.find( - (op: any) => op.operation === CHILD_CACHE_OP, + (op: BreakdownOperation) => op.operation === CHILD_CACHE_OP, ); expect(dbOp).toBeDefined(); expect(cacheOp).toBeDefined(); @@ -1250,7 +1261,7 @@ describe('MCP Trace Tools', () => { // Only trace 2's children should be included (1 call each) const dbOp = output.operations.find( - (op: any) => op.operation === CHILD_DB_OP, + (op: BreakdownOperation) => op.operation === CHILD_DB_OP, ); expect(dbOp).toBeDefined(); expect(dbOp.inParents).toBe(1); @@ -1301,6 +1312,31 @@ describe('MCP Trace Tools', () => { expect(output.summary.grandTotalTimeMs).toBeGreaterThan(0); }); + it('handles a multi-column timestampValueExpression with a leading Date column', async () => { + await Source.updateOne( + { _id: traceSource._id }, + { timestampValueExpression: 'toDate(Timestamp), Timestamp' }, + ); + + const result = await callTool( + client, + 'clickstack_trace_top_time_consuming_operations', + { + sourceId: traceSource._id.toString(), + parentFilter: `ServiceName = '${PARENT_SVC}' AND SpanName = '${PARENT_OP}'`, + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }, + ); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + const dbOp = output.operations.find( + (op: BreakdownOperation) => op.operation === CHILD_DB_OP, + ); + expect(dbOp?.inParents).toBe(2); + }); + it('should return error for invalid parentFilter SQL', async () => { const result = await callTool( client, @@ -1314,8 +1350,108 @@ describe('MCP Trace Tools', () => { ); expect(result.isError).toBe(true); - expect(getFirstText(result)).toContain('Failed to compute breakdown'); + const text = getFirstText(result); + expect(text).toContain('Failed to compute breakdown'); + expect(text).toContain('must be valid ClickHouse SQL'); + expect(text).not.toContain('shorter startTime/endTime window'); }); + + it('applies the MCP time limits even when the source sets max_execution_time', async () => { + await Source.updateOne( + { _id: traceSource._id }, + { + querySettings: [ + { setting: 'max_execution_time', value: '3600' }, + { setting: 'max_threads', value: '2' }, + ], + }, + ); + const querySpy = jest.spyOn(ClickhouseClient.prototype, 'query'); + try { + const result = await callTool( + client, + 'clickstack_trace_top_time_consuming_operations', + { + sourceId: traceSource._id.toString(), + parentFilter: `ServiceName = '${PARENT_SVC}' AND SpanName = '${PARENT_OP}'`, + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }, + ); + + expect(result.isError).toBeFalsy(); + const call = querySpy.mock.calls.find(([args]) => + args.query.includes('parent_traces'), + ); + expect(call?.[0].clickhouse_settings).toMatchObject({ + max_execution_time: 30, + readonly: '2', + max_threads: '2', + }); + const ctx: ClickhouseClient = + querySpy.mock.contexts[querySpy.mock.calls.indexOf(call!)]; + expect(ctx.requestTimeoutMs).toBe(MCP_REQUEST_TIMEOUT); + } finally { + querySpy.mockRestore(); + } + }); + + it.each([ + ['client request timeout', new Error('Timeout error.')], + [ + 'ClickHouse TIMEOUT_EXCEEDED', + new ClickHouseError({ + code: '159', + type: 'TIMEOUT_EXCEEDED', + message: 'Timeout exceeded: elapsed 30.001 seconds, maximum: 30', + }), + ], + ])('returns actionable guidance on a %s', async (_label, error) => { + const text = await callBreakdownWithFailingQuery(error); + expect(text).toContain('Failed to compute breakdown'); + expect(text).toContain('execution-time limit'); + expect(text).toContain('shorter startTime/endTime window'); + expect(text).toContain('minParentDurationMs'); + expect(text).not.toContain('must be valid ClickHouse SQL'); + }); + + it('gives no query-tuning or invalid-SQL advice on a socket timeout', async () => { + const error: NodeJS.ErrnoException = new Error('connect ETIMEDOUT'); + error.code = 'ETIMEDOUT'; + const text = await callBreakdownWithFailingQuery(error); + expect(text).toContain('Failed to compute breakdown'); + expect(text).not.toContain('execution-time limit'); + expect(text).not.toContain('shorter startTime/endTime window'); + expect(text).not.toContain('must be valid ClickHouse SQL'); + }); + + async function callBreakdownWithFailingQuery(error: Error) { + const original = ClickhouseClient.prototype.query; + const querySpy = jest + .spyOn(ClickhouseClient.prototype, 'query') + .mockImplementation(function (this: ClickhouseClient, args) { + if (args.query.includes('parent_traces')) { + return Promise.reject(error); + } + return original.call(this, args); + }); + try { + const result = await callTool( + client, + 'clickstack_trace_top_time_consuming_operations', + { + sourceId: traceSource._id.toString(), + parentFilter: `ServiceName = '${PARENT_SVC}' AND SpanName = '${PARENT_OP}'`, + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }, + ); + expect(result.isError).toBe(true); + return getFirstText(result); + } finally { + querySpy.mockRestore(); + } + } }); }); }); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 3dcdcb0d52..e04de47410 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -32,6 +32,7 @@ import { convertToInternalTileConfig, isConfigTile, } from '@/routers/external-api/v2/utils/dashboards'; +import { isQueryTimeoutError } from '@/tasks/checkAlerts/errors'; import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; @@ -869,6 +870,20 @@ function findCause( return undefined; } +const SOCKET_TIMEOUT_CODES: ReadonlySet = new Set(['ETIMEDOUT']); + +/** + * True when a query ran out of time: ClickHouse's max_execution_time or the + * client request timeout. A socket ETIMEDOUT is excluded because ClickHouse + * was unreachable, so narrowing the query won't help. + */ +export function isQueryOutOfTime(e: unknown): boolean { + const socketTimeout = findCause(e, (c): c is Error => + hasNodeErrorCode(c, SOCKET_TIMEOUT_CODES), + ); + return !socketTimeout && isQueryTimeoutError(e); +} + /** @internal Exported for testing only. */ export function errorHint(msg: string, error?: unknown): string | null { const unknownVariableError = findCause( @@ -911,7 +926,10 @@ export function errorHint(msg: string, error?: unknown): string | null { // Match only real timeouts. A bare `max_execution_time` substring would also // hijack SETTING_CONSTRAINT_VIOLATION / readonly errors ("Setting // max_execution_time shouldn't be greater than…"), which need a different fix. - if (/TIMEOUT_EXCEEDED|Timeout exceeded/i.test(msg)) { + if ( + /TIMEOUT_EXCEEDED|Timeout exceeded/i.test(msg) || + isQueryOutOfTime(error) + ) { return ( 'The query exceeded its execution-time limit. Narrow the time range so ' + 'ClickHouse can prune partitions, add filters to reduce the rows scanned, ' + diff --git a/packages/api/src/mcp/tools/trace/breakdown.ts b/packages/api/src/mcp/tools/trace/breakdown.ts index a0c6fc9e57..9a7263097d 100644 --- a/packages/api/src/mcp/tools/trace/breakdown.ts +++ b/packages/api/src/mcp/tools/trace/breakdown.ts @@ -15,89 +15,44 @@ * endpoint" without dropping into raw SQL with self-JOINs. The builder * tools (table / timeseries / search) can't express the TraceId subselect. */ +import type { ChSql } from '@hyperdx/common-utils/dist/clickhouse'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; -import { z } from 'zod'; -import { ClickhouseClient } from '@/clickhouse'; -import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; import { clickHouseErrorResult, + isQueryOutOfTime, + isServerError, parseTimeRange, } from '@/mcp/tools/query/helpers'; +import { + TRACE_BREAKDOWN_DESCRIPTION, + type TraceBreakdownInput, + traceBreakdownSchema, +} from '@/mcp/tools/trace/breakdownSchema'; +import { + durationDivisor, + getMcpClickhouseClient, + inclusiveTimeFilter, + mcpQuerySettings, +} from '@/mcp/tools/trace/shared'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; -// ─── Schema ────────────────────────────────────────────────────────────────── - -const traceBreakdownSchema = z.object({ - sourceId: z - .string() - .describe( - 'Trace source ID. Must be a source of kind="trace". ' + - 'Call clickstack_list_sources to find available sources.', - ), - parentFilter: z - .string() - .max(4096) - .describe( - 'SQL WHERE clause that selects the PARENT spans you want to break ' + - 'down. The tool finds the distinct TraceIds matching this filter, ' + - 'then aggregates all spans within those traces. ' + - 'Use SQL syntax (not Lucene). Example: ' + - "\"ServiceName = '' AND SpanName = ''\".\n\n" + - 'SCOPE TO A SPECIFIC OPERATION, not just a service. A service-only ' + - 'filter mixes every endpoint together and returns whatever has the ' + - "loudest child-time across all parents, which usually isn't what " + - 'you want. Always include a SpanName (or another operation-level ' + - 'discriminator). To compare two slow operations, call this tool ' + - 'once per operation and diff the results — two operations with ' + - "similar p99 don't necessarily share a slow child.", - ), - startTime: z - .string() - .describe('Start of the parent-filter time window as ISO 8601. REQUIRED.'), - endTime: z - .string() - .describe('End of the parent-filter time window as ISO 8601. REQUIRED.'), - minParentDurationMs: z - .number() - .min(0) - .optional() - .describe( - 'When set, only break down parent spans whose Duration ≥ this value. ' + - 'Use to focus the breakdown on slow parents (e.g. 1000 to only look ' + - 'at parents that took ≥ 1 second).', - ), - topN: z - .number() - .min(1) - .max(100) - .optional() - .default(20) - .describe( - 'Number of top operations to return, ranked by total_time_ms DESC. Default 20.', - ), - maxParentTraces: z - .number() - .min(100) - .max(1_000_000) - .optional() - .default(100_000) - .describe( - 'Safety cap on the number of distinct parent TraceIds considered. ' + - 'A breakdown over more than this many parent traces gets ' + - 'truncated to the first N. Default 100000.', - ), -}); +// The child stage looks spans up by TraceId, which is not in the sort key, so +// it reads every span in the window regardless of how few parents matched. +// Cost tracks the window length, so that is the first thing to shrink. +const TIMEOUT_SUFFIX = + 'This tool reads every span in the time window to find the children of ' + + 'matching traces, so cost grows with the window length. Retry with a ' + + 'shorter startTime/endTime window (e.g. 15-60 minutes around the ' + + 'incident), scope parentFilter to one ServiceName AND SpanName, set ' + + 'minParentDurationMs to break down only slow parents, or lower ' + + 'maxParentTraces.'; -type TraceBreakdownInput = z.infer; - -// ─── Tool ──────────────────────────────────────────────────────────────────── - -function durationDivisor(precision: number): number { - return Math.pow(10, Math.max(0, precision - 3)); -} +const INVALID_FILTER_SUFFIX = + "The parentFilter must be valid ClickHouse SQL referencing columns on the trace table (e.g. ServiceName = 'X' AND SpanName = 'Y')."; export function registerTraceBreakdown({ context, @@ -110,51 +65,7 @@ export function registerTraceBreakdown({ { title: 'Top Time-Consuming Operations Across Matching Traces', annotations: { readOnlyHint: true }, - description: - 'Given a parent-span filter and a time window, return the child ' + - 'operations contributing the most cumulative time across all traces ' + - 'matching the parent filter. Same algorithm as the in-app ' + - '"Top Most Time Consuming Operations" chart on the service dashboard.\n\n' + - 'WHAT IT DOES (two-stage, runs as one SQL):\n' + - ' 1. Pick distinct TraceIds where the parent span matches ' + - '`parentFilter` in the window. Optionally restrict to ' + - '`minParentDurationMs` to focus on slow parents.\n' + - ' 2. Aggregate ALL spans across those traces (excluding the ' + - 'matching root span itself) by (ServiceName, SpanName), ranked by ' + - '`total_time_ms` DESC.\n\n' + - 'USE WHEN: investigating "where is the time going" for a slow ' + - 'operation. Filter to a specific (ServiceName, SpanName) pair and ' + - 'set `minParentDurationMs` to the threshold above which a parent ' + - 'span counts as "slow" for your investigation.\n\n' + - 'MULTIPLE OPERATIONS SLOW: when more than one operation shows ' + - 'elevated latency, call this tool ONCE PER (service, operation) ' + - 'and compare the top child rows across the result sets. Operations ' + - 'with the same top child likely share a cause; operations with ' + - 'different top children are independent regressions that happen ' + - 'to co-occur. DO NOT merge multiple operations into a single ' + - 'parentFilter — the cumulative rank then conflates independent ' + - 'investigations into one noisy answer.\n\n' + - 'RANKING METRIC: `total_time_ms = sum(Duration)` across all matching ' + - 'child spans. This captures the true contribution to elapsed time — ' + - 'a fast-but-frequent child can dominate the latency even if its p99 ' + - 'is unremarkable.\n\n' + - 'RETURNS: array of rows, each with `service`, `operation`, ' + - '`total_time_ms`, `calls`, `in_parents` (how many parent traces ' + - 'contained at least one such span), `p50_ms`, `p99_ms`. Plus a ' + - '`summary` block with the matched-parent count.\n\n' + - 'NEXT STEP after this tool: once a dominant slow child operation is ' + - 'identified, the canonical follow-up is clickstack_event_deltas with ' + - 'slow-vs-fast spans of THAT child operation as target/baseline ' + - "(target = {where: SpanName='' AND Duration > X}, " + - "baseline = {where: SpanName='' AND Duration <= Y}). " + - 'The ranked attributes surface what distinguishes slow invocations ' + - 'of the child operation from fast ones.\n\n' + - 'CROSS-SERVICE BREAKDOWN: this tool does NOT scope children to the ' + - "parent's service. Slow cross-service calls (database, cache, " + - 'upstream HTTP) surface naturally — useful for triage.\n\n' + - 'PAIR TOOL: clickstack_trace_waterfall returns ONE concrete trace as a ' + - "parent/child tree. Use it for an example after this tool's " + - 'aggregate breakdown has pointed you at the slow downstream operation.', + description: TRACE_BREAKDOWN_DESCRIPTION, inputSchema: traceBreakdownSchema, }, async (rawInput: TraceBreakdownInput) => { @@ -178,21 +89,47 @@ export function registerTraceBreakdown({ ); } - const connection = await getConnectionById( + const clickhouseClient = await getMcpClickhouseClient( teamId.toString(), source.connection.toString(), - true, ); - if (!connection) { + if (!clickhouseClient) { return mcpUserError( `Connection not found for source: ${input.sourceId}`, ); } + const timeFilterParams = { + connectionId: source.connection.toString(), + databaseName: source.from.databaseName, + tableName: source.from.tableName, + timestampValueExpression: source.timestampValueExpression, + metadata: getMetadata(clickhouseClient), + }; + const renderTimeFilter = (start: number, end: number) => + inclusiveTimeFilter(timeFilterParams, new Date(start), new Date(end)); + let parentTimeFilter: ChSql; + let childTimeFilter: ChSql; + try { + [parentTimeFilter, childTimeFilter] = await Promise.all([ + renderTimeFilter(startDate.getTime(), endDate.getTime()), + // Widen the child window by 60s on each side to catch children + // that started slightly before / ended slightly after the parent + // sampling window. + renderTimeFilter( + startDate.getTime() - 60_000, + endDate.getTime() + 60_000, + ), + ]); + } catch (e) { + // Only the DESCRIBE lookups run here; they don't depend on the window or + // filters, so neither suffix applies. + return clickHouseErrorResult(e, 'Failed to compute breakdown'); + } + // Source-configured SQL expressions. These are trusted (set by the // team admin in source config) and we substitute them into the // generated SQL. - const tsExpr = source.timestampValueExpression; const traceIdExpr = source.traceIdExpression; const spanNameExpr = source.spanNameExpression ?? "''"; const serviceNameExpr = source.serviceNameExpression ?? "''"; @@ -215,8 +152,7 @@ export function registerTraceBreakdown({ WITH parent_traces AS ( SELECT DISTINCT ${traceIdExpr} AS _trace_id FROM \`${dbName}\`.\`${tableName}\` - WHERE ${tsExpr} >= fromUnixTimestamp64Milli({startMs:Int64}) - AND ${tsExpr} <= fromUnixTimestamp64Milli({endMs:Int64}) + WHERE ${parentTimeFilter.sql} AND (${input.parentFilter}) ${minDurationClause} LIMIT {maxParentTraces:UInt32} @@ -231,8 +167,7 @@ SELECT quantile(0.99)(${durationExpr}) / {divisor:Float64} AS p99_ms FROM \`${dbName}\`.\`${tableName}\` WHERE ${traceIdExpr} IN (SELECT _trace_id FROM parent_traces) - AND ${tsExpr} >= fromUnixTimestamp64Milli({wideStartMs:Int64}) - AND ${tsExpr} <= fromUnixTimestamp64Milli({wideEndMs:Int64}) + AND ${childTimeFilter.sql} AND NOT (${input.parentFilter}) GROUP BY service, operation ORDER BY total_time_ms DESC @@ -240,13 +175,8 @@ LIMIT {topN:UInt32} `; const params: Record = { - startMs: startDate.getTime(), - endMs: endDate.getTime(), - // Widen the child window by 60s on each side to catch children - // that started slightly before / ended slightly after the parent - // sampling window. - wideStartMs: startDate.getTime() - 60_000, - wideEndMs: endDate.getTime() + 60_000, + ...parentTimeFilter.params, + ...childTimeFilter.params, maxParentTraces: input.maxParentTraces, topN: input.topN, divisor, @@ -256,12 +186,6 @@ LIMIT {topN:UInt32} params.minParentDurationStored = input.minParentDurationMs * divisor; } - const clickhouseClient = new ClickhouseClient({ - host: connection.host, - username: connection.username, - password: connection.password, - }); - type Row = { service: string; operation: string; @@ -279,15 +203,7 @@ LIMIT {topN:UInt32} query_params: params, format: 'JSON', connectionId: source.connection.toString(), - clickhouse_settings: { - // Prevent DDL/DML injection via parentFilter — only SELECTs allowed. - readonly: '1', - ...(source.querySettings - ? Object.fromEntries( - source.querySettings.map(s => [s.setting, s.value]), - ) - : {}), - }, + clickhouse_settings: mcpQuerySettings(source.querySettings), }); const json = (await ( result as { json: () => Promise<{ data: Row[] }> } @@ -296,8 +212,11 @@ LIMIT {topN:UInt32} } catch (e) { return clickHouseErrorResult(e, { prefix: 'Failed to compute breakdown', - suffix: - "The parentFilter must be valid ClickHouse SQL referencing columns on the trace table (e.g. ServiceName = 'X' AND SpanName = 'Y').", + suffix: isQueryOutOfTime(e) + ? TIMEOUT_SUFFIX + : isServerError(e) + ? undefined + : INVALID_FILTER_SUFFIX, }); } diff --git a/packages/api/src/mcp/tools/trace/breakdownSchema.ts b/packages/api/src/mcp/tools/trace/breakdownSchema.ts new file mode 100644 index 0000000000..097608422c --- /dev/null +++ b/packages/api/src/mcp/tools/trace/breakdownSchema.ts @@ -0,0 +1,110 @@ +import { z } from 'zod'; + +export const traceBreakdownSchema = z.object({ + sourceId: z + .string() + .describe( + 'Trace source ID. Must be a source of kind="trace". ' + + 'Call clickstack_list_sources to find available sources.', + ), + parentFilter: z + .string() + .max(4096) + .describe( + 'SQL WHERE clause that selects the PARENT spans you want to break ' + + 'down. The tool finds the distinct TraceIds matching this filter, ' + + 'then aggregates all spans within those traces. ' + + 'Use SQL syntax (not Lucene). Example: ' + + "\"ServiceName = '' AND SpanName = ''\".\n\n" + + 'SCOPE TO A SPECIFIC OPERATION, not just a service. A service-only ' + + 'filter mixes every endpoint together and returns whatever has the ' + + "loudest child-time across all parents, which usually isn't what " + + 'you want. Always include a SpanName (or another operation-level ' + + 'discriminator). To compare two slow operations, call this tool ' + + 'once per operation and diff the results — two operations with ' + + "similar p99 don't necessarily share a slow child.", + ), + startTime: z + .string() + .describe('Start of the parent-filter time window as ISO 8601. REQUIRED.'), + endTime: z + .string() + .describe('End of the parent-filter time window as ISO 8601. REQUIRED.'), + minParentDurationMs: z + .number() + .min(0) + .optional() + .describe( + 'When set, only break down parent spans whose Duration ≥ this value. ' + + 'Use to focus the breakdown on slow parents (e.g. 1000 to only look ' + + 'at parents that took ≥ 1 second).', + ), + topN: z + .number() + .min(1) + .max(100) + .optional() + .default(20) + .describe( + 'Number of top operations to return, ranked by total_time_ms DESC. Default 20.', + ), + maxParentTraces: z + .number() + .min(100) + .max(1_000_000) + .optional() + .default(100_000) + .describe( + 'Safety cap on the number of distinct parent TraceIds considered. ' + + 'A breakdown over more than this many parent traces gets ' + + 'truncated to the first N. Default 100000.', + ), +}); + +export type TraceBreakdownInput = z.infer; + +export const TRACE_BREAKDOWN_DESCRIPTION = + 'Given a parent-span filter and a time window, return the child ' + + 'operations contributing the most cumulative time across all traces ' + + 'matching the parent filter. Same algorithm as the in-app ' + + '"Top Most Time Consuming Operations" chart on the service dashboard.\n\n' + + 'WHAT IT DOES (two-stage, runs as one SQL):\n' + + ' 1. Pick distinct TraceIds where the parent span matches ' + + '`parentFilter` in the window. Optionally restrict to ' + + '`minParentDurationMs` to focus on slow parents.\n' + + ' 2. Aggregate ALL spans across those traces (excluding the ' + + 'matching root span itself) by (ServiceName, SpanName), ranked by ' + + '`total_time_ms` DESC.\n\n' + + 'USE WHEN: investigating "where is the time going" for a slow ' + + 'operation. Filter to a specific (ServiceName, SpanName) pair and ' + + 'set `minParentDurationMs` to the threshold above which a parent ' + + 'span counts as "slow" for your investigation.\n\n' + + 'MULTIPLE OPERATIONS SLOW: when more than one operation shows ' + + 'elevated latency, call this tool ONCE PER (service, operation) ' + + 'and compare the top child rows across the result sets. Operations ' + + 'with the same top child likely share a cause; operations with ' + + 'different top children are independent regressions that happen ' + + 'to co-occur. DO NOT merge multiple operations into a single ' + + 'parentFilter — the cumulative rank then conflates independent ' + + 'investigations into one noisy answer.\n\n' + + 'RANKING METRIC: `total_time_ms = sum(Duration)` across all matching ' + + 'child spans. This captures the true contribution to elapsed time — ' + + 'a fast-but-frequent child can dominate the latency even if its p99 ' + + 'is unremarkable.\n\n' + + 'RETURNS: array of rows, each with `service`, `operation`, ' + + '`total_time_ms`, `calls`, `in_parents` (how many parent traces ' + + 'contained at least one such span), `p50_ms`, `p99_ms`. Plus a ' + + '`summary` block with the matched-parent count.\n\n' + + 'NEXT STEP after this tool: once a dominant slow child operation is ' + + 'identified, the canonical follow-up is clickstack_event_deltas with ' + + 'slow-vs-fast spans of THAT child operation as target/baseline ' + + "(target = {where: SpanName='' AND Duration > X}, " + + "baseline = {where: SpanName='' AND Duration <= Y}). " + + 'The ranked attributes surface what distinguishes slow invocations ' + + 'of the child operation from fast ones.\n\n' + + 'CROSS-SERVICE BREAKDOWN: this tool does NOT scope children to the ' + + "parent's service. Slow cross-service calls (database, cache, " + + 'upstream HTTP) surface naturally — useful for triage.\n\n' + + 'PAIR TOOL: clickstack_trace_waterfall returns ONE concrete trace as a ' + + "parent/child tree. Use it for an example after this tool's " + + 'aggregate breakdown has pointed you at the slow downstream operation.'; diff --git a/packages/api/src/mcp/tools/trace/shared.ts b/packages/api/src/mcp/tools/trace/shared.ts new file mode 100644 index 0000000000..499980c7cf --- /dev/null +++ b/packages/api/src/mcp/tools/trace/shared.ts @@ -0,0 +1,68 @@ +import type { ChSql } from '@hyperdx/common-utils/dist/clickhouse'; +import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { timeFilterExpr } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import type { QuerySettings } from '@hyperdx/common-utils/dist/types'; + +import { ClickhouseClient } from '@/clickhouse'; +import { getConnectionById } from '@/controllers/connection'; +import { + MCP_CLICKHOUSE_SETTINGS, + MCP_REQUEST_TIMEOUT, +} from '@/mcp/tools/query/helpers'; + +export function durationDivisor(precision: number): number { + // durationPrecision is the number of decimal digits in the stored value. + // precision=9 → ns (divide by 1e6 for ms), precision=6 → µs (divide by 1e3), + // precision=3 → already ms (divide by 1). + return Math.pow(10, Math.max(0, precision - 3)); +} + +export async function getMcpClickhouseClient( + teamId: string, + connectionId: string, +): Promise { + const connection = await getConnectionById(teamId, connectionId, true); + if (!connection) return null; + return new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + requestTimeout: MCP_REQUEST_TIMEOUT, + }); +} + +// MCP settings win over source.querySettings so a source can't relax the +// max_execution_time ceiling, or the readonly guard that blocks DDL/DML +// injected through raw SQL filters. +export function mcpQuerySettings(querySettings: QuerySettings | undefined) { + return { + ...(querySettings + ? Object.fromEntries(querySettings.map(s => [s.setting, s.value])) + : {}), + ...MCP_CLICKHOUSE_SETTINGS, + }; +} + +export type TimeFilterParams = { + timestampValueExpression: string; + metadata: Metadata; + databaseName: string; + tableName: string; + connectionId: string; +}; + +// Filters on every column of a multi-column timestampValueExpression and +// compares Date columns at day precision, so a composite "EventDate, EventTime" +// source prunes on its Date column without dropping same-day rows. +export function inclusiveTimeFilter( + params: TimeFilterParams, + start: Date, + end: Date, +): Promise { + return timeFilterExpr({ + ...params, + dateRange: [start, end], + dateRangeStartInclusive: true, + dateRangeEndInclusive: true, + }); +} diff --git a/packages/api/src/mcp/tools/trace/waterfall.ts b/packages/api/src/mcp/tools/trace/waterfall.ts index cd79e7a498..f59cb1eb18 100644 --- a/packages/api/src/mcp/tools/trace/waterfall.ts +++ b/packages/api/src/mcp/tools/trace/waterfall.ts @@ -1,9 +1,5 @@ import type { ChSql } from '@hyperdx/common-utils/dist/clickhouse'; -import { - getMetadata, - type Metadata, -} from '@hyperdx/common-utils/dist/core/metadata'; -import { timeFilterExpr } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { pickBucketTimestampColumn } from '@hyperdx/common-utils/dist/core/utils'; import { type BuilderChartConfigWithDateRange, @@ -14,15 +10,20 @@ import { } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; -import { ClickhouseClient } from '@/clickhouse'; -import { getConnectionById } from '@/controllers/connection'; +import type { ClickhouseClient } from '@/clickhouse'; import { getSource } from '@/controllers/sources'; import { clickHouseErrorResult, MCP_CLICKHOUSE_SETTINGS, - MCP_REQUEST_TIMEOUT, parseTimeRange, } from '@/mcp/tools/query/helpers'; +import { + durationDivisor, + getMcpClickhouseClient, + inclusiveTimeFilter, + mcpQuerySettings, + type TimeFilterParams, +} from '@/mcp/tools/trace/shared'; import type { ToolRegistrar } from '@/mcp/tools/types'; import { mcpUserError } from '@/mcp/utils/errors'; import logger from '@/utils/logger'; @@ -190,13 +191,6 @@ function buildPreOrderTree(spans: SpanRow[]): TreeSpan[] { return ordered; } -function durationDivisor(precision: number): number { - // durationPrecision is the number of decimal digits in the stored value. - // precision=9 → ns (divide by 1e6 for ms), precision=6 → µs (divide by 1e3), - // precision=3 → already ms (divide by 1). - return Math.pow(10, Math.max(0, precision - 3)); -} - // min()/max() over an empty set yields the epoch, i.e. 0. function parseProbeTimestamp( value: string | number | undefined, @@ -206,31 +200,6 @@ function parseProbeTimestamp( return isNaN(ms) || ms <= 0 ? null : ms; } -async function getMcpClickhouseClient( - teamId: string, - connectionId: string, -): Promise { - const connection = await getConnectionById(teamId, connectionId, true); - if (!connection) return null; - return new ClickhouseClient({ - host: connection.host, - username: connection.username, - password: connection.password, - requestTimeout: MCP_REQUEST_TIMEOUT, - }); -} - -// MCP settings win over source.querySettings so a source can't relax the -// max_execution_time / readonly ceiling this tool depends on. -function mcpQuerySettings(querySettings: QuerySettings | undefined) { - return { - ...(querySettings - ? Object.fromEntries(querySettings.map(s => [s.setting, s.value])) - : {}), - ...MCP_CLICKHOUSE_SETTINGS, - }; -} - type TimestampExprs = { /** Highest-precision DateTime column, for event-time SELECT/ORDER BY. */ eventTs: string; @@ -239,23 +208,13 @@ type TimestampExprs = { timeFilter: (start: Date, end: Date) => Promise; }; -async function resolveTimestampExprs(params: { - timestampValueExpression: string; - metadata: Metadata; - databaseName: string; - tableName: string; - connectionId: string; -}): Promise { +async function resolveTimestampExprs( + params: TimeFilterParams, +): Promise { const eventTs = await pickBucketTimestampColumn(params); return { eventTs, - timeFilter: (start, end) => - timeFilterExpr({ - ...params, - dateRange: [start, end], - dateRangeStartInclusive: true, - dateRangeEndInclusive: true, - }), + timeFilter: (start, end) => inclusiveTimeFilter(params, start, end), }; }