diff --git a/.changeset/dry-donkeys-add.md b/.changeset/dry-donkeys-add.md new file mode 100644 index 0000000000..d57841aeee --- /dev/null +++ b/.changeset/dry-donkeys-add.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/api": minor +"@hyperdx/common-utils": patch +"@hyperdx/cli": patch +--- + +Add support for event patterns in MCP server, reduce code duplication diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index 51890b0cc4..fbb5cb5967 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -3,13 +3,15 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import * as config from '@/config'; import { + bulkInsertLogs, DEFAULT_DATABASE, + DEFAULT_LOGS_TABLE, DEFAULT_TRACES_TABLE, getLoggedInAgent, getServer, } from '@/fixtures'; import Connection from '@/models/connection'; -import { Source } from '@/models/source'; +import { Source, type SourceDocument } from '@/models/source'; import { McpContext } from '../tools/types'; import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; @@ -18,7 +20,7 @@ describe('MCP Query Tool', () => { const server = getServer(); let team: any; let user: any; - let traceSource: any; + let traceSource: SourceDocument; let connection: any; let client: Client; @@ -244,6 +246,225 @@ describe('MCP Query Tool', () => { }); }); + describe('event_patterns queries', () => { + let logSource: SourceDocument; + + beforeEach(async () => { + logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_LOGS_TABLE, + }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + bodyExpression: 'Body', + severityTextExpression: 'SeverityText', + }); + }); + + it('should execute an event_patterns query on a log source', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + sourceId: logSource._id.toString(), + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveProperty('result'); + expect(output.result).toHaveProperty('patterns'); + expect(output.result).toHaveProperty('totalCount'); + expect(output.result).toHaveProperty('sampledRows'); + expect(output.result).toHaveProperty('sampleMultiplier'); + expect(output.result).toHaveProperty('bodyColumn', 'Body'); + expect(output.result).toHaveProperty('timeRange'); + expect(Array.isArray(output.result.patterns)).toBe(true); + }); + + it('should execute an event_patterns query on a trace source with explicit bodyExpression', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + sourceId: traceSource._id.toString(), + bodyExpression: 'SpanName', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveProperty('result'); + expect(Array.isArray(output.result.patterns)).toBe(true); + }); + + it('should respect sampleSize parameter', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + sourceId: logSource._id.toString(), + sampleSize: 100, + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should respect where filter', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + sourceId: logSource._id.toString(), + where: "SeverityText = 'ERROR'", + whereLanguage: 'sql', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + }); + + it('should accept custom bodyExpression', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + sourceId: logSource._id.toString(), + bodyExpression: 'SeverityText', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + const output = JSON.parse(getFirstText(result)); + expect(output.result).toHaveProperty('bodyColumn', 'SeverityText'); + }); + + it('should require sourceId', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('sourceId is required'); + }); + + it('should return error for non-existent source', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + sourceId: '000000000000000000000000', + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Source not found'); + }); + + it('should include event_patterns in the schema displayType enum', async () => { + const { tools } = await client.listTools(); + const queryTool = tools.find(t => t.name === 'hyperdx_query'); + expect(queryTool).toBeDefined(); + + const schema = queryTool!.inputSchema; + const displayTypeSchema = (schema.properties as any)?.displayType; + expect(displayTypeSchema?.enum).toContain('event_patterns'); + + // Verify new fields are present + const props = Object.keys(schema.properties ?? {}); + expect(props).toContain('sampleSize'); + expect(props).toContain('bodyExpression'); + }); + + describe('with seeded data', () => { + const now = new Date(); + const fiveMinAgo = new Date(now.getTime() - 5 * 60 * 1000); + + beforeEach(async () => { + // Seed several distinct log templates so Drain produces multiple patterns + const logs: Parameters[0] = []; + + // Template A: "User logged in from " — 20 rows + for (let i = 0; i < 20; i++) { + logs.push({ + Body: `User user_${i} logged in from 10.0.0.${i % 256}`, + ServiceName: 'auth-service', + SeverityText: 'INFO', + Timestamp: new Date(fiveMinAgo.getTime() + i * 1000), + }); + } + + // Template B: "Payment processed for order " — 10 rows + for (let i = 0; i < 10; i++) { + logs.push({ + Body: `Payment processed for order ${1000 + i}`, + ServiceName: 'payment-service', + SeverityText: 'INFO', + Timestamp: new Date(fiveMinAgo.getTime() + (20 + i) * 1000), + }); + } + + // Template C: unique single message + logs.push({ + Body: 'System startup complete, all services healthy', + ServiceName: 'system', + SeverityText: 'INFO', + Timestamp: new Date(fiveMinAgo.getTime() + 31000), + }); + + await bulkInsertLogs(logs); + }); + + it('should mine patterns from seeded data and return non-empty results', async () => { + const result = await callTool(client, 'hyperdx_query', { + displayType: 'event_patterns', + sourceId: logSource._id.toString(), + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + const { patterns, totalCount, sampledRows } = output.result; + + // Should have found data + expect(totalCount).toBeGreaterThanOrEqual(31); + expect(sampledRows).toBeGreaterThanOrEqual(31); + + // Should have mined at least one pattern + expect(patterns.length).toBeGreaterThanOrEqual(1); + + // The most common pattern should contain <*> placeholders + const topPattern = patterns[0]; + expect(topPattern.pattern).toContain('<*>'); + + // Patterns should be sorted by estimatedCount descending (monotonic) + for (let i = 1; i < patterns.length; i++) { + expect(patterns[i - 1].estimatedCount).toBeGreaterThanOrEqual( + patterns[i].estimatedCount, + ); + } + + // Each pattern should have trend data with at least some non-zero buckets + for (const p of patterns) { + expect(p.trend.length).toBeGreaterThan(0); + } + + // At least one pattern should have a non-zero trend bucket count + const hasNonZeroTrend = patterns.some((p: any) => + p.trend.some((t: any) => t.count > 0), + ); + expect(hasNonZeroTrend).toBe(true); + }); + }); + }); + describe('error handling', () => { it('should return error for invalid time range', async () => { const result = await callTool(client, 'hyperdx_query', { diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts new file mode 100644 index 0000000000..ab380ce8d4 --- /dev/null +++ b/packages/api/src/mcp/tools/query/eventPatterns.ts @@ -0,0 +1,346 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + getFirstTimestampValueExpression, + splitAndTrimWithBracket, +} from '@hyperdx/common-utils/dist/core/utils'; +import { minePatterns } from '@hyperdx/common-utils/dist/drain'; +import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + +// ─── Source helpers ────────────────────────────────────────────────────────── + +interface SourceBodyFields { + kind: string; + spanNameExpression?: string; + bodyExpression?: string; + implicitColumnExpression?: string; +} + +/** + * Resolve the body column expression for pattern mining from a source. + * Mirrors the web app's getEventBody() logic (packages/app/src/source.ts). + */ +function resolveBodyExpression(source: SourceBodyFields): string | undefined { + let expression: string | undefined; + if (source.kind === SourceKind.Trace) { + expression = source.spanNameExpression; + } else if (source.kind === SourceKind.Log) { + expression = source.bodyExpression ?? source.implicitColumnExpression; + } + if (!expression) return undefined; + const multiExpr = splitAndTrimWithBracket(expression); + return multiExpr.length === 1 ? expression : multiExpr[0]; +} + +/** Reject bodyExpression values containing SQL-unsafe characters. */ +// eslint-disable-next-line no-useless-escape +const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; + +// ─── Event pattern mining ──────────────────────────────────────────────────── + +export async function runEventPatterns( + teamId: string, + sourceId: string, + startDate: Date, + endDate: Date, + options?: { + where?: string; + whereLanguage?: 'lucene' | 'sql'; + bodyExpression?: string; + sampleSize?: number; + }, +) { + const sampleSize = options?.sampleSize ?? 10_000; + + // ── Resolve source & connection ── + const source = await getSource(teamId, sourceId); + if (!source) { + return { + isError: true as const, + content: [ + { type: 'text' as const, text: `Source not found: ${sourceId}` }, + ], + }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `Connection not found for source: ${sourceId}`, + }, + ], + }; + } + + // ── Determine body column ── + // Sanitize caller-supplied bodyExpression: must be a single column reference + // matching the documented format (e.g. "Body", "SpanAttributes['http.url']"). + // The allowlist rejects injection attempts like "Body) OR (1=1". + let bodyColumn: string | undefined; + if (options?.bodyExpression) { + const parts = splitAndTrimWithBracket(options.bodyExpression); + if (parts.length !== 1 || !SAFE_BODY_EXPR_CHARS.test(parts[0])) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + 'bodyExpression must be a single column expression ' + + '(e.g. "Body", "SpanName", "SpanAttributes[\'http.url\']"). ' + + 'Multiple expressions, function calls, or sub-queries are not allowed.', + }, + ], + }; + } + bodyColumn = parts[0]; + } else { + bodyColumn = resolveBodyExpression(source); + } + if (!bodyColumn) { + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: + 'Could not determine body column for pattern mining. ' + + 'This source may not have a body/spanName expression configured. ' + + 'Try specifying bodyExpression explicitly.', + }, + ], + }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + + const tsExpr = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + const implicitColumn = + 'implicitColumnExpression' in source + ? source.implicitColumnExpression + : undefined; + + // ── Query 1: Random sample of events ── + const sampleConfig = { + displayType: DisplayType.Search, + source: source._id.toString(), + select: `${bodyColumn} as __hdx_pattern_body, ${tsExpr} as __hdx_pattern_ts`, + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + orderBy: [{ ordering: 'DESC' as const, valueExpression: 'rand()' }], + limit: { limit: sampleSize, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + // ── Query 2: Total count ── + const countConfig = { + displayType: DisplayType.Table, + source: source._id.toString(), + select: 'count() as total', + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + limit: { limit: 1, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + // Fire both queries in parallel + let sampleResult: Awaited< + ReturnType + >; + let countResult: Awaited< + ReturnType + >; + try { + [sampleResult, countResult] = await Promise.all([ + clickhouseClient.queryChartConfig({ + config: sampleConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + clickhouseClient.queryChartConfig({ + config: countConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + ]); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `ClickHouse query failed: ${message}`, + }, + ], + }; + } + + const sampleRows = sampleResult.data; + const totalCount = Number(countResult.data?.[0]?.total ?? 0); + + if (!sampleRows || sampleRows.length === 0) { + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: { + patterns: [], + totalCount, + sampledRows: 0, + sampleMultiplier: 1, + bodyColumn, + timeRange: { + start: startDate.toISOString(), + end: endDate.toISOString(), + }, + }, + hint: 'No data found in the queried time range. Try setting startTime to a wider window (e.g. 24 hours ago) or check that filters match existing data.', + }, + null, + 2, + ), + }, + ], + }; + } + + // ── Mine patterns using the shared Drain pipeline ── + let rawPatterns: ReturnType['patterns']; + let sampleMultiplier: number; + try { + ({ patterns: rawPatterns, sampleMultiplier } = minePatterns(sampleRows, { + totalCount, + startDate, + endDate, + maxSamples: 5, + getBody: row => { + const raw = row.__hdx_pattern_body; + return raw != null ? String(raw) : ''; + }, + getTimestamp: row => { + const tsRaw = row.__hdx_pattern_ts; + return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; + }, + })); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + isError: true as const, + content: [ + { + type: 'text' as const, + text: `Pattern mining failed: ${message}`, + }, + ], + }; + } + + // ── Format response ── + // Convert trend timestamps to ISO strings, extract sample body texts, + // and build a whereSnippet per pattern so the agent can drill into + // matching events via a follow-up displayType:"search" query. + const patterns = rawPatterns.map(p => { + // Build a Lucene-compatible where clause from the pattern's literal + // (non-<*>) tokens. This lets agents chain: pattern → search. + // Escape \ and " (the phrase-query metachars) in each token. + const literalTokens = p.pattern + .split(/\s+/) + .filter(t => t !== '<*>' && t.length > 0) + .map(t => t.replace(/[\\"]/g, '\\$&')); + const whereSnippet = + literalTokens.length > 0 + ? `${bodyColumn}:"${literalTokens.join(' ')}"` + : ''; + + return { + id: p.id, + pattern: p.pattern, + estimatedCount: p.estimatedCount, + sampleCount: p.sampleCount, + whereSnippet, + trend: p.trend.map(t => ({ + ts: new Date(t.ts).toISOString(), + count: t.count, + })), + samples: p.samples.map(row => { + const raw = row.__hdx_pattern_body; + return raw != null ? String(raw) : ''; + }), + }; + }); + + const output = { + patterns, + totalCount, + sampledRows: sampleRows.length, + sampleMultiplier: Math.round(sampleMultiplier * 100) / 100, + bodyColumn, + timeRange: { + start: startDate.toISOString(), + end: endDate.toISOString(), + }, + }; + + const trimmedOutput = trimToolResponse(output); + const isTrimmed = + JSON.stringify(trimmedOutput).length < JSON.stringify(output).length; + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: trimmedOutput, + ...(isTrimmed + ? { + note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', + } + : {}), + }, + null, + 2, + ), + }, + ], + }; +} diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index efb44fcfda..aea5a2c89d 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -5,6 +5,7 @@ import { externalDashboardTileSchemaWithId } from '@/utils/zod'; import { withToolTracing } from '../../utils/tracing'; import type { ToolDefinition } from '../types'; +import { runEventPatterns } from './eventPatterns'; import { parseTimeRange, runConfigTile } from './helpers'; import { hyperdxQuerySchema, validateQueryInput } from './schemas'; @@ -24,6 +25,12 @@ const queryTools: ToolDefinition = (server, context) => { 'PREFERRED: Use the builder display types (line, stacked_bar, table, number, pie) ' + 'for aggregated metrics, or "search" for browsing individual log/event rows. ' + 'These are safer, easier to construct, and cover most use cases.\n\n' + + 'PATTERN MINING: Use "event_patterns" to discover recurring message templates ' + + 'in logs or traces. Samples random events, clusters them using the Drain algorithm, ' + + 'and returns patterns sorted by frequency with estimated counts and time trends. ' + + 'Useful for identifying the most common log messages, spotting anomalies, and understanding noise. ' + + 'Each pattern includes a "whereSnippet" field — use it as the "where" parameter in a follow-up ' + + '"search" query on the same source to browse the matching raw events.\n\n' + 'ADVANCED: Use displayType "sql" only when you need capabilities the builder cannot express, ' + 'such as JOINs, sub-queries, CTEs, or querying tables not registered as sources. ' + 'Raw SQL requires a connectionId (not sourceId) and a hand-written ClickHouse SQL query.\n\n' + @@ -53,6 +60,23 @@ const queryTools: ToolDefinition = (server, context) => { } const { startDate, endDate } = timeRange; + // Event patterns has its own execution path (sample + mine), + // so it returns early without going through runConfigTile. + if (input.displayType === 'event_patterns') { + return runEventPatterns( + teamId.toString(), + input.sourceId!, + startDate, + endDate, + { + where: input.where, + whereLanguage: input.whereLanguage, + bodyExpression: input.bodyExpression, + sampleSize: input.sampleSize, + }, + ); + } + let tile: ExternalDashboardTileWithId; if (input.displayType === 'sql') { diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index f532528ead..73dad18739 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -91,7 +91,16 @@ function isBuilderDisplayType(dt: string): dt is BuilderDisplayType { export const hyperdxQuerySchema = z.object({ // ── Shared fields (all display types) ── displayType: z - .enum(['line', 'stacked_bar', 'table', 'number', 'pie', 'search', 'sql']) + .enum([ + 'line', + 'stacked_bar', + 'table', + 'number', + 'pie', + 'search', + 'event_patterns', + 'sql', + ]) .describe( 'How to query and visualize the data:\n' + ' line – time-series line chart (builder)\n' + @@ -100,6 +109,8 @@ export const hyperdxQuerySchema = z.object({ ' number – single aggregate scalar (builder)\n' + ' pie – pie chart, one metric grouped (builder)\n' + ' search – browse individual log/event rows\n' + + ' event_patterns – mine recurring patterns from log/event bodies using the Drain algorithm. ' + + 'Returns the most common message templates, estimated counts, and time-bucketed trends.\n' + ' sql – ADVANCED: raw ClickHouse SQL query', ), startTime: z @@ -119,8 +130,8 @@ export const hyperdxQuerySchema = z.object({ .string() .optional() .describe( - 'Source ID — required for builder display types (line, stacked_bar, table, number, pie) ' + - 'and for "search". Call hyperdx_list_sources to find available sources.', + 'Source ID — required for builder display types (line, stacked_bar, table, number, pie), ' + + '"search", and "event_patterns". Call hyperdx_list_sources to find available sources.', ), select: z .array(mcpSelectItemSchema) @@ -164,7 +175,7 @@ export const hyperdxQuerySchema = z.object({ .optional() .default('') .describe( - 'Row filter for "search" display type. ' + + 'Row filter for "search" and "event_patterns" display types. ' + 'Examples: "level:error", "service.name:api AND duration:>500"', ), whereLanguage: z @@ -172,7 +183,7 @@ export const hyperdxQuerySchema = z.object({ .optional() .default('lucene') .describe( - 'Query language for the "where" filter ("search" display type only). Default: lucene', + 'Query language for the "where" filter ("search" and "event_patterns" display types). Default: lucene', ), columns: z .string() @@ -193,6 +204,26 @@ export const hyperdxQuerySchema = z.object({ 'Use smaller values to reduce response size.', ), + // ── Event patterns fields ── + // TODO: explore whether we can safely increase the max beyond 25_000 + sampleSize: z + .number() + .min(1) + .max(25_000) + .optional() + .describe( + 'Number of random rows to sample for pattern mining ("event_patterns" only). ' + + 'Default: 10000. Higher values produce more accurate patterns but take longer.', + ), + bodyExpression: z + .string() + .optional() + .describe( + 'Column expression to mine patterns from ("event_patterns" only). ' + + 'Auto-detected from the source if omitted (Body for logs, SpanName for traces). ' + + 'Example: "Body", "SpanName", "SpanAttributes[\'http.url\']"', + ), + // ── SQL-only fields ── connectionId: z .string() @@ -256,6 +287,10 @@ export function validateQueryInput( if (!data.sourceId) { return 'sourceId is required when displayType is "search"'; } + } else if (displayType === 'event_patterns') { + if (!data.sourceId) { + return 'sourceId is required when displayType is "event_patterns"'; + } } else if (displayType === 'sql') { if (!data.connectionId) { return 'connectionId is required when displayType is "sql"'; @@ -265,5 +300,15 @@ export function validateQueryInput( } } + // Reject event_patterns-only fields on other display types + if (displayType !== 'event_patterns') { + if (data.sampleSize != null) { + return `sampleSize is only valid when displayType is "event_patterns"`; + } + if (data.bodyExpression != null) { + return `bodyExpression is only valid when displayType is "event_patterns"`; + } + } + return null; } diff --git a/packages/cli/src/components/EventViewer/usePatternData.ts b/packages/cli/src/components/EventViewer/usePatternData.ts index 42528ca743..65ec57d98c 100644 --- a/packages/cli/src/components/EventViewer/usePatternData.ts +++ b/packages/cli/src/components/EventViewer/usePatternData.ts @@ -5,12 +5,11 @@ */ import { useState, useEffect, useCallback, useRef } from 'react'; +import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { - TemplateMiner, - TemplateMinerConfig, + minePatterns, + type TrendBucket, } from '@hyperdx/common-utils/dist/drain'; -import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import type { SourceResponse, ProxyClickhouseClient } from '@/api/client'; import { @@ -20,59 +19,16 @@ import { import { getEventBody } from '@/shared/source'; import type { EventRow } from './types'; -import { flatten } from './utils'; // ---- Constants ----------------------------------------------------- const SAMPLES = 10_000; -// ---- Time bucketing utilities -------------------------------------- - -/** Parse a granularity string like "5 minute" into seconds. */ -function granularityToSeconds(granularity: string): number { - const [num, unit] = granularity.split(' '); - const n = parseInt(num, 10); - switch (unit) { - case 'second': - return n; - case 'minute': - return n * 60; - case 'hour': - return n * 3600; - case 'day': - return n * 86400; - default: - return n * 60; - } -} - -/** Round a timestamp down to the start of its granularity bucket. */ -function toStartOfBucket(ts: number, granularityMs: number): number { - return Math.floor(ts / granularityMs) * granularityMs; -} - -/** Generate all bucket start timestamps between start and end. */ -function generateBuckets( - startMs: number, - endMs: number, - granularityMs: number, -): number[] { - const buckets: number[] = []; - let current = toStartOfBucket(startMs, granularityMs); - while (current < endMs) { - buckets.push(current); - current += granularityMs; - } - return buckets; -} - // ---- Types --------------------------------------------------------- -export interface TrendBucket { - ts: number; - count: number; -} +export type { TrendBucket }; +/** CLI-specific pattern group that preserves full EventRow samples and uses `count` for backward compat. */ export interface PatternGroup { id: string; pattern: string; @@ -204,100 +160,33 @@ export function usePatternData({ const tsExpr = source.timestampValueExpression ?? 'TimestampTime'; const tsColumn = resultKeys.find(k => k === tsExpr) ?? resultKeys[0]; - // Compute granularity for trend buckets - const granularity = convertDateRangeToGranularityString( - [startTime, endTime], - 24, - ); - const granularityMs = granularityToSeconds(granularity) * 1000; - const allBuckets = generateBuckets( - startTime.getTime(), - endTime.getTime(), - granularityMs, - ); - - // Mine patterns - const config = new TemplateMinerConfig(); - const miner = new TemplateMiner(config); - - const clustered: Array<{ - clusterId: number; - row: EventRow; - tsMs: number; - }> = []; - for (const row of sampleRows) { - const body = row[effectiveBodyColumn]; - const text = body != null ? flatten(String(body)) : ''; - const result = miner.addLogMessage(text); - const tsRaw = row[tsColumn]; - const tsMs = - tsRaw != null - ? new Date(String(tsRaw)).getTime() - : startTime.getTime(); - clustered.push({ clusterId: result.clusterId, row, tsMs }); - } - - // Group by cluster ID - const groups = new Map< - number, - { - rows: EventRow[]; - template: string; - bucketCounts: Map; - } - >(); - - for (const { clusterId, row, tsMs } of clustered) { - const bucket = toStartOfBucket(tsMs, granularityMs); - const existing = groups.get(clusterId); - if (existing) { - existing.rows.push(row); - existing.bucketCounts.set( - bucket, - (existing.bucketCounts.get(bucket) ?? 0) + 1, - ); - } else { + // Mine patterns using the shared Drain pipeline + const { patterns: rawPatterns } = minePatterns(sampleRows, { + totalCount: total, + startDate: startTime, + endDate: endTime, + trendBuckets: 24, + maxSamples: sampleRows.length, + getBody: row => { const body = row[effectiveBodyColumn]; - const text = body != null ? flatten(String(body)) : ''; - const match = miner.match(text, 'fallback'); - const bucketCounts = new Map(); - bucketCounts.set(bucket, 1); - groups.set(clusterId, { - rows: [row], - template: match?.getTemplate() ?? text, - bucketCounts, - }); - } - } - - // Compute sampleMultiplier - const sampleMultiplier = - total > 0 && sampleRows.length > 0 ? total / sampleRows.length : 1; - - // Convert to sorted array with estimated counts and trend data - const result: PatternGroup[] = []; - for (const [id, { rows, template, bucketCounts }] of groups) { - const trend: TrendBucket[] = allBuckets.map(bucketTs => ({ - ts: bucketTs, - count: Math.round( - (bucketCounts.get(bucketTs) ?? 0) * sampleMultiplier, - ), - })); - - result.push({ - id: String(id), - pattern: template, - count: rows.length, - estimatedCount: Math.max( - Math.round(rows.length * sampleMultiplier), - 1, - ), - samples: rows, - trend, - }); - } + return body != null ? String(body) : ''; + }, + getTimestamp: row => { + const tsRaw = row[tsColumn]; + return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; + }, + }); + + // Convert to CLI pattern shape (preserves full EventRow samples) + const result: PatternGroup[] = rawPatterns.map(p => ({ + id: p.id, + pattern: p.pattern, + count: p.sampleCount, + estimatedCount: p.estimatedCount, + samples: p.samples, + trend: p.trend, + })); - result.sort((a, b) => b.estimatedCount - a.estimatedCount); setPatterns(result); } catch (err: unknown) { setError(err instanceof Error ? err : new Error(String(err))); diff --git a/packages/common-utils/src/__tests__/mine-patterns.test.ts b/packages/common-utils/src/__tests__/mine-patterns.test.ts new file mode 100644 index 0000000000..9079101a2e --- /dev/null +++ b/packages/common-utils/src/__tests__/mine-patterns.test.ts @@ -0,0 +1,392 @@ +import { flattenBody, minePatterns } from '../drain/mine-patterns'; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +type Row = Record; + +const baseDate = new Date('2024-01-01T00:00:00Z'); +const endDate = new Date('2024-01-02T00:00:00Z'); // 24h window + +function makeRow(body: string, tsOffset = 0): Row { + return { + body, + ts: new Date(baseDate.getTime() + tsOffset).toISOString(), + }; +} + +const defaultCallbacks = { + getBody: (row: Row) => String(row.body ?? ''), + getTimestamp: (row: Row) => { + const ts = row.ts; + return ts != null ? new Date(String(ts)).getTime() : null; + }, +}; + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +describe('minePatterns', () => { + describe('empty rows', () => { + it('should return empty patterns and sampleMultiplier 1', () => { + const result = minePatterns([], { + totalCount: 100, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + expect(result.patterns).toEqual([]); + expect(result.sampleMultiplier).toBe(1); + }); + }); + + describe('single row', () => { + it('should return one pattern with the row as a sample', () => { + const rows = [makeRow('hello world')]; + const result = minePatterns(rows, { + totalCount: 1, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + expect(result.patterns).toHaveLength(1); + expect(result.patterns[0].sampleCount).toBe(1); + expect(result.patterns[0].samples).toHaveLength(1); + expect(result.patterns[0].samples[0]).toBe(rows[0]); + expect(result.patterns[0].estimatedCount).toBe(1); + }); + }); + + describe('multi-cluster grouping and sort order', () => { + it('should group rows into clusters and sort by estimatedCount descending', () => { + // Create rows that will form two distinct patterns + const rows = [ + // Pattern A: 3 rows + makeRow('user alice logged in', 0), + makeRow('user bob logged in', 1000), + makeRow('user charlie logged in', 2000), + // Pattern B: 1 row + makeRow('payment processed for order 123', 3000), + ]; + + const result = minePatterns(rows, { + totalCount: 4, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + expect(result.patterns.length).toBeGreaterThanOrEqual(1); + + // Verify descending sort order + for (let i = 1; i < result.patterns.length; i++) { + expect(result.patterns[i - 1].estimatedCount).toBeGreaterThanOrEqual( + result.patterns[i].estimatedCount, + ); + } + + // Total sampleCount across all patterns should equal total rows + const totalSampled = result.patterns.reduce( + (sum, p) => sum + p.sampleCount, + 0, + ); + expect(totalSampled).toBe(4); + }); + + it('should produce <*> placeholders in patterns with variable tokens', () => { + const rows = [ + makeRow('user alice logged in'), + makeRow('user bob logged in'), + makeRow('user charlie logged in'), + ]; + + const result = minePatterns(rows, { + totalCount: 3, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + expect(result.patterns).toHaveLength(1); + expect(result.patterns[0].pattern).toContain('<*>'); + }); + }); + + describe('sampleMultiplier math', () => { + it('should scale estimatedCount when totalCount > rows.length', () => { + // 2 sample rows representing 200 total events → multiplier = 100 + const rows = [ + makeRow('request to /api/users', 0), + makeRow('request to /api/users', 1000), + ]; + + const result = minePatterns(rows, { + totalCount: 200, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + expect(result.sampleMultiplier).toBe(100); + expect(result.patterns[0].estimatedCount).toBe(200); + }); + + it('should use sampleMultiplier 1 when totalCount is 0', () => { + const rows = [makeRow('hello')]; + const result = minePatterns(rows, { + totalCount: 0, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + expect(result.sampleMultiplier).toBe(1); + }); + + it('should use sampleMultiplier 1 when totalCount equals row count', () => { + const rows = [makeRow('hello'), makeRow('world')]; + const result = minePatterns(rows, { + totalCount: 2, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + expect(result.sampleMultiplier).toBe(1); + }); + }); + + describe('maxSamples cap', () => { + it('should cap samples per pattern to maxSamples', () => { + // All rows match the same pattern + const rows = Array.from({ length: 10 }, (_, i) => + makeRow(`request to /api/endpoint`, i * 1000), + ); + + const result = minePatterns(rows, { + totalCount: 10, + startDate: baseDate, + endDate, + maxSamples: 3, + ...defaultCallbacks, + }); + + expect(result.patterns).toHaveLength(1); + expect(result.patterns[0].samples).toHaveLength(3); + expect(result.patterns[0].sampleCount).toBe(10); + }); + + it('should keep all samples when maxSamples is larger than row count', () => { + const rows = Array.from({ length: 5 }, (_, i) => + makeRow(`request to /api/endpoint`, i * 1000), + ); + + const result = minePatterns(rows, { + totalCount: 5, + startDate: baseDate, + endDate, + maxSamples: 100, + ...defaultCallbacks, + }); + + expect(result.patterns[0].samples).toHaveLength(5); + }); + + it('should default maxSamples to 5', () => { + const rows = Array.from({ length: 10 }, (_, i) => + makeRow(`request to /api/endpoint`, i * 1000), + ); + + const result = minePatterns(rows, { + totalCount: 10, + startDate: baseDate, + endDate, + // maxSamples not set — should default to 5 + ...defaultCallbacks, + }); + + expect(result.patterns[0].samples).toHaveLength(5); + }); + }); + + describe('trend bucket alignment', () => { + it('should generate trend buckets covering the time range', () => { + const start = new Date('2024-01-01T00:00:00Z'); + const end = new Date('2024-01-01T01:00:00Z'); // 1 hour + + const rows = [ + { body: 'event', ts: '2024-01-01T00:15:00Z' }, + { body: 'event', ts: '2024-01-01T00:45:00Z' }, + ]; + + const result = minePatterns(rows, { + totalCount: 2, + startDate: start, + endDate: end, + trendBuckets: 4, + ...defaultCallbacks, + }); + + expect(result.patterns).toHaveLength(1); + const trend = result.patterns[0].trend; + + // Should have multiple buckets + expect(trend.length).toBeGreaterThan(0); + + // All bucket timestamps should be within [start, end) + for (const bucket of trend) { + expect(bucket.ts).toBeGreaterThanOrEqual(start.getTime()); + expect(bucket.ts).toBeLessThan(end.getTime()); + } + + // At least one bucket should have a non-zero count + const nonZero = trend.filter(b => b.count > 0); + expect(nonZero.length).toBeGreaterThan(0); + }); + + it('should scale trend bucket counts by sampleMultiplier', () => { + const start = new Date('2024-01-01T00:00:00Z'); + const end = new Date('2024-01-01T01:00:00Z'); + + const rows = [{ body: 'event', ts: '2024-01-01T00:15:00Z' }]; + + const result = minePatterns(rows, { + totalCount: 100, + startDate: start, + endDate: end, + trendBuckets: 4, + ...defaultCallbacks, + }); + + // With 1 sample row and totalCount 100, multiplier is 100 + const nonZeroBuckets = result.patterns[0].trend.filter(b => b.count > 0); + expect(nonZeroBuckets.length).toBeGreaterThan(0); + // The bucket containing the event should have count = 1 * 100 = 100 + expect(nonZeroBuckets[0].count).toBe(100); + }); + }); + + describe('NaN/null timestamp handling', () => { + it('should fall back to startDate for null timestamps', () => { + const rows = [{ body: 'no timestamp', ts: null }]; + const result = minePatterns(rows, { + totalCount: 1, + startDate: baseDate, + endDate, + getBody: (row: Row) => String(row.body ?? ''), + getTimestamp: () => null, + }); + + expect(result.patterns).toHaveLength(1); + // The event should be bucketed — at least one trend bucket should have count > 0 + const nonZero = result.patterns[0].trend.filter(b => b.count > 0); + expect(nonZero.length).toBeGreaterThan(0); + }); + + it('should fall back to startDate for undefined timestamps', () => { + const rows = [{ body: 'no timestamp' }]; + const result = minePatterns(rows, { + totalCount: 1, + startDate: baseDate, + endDate, + getBody: (row: Row) => String(row.body ?? ''), + getTimestamp: () => undefined, + }); + + expect(result.patterns).toHaveLength(1); + const nonZero = result.patterns[0].trend.filter(b => b.count > 0); + expect(nonZero.length).toBeGreaterThan(0); + }); + }); + + describe('estimatedCount floor', () => { + it('should floor estimatedCount at 1 via Math.max', () => { + // With sampleMultiplier < 1 (totalCount < rows.length is unusual but possible) + // Each pattern with sampleCount=1 would get estimatedCount = round(1 * 0.5) = 1 + // But test with very small multiplier scenario: totalCount=1, 10 rows + const rows = Array.from({ length: 10 }, (_, i) => + makeRow(`unique message ${i}`, i * 1000), + ); + + const result = minePatterns(rows, { + totalCount: 1, // multiplier = 0.1 + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + // Every pattern should have estimatedCount >= 1 + for (const p of result.patterns) { + expect(p.estimatedCount).toBeGreaterThanOrEqual(1); + } + }); + }); + + describe('TRow generic typing', () => { + it('should preserve row type in pattern samples', () => { + interface MyRow extends Record { + body: string; + ts: string; + extra: number; + } + + const rows: MyRow[] = [ + { body: 'hello world', ts: '2024-01-01T00:00:00Z', extra: 42 }, + ]; + + const result = minePatterns(rows, { + totalCount: 1, + startDate: baseDate, + endDate, + getBody: (row: MyRow) => row.body, + getTimestamp: (row: MyRow) => new Date(row.ts).getTime(), + }); + + // TypeScript should infer samples as MyRow[] + const sample = result.patterns[0].samples[0]; + expect(sample.extra).toBe(42); + }); + }); + + describe('body normalization (flattenBody)', () => { + it('should collapse newlines into spaces', () => { + expect(flattenBody('line1\nline2\nline3')).toBe('line1 line2 line3'); + }); + + it('should collapse runs of whitespace into single spaces', () => { + expect(flattenBody('hello world')).toBe('hello world'); + }); + + it('should trim leading and trailing whitespace', () => { + expect(flattenBody(' hello ')).toBe('hello'); + }); + + it('should handle combined newlines and whitespace', () => { + expect(flattenBody(' line1\n line2\n\n line3 ')).toBe( + 'line1 line2 line3', + ); + }); + + it('should return empty string for whitespace-only input', () => { + expect(flattenBody(' \n\n ')).toBe(''); + }); + + it('should group multiline bodies into the same pattern', () => { + const rows = [ + makeRow('error occurred\n at line 1\n at line 2'), + makeRow('error occurred\n at line 5\n at line 10'), + ]; + + const result = minePatterns(rows, { + totalCount: 2, + startDate: baseDate, + endDate, + ...defaultCallbacks, + }); + + // Both multiline messages should cluster together + expect(result.patterns).toHaveLength(1); + expect(result.patterns[0].sampleCount).toBe(2); + }); + }); +}); diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index 1a0faf02a9..937aaa82da 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -4,6 +4,13 @@ export { Drain } from './drain'; export { LogCluster } from './log-cluster'; export { LruCache } from './lru-cache'; export { LogMasker, MaskingInstruction } from './masking'; +export type { + MinePatternOptions, + MinePatternResult, + PatternGroup, + TrendBucket, +} from './mine-patterns'; +export { flattenBody, minePatterns } from './mine-patterns'; export { Node } from './node'; export type { AddLogMessageResult, ExtractedParameter } from './template-miner'; export { TemplateMiner } from './template-miner'; diff --git a/packages/common-utils/src/drain/mine-patterns.ts b/packages/common-utils/src/drain/mine-patterns.ts new file mode 100644 index 0000000000..82c4abd9ce --- /dev/null +++ b/packages/common-utils/src/drain/mine-patterns.ts @@ -0,0 +1,194 @@ +import { + convertDateRangeToGranularityString, + timeBucketByGranularity, + toStartOfInterval, +} from '../core/utils'; +import { TemplateMinerConfig } from './config'; +import { TemplateMiner } from './template-miner'; + +// ─── Body normalization ────────────────────────────────────────────────────── + +/** Collapse newlines and runs of whitespace into single spaces. */ +export function flattenBody(s: string): string { + return s + .replace(/\n/g, ' ') + .replace(/\s{2,}/g, ' ') + .trim(); +} + +// ─── Types ─────────────────────────────────────────────────────────────────── + +export interface TrendBucket { + /** Bucket start timestamp in milliseconds. */ + ts: number; + /** Estimated count for this bucket (scaled by sampleMultiplier). */ + count: number; +} + +export interface PatternGroup> { + /** Cluster ID from the Drain algorithm. */ + id: string; + /** Generalized template string with <*> placeholders. */ + pattern: string; + /** Raw count of matching rows within the sample. */ + sampleCount: number; + /** Estimated total count (sampleCount * sampleMultiplier). */ + estimatedCount: number; + /** Sample rows that matched this pattern (capped by maxSamples). */ + samples: TRow[]; + /** Time-bucketed trend data. */ + trend: TrendBucket[]; +} + +export interface MinePatternResult> { + patterns: PatternGroup[]; + sampleMultiplier: number; +} + +export interface MinePatternOptions> { + /** Total event count for the full (unsampled) time range. Used to compute sampleMultiplier. */ + totalCount: number; + /** Start of the query time range. */ + startDate: Date; + /** End of the query time range. */ + endDate: Date; + /** Max number of trend time buckets to generate. Default: 24. */ + trendBuckets?: number; + /** Max number of sample rows to keep per pattern. Default: 5. */ + maxSamples?: number; + /** + * Extract the body text from a row. The returned string is normalized + * (newlines collapsed, whitespace trimmed) before being fed to Drain. + */ + getBody: (row: TRow) => string; + /** + * Extract the timestamp (as epoch ms) from a row. + * Falls back to startDate if the extractor returns null/undefined. + */ + getTimestamp: (row: TRow) => number | null | undefined; +} + +// ─── Core mining function ──────────────────────────────────────────────────── + +/** + * Run the Drain log-template mining algorithm over a set of sampled rows. + * + * This is a pure function: it takes already-fetched rows, mines patterns, + * groups them, computes estimated counts and trend data, and returns sorted + * pattern groups. Both the MCP server and the CLI use this to avoid duplicating + * the Drain + bucketing + grouping logic. + */ +export function minePatterns>( + rows: TRow[], + options: MinePatternOptions, +): MinePatternResult { + const { + totalCount, + startDate, + endDate, + trendBuckets = 24, + maxSamples = 5, + getBody, + getTimestamp, + } = options; + + if (rows.length === 0) { + return { patterns: [], sampleMultiplier: 1 }; + } + + // ── Set up Drain miner ── + const drainConfig = new TemplateMinerConfig(); + const miner = new TemplateMiner(drainConfig); + + // ── Compute time buckets using shared helpers ── + const granularity = convertDateRangeToGranularityString( + [startDate, endDate], + trendBuckets, + ); + const allBucketDates = timeBucketByGranularity( + startDate, + endDate, + granularity, + ); + const allBuckets = allBucketDates.map(d => d.getTime()); + + // ── Process each row through Drain ── + const clustered: Array<{ + clusterId: number; + row: TRow; + bodyText: string; + tsMs: number; + }> = []; + for (const row of rows) { + const bodyText = flattenBody(getBody(row)); + const result = miner.addLogMessage(bodyText); + const tsMs = getTimestamp(row) ?? startDate.getTime(); + clustered.push({ clusterId: result.clusterId, row, bodyText, tsMs }); + } + + // ── Group by cluster ID ── + const groups = new Map< + number, + { + samples: TRow[]; + template: string; + bucketCounts: Map; + } + >(); + + for (const { clusterId, row, bodyText, tsMs } of clustered) { + const bucket = toStartOfInterval(new Date(tsMs), granularity).getTime(); + const existing = groups.get(clusterId); + if (existing) { + if (existing.samples.length < maxSamples) { + existing.samples.push(row); + } + existing.bucketCounts.set( + bucket, + (existing.bucketCounts.get(bucket) ?? 0) + 1, + ); + } else { + const match = miner.match(bodyText, 'fallback'); + const bucketCounts = new Map(); + bucketCounts.set(bucket, 1); + groups.set(clusterId, { + samples: [row], + template: match?.getTemplate() ?? bodyText, + bucketCounts, + }); + } + } + + // ── Build result with estimated counts ── + const sampleMultiplier = + totalCount > 0 && rows.length > 0 ? totalCount / rows.length : 1; + + // Pre-compute per-cluster counts + const clusterCounts = new Map(); + for (const { clusterId } of clustered) { + clusterCounts.set(clusterId, (clusterCounts.get(clusterId) ?? 0) + 1); + } + + const patterns: PatternGroup[] = []; + for (const [id, { samples, template, bucketCounts }] of groups) { + const sampleCount = clusterCounts.get(id) ?? 0; + + const trend: TrendBucket[] = allBuckets.map(bucketTs => ({ + ts: bucketTs, + count: Math.round((bucketCounts.get(bucketTs) ?? 0) * sampleMultiplier), + })); + + patterns.push({ + id: String(id), + pattern: template, + sampleCount, + estimatedCount: Math.max(Math.round(sampleCount * sampleMultiplier), 1), + samples, + trend, + }); + } + + patterns.sort((a, b) => b.estimatedCount - a.estimatedCount); + + return { patterns, sampleMultiplier }; +}