diff --git a/.changeset/emerging-signals-mcp-tool.md b/.changeset/emerging-signals-mcp-tool.md new file mode 100644 index 0000000000..4e788fc6ef --- /dev/null +++ b/.changeset/emerging-signals-mcp-tool.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": minor +--- + +feat(mcp): add `clickstack_emerging_signals` MCP tool — a two-window Drain pattern novelty detector that set-differences mined log/event patterns between an earlier baseline window and a current window to surface what is newly emerging or has disappeared. Shares the Drain sample-and-mine pipeline with `clickstack_event_patterns` via an extracted `mineWindowPatterns` helper, and keys patterns across windows with a `normalizeTemplate` helper. diff --git a/MCP.md b/MCP.md index 01fd972ffe..cbb8881ee9 100644 --- a/MCP.md +++ b/MCP.md @@ -131,6 +131,7 @@ with: | `clickstack_search` | Browse individual log, event, or trace rows | | `clickstack_event_patterns` | Discover the most common log messages and event patterns using Drain clustering | | `clickstack_event_deltas` | Compare two row groups and rank properties by how their value distributions differ | +| `clickstack_emerging_signals` | Detect NEW or GONE log/event patterns between a baseline and a current window (two-window Drain novelty diff) | | `clickstack_sql` | Execute raw ClickHouse SQL for advanced queries (JOINs, CTEs, sub-queries) | | `clickstack_get_dashboard` | List all dashboards or get full detail for a specific dashboard | | `clickstack_save_dashboard` | Create or update a dashboard with tiles (charts, tables, numbers, search, markdown) | diff --git a/packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts b/packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts new file mode 100644 index 0000000000..59ccb7301f --- /dev/null +++ b/packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts @@ -0,0 +1,247 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; + +import * as config from '@/config'; +import { + bulkInsertLogs, + DEFAULT_DATABASE, + DEFAULT_LOGS_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import { McpContext } from '@/mcp/tools/types'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; + +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +type EmergingPattern = { + pattern: string; + sample: string; + status: string; + currentShare: number; + baselineShare: number; +}; + +describe('MCP Emerging Signals Tool', () => { + const server = getServer(); + let team: any; + let user: any; + let logSource: any; + let connection: any; + let client: Client; + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const result = await getLoggedInAgent(server); + team = result.team; + user = result.user; + + connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Logs', + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + bodyExpression: 'Body', + defaultTableSelectExpression: + 'Timestamp, ServiceName, SeverityText, Body', + }); + + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + describe('schema serialization', () => { + it('exposes clickstack_emerging_signals via tools/list with current + baseline windows', async () => { + const { tools } = await client.listTools(); + const t = tools.find(t => t.name === 'clickstack_emerging_signals'); + expect(t).toBeDefined(); + const schema = t!.inputSchema; + const props = Object.keys(schema.properties ?? {}); + expect(props).toContain('sourceId'); + expect(props).toContain('currentStartTime'); + expect(props).toContain('currentEndTime'); + expect(props).toContain('baselineStartTime'); + expect(props).toContain('baselineEndTime'); + expect(props).toContain('minShareRatio'); + expect(props).toContain('topN'); + expect(schema.required).toContain('sourceId'); + expect(schema.required).toContain('currentStartTime'); + expect(schema.required).toContain('baselineStartTime'); + }); + }); + + describe('validation', () => { + it('rejects overlapping baseline and current windows', async () => { + const result = await callTool(client, 'clickstack_emerging_signals', { + sourceId: logSource._id.toString(), + // Baseline [00:00, 00:40) overlaps current [00:30, 01:00). + baselineStartTime: '2026-05-10T00:00:00Z', + baselineEndTime: '2026-05-10T00:40:00Z', + currentStartTime: '2026-05-10T00:30:00Z', + currentEndTime: '2026-05-10T01:00:00Z', + }); + expect(result.isError).toBeTruthy(); + expect(getFirstText(result)).toMatch(/overlap/i); + }); + + it('rejects a current window where endTime <= startTime', async () => { + const result = await callTool(client, 'clickstack_emerging_signals', { + sourceId: logSource._id.toString(), + baselineStartTime: '2026-05-10T00:00:00Z', + baselineEndTime: '2026-05-10T00:30:00Z', + currentStartTime: '2026-05-10T01:00:00Z', + currentEndTime: '2026-05-10T01:00:00Z', + }); + expect(result.isError).toBeTruthy(); + expect(getFirstText(result)).toMatch(/current window/i); + }); + }); + + describe('algorithm against ClickHouse', () => { + // Capture once so insert timestamps and query windows stay consistent. + let now: number; + + beforeEach(async () => { + now = Date.now(); + const events: Array<{ + Body: string; + ServiceName: string; + SeverityText: string; + Timestamp: Date; + }> = []; + + // Baseline window (5–10 min ago): a single steady-state log template, + // spread evenly across the whole 5-minute region so any sub-window of it + // is populated. The "checkout" line is common in BOTH windows, so it must + // NOT be reported as emerging. + const baselineStart = now - 10 * 60_000; + for (let i = 0; i < 300; i++) { + events.push({ + Body: `checkout completed order=${i} amount=${i % 50}`, + ServiceName: 'checkout', + SeverityText: 'INFO', + // Spread across ~5 min (300 × 1s), i.e. now-10min .. now-5min. + Timestamp: new Date(baselineStart + i * 1000), + }); + } + + // Current window (last 5 min): the same steady-state template PLUS a + // brand-new template that never appeared in baseline — a feature-flag + // shadow-eval line. Drain clusters each into its own template, and the + // shadow-eval one has no baseline counterpart → status "new". + const currentStart = now - 5 * 60_000; + for (let i = 0; i < 300; i++) { + events.push({ + Body: `checkout completed order=${i} amount=${i % 50}`, + ServiceName: 'checkout', + SeverityText: 'INFO', + Timestamp: new Date(currentStart + i * 1000), + }); + } + for (let i = 0; i < 150; i++) { + events.push({ + Body: `shadow-eval flag=new_pricing variant=${i % 3} latency_ms=${i}`, + ServiceName: 'checkout', + SeverityText: 'INFO', + Timestamp: new Date(currentStart + i * 1000), + }); + } + + await bulkInsertLogs(events); + }); + + it('surfaces a brand-new log template as an emerging "new" signal', async () => { + const result = await callTool(client, 'clickstack_emerging_signals', { + sourceId: logSource._id.toString(), + currentStartTime: new Date(now - 5 * 60_000).toISOString(), + currentEndTime: new Date(now).toISOString(), + baselineStartTime: new Date(now - 10 * 60_000).toISOString(), + baselineEndTime: new Date(now - 5 * 60_000).toISOString(), + sampleSize: 2000, + topN: 10, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + + // Both windows sampled real data. + expect(output.summary.currentWindow.sampled).toBeGreaterThan(0); + expect(output.summary.baselineWindow.sampled).toBeGreaterThan(0); + expect(output.summary.warning).toBeUndefined(); + + // The shadow-eval template must appear in the emerging list as "new". + expect(output.emerging.length).toBeGreaterThan(0); + const emerging: EmergingPattern[] = output.emerging; + const shadow = emerging.find( + p => /shadow-eval/.test(p.pattern) || /shadow-eval/.test(p.sample), + ); + expect(shadow).toBeDefined(); + expect(shadow!.status).toBe('new'); + expect(shadow!.baselineShare).toBe(0); + expect(shadow!.currentShare).toBeGreaterThan(0); + + // The steady-state checkout template is present in both windows, so it + // must NOT be reported as emerging. + const checkoutEmerging = emerging.find( + p => + /checkout completed/.test(p.pattern) || + /checkout completed/.test(p.sample), + ); + expect(checkoutEmerging).toBeUndefined(); + }); + + it('reports nothing novel when both windows share the same patterns', async () => { + // Two non-overlapping halves of the baseline-only region (now-10min .. + // now-5min). Both contain only the steady-state checkout template and + // neither contains the shadow-eval template, so nothing should emerge. + const result = await callTool(client, 'clickstack_emerging_signals', { + sourceId: logSource._id.toString(), + currentStartTime: new Date(now - 7.5 * 60_000).toISOString(), + currentEndTime: new Date(now - 5 * 60_000).toISOString(), + baselineStartTime: new Date(now - 10 * 60_000).toISOString(), + baselineEndTime: new Date(now - 7.5 * 60_000).toISOString(), + sampleSize: 2000, + topN: 10, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.summary.currentWindow.sampled).toBeGreaterThan(0); + expect(output.summary.baselineWindow.sampled).toBeGreaterThan(0); + // Neither window contains the shadow-eval template, so it must not show up. + const emerging: EmergingPattern[] = output.emerging; + const shadow = emerging.find( + p => /shadow-eval/.test(p.pattern) || /shadow-eval/.test(p.sample), + ); + expect(shadow).toBeUndefined(); + }); + }); +}); diff --git a/packages/api/src/mcp/tools/query/__tests__/emergingSignalsClassify.test.ts b/packages/api/src/mcp/tools/query/__tests__/emergingSignalsClassify.test.ts new file mode 100644 index 0000000000..26bb84251e --- /dev/null +++ b/packages/api/src/mcp/tools/query/__tests__/emergingSignalsClassify.test.ts @@ -0,0 +1,134 @@ +import { classifyShift } from '@/mcp/tools/query/emergingSignals'; + +describe('classifyShift', () => { + const RATIO = 3; + // Two sampled rows' worth of share for a 10k-row window (the tool's default + // brand-new floor: 2 / sampledCount). + const FLOOR = 2 / 10_000; + + describe('brand-new patterns (baseShare === 0)', () => { + it('emerges when current share clears the floor', () => { + expect( + classifyShift({ curShare: FLOOR, baseShare: 0 }, RATIO, FLOOR), + ).toBe('emerging'); + expect( + classifyShift({ curShare: 0.05, baseShare: 0 }, RATIO, FLOOR), + ).toBe('emerging'); + }); + + it('is ignored when current share is a lone row below the floor', () => { + expect( + classifyShift({ curShare: FLOOR / 2, baseShare: 0 }, RATIO, FLOOR), + ).toBeNull(); + }); + }); + + describe('emerging via ratio', () => { + it('reports an EXACTLY 3× shift at a 10k sample (float-boundary case)', () => { + // At N=10000, 3 * (1/10000) rounds just above 3/10000, so a naive + // cross-product drops this genuine 1→3 shift. The relative tolerance + // admits it. This is the exact case Greptile flagged. + const baseShare = 1 / 10_000; + const curShare = 3 / 10_000; + expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBe( + 'emerging', + ); + }); + + it('reports an exact 3× shift at a small sample too', () => { + expect( + classifyShift({ curShare: 3 / 500, baseShare: 1 / 500 }, RATIO, FLOOR), + ).toBe('emerging'); + }); + + it('does NOT report a pattern clearly under ratio×', () => { + const baseShare = 1 / 500; + const curShare = 2 / 500; // 2× < 3× + expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBeNull(); + }); + + it('does NOT over-admit just-below-threshold shifts (tolerance is tiny)', () => { + // 2.9× must stay out — the relative tolerance is 1e-9, not a fudge factor. + const baseShare = 1 / 10_000; + const curShare = 2.9 / 10_000; + expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBeNull(); + }); + + it('reports a pattern well above ratio×', () => { + expect( + classifyShift( + { curShare: 10 / 10_000, baseShare: 1 / 10_000 }, + RATIO, + FLOOR, + ), + ).toBe('emerging'); + }); + + it('does not carry the old epsilon penalty (a clean shift is not suppressed)', () => { + // Before the fix, `curShare / (baseShare + EPS)` pushed borderline shifts + // just under the threshold. Cross-multiplication has no such bias: a + // comfortably-above-3× shift always qualifies regardless of magnitude. + expect( + classifyShift( + { curShare: 3.01 / 10_000, baseShare: 1 / 10_000 }, + RATIO, + FLOOR, + ), + ).toBe('emerging'); + }); + }); + + describe('disappeared', () => { + it('reports a pattern absent from the current window', () => { + expect( + classifyShift({ curShare: 0, baseShare: 0.02 }, RATIO, FLOOR), + ).toBe('disappeared'); + }); + + it('reports a pattern that dropped at/above ratio× rarer', () => { + // 3 baseline rows vs 1 current row → 3× rarer now. + const baseShare = 3 / 500; + const curShare = 1 / 500; + expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBe( + 'disappeared', + ); + }); + + it('reports an EXACTLY 3× drop at a 10k sample (float-boundary case)', () => { + const baseShare = 3 / 10_000; + const curShare = 1 / 10_000; + expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBe( + 'disappeared', + ); + }); + + it('does NOT report a pattern clearly under ratio× rarer', () => { + const baseShare = 2 / 500; // 2× < 3× + const curShare = 1 / 500; + expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBeNull(); + }); + }); + + describe('stable patterns', () => { + it('returns null for a pattern with similar share in both windows', () => { + expect( + classifyShift({ curShare: 0.01, baseShare: 0.01 }, RATIO, FLOOR), + ).toBeNull(); + }); + + it('does not report an equal-share pattern as emerging at minShareRatio=1', () => { + // ratio=1 is schema-allowed. Without the directional guard, the relative + // tolerance would make curShare >= 1 * baseShare * (1 - eps) true for a + // stable pattern and flood the report with steady-state templates. + expect( + classifyShift({ curShare: 0.01, baseShare: 0.01 }, 1, FLOOR), + ).toBeNull(); + }); + + it('still reports a genuine increase at minShareRatio=1', () => { + expect(classifyShift({ curShare: 0.02, baseShare: 0.01 }, 1, FLOOR)).toBe( + 'emerging', + ); + }); + }); +}); diff --git a/packages/api/src/mcp/tools/query/emergingSignals.ts b/packages/api/src/mcp/tools/query/emergingSignals.ts new file mode 100644 index 0000000000..f80df0db03 --- /dev/null +++ b/packages/api/src/mcp/tools/query/emergingSignals.ts @@ -0,0 +1,363 @@ +import { z } from 'zod'; + +import type { ToolRegistrar } from '@/mcp/tools/types'; +import { mcpUserError } from '@/mcp/utils/errors'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + +import { parseTimeRange } from './helpers'; +import { mineWindowPatterns, normalizeTemplate } from './runEventPatterns'; +import { sourceIdSchema, whereLanguageSchema, whereSchema } from './schemas'; + +// ─── Classification ──────────────────────────────────────────────────────── + +/** + * Decide whether a pattern's share swing between the baseline and current + * windows makes it emerging, disappeared, or neither. Pure so the exact + * threshold behavior is unit-testable. + * + * - emerging: absent from baseline and clears `newPatternShareFloor`, OR + * `curShare / baseShare >= ratio`. + * - disappeared: absent from the current window, OR + * `baseShare / curShare >= ratio`. + * + * Comparisons are inclusive (`>=`) and cross-multiplied rather than divided: + * `curShare >= ratio * baseShare` instead of `curShare / baseShare >= ratio` + * (drops the old divide-by-`(x + EPS)` hack that always nudged the ratio the + * wrong way and could suppress a genuine shift, and needs no divide-by-zero + * guard). + * + * Shares are floating-point, so a shift that is mathematically exactly `ratio`× + * can land a hair above the cross-product for some sample sizes (e.g. at a 10k + * sample, `3 * (1/10000)` rounds just above `3/10000`) and be dropped. A tiny + * RELATIVE tolerance biased toward qualifying admits the exact boundary while + * still rejecting anything meaningfully below it (2.9× stays out). + */ +const RATIO_REL_TOLERANCE = 1e-9; + +export function classifyShift( + shares: { curShare: number; baseShare: number }, + ratio: number, + newPatternShareFloor: number, +): 'emerging' | 'disappeared' | null { + const { curShare, baseShare } = shares; + if (baseShare === 0) { + return curShare >= newPatternShareFloor ? 'emerging' : null; + } + const tol = 1 - RATIO_REL_TOLERANCE; + // Require a real directional change in addition to meeting the ratio. Without + // the `curShare > baseShare` / `baseShare > curShare` guards, a caller-allowed + // `minShareRatio: 1` would classify every stable (equal-share) pattern as + // emerging, flooding the novelty report with steady-state templates. + if (curShare > baseShare && curShare >= ratio * baseShare * tol) { + return 'emerging'; + } + if ( + curShare === 0 || + (baseShare > curShare && baseShare >= ratio * curShare * tol) + ) { + return 'disappeared'; + } + return null; +} + +// ─── Schema ────────────────────────────────────────────────────────────────── + +const emergingSignalsSchema = z.object({ + sourceId: sourceIdSchema, + where: whereSchema, + whereLanguage: whereLanguageSchema, + bodyExpression: z + .string() + .optional() + .describe( + 'Column expression to mine patterns from. Auto-detected from the source ' + + 'if omitted (Body for logs, SpanName for traces). ' + + 'Example: "Body", "SpanName", "SpanAttributes[\'http.url\']".', + ), + currentStartTime: z + .string() + .describe( + 'Start of the CURRENT / report window (ISO 8601). This is the window you ' + + 'want to characterize (e.g. the last hour).', + ), + currentEndTime: z + .string() + .describe( + 'End of the CURRENT / report window (ISO 8601). Required — pass the ' + + 'current timestamp for a window ending now.', + ), + baselineStartTime: z + .string() + .describe( + 'Start of the BASELINE window (ISO 8601) — an earlier "normal" period to ' + + 'compare against. Should NOT overlap the current window.', + ), + baselineEndTime: z + .string() + .describe( + 'End of the BASELINE window (ISO 8601). Typically equals currentStartTime ' + + '(the baseline ends where the current window begins).', + ), + sampleSize: z + .number() + .min(1) + .max(25_000) + .optional() + .default(10_000) + .describe('Rows sampled per window for pattern mining. Default 10000.'), + topN: z + .number() + .min(1) + .max(100) + .optional() + .default(20) + .describe( + 'Max emerging + disappeared patterns to return each. Default 20.', + ), + minShareRatio: z + .number() + .min(1) + .optional() + .default(3) + .describe( + 'How many times more frequent (by share of window) a pattern must be in ' + + 'the current window vs baseline to count as EMERGING (and vice-versa ' + + 'for DISAPPEARED). Default 3× — a pattern must have tripled its share to ' + + 'qualify as a shift, not just noise.', + ), +}); + +// ─── Tool registration ─────────────────────────────────────────────────────── + +export function registerEmergingSignals({ + context, + registerTool, +}: ToolRegistrar) { + const { teamId } = context; + + registerTool( + 'clickstack_emerging_signals', + { + title: 'Emerging & Disappeared Signals', + description: + 'Detect what is NEW or GONE between an earlier baseline window and a ' + + 'current window — log/event patterns that emerged, ramped up, or ' + + 'stopped. This answers "what changed / what is novel?" — NOT "what ' + + 'attribute value differs?".\n\n' + + 'USE THIS for status checks, health reports, post-deploy diffs, and any ' + + '"call out anything new or worth a closer look" question. It mines ' + + 'event patterns (Drain) in BOTH windows and set-differences them:\n' + + ' - emerging: patterns whose share of the window is >= minShareRatio× ' + + 'higher now than in baseline (includes brand-new templates absent from ' + + 'baseline)\n' + + ' - disappeared: patterns that were common in baseline but >= ' + + 'minShareRatio× rarer (or absent) now\n\n' + + 'WHY NOT clickstack_event_deltas: event_deltas compares ATTRIBUTE VALUE ' + + 'DISTRIBUTIONS between two row groups (e.g. "region shifted toward ' + + 'eu-west"). It CANNOT surface a brand-new log template or a new endpoint ' + + 'that simply did not exist before — a novel signal has no baseline ' + + 'distribution to shift. Use emerging_signals for novelty/emergence ' + + '(set membership over time); use event_deltas for "what is different ' + + 'about these rows" (distribution shift within a shared population).\n\n' + + 'Requires sourceId — call clickstack_list_sources / ' + + 'clickstack_describe_source first. Provide two non-overlapping windows: ' + + 'the current window to characterize and an earlier baseline. Typically ' + + 'baselineEndTime == currentStartTime.\n\n' + + 'CALIBRATION: routine variance is not novelty. A pattern that merely ' + + 'wobbled in volume is NOT emerging; only report shifts past ' + + 'minShareRatio. An empty emerging list is a valid, informative answer ' + + '("nothing novel") — do not manufacture findings.', + inputSchema: emergingSignalsSchema, + }, + async input => { + const cur = parseTimeRange(input.currentStartTime, input.currentEndTime); + if ('error' in cur) return mcpUserError(`current window: ${cur.error}`); + const base = parseTimeRange( + input.baselineStartTime, + input.baselineEndTime, + ); + if ('error' in base) + return mcpUserError(`baseline window: ${base.error}`); + + // Reject overlapping windows outright. An overlapping baseline shares + // rows with the current window, so the set-difference is contaminated and + // "emerging"/"disappeared" results are actively misleading rather than + // just noisy. (Inverted / zero-length windows are already caught by + // parseTimeRange above.) + if (base.endDate > cur.startDate && base.startDate < cur.endDate) { + return mcpUserError( + 'baseline and current windows overlap — the baseline must end at or ' + + 'before the current window starts (typically baselineEndTime == ' + + 'currentStartTime). Overlapping windows share rows and produce a ' + + 'misleading novelty diff. Adjust the windows so they do not overlap.', + ); + } + + const opts = { + where: input.where, + whereLanguage: input.whereLanguage, + bodyExpression: input.bodyExpression, + sampleSize: input.sampleSize, + trendBuckets: 0, + }; + + const [curRes, baseRes] = await Promise.all([ + mineWindowPatterns( + teamId.toString(), + input.sourceId, + cur.startDate, + cur.endDate, + opts, + ), + mineWindowPatterns( + teamId.toString(), + input.sourceId, + base.startDate, + base.endDate, + opts, + ), + ]); + if ('error' in curRes) return curRes.error; + if ('error' in baseRes) return baseRes.error; + + // Build share-of-window maps keyed by normalized template. + type Agg = { + pattern: string; + curShare: number; + baseShare: number; + curCount: number; + baseCount: number; + sample: string; + }; + const byKey = new Map(); + + const ingest = (res: typeof curRes, which: 'cur' | 'base') => { + const sampled = res.sampledCount || 1; + for (const p of res.patterns) { + const key = normalizeTemplate(p.pattern); + let a = byKey.get(key); + if (!a) { + a = { + pattern: p.pattern, + curShare: 0, + baseShare: 0, + curCount: 0, + baseCount: 0, + sample: p.samples?.[0] + ? String(p.samples[0].__hdx_pattern_body ?? '') + : p.pattern, + }; + byKey.set(key, a); + } + const share = p.sampleCount / sampled; + // Accumulate rather than overwrite: if two Drain clusters in ONE + // window normalize to the same key, both contribute their share/count + // instead of the second silently clobbering the first. Each window is + // ingested exactly once and all fields init to 0, so additive merge is + // correct for both colliding and non-colliding keys. + if (which === 'cur') { + a.curShare += share; + a.curCount += p.estimatedCount; + } else { + a.baseShare += share; + a.baseCount += p.estimatedCount; + } + } + }; + ingest(curRes, 'cur'); + ingest(baseRes, 'base'); + + const ratio = input.minShareRatio; + // Minimum share for a BRAND-NEW pattern (absent from baseline) to count + // as emerging. A single sampled row that Drain happened to cluster on its + // own is not a signal — require roughly two sampled rows' worth of share + // so one-off log lines don't get reported as novel "new" patterns. Uses + // the current window's sample size; falls back to 0 (no floor) when the + // window sampled nothing so we never divide by zero. + const newPatternShareFloor = + curRes.sampledCount > 0 ? 2 / curRes.sampledCount : 0; + const emerging: Agg[] = []; + const disappeared: Agg[] = []; + for (const a of byKey.values()) { + const verdict = classifyShift(a, ratio, newPatternShareFloor); + if (verdict === 'emerging') emerging.push(a); + else if (verdict === 'disappeared') disappeared.push(a); + } + // Rank by absolute share swing. + emerging.sort( + (x, y) => y.curShare - y.baseShare - (x.curShare - x.baseShare), + ); + disappeared.sort( + (x, y) => y.baseShare - y.curShare - (x.baseShare - x.curShare), + ); + + const fmt = (a: Agg) => ({ + pattern: a.pattern, + currentShare: Math.round(a.curShare * 10000) / 10000, + baselineShare: Math.round(a.baseShare * 10000) / 10000, + currentEstimatedCount: a.curCount, + baselineEstimatedCount: a.baseCount, + status: + a.baseShare === 0 ? 'new' : a.curShare === 0 ? 'gone' : 'shifted', + sample: a.sample.slice(0, 300), + }); + + const output = { + summary: { + currentWindow: { + start: cur.startDate.toISOString(), + end: cur.endDate.toISOString(), + sampled: curRes.sampledCount, + total: curRes.totalCount, + }, + baselineWindow: { + start: base.startDate.toISOString(), + end: base.endDate.toISOString(), + sampled: baseRes.sampledCount, + total: baseRes.totalCount, + }, + minShareRatio: ratio, + emergingCount: emerging.length, + disappearedCount: disappeared.length, + bodyColumn: curRes.bodyColumn, + ...(curRes.sampledCount === 0 || baseRes.sampledCount === 0 + ? { + warning: + 'One or both windows sampled 0 rows — an empty emerging/' + + 'disappeared result here means NO DATA, not "nothing novel". ' + + 'Verify the time windows and where filter via clickstack_search.', + } + : {}), + }, + emerging: emerging.slice(0, input.topN).map(fmt), + disappeared: disappeared.slice(0, input.topN).map(fmt), + usage: + 'emerging = patterns newly present or >= minShareRatio× more frequent ' + + '(by share of window) now vs baseline; status "new" means absent from ' + + 'baseline entirely. disappeared = the reverse. Shares are fractions of ' + + "each window's sampled rows, so they are comparable across windows of " + + 'different volume. An empty emerging list means nothing novel crossed ' + + 'the threshold — a valid finding.', + }; + + const { data, isTrimmed } = trimToolResponse(output); + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + isTrimmed + ? { + ...data, + note: 'Trimmed for context size — lower topN or narrow windows.', + } + : data, + null, + 2, + ), + }, + ], + }; + }, + ); +} diff --git a/packages/api/src/mcp/tools/query/index.ts b/packages/api/src/mcp/tools/query/index.ts index a1ba01d5dc..700a8a24ca 100644 --- a/packages/api/src/mcp/tools/query/index.ts +++ b/packages/api/src/mcp/tools/query/index.ts @@ -1,5 +1,6 @@ import type { ToolDefinition, ToolRegistrar } from '@/mcp/tools/types'; +import { registerEmergingSignals } from './emergingSignals'; import { registerEventDeltas } from './eventDeltas'; import { registerEventPatterns } from './eventPatterns'; import { registerSearch } from './search'; @@ -13,6 +14,7 @@ const queryTools: ToolDefinition = (registrar: ToolRegistrar) => { registerSearch(registrar); registerEventPatterns(registrar); registerEventDeltas(registrar); + registerEmergingSignals(registrar); registerSql(registrar); }; diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts index 3f22ee7736..d416ab5b6d 100644 --- a/packages/api/src/mcp/tools/query/runEventPatterns.ts +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -10,7 +10,11 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; -import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; +import { + type McpErrorResult, + mcpServerError, + mcpUserError, +} from '@/mcp/utils/errors'; import { trimToolResponse } from '@/utils/trimToolResponse'; import { @@ -19,9 +23,16 @@ import { SAFE_BODY_EXPR_CHARS, } from './helpers'; -// ─── Event pattern mining ──────────────────────────────────────────────────── +// ─── Reusable window mining ────────────────────────────────────────────────── -export async function runEventPatterns( +/** + * Sample + Drain-mine one time window and return the raw pattern groups + * (template + counts), without MCP response formatting. Shared by + * runEventPatterns (single window) and emerging_signals (two-window diff). + * + * Returns either an error result (MCP shape) or the mined patterns. + */ +export async function mineWindowPatterns( teamId: string, sourceId: string, startDate: Date, @@ -31,34 +42,42 @@ export async function runEventPatterns( whereLanguage?: 'lucene' | 'sql'; bodyExpression?: string; sampleSize?: number; - topN?: number; trendBuckets?: number; }, -) { +): Promise< + | { error: McpErrorResult } + | { + patterns: ReturnType['patterns']; + sampleMultiplier: number; + totalCount: number; + sampledCount: number; + bodyColumn: string; + } +> { const sampleSize = options?.sampleSize ?? 10_000; - const topN = options?.topN ?? 20; - const trendBuckets = options?.trendBuckets ?? 24; + const trendBuckets = options?.trendBuckets ?? 0; - // ── Resolve source & connection ── const source = await getSource(teamId, sourceId); if (!source) { - return mcpUserError( - `Source not found: ${sourceId}. Call clickstack_list_sources to discover available source IDs.`, - ); + return { + error: mcpUserError( + `Source not found: ${sourceId}. Call clickstack_list_sources to discover available source IDs.`, + ), + }; } - const connection = await getConnectionById( teamId, source.connection.toString(), true, ); if (!connection) { - return mcpUserError( - `Connection not found for source: ${sourceId}. Call clickstack_list_sources to discover available source IDs.`, - ); + return { + error: mcpUserError( + `Connection not found for source: ${sourceId}. Call clickstack_list_sources to discover available source IDs.`, + ), + }; } - // ── Determine body column ── // Sanitize caller-supplied bodyExpression: must be a single column reference // matching the documented format (e.g. "Body", "SpanAttributes['http.url']"). // The allowlist rejects injection attempts like "Body) OR (1=1". @@ -66,22 +85,26 @@ export async function runEventPatterns( if (options?.bodyExpression) { const parts = splitAndTrimWithBracket(options.bodyExpression); if (parts.length !== 1 || !SAFE_BODY_EXPR_CHARS.test(parts[0])) { - return mcpUserError( - 'bodyExpression must be a single column expression ' + - '(e.g. "Body", "SpanName", "SpanAttributes[\'http.url\']"). ' + - 'Multiple expressions, function calls, or sub-queries are not allowed.', - ); + return { + error: mcpUserError( + 'bodyExpression must be a single column expression ' + + '(e.g. "Body", "SpanName", "SpanAttributes[\'http.url\']"). ' + + 'Multiple expressions, function calls, or sub-queries are not allowed.', + ), + }; } bodyColumn = parts[0]; } else { bodyColumn = resolveBodyExpression(source); } if (!bodyColumn) { - return mcpUserError( - 'Could not determine body column for pattern mining. ' + - 'This source may not have a body/spanName expression configured. ' + - 'Try specifying bodyExpression explicitly.', - ); + return { + error: mcpUserError( + 'Could not determine body column for pattern mining. ' + + 'This source may not have a body/spanName expression configured. ' + + 'Try specifying bodyExpression explicitly.', + ), + }; } const clickhouseClient = new ClickhouseClient({ @@ -90,7 +113,6 @@ export async function runEventPatterns( password: connection.password, }); const metadata = getMetadata(clickhouseClient); - const tsExpr = getFirstTimestampValueExpression( source.timestampValueExpression, ); @@ -103,7 +125,6 @@ export async function runEventPatterns( ? source.useTextIndexForImplicitColumn : undefined; - // ── Query 1: Random sample of events ── const sampleConfig = { displayType: DisplayType.Search, source: source._id.toString(), @@ -123,7 +144,6 @@ export async function runEventPatterns( dateRange: [startDate, endDate] as [Date, Date], } satisfies ChartConfigWithDateRange; - // ── Query 2: Total count ── const countConfig = { displayType: DisplayType.Table, source: source._id.toString(), @@ -142,13 +162,12 @@ export async function runEventPatterns( dateRange: [startDate, endDate] as [Date, Date], } satisfies ChartConfigWithDateRange; - // Fire both queries in parallel - let sampleResult: Awaited< - ReturnType - >; - let countResult: Awaited< - ReturnType - >; + // The client is constructed fresh per call (not pooled) and is only used for + // the two queries below — the Drain mining afterward never touches it. Close + // it in a finally so the underlying HTTP agent's sockets are released whether + // the queries succeed or throw, rather than leaking one client per call + // (emerging_signals opens two per invocation). + let sampleResult, countResult; try { [sampleResult, countResult] = await Promise.all([ clickhouseClient.queryChartConfig({ @@ -165,13 +184,125 @@ export async function runEventPatterns( }), ]); } catch (err) { - return clickHouseErrorResult(err); + return { error: clickHouseErrorResult(err) }; + } finally { + await clickhouseClient.close().catch(() => {}); } - const sampleRows = sampleResult.data; + const sampleRows = sampleResult.data ?? []; const totalCount = Number(countResult.data?.[0]?.total ?? 0); + if (sampleRows.length === 0) { + return { + patterns: [], + sampleMultiplier: 1, + totalCount, + sampledCount: 0, + bodyColumn, + }; + } + + // Wrap the synchronous Drain mining in try/catch. Without this a thrown + // Drain error escapes the helper — and since callers (emerging_signals) + // await it inside Promise.all and only inspect the returned {error} union + // AFTER the await, an unwrapped throw becomes an unhandled rejection / 500 + // instead of a clean MCP error result. + let patterns: ReturnType['patterns']; + let sampleMultiplier: number; + try { + ({ patterns, sampleMultiplier } = minePatterns(sampleRows, { + totalCount, + startDate, + endDate, + trendBuckets, + maxSamples: 5, + getBody: row => { + const raw = row.__hdx_pattern_body; + return raw != null ? String(raw) : ''; + }, + getTimestamp: row => { + const tsRaw = row.__hdx_pattern_ts; + return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; + }, + })); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { error: mcpServerError(`Pattern mining failed: ${message}`) }; + } + + return { + patterns, + sampleMultiplier, + totalCount, + sampledCount: sampleRows.length, + bodyColumn, + }; +} - if (!sampleRows || sampleRows.length === 0) { +// ─── Drain template normalization (emerging_signals cross-window keying) ───── + +/** + * Normalize a Drain template so the same logical pattern matches across + * windows despite Drain assigning different cluster ids each run. We collapse + * runs of whitespace and treat the placeholder token uniformly. Distinct + * placeholder PLACEMENTS remain distinct (only the placeholder glyph is + * unified, its position is preserved), so unrelated templates do not collapse. + * + * Pure string logic — used by the emerging_signals MCP tool. + */ +export function normalizeTemplate(pattern: string): string { + return pattern + .replace(/<\*>/g, '\u0001') // stable placeholder marker + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); +} + +// ─── Event pattern mining ──────────────────────────────────────────────────── + +export async function runEventPatterns( + teamId: string, + sourceId: string, + startDate: Date, + endDate: Date, + options?: { + where?: string; + whereLanguage?: 'lucene' | 'sql'; + bodyExpression?: string; + sampleSize?: number; + topN?: number; + trendBuckets?: number; + }, +) { + const topN = options?.topN ?? 20; + // runEventPatterns defaults trendBuckets to 24 (single-window view wants a + // per-pattern trend sparkline); the shared helper defaults to 0. Thread the + // resolved value through so the delegated mining uses the same default. + const trendBuckets = options?.trendBuckets ?? 24; + + // Delegate source/connection/body resolution + sample/count querying + Drain + // mining to the shared helper. runEventPatterns keeps ONLY its richer + // single-window response formatting (topN slicing, whereSnippet, trend ISO + // conversion, sampleMultiplier usage). The helper wraps minePatterns in + // try/catch and returns an {error} union, so a thrown Drain error surfaces as + // a clean MCP error here rather than escaping. + const mined = await mineWindowPatterns(teamId, sourceId, startDate, endDate, { + where: options?.where, + whereLanguage: options?.whereLanguage, + bodyExpression: options?.bodyExpression, + sampleSize: options?.sampleSize, + trendBuckets, + }); + if ('error' in mined) return mined.error; + + const { + patterns: rawPatterns, + sampleMultiplier, + totalCount, + sampledCount, + bodyColumn, + } = mined; + + if (sampledCount === 0) { return { content: [ { @@ -201,35 +332,10 @@ export async function runEventPatterns( }; } - // ── Mine patterns using the shared Drain pipeline ── - let rawPatterns: ReturnType['patterns']; - let sampleMultiplier: number; - try { - ({ patterns: rawPatterns, sampleMultiplier } = minePatterns(sampleRows, { - totalCount, - startDate, - endDate, - trendBuckets, - maxSamples: 5, - getBody: row => { - const raw = row.__hdx_pattern_body; - return raw != null ? String(raw) : ''; - }, - getTimestamp: row => { - const tsRaw = row.__hdx_pattern_ts; - return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; - }, - })); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - return mcpServerError(`Pattern mining failed: ${message}`); - } - // ── Format response ── // Convert trend timestamps to ISO strings, extract sample body texts, // and build a whereSnippet per pattern so the agent can drill into // matching events via a follow-up clickstack_search query. - const sampledCount = sampleRows.length; const slicedPatterns = rawPatterns.slice(0, topN); const patterns = slicedPatterns.map((p, i) => {