feat(mcp): add clickstack_emerging_signals tool (+7% on service-health-check)

.changeset/emerging-signals-mcp-tool.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": minor
+---
+
+feat(mcp): add `clickstack_emerging_signals` MCP tool — a two-window Drain pattern novelty detector that set-differences mined log/event patterns between an earlier baseline window and a current window to surface what is newly emerging or has disappeared. Shares the Drain sample-and-mine pipeline with `clickstack_event_patterns` via an extracted `mineWindowPatterns` helper, and keys patterns across windows with a `normalizeTemplate` helper.

MCP.md

@@ -131,6 +131,7 @@ with:
 | `clickstack_search`              | Browse individual log, event, or trace rows                                                  |
 | `clickstack_event_patterns`      | Discover the most common log messages and event patterns using Drain clustering               |
 | `clickstack_event_deltas`        | Compare two row groups and rank properties by how their value distributions differ            |
+| `clickstack_emerging_signals`    | Detect NEW or GONE log/event patterns between a baseline and a current window (two-window Drain novelty diff) |
 | `clickstack_sql`                 | Execute raw ClickHouse SQL for advanced queries (JOINs, CTEs, sub-queries)                   |
 | `clickstack_get_dashboard`       | List all dashboards or get full detail for a specific dashboard                              |
 | `clickstack_save_dashboard`      | Create or update a dashboard with tiles (charts, tables, numbers, search, markdown)          |

packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts

@@ -0,0 +1,247 @@
+import { SourceKind } from '@hyperdx/common-utils/dist/types';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+
+import * as config from '@/config';
+import {
+  bulkInsertLogs,
+  DEFAULT_DATABASE,
+  DEFAULT_LOGS_TABLE,
+  getLoggedInAgent,
+  getServer,
+} from '@/fixtures';
+import { McpContext } from '@/mcp/tools/types';
+import Connection from '@/models/connection';
+import { Source } from '@/models/source';
+
+import { callTool, createTestClient, getFirstText } from './mcpTestUtils';
+
+type EmergingPattern = {
+  pattern: string;
+  sample: string;
+  status: string;
+  currentShare: number;
+  baselineShare: number;
+};
+
+describe('MCP Emerging Signals Tool', () => {
+  const server = getServer();
+  let team: any;
+  let user: any;
+  let logSource: any;
+  let connection: any;
+  let client: Client;
+
+  beforeAll(async () => {
+    await server.start();
+  });
+
+  beforeEach(async () => {
+    const result = await getLoggedInAgent(server);
+    team = result.team;
+    user = result.user;
+
+    connection = await Connection.create({
+      team: team._id,
+      name: 'Default',
+      host: config.CLICKHOUSE_HOST,
+      username: config.CLICKHOUSE_USER,
+      password: config.CLICKHOUSE_PASSWORD,
+    });
+
+    logSource = await Source.create({
+      kind: SourceKind.Log,
+      team: team._id,
+      from: { databaseName: DEFAULT_DATABASE, tableName: DEFAULT_LOGS_TABLE },
+      timestampValueExpression: 'Timestamp',
+      connection: connection._id,
+      name: 'Logs',
+      serviceNameExpression: 'ServiceName',
+      severityTextExpression: 'SeverityText',
+      bodyExpression: 'Body',
+      defaultTableSelectExpression:
+        'Timestamp, ServiceName, SeverityText, Body',
+    });
+
+    const context: McpContext = {
+      teamId: team._id.toString(),
+      userId: user._id.toString(),
+    };
+    client = await createTestClient(context);
+  });
+
+  afterEach(async () => {
+    await client.close();
+    await server.clearDBs();
+  });
+
+  afterAll(async () => {
+    await server.stop();
+  });
+
+  describe('schema serialization', () => {
+    it('exposes clickstack_emerging_signals via tools/list with current + baseline windows', async () => {
+      const { tools } = await client.listTools();
+      const t = tools.find(t => t.name === 'clickstack_emerging_signals');
+      expect(t).toBeDefined();
+      const schema = t!.inputSchema;
+      const props = Object.keys(schema.properties ?? {});
+      expect(props).toContain('sourceId');
+      expect(props).toContain('currentStartTime');
+      expect(props).toContain('currentEndTime');
+      expect(props).toContain('baselineStartTime');
+      expect(props).toContain('baselineEndTime');
+      expect(props).toContain('minShareRatio');
+      expect(props).toContain('topN');
+      expect(schema.required).toContain('sourceId');
+      expect(schema.required).toContain('currentStartTime');
+      expect(schema.required).toContain('baselineStartTime');
+    });
+  });
+
+  describe('validation', () => {
+    it('rejects overlapping baseline and current windows', async () => {
+      const result = await callTool(client, 'clickstack_emerging_signals', {
+        sourceId: logSource._id.toString(),
+        // Baseline [00:00, 00:40) overlaps current [00:30, 01:00).
+        baselineStartTime: '2026-05-10T00:00:00Z',
+        baselineEndTime: '2026-05-10T00:40:00Z',
+        currentStartTime: '2026-05-10T00:30:00Z',
+        currentEndTime: '2026-05-10T01:00:00Z',
+      });
+      expect(result.isError).toBeTruthy();
+      expect(getFirstText(result)).toMatch(/overlap/i);
+    });
+
+    it('rejects a current window where endTime <= startTime', async () => {
+      const result = await callTool(client, 'clickstack_emerging_signals', {
+        sourceId: logSource._id.toString(),
+        baselineStartTime: '2026-05-10T00:00:00Z',
+        baselineEndTime: '2026-05-10T00:30:00Z',
+        currentStartTime: '2026-05-10T01:00:00Z',
+        currentEndTime: '2026-05-10T01:00:00Z',
+      });
+      expect(result.isError).toBeTruthy();
+      expect(getFirstText(result)).toMatch(/current window/i);
+    });
+  });
+
+  describe('algorithm against ClickHouse', () => {
+    // Capture once so insert timestamps and query windows stay consistent.
+    let now: number;
+
+    beforeEach(async () => {
+      now = Date.now();
+      const events: Array<{
+        Body: string;
+        ServiceName: string;
+        SeverityText: string;
+        Timestamp: Date;
+      }> = [];
+
+      // Baseline window (5–10 min ago): a single steady-state log template,
+      // spread evenly across the whole 5-minute region so any sub-window of it
+      // is populated. The "checkout" line is common in BOTH windows, so it must
+      // NOT be reported as emerging.
+      const baselineStart = now - 10 * 60_000;
+      for (let i = 0; i < 300; i++) {
+        events.push({
+          Body: `checkout completed order=${i} amount=${i % 50}`,
+          ServiceName: 'checkout',
+          SeverityText: 'INFO',
+          // Spread across ~5 min (300 × 1s), i.e. now-10min .. now-5min.
+          Timestamp: new Date(baselineStart + i * 1000),
+        });
+      }
+
+      // Current window (last 5 min): the same steady-state template PLUS a
+      // brand-new template that never appeared in baseline — a feature-flag
+      // shadow-eval line. Drain clusters each into its own template, and the
+      // shadow-eval one has no baseline counterpart → status "new".
+      const currentStart = now - 5 * 60_000;
+      for (let i = 0; i < 300; i++) {
+        events.push({
+          Body: `checkout completed order=${i} amount=${i % 50}`,
+          ServiceName: 'checkout',
+          SeverityText: 'INFO',
+          Timestamp: new Date(currentStart + i * 1000),
+        });
+      }
+      for (let i = 0; i < 150; i++) {
+        events.push({
+          Body: `shadow-eval flag=new_pricing variant=${i % 3} latency_ms=${i}`,
+          ServiceName: 'checkout',
+          SeverityText: 'INFO',
+          Timestamp: new Date(currentStart + i * 1000),
+        });
+      }
+
+      await bulkInsertLogs(events);
+    });
+
+    it('surfaces a brand-new log template as an emerging "new" signal', async () => {
+      const result = await callTool(client, 'clickstack_emerging_signals', {
+        sourceId: logSource._id.toString(),
+        currentStartTime: new Date(now - 5 * 60_000).toISOString(),
+        currentEndTime: new Date(now).toISOString(),
+        baselineStartTime: new Date(now - 10 * 60_000).toISOString(),
+        baselineEndTime: new Date(now - 5 * 60_000).toISOString(),
+        sampleSize: 2000,
+        topN: 10,
+      });
+
+      expect(result.isError).toBeFalsy();
+      const output = JSON.parse(getFirstText(result));
+
+      // Both windows sampled real data.
+      expect(output.summary.currentWindow.sampled).toBeGreaterThan(0);
+      expect(output.summary.baselineWindow.sampled).toBeGreaterThan(0);
+      expect(output.summary.warning).toBeUndefined();
+
+      // The shadow-eval template must appear in the emerging list as "new".
+      expect(output.emerging.length).toBeGreaterThan(0);
+      const emerging: EmergingPattern[] = output.emerging;
+      const shadow = emerging.find(
+        p => /shadow-eval/.test(p.pattern) || /shadow-eval/.test(p.sample),
+      );
+      expect(shadow).toBeDefined();
+      expect(shadow!.status).toBe('new');
+      expect(shadow!.baselineShare).toBe(0);
+      expect(shadow!.currentShare).toBeGreaterThan(0);
+
+      // The steady-state checkout template is present in both windows, so it
+      // must NOT be reported as emerging.
+      const checkoutEmerging = emerging.find(
+        p =>
+          /checkout completed/.test(p.pattern) ||
+          /checkout completed/.test(p.sample),
+      );
+      expect(checkoutEmerging).toBeUndefined();
+    });
+
+    it('reports nothing novel when both windows share the same patterns', async () => {
+      // Two non-overlapping halves of the baseline-only region (now-10min ..
+      // now-5min). Both contain only the steady-state checkout template and
+      // neither contains the shadow-eval template, so nothing should emerge.
+      const result = await callTool(client, 'clickstack_emerging_signals', {
+        sourceId: logSource._id.toString(),
+        currentStartTime: new Date(now - 7.5 * 60_000).toISOString(),
+        currentEndTime: new Date(now - 5 * 60_000).toISOString(),
+        baselineStartTime: new Date(now - 10 * 60_000).toISOString(),
+        baselineEndTime: new Date(now - 7.5 * 60_000).toISOString(),
+        sampleSize: 2000,
+        topN: 10,
+      });
+
+      expect(result.isError).toBeFalsy();
+      const output = JSON.parse(getFirstText(result));
+      expect(output.summary.currentWindow.sampled).toBeGreaterThan(0);
+      expect(output.summary.baselineWindow.sampled).toBeGreaterThan(0);
+      // Neither window contains the shadow-eval template, so it must not show up.
+      const emerging: EmergingPattern[] = output.emerging;
+      const shadow = emerging.find(
+        p => /shadow-eval/.test(p.pattern) || /shadow-eval/.test(p.sample),
+      );
+      expect(shadow).toBeUndefined();
+    });
+  });
+});

packages/api/src/mcp/tools/query/__tests__/emergingSignalsClassify.test.ts

@@ -0,0 +1,134 @@
+import { classifyShift } from '@/mcp/tools/query/emergingSignals';
+
+describe('classifyShift', () => {
+  const RATIO = 3;
+  // Two sampled rows' worth of share for a 10k-row window (the tool's default
+  // brand-new floor: 2 / sampledCount).
+  const FLOOR = 2 / 10_000;
+
+  describe('brand-new patterns (baseShare === 0)', () => {
+    it('emerges when current share clears the floor', () => {
+      expect(
+        classifyShift({ curShare: FLOOR, baseShare: 0 }, RATIO, FLOOR),
+      ).toBe('emerging');
+      expect(
+        classifyShift({ curShare: 0.05, baseShare: 0 }, RATIO, FLOOR),
+      ).toBe('emerging');
+    });
+
+    it('is ignored when current share is a lone row below the floor', () => {
+      expect(
+        classifyShift({ curShare: FLOOR / 2, baseShare: 0 }, RATIO, FLOOR),
+      ).toBeNull();
+    });
+  });
+
+  describe('emerging via ratio', () => {
+    it('reports an EXACTLY 3× shift at a 10k sample (float-boundary case)', () => {
+      // At N=10000, 3 * (1/10000) rounds just above 3/10000, so a naive
+      // cross-product drops this genuine 1→3 shift. The relative tolerance
+      // admits it. This is the exact case Greptile flagged.
+      const baseShare = 1 / 10_000;
+      const curShare = 3 / 10_000;
+      expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBe(
+        'emerging',
+      );
+    });
+
+    it('reports an exact 3× shift at a small sample too', () => {
+      expect(
+        classifyShift({ curShare: 3 / 500, baseShare: 1 / 500 }, RATIO, FLOOR),
+      ).toBe('emerging');
+    });
+
+    it('does NOT report a pattern clearly under ratio×', () => {
+      const baseShare = 1 / 500;
+      const curShare = 2 / 500; // 2× < 3×
+      expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBeNull();
+    });
+
+    it('does NOT over-admit just-below-threshold shifts (tolerance is tiny)', () => {
+      // 2.9× must stay out — the relative tolerance is 1e-9, not a fudge factor.
+      const baseShare = 1 / 10_000;
+      const curShare = 2.9 / 10_000;
+      expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBeNull();
+    });
+
+    it('reports a pattern well above ratio×', () => {
+      expect(
+        classifyShift(
+          { curShare: 10 / 10_000, baseShare: 1 / 10_000 },
+          RATIO,
+          FLOOR,
+        ),
+      ).toBe('emerging');
+    });
+
+    it('does not carry the old epsilon penalty (a clean shift is not suppressed)', () => {
+      // Before the fix, `curShare / (baseShare + EPS)` pushed borderline shifts
+      // just under the threshold. Cross-multiplication has no such bias: a
+      // comfortably-above-3× shift always qualifies regardless of magnitude.
+      expect(
+        classifyShift(
+          { curShare: 3.01 / 10_000, baseShare: 1 / 10_000 },
+          RATIO,
+          FLOOR,
+        ),
+      ).toBe('emerging');
+    });
+  });
+
+  describe('disappeared', () => {
+    it('reports a pattern absent from the current window', () => {
+      expect(
+        classifyShift({ curShare: 0, baseShare: 0.02 }, RATIO, FLOOR),
+      ).toBe('disappeared');
+    });
+
+    it('reports a pattern that dropped at/above ratio× rarer', () => {
+      // 3 baseline rows vs 1 current row → 3× rarer now.
+      const baseShare = 3 / 500;
+      const curShare = 1 / 500;
+      expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBe(
+        'disappeared',
+      );
+    });
+
+    it('reports an EXACTLY 3× drop at a 10k sample (float-boundary case)', () => {
+      const baseShare = 3 / 10_000;
+      const curShare = 1 / 10_000;
+      expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBe(
+        'disappeared',
+      );
+    });
+
+    it('does NOT report a pattern clearly under ratio× rarer', () => {
+      const baseShare = 2 / 500; // 2× < 3×
+      const curShare = 1 / 500;
+      expect(classifyShift({ curShare, baseShare }, RATIO, FLOOR)).toBeNull();
+    });
+  });
+
+  describe('stable patterns', () => {
+    it('returns null for a pattern with similar share in both windows', () => {
+      expect(
+        classifyShift({ curShare: 0.01, baseShare: 0.01 }, RATIO, FLOOR),
+      ).toBeNull();
+    });
+
+    it('does not report an equal-share pattern as emerging at minShareRatio=1', () => {
+      // ratio=1 is schema-allowed. Without the directional guard, the relative
+      // tolerance would make curShare >= 1 * baseShare * (1 - eps) true for a
+      // stable pattern and flood the report with steady-state templates.
+      expect(
+        classifyShift({ curShare: 0.01, baseShare: 0.01 }, 1, FLOOR),
+      ).toBeNull();
+    });
+
+    it('still reports a genuine increase at minShareRatio=1', () => {
+      expect(classifyShift({ curShare: 0.02, baseShare: 0.01 }, 1, FLOOR)).toBe(
+        'emerging',
+      );
+    });
+  });
+});