From b22475a64192dd3a904f66888c857dc2c2ab4797 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 29 May 2026 08:42:21 -0700 Subject: [PATCH 1/4] feat(mcp): add denoise option to hyperdx_search tool Add a `denoise` boolean parameter to the MCP `hyperdx_search` tool that automatically filters out high-frequency repetitive event patterns from search results, mirroring the web app's "Denoise Results" feature. When `denoise=true`: - Samples 10k random events from the same source/time range - Mines patterns using the shared Drain algorithm (common-utils) - Identifies "noisy" patterns (>10% of sampled events) - Matches each search result row against learned patterns - Filters out rows matching noisy patterns - Returns filtered rows plus metadata listing removed patterns Also extracts `resolveBodyExpression()` and `SAFE_BODY_EXPR_CHARS` from runEventPatterns.ts into helpers.ts for reuse. --- packages/api/src/mcp/tools/query/denoise.ts | 274 ++++++++++++++++++ packages/api/src/mcp/tools/query/helpers.ts | 36 ++- .../src/mcp/tools/query/runEventPatterns.ts | 29 +- packages/api/src/mcp/tools/query/search.ts | 108 ++++++- 4 files changed, 415 insertions(+), 32 deletions(-) create mode 100644 packages/api/src/mcp/tools/query/denoise.ts diff --git a/packages/api/src/mcp/tools/query/denoise.ts b/packages/api/src/mcp/tools/query/denoise.ts new file mode 100644 index 0000000000..234a4b4099 --- /dev/null +++ b/packages/api/src/mcp/tools/query/denoise.ts @@ -0,0 +1,274 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; +import { + flattenBody, + minePatterns, + TemplateMiner, + TemplateMinerConfig, +} from '@hyperdx/common-utils/dist/drain'; +import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { DisplayType } from '@hyperdx/common-utils/dist/types'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; + +import { resolveBodyExpression } from './helpers'; + +// ─── Constants ─────────────────────────────────────────────────────────────── + +/** Patterns matching more than this fraction of sampled events are "noisy". */ +const NOISE_THRESHOLD = 0.1; + +/** Number of random rows to sample for pattern learning. */ +const DENOISE_SAMPLE_SIZE = 10_000; + +// ─── Types ─────────────────────────────────────────────────────────────────── + +export interface DenoiseResult { + /** Filtered rows with noisy patterns removed. */ + rows: Record[]; + /** Patterns that were identified as noisy and removed. */ + removedPatterns: Array<{ + pattern: string; + estimatedCount: number; + sampleCount: number; + }>; +} + +// ─── Core denoising function ───────────────────────────────────────────────── + +/** + * Denoise search results by mining patterns from a random sample, identifying + * "noisy" patterns (those accounting for >10% of the sample), and filtering + * them out of the result rows. + * + * This mirrors the web app's "Denoise Results" feature + * (packages/app/src/components/DBRowTable.tsx) but runs server-side using + * the shared TypeScript Drain implementation. + */ +export async function denoiseSearchResults( + teamId: string, + sourceId: string, + startDate: Date, + endDate: Date, + rows: Record[], + options?: { + where?: string; + whereLanguage?: 'lucene' | 'sql'; + }, +): Promise { + if (rows.length === 0) { + return { rows, removedPatterns: [] }; + } + + // ── Resolve source & connection ── + const source = await getSource(teamId, sourceId); + if (!source) { + // Can't denoise without source info — return rows unmodified + return { rows, removedPatterns: [] }; + } + + const bodyColumn = resolveBodyExpression(source); + if (!bodyColumn) { + // Source doesn't have a body column — can't mine patterns + return { rows, removedPatterns: [] }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { rows, removedPatterns: [] }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + + const tsExpr = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + const implicitColumn = + 'implicitColumnExpression' in source + ? source.implicitColumnExpression + : undefined; + const useTextIndexForImplicitColumn = + 'useTextIndexForImplicitColumn' in source + ? source.useTextIndexForImplicitColumn + : undefined; + + // ── Query: Random sample of events for pattern learning ── + const sampleConfig = { + displayType: DisplayType.Search, + source: source._id.toString(), + select: `${bodyColumn} as __hdx_pattern_body, ${tsExpr} as __hdx_pattern_ts`, + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + useTextIndexForImplicitColumn, + orderBy: [{ ordering: 'DESC' as const, valueExpression: 'rand()' }], + limit: { limit: DENOISE_SAMPLE_SIZE, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + // ── Query: Total count for sample multiplier ── + const countConfig = { + displayType: DisplayType.Table, + source: source._id.toString(), + select: 'count() as total', + from: { + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }, + where: options?.where ?? '', + whereLanguage: options?.whereLanguage ?? ('lucene' as const), + connection: source.connection.toString(), + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: implicitColumn, + useTextIndexForImplicitColumn, + limit: { limit: 1, offset: 0 }, + dateRange: [startDate, endDate] as [Date, Date], + } satisfies ChartConfigWithDateRange; + + let sampleResult: Awaited< + ReturnType + >; + let countResult: Awaited< + ReturnType + >; + try { + [sampleResult, countResult] = await Promise.all([ + clickhouseClient.queryChartConfig({ + config: sampleConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + clickhouseClient.queryChartConfig({ + config: countConfig, + metadata, + querySettings: source.querySettings, + opts: { clickhouse_settings: { max_execution_time: 30 } }, + }), + ]); + } catch { + // If sampling fails, return rows unmodified rather than failing the search + return { rows, removedPatterns: [] }; + } + + const sampleRows = sampleResult.data; + const totalCount = Number(countResult.data?.[0]?.total ?? 0); + + if (!sampleRows || sampleRows.length === 0) { + return { rows, removedPatterns: [] }; + } + + // ── Mine patterns from the sample ── + const { patterns, sampleMultiplier } = minePatterns(sampleRows, { + totalCount, + startDate, + endDate, + maxSamples: 0, // We don't need samples for denoising + getBody: row => { + const raw = row.__hdx_pattern_body; + return raw != null ? String(raw) : ''; + }, + getTimestamp: row => { + const tsRaw = row.__hdx_pattern_ts; + return tsRaw != null ? new Date(String(tsRaw)).getTime() : null; + }, + }); + + if (patterns.length === 0) { + return { rows, removedPatterns: [] }; + } + + // ── Identify noisy patterns (>10% of sampled events) ── + const sampledRowCount = sampleRows.length; + const noisyPatternIds = new Set(); + const removedPatterns: DenoiseResult['removedPatterns'] = []; + + for (const p of patterns) { + if (p.sampleCount / sampledRowCount > NOISE_THRESHOLD) { + noisyPatternIds.add(p.id); + removedPatterns.push({ + pattern: p.pattern, + estimatedCount: p.estimatedCount, + sampleCount: p.sampleCount, + }); + } + } + + if (noisyPatternIds.size === 0) { + return { rows, removedPatterns: [] }; + } + + // ── Re-create the same Drain miner and train it on the same sample ── + // We need a fresh miner to match result rows against the learned patterns. + // The minePatterns() function doesn't expose the miner, so we rebuild it. + const drainConfig = new TemplateMinerConfig(); + const miner = new TemplateMiner(drainConfig); + for (const row of sampleRows) { + const raw = row.__hdx_pattern_body; + const bodyText = flattenBody(raw != null ? String(raw) : ''); + miner.addLogMessage(bodyText); + } + + // ── Match each result row and filter out noisy ones ── + // The body column in result rows may be named differently from the + // pattern mining column. We try the exact bodyColumn name first, + // then fall back to common column names. + const bodyColumnKey = findBodyColumnKey(rows[0], bodyColumn); + if (!bodyColumnKey) { + // Can't find the body column in result rows — return unmodified + return { rows, removedPatterns: [] }; + } + + const filteredRows = rows.filter(row => { + const bodyValue = row[bodyColumnKey]; + if (bodyValue == null) return true; // Keep rows with no body + const bodyText = flattenBody(String(bodyValue)); + const match = miner.match(bodyText, 'fallback'); + if (!match) return true; // No pattern match — keep the row + return !noisyPatternIds.has(String(match.clusterId)); + }); + + return { + rows: filteredRows, + removedPatterns, + }; +} + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +/** + * Find the key in a result row that corresponds to the body column expression. + * ClickHouse may return the column under its expression name or a simplified alias. + */ +function findBodyColumnKey( + row: Record, + bodyColumn: string, +): string | null { + // Direct match (e.g. "Body", "SpanName") + if (bodyColumn in row) return bodyColumn; + + // Case-insensitive match + const lowerBody = bodyColumn.toLowerCase(); + for (const key of Object.keys(row)) { + if (key.toLowerCase() === lowerBody) return key; + } + + return null; +} diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 4ecc9a2b2e..1516457d43 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -1,6 +1,9 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; +import { + getFirstTimestampValueExpression, + splitAndTrimWithBracket, +} from '@hyperdx/common-utils/dist/core/utils'; import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import type { ChartConfigWithDateRange, @@ -24,6 +27,37 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; +// ─── Source body expression helpers ────────────────────────────────────────── + +export interface SourceBodyFields { + kind: string; + spanNameExpression?: string; + bodyExpression?: string; + implicitColumnExpression?: string; +} + +/** + * Resolve the body column expression for pattern mining from a source. + * Mirrors the web app's getEventBody() logic (packages/app/src/source.ts). + */ +export function resolveBodyExpression( + source: SourceBodyFields, +): string | undefined { + let expression: string | undefined; + if (source.kind === SourceKind.Trace) { + expression = source.spanNameExpression; + } else if (source.kind === SourceKind.Log) { + expression = source.bodyExpression ?? source.implicitColumnExpression; + } + if (!expression) return undefined; + const multiExpr = splitAndTrimWithBracket(expression); + return multiExpr.length === 1 ? expression : multiExpr[0]; +} + +/** Reject bodyExpression values containing SQL-unsafe characters. */ +// eslint-disable-next-line no-useless-escape +export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; + // ─── Tile construction ─────────────────────────────────────────────────────── /** diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts index 6a460f51f2..487d6119bb 100644 --- a/packages/api/src/mcp/tools/query/runEventPatterns.ts +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -12,34 +12,7 @@ import { getConnectionById } from '@/controllers/connection'; import { getSource } from '@/controllers/sources'; import { trimToolResponse } from '@/utils/trimToolResponse'; -// ─── Source helpers ────────────────────────────────────────────────────────── - -interface SourceBodyFields { - kind: string; - spanNameExpression?: string; - bodyExpression?: string; - implicitColumnExpression?: string; -} - -/** - * Resolve the body column expression for pattern mining from a source. - * Mirrors the web app's getEventBody() logic (packages/app/src/source.ts). - */ -function resolveBodyExpression(source: SourceBodyFields): string | undefined { - let expression: string | undefined; - if (source.kind === SourceKind.Trace) { - expression = source.spanNameExpression; - } else if (source.kind === SourceKind.Log) { - expression = source.bodyExpression ?? source.implicitColumnExpression; - } - if (!expression) return undefined; - const multiExpr = splitAndTrimWithBracket(expression); - return multiExpr.length === 1 ? expression : multiExpr[0]; -} - -/** Reject bodyExpression values containing SQL-unsafe characters. */ -// eslint-disable-next-line no-useless-escape -const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/; +import { resolveBodyExpression, SAFE_BODY_EXPR_CHARS } from './helpers'; // ─── Event pattern mining ──────────────────────────────────────────────────── diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 272a757a54..3fe43166bb 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -1,8 +1,11 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + import { withToolTracing } from '../../utils/tracing'; import type { McpContext } from '../types'; +import { denoiseSearchResults } from './denoise'; import { buildTile, parseTimeRange, runConfigTile } from './helpers'; import { endTimeSchema, @@ -36,6 +39,16 @@ const searchSchema = z.object({ 'Maximum number of rows to return (1-200). Default: 50. ' + 'Use smaller values to reduce response size.', ), + denoise: z + .boolean() + .optional() + .default(false) + .describe( + 'When true, automatically removes events matching high-frequency patterns ' + + '(those accounting for >10% of sampled events) from the results. ' + + 'Useful for cutting through log noise to find unusual or interesting events. ' + + 'Adds ~1-2s of latency for the pattern sampling queries.', + ), startTime: startTimeSchema, endTime: endTimeSchema, }); @@ -56,6 +69,8 @@ export function registerSearch(server: McpServer, context: McpContext) { 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first.\n\n' + 'For aggregated metrics, use hyperdx_table instead. ' + 'For pattern discovery, use hyperdx_event_patterns instead.\n\n' + + 'Set denoise=true to automatically filter out high-frequency repetitive patterns, ' + + 'surfacing only unusual or interesting events.\n\n' + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + "Map attributes use bracket syntax: SpanAttributes['http.method'].", inputSchema: searchSchema, @@ -78,9 +93,96 @@ export function registerSearch(server: McpServer, context: McpContext) { whereLanguage: input.whereLanguage, }); - return runConfigTile(teamId.toString(), tile, startDate, endDate, { - maxResults: input.maxResults, - }); + const result = await runConfigTile( + teamId.toString(), + tile, + startDate, + endDate, + { + maxResults: input.maxResults, + }, + ); + + // ── Denoising post-processing ── + if (!input.denoise || ('isError' in result && result.isError)) { + return result; + } + + // Extract the raw result data from the formatted response. + // runConfigTile returns { content: [{ type: "text", text: JSON }] }. + const resultText = result.content?.[0]?.text; + if (!resultText) return result; + + let parsed: { result?: { data?: Record[] } }; + try { + parsed = JSON.parse(resultText); + } catch { + return result; + } + + const resultData = parsed.result; + const rows = (resultData as Record | undefined)?.data as + | Record[] + | undefined; + if (!rows || !Array.isArray(rows) || rows.length === 0) { + return result; + } + + const denoised = await denoiseSearchResults( + teamId.toString(), + input.sourceId, + startDate, + endDate, + rows, + { + where: input.where, + whereLanguage: input.whereLanguage, + }, + ); + + // Replace rows in the result with denoised rows and add metadata + const denoisedResult = { + ...resultData, + data: denoised.rows, + }; + const trimmedResult = trimToolResponse(denoisedResult); + const isTrimmed = + JSON.stringify(trimmedResult).length < + JSON.stringify(denoisedResult).length; + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: trimmedResult, + ...(denoised.removedPatterns.length > 0 + ? { + denoised: { + removedPatterns: denoised.removedPatterns, + originalRowCount: rows.length, + filteredRowCount: denoised.rows.length, + }, + } + : {}), + ...(isTrimmed + ? { + note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', + } + : {}), + ...(denoised.rows.length === 0 + ? { + hint: 'All events matched noisy patterns and were removed. Try narrowing filters or disabling denoise to see all events.', + } + : {}), + }, + null, + 2, + ), + }, + ], + }; }), ); } From ee722c96a49b31e2a2eedd875a48a16aa6c62a14 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 29 May 2026 10:45:21 -0700 Subject: [PATCH 2/4] fix: address review feedback on denoise feature - Key noisy patterns by template string (p.pattern / match.getTemplate()) instead of per-instance cluster ID, eliminating fragile coupling to minePatterns() internal auto-increment ordering - Always emit a 'denoised' metadata block when denoise=true, including a 'skipped' field with the reason when denoising cannot proceed (source_not_found, no_body_column, body_column_not_in_results, connection_not_found, sampling_failed, no_sample_data, no_rows) - Rename originalRowCount to returnedRowCountBeforeDenoise to make the post-trim semantics explicit - Fix misleading maxSamples:0 comment (minePatterns always keeps at least one sample per cluster); use maxSamples:1 instead - Add integration tests for denoise=true: schema exposure, empty results handling, noisy pattern filtering with seeded data, denoised metadata shape assertions, and denoise=false control case --- .../api/src/mcp/__tests__/queryTool.test.ts | 104 ++++++++++++++++++ packages/api/src/mcp/tools/query/denoise.ts | 50 +++++---- packages/api/src/mcp/tools/query/search.ts | 23 ++-- 3 files changed, 144 insertions(+), 33 deletions(-) diff --git a/packages/api/src/mcp/__tests__/queryTool.test.ts b/packages/api/src/mcp/__tests__/queryTool.test.ts index d51a6f92a0..d8400013c2 100644 --- a/packages/api/src/mcp/__tests__/queryTool.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.test.ts @@ -356,6 +356,110 @@ describe('MCP Query Tools', () => { expect(result.isError).toBe(true); expect(getFirstText(result)).toMatch(/sourceId/i); }); + + it('should expose denoise property in schema', async () => { + const { tools } = await client.listTools(); + const tool = tools.find(t => t.name === 'hyperdx_search'); + expect(tool).toBeDefined(); + const props = Object.keys(tool!.inputSchema.properties ?? {}); + expect(props).toContain('denoise'); + }); + + it('should emit denoised block when denoise=true on empty results', async () => { + const result = await callTool(client, 'hyperdx_search', { + sourceId: logSource._id.toString(), + denoise: true, + startTime: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + endTime: new Date().toISOString(), + }); + + expect(result.isError).toBeFalsy(); + // With no data, the denoised block should not appear because the + // search result itself has no rows to process (early return path). + const output = JSON.parse(getFirstText(result)); + expect(output).toHaveProperty('result'); + }); + + describe('denoise with seeded data', () => { + const now = new Date(); + const fiveMinAgo = new Date(now.getTime() - 5 * 60 * 1000); + + beforeEach(async () => { + const logs: Parameters[0] = []; + + // Noisy pattern: "Health check OK from " — 80 rows (>10% threshold) + for (let i = 0; i < 80; i++) { + logs.push({ + Body: `Health check OK from 10.0.${Math.floor(i / 256)}.${i % 256}`, + ServiceName: 'loadbalancer', + SeverityText: 'INFO', + Timestamp: new Date(fiveMinAgo.getTime() + i * 100), + }); + } + + // Unique/rare events — 5 rows (well below 10% threshold) + for (let i = 0; i < 5; i++) { + logs.push({ + Body: `Rare event type ${String.fromCharCode(65 + i)} occurred in subsystem`, + ServiceName: 'worker', + SeverityText: 'WARN', + Timestamp: new Date(fiveMinAgo.getTime() + (80 + i) * 1000), + }); + } + + await bulkInsertLogs(logs); + }); + + it('should filter noisy patterns and emit denoised metadata', async () => { + const result = await callTool(client, 'hyperdx_search', { + sourceId: logSource._id.toString(), + denoise: true, + maxResults: 200, + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + + // Must have a denoised block + expect(output).toHaveProperty('denoised'); + expect(output.denoised).toHaveProperty('removedPatterns'); + expect(output.denoised).toHaveProperty('returnedRowCountBeforeDenoise'); + expect(output.denoised).toHaveProperty('filteredRowCount'); + + // Should not have a skipped reason + expect(output.denoised.skipped).toBeUndefined(); + + // The noisy health check pattern should be in removedPatterns + expect(output.denoised.removedPatterns.length).toBeGreaterThanOrEqual( + 1, + ); + const healthPattern = output.denoised.removedPatterns.find( + (p: { pattern: string }) => p.pattern.includes('Health check'), + ); + expect(healthPattern).toBeDefined(); + + // Filtered count should be less than original + expect(output.denoised.filteredRowCount).toBeLessThan( + output.denoised.returnedRowCountBeforeDenoise, + ); + }); + + it('should return results without denoised block when denoise=false', async () => { + const result = await callTool(client, 'hyperdx_search', { + sourceId: logSource._id.toString(), + denoise: false, + maxResults: 200, + startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(), + endTime: new Date(now.getTime() + 60 * 1000).toISOString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output).not.toHaveProperty('denoised'); + }); + }); }); // ─── hyperdx_event_patterns ───────────────────────────────────────────────── diff --git a/packages/api/src/mcp/tools/query/denoise.ts b/packages/api/src/mcp/tools/query/denoise.ts index 234a4b4099..a795bd0a0e 100644 --- a/packages/api/src/mcp/tools/query/denoise.ts +++ b/packages/api/src/mcp/tools/query/denoise.ts @@ -34,6 +34,11 @@ export interface DenoiseResult { estimatedCount: number; sampleCount: number; }>; + /** + * When non-null, denoising was skipped and rows are returned unmodified. + * The value describes why (e.g. "body_column_not_in_results"). + */ + skipped?: string; } // ─── Core denoising function ───────────────────────────────────────────────── @@ -59,20 +64,18 @@ export async function denoiseSearchResults( }, ): Promise { if (rows.length === 0) { - return { rows, removedPatterns: [] }; + return { rows, removedPatterns: [], skipped: 'no_rows' }; } // ── Resolve source & connection ── const source = await getSource(teamId, sourceId); if (!source) { - // Can't denoise without source info — return rows unmodified - return { rows, removedPatterns: [] }; + return { rows, removedPatterns: [], skipped: 'source_not_found' }; } const bodyColumn = resolveBodyExpression(source); if (!bodyColumn) { - // Source doesn't have a body column — can't mine patterns - return { rows, removedPatterns: [] }; + return { rows, removedPatterns: [], skipped: 'no_body_column' }; } const connection = await getConnectionById( @@ -81,7 +84,7 @@ export async function denoiseSearchResults( true, ); if (!connection) { - return { rows, removedPatterns: [] }; + return { rows, removedPatterns: [], skipped: 'connection_not_found' }; } const clickhouseClient = new ClickhouseClient({ @@ -165,22 +168,24 @@ export async function denoiseSearchResults( ]); } catch { // If sampling fails, return rows unmodified rather than failing the search - return { rows, removedPatterns: [] }; + return { rows, removedPatterns: [], skipped: 'sampling_failed' }; } const sampleRows = sampleResult.data; const totalCount = Number(countResult.data?.[0]?.total ?? 0); if (!sampleRows || sampleRows.length === 0) { - return { rows, removedPatterns: [] }; + return { rows, removedPatterns: [], skipped: 'no_sample_data' }; } // ── Mine patterns from the sample ── - const { patterns, sampleMultiplier } = minePatterns(sampleRows, { + // Note: maxSamples: 1 — minePatterns always keeps at least one sample per + // cluster internally; we just minimize memory overhead. + const { patterns } = minePatterns(sampleRows, { totalCount, startDate, endDate, - maxSamples: 0, // We don't need samples for denoising + maxSamples: 1, getBody: row => { const raw = row.__hdx_pattern_body; return raw != null ? String(raw) : ''; @@ -196,13 +201,16 @@ export async function denoiseSearchResults( } // ── Identify noisy patterns (>10% of sampled events) ── + // Key by template string rather than cluster ID so we are not coupled to + // the auto-incrementing IDs generated inside minePatterns(). The matching + // miner below produces its own IDs; comparing template strings is stable. const sampledRowCount = sampleRows.length; - const noisyPatternIds = new Set(); + const noisyTemplates = new Set(); const removedPatterns: DenoiseResult['removedPatterns'] = []; for (const p of patterns) { if (p.sampleCount / sampledRowCount > NOISE_THRESHOLD) { - noisyPatternIds.add(p.id); + noisyTemplates.add(p.pattern); removedPatterns.push({ pattern: p.pattern, estimatedCount: p.estimatedCount, @@ -211,13 +219,11 @@ export async function denoiseSearchResults( } } - if (noisyPatternIds.size === 0) { + if (noisyTemplates.size === 0) { return { rows, removedPatterns: [] }; } - // ── Re-create the same Drain miner and train it on the same sample ── - // We need a fresh miner to match result rows against the learned patterns. - // The minePatterns() function doesn't expose the miner, so we rebuild it. + // ── Build a miner trained on the same sample for row matching ── const drainConfig = new TemplateMinerConfig(); const miner = new TemplateMiner(drainConfig); for (const row of sampleRows) { @@ -227,13 +233,13 @@ export async function denoiseSearchResults( } // ── Match each result row and filter out noisy ones ── - // The body column in result rows may be named differently from the - // pattern mining column. We try the exact bodyColumn name first, - // then fall back to common column names. const bodyColumnKey = findBodyColumnKey(rows[0], bodyColumn); if (!bodyColumnKey) { - // Can't find the body column in result rows — return unmodified - return { rows, removedPatterns: [] }; + return { + rows, + removedPatterns: [], + skipped: 'body_column_not_in_results', + }; } const filteredRows = rows.filter(row => { @@ -242,7 +248,7 @@ export async function denoiseSearchResults( const bodyText = flattenBody(String(bodyValue)); const match = miner.match(bodyText, 'fallback'); if (!match) return true; // No pattern match — keep the row - return !noisyPatternIds.has(String(match.clusterId)); + return !noisyTemplates.has(match.getTemplate()); }); return { diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 9dee71619b..8accdf971a 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -140,7 +140,9 @@ export function registerSearch(server: McpServer, context: McpContext) { }, ); - // Replace rows in the result with denoised rows and add metadata + // Replace rows in the result with denoised rows and add metadata. + // Always emit a `denoised` block when denoise=true so callers can + // distinguish "no noisy patterns" from "denoise was not requested". const denoisedResult = { ...resultData, data: denoised.rows, @@ -155,21 +157,20 @@ export function registerSearch(server: McpServer, context: McpContext) { text: JSON.stringify( { result: trimmedResult, - ...(denoised.removedPatterns.length > 0 - ? { - denoised: { - removedPatterns: denoised.removedPatterns, - originalRowCount: rows.length, - filteredRowCount: denoised.rows.length, - }, - } - : {}), + denoised: { + removedPatterns: denoised.removedPatterns, + // rows.length is the count returned by runConfigTile + // (already subject to maxResults and trim limits). + returnedRowCountBeforeDenoise: rows.length, + filteredRowCount: denoised.rows.length, + ...(denoised.skipped ? { skipped: denoised.skipped } : {}), + }, ...(isTrimmed ? { note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', } : {}), - ...(denoised.rows.length === 0 + ...(denoised.rows.length === 0 && !denoised.skipped ? { hint: 'All events matched noisy patterns and were removed. Try narrowing filters or disabling denoise to see all events.', } From acbb053581ac23315b701eb114f3b862354a4187 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 3 Jun 2026 15:17:17 -0600 Subject: [PATCH 3/4] chore: update changeset for clickstack_ rename and shared constants --- .changeset/mcp-search-denoise.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.changeset/mcp-search-denoise.md b/.changeset/mcp-search-denoise.md index 3863506acb..53b5495980 100644 --- a/.changeset/mcp-search-denoise.md +++ b/.changeset/mcp-search-denoise.md @@ -1,10 +1,12 @@ --- '@hyperdx/api': patch +'@hyperdx/app': patch +'@hyperdx/common-utils': patch --- -feat(mcp): add denoise option to hyperdx_search tool +feat(mcp): add denoise option to clickstack_search tool -Add a `denoise` boolean parameter to the MCP `hyperdx_search` tool that +Add a `denoise` boolean parameter to the MCP `clickstack_search` tool that automatically filters out high-frequency repetitive event patterns from search results, mirroring the web app's "Denoise Results" feature. @@ -12,3 +14,7 @@ When enabled, the tool samples 10k random events, mines patterns using the Drain algorithm, identifies noisy patterns (>10% of sample), and filters them out of result rows. Returns filtered rows plus metadata listing removed patterns with estimated counts. + +Extracts shared denoise constants (`DENOISE_SAMPLE_SIZE`, +`DENOISE_NOISE_THRESHOLD`) into `@hyperdx/common-utils` so the web app +and MCP server use the same values. From 10b0748568d781a8f6a526c764bc69b57de37743 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 3 Jun 2026 16:39:53 -0600 Subject: [PATCH 4/4] fix: gracefully degrade when denoiseSearchResults throws Wrap the denoiseSearchResults call in a try/catch so that a failure in the denoise post-processing step (getSource, getConnectionById, pattern mining, etc.) does not discard an already-successful search result. On catch, return the original rows with a denoised metadata block containing skipped: 'denoise_failed', following the same graceful- degradation convention used by other failure paths in denoise.ts. --- packages/api/src/mcp/tools/query/search.ts | 60 ++++++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 39680d9140..6625b883dd 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -1,6 +1,7 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; +import logger from '@/utils/logger'; import { trimToolResponse } from '@/utils/trimToolResponse'; import { withToolTracing } from '../../utils/tracing'; @@ -128,17 +129,54 @@ export function registerSearch(server: McpServer, context: McpContext) { return result; } - const denoised = await denoiseSearchResults( - teamId.toString(), - input.sourceId, - startDate, - endDate, - rows, - { - where: input.where, - whereLanguage: input.whereLanguage, - }, - ); + let denoised; + try { + denoised = await denoiseSearchResults( + teamId.toString(), + input.sourceId, + startDate, + endDate, + rows, + { + where: input.where, + whereLanguage: input.whereLanguage, + }, + ); + } catch (err) { + // Denoise is a post-processing enhancement — a failure here must + // never discard the already-successful search result. + logger.warn( + { err, sourceId: input.sourceId }, + 'denoiseSearchResults failed; returning raw results', + ); + + const { data: trimmedResult, isTrimmed } = trimToolResponse(resultData); + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + result: trimmedResult, + denoised: { + removedPatterns: [], + returnedRowCountBeforeDenoise: rows.length, + filteredRowCount: rows.length, + skipped: 'denoise_failed', + }, + ...(isTrimmed + ? { + note: 'Result was trimmed for context size. Narrow the time range or add filters to reduce data.', + } + : {}), + }, + null, + 2, + ), + }, + ], + }; + } // Replace rows in the result with denoised rows and add metadata. // Always emit a `denoised` block when denoise=true so callers can