diff --git a/.changeset/mean-spoons-poke.md b/.changeset/mean-spoons-poke.md new file mode 100644 index 0000000000..c5f646dcb8 --- /dev/null +++ b/.changeset/mean-spoons-poke.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/hdx-eval": patch +"@hyperdx/api": patch +--- + +feat: expose summary metrics through the mcp diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index ec42edc5b0..caca6da6e5 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -404,6 +404,27 @@ export const bulkInsertMetricsHistogram = async ( ); }; +export const bulkInsertMetricsSummary = async ( + metrics: { + MetricName: string; + ResourceAttributes: Record; + ScopeAttributes?: Record; + Attributes?: Record; + ServiceName?: string; + TimeUnix: Date; + Count?: number; + Sum?: number; + }[], +) => { + if (!config.IS_CI) { + throw new Error('ONLY execute this in CI env 😈 !!!'); + } + await bulkInsertData( + `${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.SUMMARY}`, + metrics, + ); +}; + type ExponentialHistogramMetricPoint = { TimeUnix: Date; ServiceName?: string; diff --git a/packages/api/src/mcp/__tests__/listMetricsCursor.test.ts b/packages/api/src/mcp/__tests__/listMetricsCursor.test.ts index aca86c3d36..150f677e28 100644 --- a/packages/api/src/mcp/__tests__/listMetricsCursor.test.ts +++ b/packages/api/src/mcp/__tests__/listMetricsCursor.test.ts @@ -41,6 +41,16 @@ describe('listMetrics cursor', () => { expect(decodeCursor(encodeCursor(payload))).toEqual(payload); }); + it('round-trips a summary cursor', () => { + // Summary is discoverable (listed) even though it is not queryable + // by the builder tools. + const payload = { + kind: 'summary' as const, + lastName: 'http_request_duration_seconds', + }; + expect(decodeCursor(encodeCursor(payload))).toEqual(payload); + }); + it('round-trips metric names with dots, dashes, and unicode', () => { const payload = { kind: 'gauge' as const, @@ -74,12 +84,7 @@ describe('listMetrics cursor', () => { expect(decodeCursor(malformed)).toBeNull(); }); - it('returns null when kind is not a queryable metric kind', () => { - const summaryCursor = Buffer.from( - JSON.stringify({ kind: 'summary', lastName: 'x' }), - ).toString('base64'); - expect(decodeCursor(summaryCursor)).toBeNull(); - + it('returns null when kind is not a discoverable metric kind', () => { const bogusCursor = Buffer.from( JSON.stringify({ kind: 'bogus', lastName: 'x' }), ).toString('base64'); diff --git a/packages/api/src/mcp/__tests__/metricKinds.test.ts b/packages/api/src/mcp/__tests__/metricKinds.test.ts index c415daccd3..2feba8c0f0 100644 --- a/packages/api/src/mcp/__tests__/metricKinds.test.ts +++ b/packages/api/src/mcp/__tests__/metricKinds.test.ts @@ -1,4 +1,22 @@ -import { sanitizeMetricTables } from '@/mcp/tools/sources/metricKinds'; +import { + DISCOVERABLE_METRIC_KINDS, + QUERYABLE_METRIC_KINDS, + sanitizeMetricTables, +} from '@/mcp/tools/sources/metricKinds'; + +describe('DISCOVERABLE_METRIC_KINDS', () => { + it('is the queryable kinds plus summary, with summary last', () => { + // Discovery tools list summary metrics, but the query renderer cannot + // translate them — the query/dashboard schemas must stay on the + // queryable list. Summary is last so paginated scans hit it after + // every queryable kind. + expect(DISCOVERABLE_METRIC_KINDS).toEqual([ + ...QUERYABLE_METRIC_KINDS, + 'summary', + ]); + expect(QUERYABLE_METRIC_KINDS).not.toContain('summary'); + }); +}); describe('sanitizeMetricTables', () => { it('returns undefined for null / undefined input', () => { diff --git a/packages/api/src/mcp/__tests__/sources.int.test.ts b/packages/api/src/mcp/__tests__/sources.int.test.ts index 1130ec3a5b..4a442bd9eb 100644 --- a/packages/api/src/mcp/__tests__/sources.int.test.ts +++ b/packages/api/src/mcp/__tests__/sources.int.test.ts @@ -7,6 +7,7 @@ import { bulkInsertMetricsGauge, bulkInsertMetricsHistogram, bulkInsertMetricsSum, + bulkInsertMetricsSummary, DEFAULT_DATABASE, DEFAULT_LOGS_TABLE, DEFAULT_METRICS_TABLE, @@ -429,6 +430,101 @@ describe('MCP Source Tools', () => { ); }); + it('discovers summary metrics and directs querying to clickstack_sql', async () => { + // Summary metrics are discoverable (list/describe) but not queryable: + // the query renderer has no summary translation, so every discovery + // surface must redirect the agent to clickstack_sql. + const metricSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { databaseName: DEFAULT_DATABASE, tableName: '' }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: DEFAULT_METRICS_TABLE.GAUGE, + [MetricsDataType.Summary.toLowerCase()]: + DEFAULT_METRICS_TABLE.SUMMARY, + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'Metrics With Summary', + }); + const now = new Date(); + await bulkInsertMetricsSummary([ + { + MetricName: 'http_request_duration_seconds', + ResourceAttributes: { 'service.name': 'svc-a' }, + Attributes: { endpoint: '/api/v1/users' }, + ServiceName: 'svc-a', + TimeUnix: now, + Count: 10, + Sum: 1.5, + }, + ]); + + const describeSourceResult = await callTool( + client, + 'clickstack_describe_source', + { sourceId: metricSource._id.toString() }, + ); + expect(describeSourceResult.isError).toBeFalsy(); + const describedSource = JSON.parse(getFirstText(describeSourceResult)); + // The summary table is listed and its metric names are sampled, but + // the representative discovery table stays a queryable kind. + expect(describedSource.source.metricTables.summary).toBe( + DEFAULT_METRICS_TABLE.SUMMARY, + ); + expect(describedSource.source.discoveryMetricKind).toBe('gauge'); + expect(describedSource.source.metricNames.summary).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'http_request_duration_seconds' }), + ]), + ); + expect(describedSource.usage.metricNames).toContain('clickstack_sql'); + + const listResult = await callTool(client, 'clickstack_list_metrics', { + sourceId: metricSource._id.toString(), + kind: 'summary', + }); + expect(listResult.isError).toBeFalsy(); + const listedMetrics = JSON.parse(getFirstText(listResult)); + expect(listedMetrics.metrics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'http_request_duration_seconds', + kind: 'summary', + }), + ]), + ); + expect(listedMetrics.summaryNote).toContain('clickstack_sql'); + + const describeMetricResult = await callTool( + client, + 'clickstack_describe_metric', + { + sourceId: metricSource._id.toString(), + metricName: 'http_request_duration_seconds', + kind: 'summary', + }, + ); + expect(describeMetricResult.isError).toBeFalsy(); + const describedMetric = JSON.parse(getFirstText(describeMetricResult)); + expect(describedMetric.kinds[0]).toMatchObject({ kind: 'summary' }); + // Attribute discovery is table-generic and must work on the summary + // table like any other kind. + expect(describedMetric.kinds[0].attributeKeys.Attributes).toContain( + 'endpoint', + ); + expect(describedMetric.kinds[0].usage).toContain('clickstack_sql'); + expect(describedMetric.nextSteps.query).toContain('clickstack_sql'); + // The redirect must name the database-qualified table — clickstack_sql + // has no source context to resolve a bare table name against. + expect(describedMetric.nextSteps.query).toContain( + `${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.SUMMARY}`, + ); + expect(describedMetric.nextSteps.query).not.toContain( + 'clickstack_timeseries({', + ); + }); + it('should include usage guidance and nextSteps', async () => { const result = await callTool(client, 'clickstack_describe_source', { sourceId: traceSource._id.toString(), diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 7c9525b4ec..c8b510d0d5 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -964,7 +964,7 @@ For configType: "sql" tiles, write ClickHouse SQL with template macros: == METRIC SOURCES == -Builder tiles work on metric sources. Each select item on a metric tile MUST set metricType ("gauge" | "sum" | "histogram" | "exponential histogram") and metricName (the OTel metric name, e.g. "system.cpu.utilization"). valueExpression defaults to "Value" when omitted, so a typical metric series is { aggFn: "", metricType: "", metricName: "" }. summary metrics are not supported by the renderer. +Builder tiles work on metric sources. Each select item on a metric tile MUST set metricType ("gauge" | "sum" | "histogram" | "exponential histogram") and metricName (the OTel metric name, e.g. "system.cpu.utilization"). valueExpression defaults to "Value" when omitted, so a typical metric series is { aggFn: "", metricType: "", metricName: "" }. summary metrics are not supported by the renderer; to chart them, use a raw SQL tile (configType: "sql") against the table named in the source's metricTables.summary. Per-kind aggregation guidance: gauge Use aggFn:"last_value" | "avg" | "min" | "max". Set isDelta:true for Prometheus-style delta over each bucket. diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 82e6fdb88f..f0226d3ba8 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -189,7 +189,8 @@ const mcpTileSelectItemSchema = z .describe( 'METRIC SOURCES ONLY. OTel metric kind: gauge, sum, histogram, or exponential histogram. ' + 'Required (with metricName) when the tile sourceId is a metric source. ' + - 'summary is not supported by the renderer.', + 'summary is not supported by the renderer — chart summary metrics with a raw ' + + 'SQL tile (configType:"sql") against the table in the source\'s metricTables.summary.', ), metricName: z .string() diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts index 9f853e9d18..f24707e524 100644 --- a/packages/api/src/mcp/tools/query/schemas.ts +++ b/packages/api/src/mcp/tools/query/schemas.ts @@ -88,7 +88,8 @@ const mcpMetricTypeSchema = z ' sum – cumulative or delta counters (request counts, bytes processed). Use aggFn:"increase" for counter increase, or sum/avg on the computed rate.\n' + ' histogram – bucketed distributions (request duration). Use aggFn:"quantile" with level for percentiles, or aggFn:"count" for total bucket count.\n' + ' exponential histogram – exponential-bucket distributions. Use aggFn:"quantile" with level for percentiles, or aggFn:"count" for total bucket count.\n' + - 'NOTE: summary is not supported by the query renderer.', + 'NOTE: summary is not supported by the query renderer — to look at summary ' + + "metrics, use clickstack_sql against the table in the source's metricTables.summary.", ); /** diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts index 586f0d494c..8deb369099 100644 --- a/packages/api/src/mcp/tools/query/sql.ts +++ b/packages/api/src/mcp/tools/query/sql.ts @@ -64,7 +64,9 @@ export function registerSql({ context, registerTool }: ToolRegistrar) { description: 'Execute raw ClickHouse SQL. ' + 'ADVANCED: only use this when you need capabilities the builder tools cannot express — ' + - 'JOINs, sub-queries, CTEs, or querying tables not registered as sources.\n\n' + + 'JOINs, sub-queries, CTEs, querying tables not registered as sources, or looking at ' + + 'summary-type metrics (the metricTables.summary table on a metric source, which the ' + + 'builder tools cannot query).\n\n' + 'Requires connectionId (not sourceId) — call clickstack_list_sources to find connections. ' + 'Call clickstack_describe_source to discover column names before writing SQL.\n\n' + 'Results are always returned as table rows — for time-series semantics, ' + diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index 88d2a9bdbb..69a2102492 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -229,7 +229,8 @@ export function registerTable({ context, registerTool }: ToolRegistrar) { 'Per kind: gauge uses last_value/avg/min/max; sum uses aggFn:"increase" for counter increase ' + '(top-N capped at 20 groups when combined with groupBy), or sum/avg on the rate; ' + 'histogram and exponential histogram use aggFn:"quantile" + level for percentiles, or aggFn:"count" for total bucket count.\n' + - 'summary metrics are not supported by the query renderer.', + 'summary metrics are not supported by the query renderer — query them with ' + + "clickstack_sql against the table in the source's metricTables.summary.", inputSchema: tableSchema, }, async input => { diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index c0e246da8d..4e20bce1c6 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -99,7 +99,8 @@ export function registerTimeseries({ context, registerTool }: ToolRegistrar) { 'histogram and exponential histogram use aggFn:"quantile" + level for percentiles, or aggFn:"count" for total bucket count.\n' + 'TOP-N CAP: aggFn:"increase" + groupBy is capped at 20 groups by the renderer ' + '(top by max bucket sum). Narrow with where/groupBy to see other groups.\n' + - 'summary metrics are not supported by the query renderer.', + 'summary metrics are not supported by the query renderer — query them with ' + + "clickstack_sql against the table in the source's metricTables.summary.", inputSchema: timeseriesSchema, }, async input => { diff --git a/packages/api/src/mcp/tools/sources/describeMetric.ts b/packages/api/src/mcp/tools/sources/describeMetric.ts index 3d3ae11683..db1b01d14b 100644 --- a/packages/api/src/mcp/tools/sources/describeMetric.ts +++ b/packages/api/src/mcp/tools/sources/describeMetric.ts @@ -20,8 +20,8 @@ import logger from '@/utils/logger'; import { trimToolResponse } from '@/utils/trimToolResponse'; import { - QUERYABLE_METRIC_KINDS, - type QueryableMetricKind, + DISCOVERABLE_METRIC_KINDS, + type DiscoverableMetricKind, } from './metricKinds'; const DEFAULT_LOOKBACK_MS = 24 * 60 * 60 * 1000; @@ -45,7 +45,7 @@ const MAX_ATTR_KEYS_TO_SAMPLE = 12; // Per-kind aggregation guidance baked into the response so the agent can // build a valid clickstack_timeseries / clickstack_table call without // re-reading the schemas. -const KIND_USAGE: Record = { +const KIND_USAGE: Record = { gauge: 'Gauge: use aggFn:"last_value"|"avg"|"min"|"max" on Value. Set isDelta:true for Prometheus-style delta over each bucket.', sum: 'Sum (counter): use aggFn:"increase" for the per-bucket counter increase (reset-aware), or aggFn:"sum"/"avg" on the computed rate. increase+groupBy is capped at the top 20 groups.', @@ -53,6 +53,10 @@ const KIND_USAGE: Record = { 'Histogram: use aggFn:"quantile" with level ∈ {0.5, 0.9, 0.95, 0.99} for percentiles, or aggFn:"count" for the total bucket count.', 'exponential histogram': 'Exponential histogram: use aggFn:"quantile" with level ∈ {0.5, 0.9, 0.95, 0.99} for percentiles, or aggFn:"count" for the total bucket count.', + summary: + 'Summary: client-precomputed quantiles (e.g. Prometheus summaries). Not supported by ' + + 'clickstack_timeseries / clickstack_table — query this table with clickstack_sql ' + + "using the source's connectionId.", }; // ─── Schema ────────────────────────────────────────────────────────────────── @@ -71,9 +75,9 @@ const describeMetricSchema = z.object({ 'Discover via clickstack_list_metrics or clickstack_describe_source.', ), kind: z - .enum(QUERYABLE_METRIC_KINDS) + .enum(DISCOVERABLE_METRIC_KINDS) .describe( - 'Metric kind: "gauge" | "sum" | "histogram" | "exponential histogram". Required. ' + + 'Metric kind: "gauge" | "sum" | "histogram" | "exponential histogram" | "summary". Required. ' + 'Discover via clickstack_list_metrics (which returns name + kind per entry) ' + 'or clickstack_describe_source (which groups metric-name samples by kind). ' + 'A metric name can legitimately live in more than one kind (e.g. ' + @@ -111,7 +115,7 @@ type AttributeValuesMeta = { }; type KindDetail = { - kind: QueryableMetricKind; + kind: DiscoverableMetricKind; tableName: string; unit?: string; description?: string; @@ -641,13 +645,33 @@ async function describeMetricImpl( !meta.unit && !meta.description; - const queryExample = `clickstack_timeseries({ sourceId: "${input.sourceId}", select: [{ aggFn: ${ - kind === 'sum' - ? '"increase"' - : kind === 'histogram' || kind === 'exponential histogram' - ? '"quantile", level: 0.95' - : '"avg"' - }, metricType: "${kind}", metricName: "${input.metricName}" }] })`; + const makeNextStepsQuery = () => { + if (kind === 'summary') { + // Summary metrics have no renderer support, so the next step is + // clickstack_sql rather than a builder-tool example. + // clickstack_sql runs with no source context, so qualify the table + // with the source's database — a bare table name would resolve + // against the connection user's default database. + const qualifiedTable = databaseName + ? `${databaseName}.${tableName}` + : tableName; + return ( + 'summary metrics cannot be queried with clickstack_timeseries / clickstack_table — ' + + `use clickstack_sql against the "${qualifiedTable}" table with this source's connectionId ` + + '(see clickstack_list_sources).' + ); + } + + let aggFn = '"avg"'; + + if (kind === 'sum') { + aggFn = '"increase"'; + } else if (kind === 'histogram' || kind === 'exponential histogram') { + aggFn = '"quantile", level: 0.95'; + } + + return `Example: clickstack_timeseries({ sourceId: "${input.sourceId}", select: [{ aggFn: ${aggFn}, metricType: "${kind}", metricName: "${input.metricName}" }] })`; + }; const responseObj: Record = { metricName: input.metricName, @@ -666,7 +690,7 @@ async function describeMetricImpl( 'confirm the metric name + kind combination exists.', }), nextSteps: { - query: `Example: ${queryExample}`, + query: makeNextStepsQuery(), }, }; @@ -708,10 +732,14 @@ export function registerDescribeMetric({ 'sample) to get attribute keys, sampled values, unit, and description for a ' + 'specific (metricName, kind) pair. Attribute keys vary per metric — not per source — ' + "so always call this before clickstack_timeseries / clickstack_table for any metric you've never queried.\n\n" + - 'REQUIRES `kind` — pass the gauge/sum/histogram/exponential histogram value emitted alongside the metric name by ' + + 'REQUIRES `kind` — pass the gauge/sum/histogram/exponential histogram/summary value emitted alongside the metric name by ' + 'clickstack_list_metrics or clickstack_describe_source. A metric name can legitimately ' + 'live in more than one kind (e.g. "container.cpu.usage" appears in both gauge and sum); ' + 'call this tool once per kind you care about.\n\n' + + 'kind:"summary" is accepted for discovery (attribute keys, sampled values, unit, ' + + 'description), but summary metrics cannot be queried with clickstack_timeseries / ' + + "clickstack_table — use clickstack_sql against the table in the source's " + + 'metricTables.summary.\n\n' + 'attributeValuesMeta on each kind reports sampledKeys (queried for values) and ' + 'truncatedKeys (skipped by the per-call sampling cap) — a key in truncatedKeys was ' + 'never queried, so query it directly if you need its values.\n\n' + diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 3c357332d9..9d97029424 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -20,6 +20,7 @@ import logger from '@/utils/logger'; import { trimToolResponse } from '@/utils/trimToolResponse'; import { + DISCOVERABLE_METRIC_KINDS, QUERYABLE_METRIC_KINDS, type QueryableMetricKind, sanitizeMetricTables, @@ -519,16 +520,18 @@ async function describeSourceSchema( } // ── 5. Metric name + unit + description sampling ────────────────────── - // For metric sources, sample distinct MetricName values per queryable - // kind so the agent has a starter list without needing a follow-up call - // to clickstack_list_metrics for the common case (<= 20 metrics/kind). + // For metric sources, sample distinct MetricName values per discoverable + // kind (including the non-queryable summary kind, so agents know those + // metrics exist) so the agent has a starter list without needing a + // follow-up call to clickstack_list_metrics for the common case + // (<= 20 metrics/kind). // Defensively check for MetricUnit / MetricDescription columns: they // exist on the standard OTel Collector schema but a custom metric table // may not declare them. if (source.kind === SourceKind.Metric && !signal.aborted) { const metricNames: Record = {}; await Promise.all( - QUERYABLE_METRIC_KINDS.map(async kind => { + DISCOVERABLE_METRIC_KINDS.map(async kind => { const kindTableName = source.metricTables[kind]; if (!kindTableName) return; try { @@ -582,7 +585,9 @@ async function describeSourceSchema( const isMetricSource = source.kind === SourceKind.Metric; const queryNextStep = isMetricSource - ? `Use clickstack_timeseries or clickstack_table with sourceId "${sourceId}" and metricType/metricName from above.` + ? `Use clickstack_timeseries or clickstack_table with sourceId "${sourceId}" and metricType/metricName from above. ` + + 'Summary metrics (metricTables.summary) are not supported by those tools — ' + + "query the summary table with clickstack_sql using this source's connectionId." : `Use clickstack_timeseries, clickstack_table, or clickstack_search with sourceId "${sourceId}" and the columns/attributes above.`; const discoveryNextStep = isMetricSource ? `For more metric names than the sample above, call clickstack_list_metrics with sourceId "${sourceId}". For per-metric attribute keys + sampled values, call clickstack_describe_metric with sourceId and metricName.` @@ -598,8 +603,10 @@ async function describeSourceSchema( lowCardinalityValues: lcValuesHint, ...(isMetricSource && { metricNames: - 'Each entry maps a metric kind (gauge/sum/histogram/exponential histogram) to a sample of metric names ' + - 'available on that table. Pass metricType + metricName on each select item.', + 'Each entry maps a metric kind (gauge/sum/histogram/exponential histogram/summary) to a sample of metric names ' + + 'available on that table. Pass metricType + metricName on each select item. ' + + 'EXCEPTION: summary metrics cannot be charted — query them with clickstack_sql ' + + "against the table in metricTables.summary using this source's connectionId.", }), }, nextSteps: { diff --git a/packages/api/src/mcp/tools/sources/listMetrics.ts b/packages/api/src/mcp/tools/sources/listMetrics.ts index 70615d1216..bd5704e494 100644 --- a/packages/api/src/mcp/tools/sources/listMetrics.ts +++ b/packages/api/src/mcp/tools/sources/listMetrics.ts @@ -15,8 +15,8 @@ import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; import logger from '@/utils/logger'; import { - QUERYABLE_METRIC_KINDS, - type QueryableMetricKind, + DISCOVERABLE_METRIC_KINDS, + type DiscoverableMetricKind, } from './metricKinds'; const DEFAULT_LIMIT = 50; @@ -33,7 +33,7 @@ const MAX_EXEC_SECONDS = 8; // ─── Cursor ────────────────────────────────────────────────────────────────── export type ListMetricsCursorPayload = { - kind: QueryableMetricKind; + kind: DiscoverableMetricKind; lastName: string; }; @@ -52,10 +52,10 @@ export function decodeCursor(raw: string): ListMetricsCursorPayload | null { parsed !== null && typeof parsed.kind === 'string' && typeof parsed.lastName === 'string' && - (QUERYABLE_METRIC_KINDS as readonly string[]).includes(parsed.kind) + (DISCOVERABLE_METRIC_KINDS as readonly string[]).includes(parsed.kind) ) { return { - kind: parsed.kind as QueryableMetricKind, + kind: parsed.kind as DiscoverableMetricKind, lastName: parsed.lastName, }; } @@ -74,11 +74,13 @@ const listMetricsSchema = z.object({ 'Source ID. Must reference a metric source — get IDs from clickstack_list_sources.', ), kind: z - .enum(QUERYABLE_METRIC_KINDS) + .enum(DISCOVERABLE_METRIC_KINDS) .optional() .describe( 'Optional metric kind filter. Omit to scan every populated kind on the source ' + - '(gauge, sum, histogram, exponential histogram). Set to narrow results to one kind.', + '(gauge, sum, histogram, exponential histogram, summary). Set to narrow results to one kind. ' + + 'NOTE: summary metrics are discovery-only — they cannot be passed to ' + + 'clickstack_timeseries / clickstack_table; query them with clickstack_sql.', ), namePattern: z .string() @@ -121,7 +123,7 @@ const listMetricsSchema = z.object({ type MetricEntry = { name: string; - kind: QueryableMetricKind; + kind: DiscoverableMetricKind; unit?: string; description?: string; }; @@ -161,7 +163,7 @@ async function fetchMetricsForKind({ }: { clickhouseClient: ClickhouseClient; metadata: ReturnType; - kind: QueryableMetricKind; + kind: DiscoverableMetricKind; databaseName: string; tableName: string; connectionId: string; @@ -258,9 +260,12 @@ export function registerListMetrics({ 'DISCOVERY: Use this after clickstack_describe_source when you need more metric ' + 'names than the per-kind sample shows, or when you want to narrow by ' + 'kind / name pattern / time window. ' + - 'Returns paginated metric names per kind (gauge/sum/histogram/exponential histogram) ' + + 'Returns paginated metric names per kind (gauge/sum/histogram/exponential histogram/summary) ' + 'with optional unit and description (when the OTel-default columns are present). ' + 'Pass the returned `nextCursor` back unchanged to fetch the next page.\n\n' + + 'Summary metrics are listed for discovery only — they cannot be passed to ' + + 'clickstack_timeseries / clickstack_table; query them with clickstack_sql ' + + "against the table in the source's metricTables.summary.\n\n" + 'Workflow: clickstack_list_sources → clickstack_describe_source → ' + 'clickstack_list_metrics → clickstack_describe_metric → ' + 'clickstack_timeseries|clickstack_table.', @@ -341,9 +346,9 @@ async function listMetricsImpl( // Resolve which kinds to scan, in order. When a cursor is set, // skip kinds before the cursor's kind (already returned) and start // the cursor's kind at the lastName-exclusive position. - const requestedKinds: QueryableMetricKind[] = input.kind + const requestedKinds: DiscoverableMetricKind[] = input.kind ? [input.kind] - : QUERYABLE_METRIC_KINDS.filter(k => Boolean(source.metricTables[k])); + : DISCOVERABLE_METRIC_KINDS.filter(k => Boolean(source.metricTables[k])); const startKindIdx = cursor ? requestedKinds.indexOf(cursor.kind) : 0; if (startKindIdx < 0) { // Cursor points at a kind that's not in scope for this call — @@ -434,6 +439,11 @@ async function listMetricsImpl( const responseObj: Record = { metrics, ...(nextCursor && { nextCursor }), + ...(metrics.some(m => m.kind === 'summary') && { + summaryNote: + 'summary metrics cannot be queried with clickstack_timeseries / clickstack_table — ' + + "use clickstack_sql against the table in the source's metricTables.summary.", + }), ...(partialFailure.length > 0 && { partialFailure, hint: @@ -447,7 +457,8 @@ async function listMetricsImpl( 'removing the namePattern filter, or omitting `kind` to scan every populated metric table.', }), usage: - 'Pass `metricType` + `metricName` from each entry to clickstack_timeseries / clickstack_table. ' + + 'Pass `metricType` + `metricName` from each entry to clickstack_timeseries / clickstack_table ' + + '(summary metrics excepted — query those with clickstack_sql). ' + 'For per-metric attribute keys and sampled values, call clickstack_describe_metric.', }; diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts index 8199ac8547..d4dcabeccc 100644 --- a/packages/api/src/mcp/tools/sources/listSources.ts +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -26,7 +26,9 @@ export function registerListSources({ 'This two-step approach avoids fetching expensive schema details for sources you do not need.\n\n' + 'NOTE: For most queries, use source IDs with clickstack_timeseries, clickstack_table, ' + 'clickstack_search, or clickstack_event_patterns. ' + - 'Connection IDs are only needed for clickstack_sql (raw ClickHouse SQL).', + 'Connection IDs are only needed for clickstack_sql (raw ClickHouse SQL).\n\n' + + 'Metric sources may list a "summary" table in metricTables. Summary metrics are ' + + 'not supported by the builder tools — use clickstack_sql to look at them.', inputSchema: z.object({}), }, async () => { diff --git a/packages/api/src/mcp/tools/sources/metricKinds.ts b/packages/api/src/mcp/tools/sources/metricKinds.ts index 7eeedc42b7..9df70ad590 100644 --- a/packages/api/src/mcp/tools/sources/metricKinds.ts +++ b/packages/api/src/mcp/tools/sources/metricKinds.ts @@ -39,6 +39,21 @@ const _assertKindsMatchEnum: readonly QueryableMetricKind[] = [ ]; void _assertKindsMatchEnum; +/** + * Metric kinds exposed by the discovery tools (clickstack_list_metrics, + * clickstack_describe_metric, and clickstack_describe_source's metric-name + * sampling). Superset of QUERYABLE_METRIC_KINDS: adds `summary`, whose + * metrics are discoverable but cannot be charted — the query renderer has + * no summary translation, so agents are directed to clickstack_sql for + * those. + */ +export const DISCOVERABLE_METRIC_KINDS = [ + ...QUERYABLE_METRIC_KINDS, + 'summary', +] as const; + +export type DiscoverableMetricKind = (typeof DISCOVERABLE_METRIC_KINDS)[number]; + /** * Allowed kind keys on the `metricTables` map when serialising a metric * source into an MCP response. Includes the non-queryable `summary` kind diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md index f2eb2394e4..db6f9e634a 100644 --- a/packages/hdx-eval/README.md +++ b/packages/hdx-eval/README.md @@ -418,9 +418,15 @@ tables score identically, so both arms are graded by the same rule. Tool names, tool outputs, and the prompt never count. Adoption is intentionally **excluded** from the combined score: measuring tool usage must not inflate outcome quality. `metric-saturation` (metrics load-bearing) should show -≈100% adoption; the interesting readout is `deploy-regression` (metrics -merely corroborating), where adoption measures whether the agent reaches for -metrics it doesn't strictly need. +≈100% adoption on its four core checks; the interesting readout is +`deploy-regression` (metrics merely corroborating), where adoption measures +whether the agent reaches for metrics it doesn't strictly need. + +`metric-saturation` includes two opportunistic checks (`discovered_summary_metric`, +`queried_summary_via_sql`) on the `db.client.operation.duration` Summary distractor, +which is discoverable via MCP but only queryable through raw `clickstack_sql`. +They measure whether the agent follows that discovery → SQL path; since a strong +run may justifiably skip the distractor after finding the JVM leak, sub-100% scores are expected. ### Baseline + Challengers diff --git a/packages/hdx-eval/src/__tests__/grade.test.ts b/packages/hdx-eval/src/__tests__/grade.test.ts index 6c48d92981..c951b9603c 100644 --- a/packages/hdx-eval/src/__tests__/grade.test.ts +++ b/packages/hdx-eval/src/__tests__/grade.test.ts @@ -280,19 +280,36 @@ describe('gradeBatch metric-adoption checks', () => { groupBy: ['k8s.pod.name', 'jvm.memory.pool.name'], }, }, + // Summary metrics are discoverable but not queryable by the + // builder tools — the intended path is discovery → clickstack_sql. + // One SQL call naming the summary metric satisfies BOTH summary + // checks (discovered_summary_metric needs the name anywhere; + // queried_summary_via_sql additionally needs SELECT / + // ValueAtQuantiles in the same call's args). + { + name: 'mcp__hyperdx__clickstack_sql', + isError: false, + input: { + connectionId: 'c1', + sql: "SELECT TimeUnix, ValueAtQuantiles.Value[indexOf(ValueAtQuantiles.Quantile, 0.99)] AS p99 FROM eval_metric_saturation_otel_metrics_summary WHERE MetricName = 'db.client.operation.duration' ORDER BY TimeUnix", + }, + }, ], finalAnswer: 'JVM heap leak on recommendation-service.', }), ); const result = await gradeBatch(batchDir, { skipJudge: true }); const grade: GradeRecord = result.graded[0]; - // All four adoption checks hit (queried a target metric, the JVM memory - // metric, a GC metric, and grouped memory by pod/pool in one call). + // All six adoption checks hit (queried a target metric, the JVM memory + // metric, a GC metric, grouped memory by pod/pool in one call, and + // discovered + SQL-queried the summary distractor). expect(grade.adoption).toBeDefined(); expect(grade.adoption!.hits.map(h => h.id).sort()).toEqual([ + 'discovered_summary_metric', 'grouped_memory_by_pod_or_pool', 'queried_gc_metric', 'queried_jvm_memory_metric', + 'queried_summary_via_sql', 'queried_target_metric', ]); expect(grade.adoption!.score).toBeCloseTo(1, 5); diff --git a/packages/hdx-eval/src/__tests__/metric-saturation.test.ts b/packages/hdx-eval/src/__tests__/metric-saturation.test.ts index c4bd9798c9..2e2f38e1f2 100644 --- a/packages/hdx-eval/src/__tests__/metric-saturation.test.ts +++ b/packages/hdx-eval/src/__tests__/metric-saturation.test.ts @@ -306,14 +306,19 @@ describe('metric-saturation scenario', () => { expect(twinMax).toBeLessThan(100); // the twin never develops a tail }); - it('summary — neighbor db.client latency is a stable, unrelated decoy', () => { + it('summary — db.client latency stays flat (exonerates the database)', () => { const sm = m.summary!.filter( s => s.metricName === 'db.client.operation.duration', ); expect(sm.length).toBeGreaterThan(0); - // It lives on a DIFFERENT service than the incident. - for (const s of sm) expect(s.serviceName).toBe('inventory-service'); - // p99 is flat across the whole window (not correlated with the incident). + // Emitted by a healthy client of the shared database, not the subject. + for (const s of sm) { + expect(s.serviceName).toBe('inventory-service'); + // Linkage to the database the misleading timeout logs blame. + expect(s.attributes?.['db.name']).toBe('product_catalog'); + } + // p99 is flat across the whole window — the timeouts logged by the + // subject are client-side artifacts, not database slowness. const p99 = sm.map(s => s.quantiles[2].value); expect(Math.max(...p99) - Math.min(...p99)).toBeLessThan(20); }); @@ -418,6 +423,23 @@ describe('metric-saturation scenario', () => { ); expect(restartLines.length).toBeGreaterThan(0); }); + + it('plants misleading client-side DB-timeout lines during pressure windows', () => { + const timeoutLines = result.logs.filter(l => + /feature-store query timed out/i.test(l.body), + ); + // Enough volume to dominate a WARN-level log-pattern query. + expect(timeoutLines.length).toBeGreaterThan(5); + for (const l of timeoutLines) { + // On the subject (client side), not the database's owner. + expect(l.serviceName).toBe(SUBJECT); + // Only after the leak begins — they are GC-stall artifacts. + expect(l.timestampMs).toBeGreaterThanOrEqual(LEAK_START_MS); + // They blame the shared database by name, never the real cause. + expect(l.body).toContain('product_catalog'); + expect(/heap|memory|\bgc\b|jvm/i.test(l.body)).toBe(false); + } + }); }); }); diff --git a/packages/hdx-eval/src/scenarios/metric-saturation/generate.ts b/packages/hdx-eval/src/scenarios/metric-saturation/generate.ts index 21c631406d..6f1fb15ad9 100644 --- a/packages/hdx-eval/src/scenarios/metric-saturation/generate.ts +++ b/packages/hdx-eval/src/scenarios/metric-saturation/generate.ts @@ -25,7 +25,11 @@ * - Sum jvm.gc.collections : cumulative; Old-Gen accelerates. * - Histogram http.server.request.duration : latency shifts up. * - ExponentialHistogram jvm.gc.pause : heavy tail (mechanism). - * - Summary db.client.operation.duration (inventory-service) : stable decoy. + * - Summary db.client.operation.duration (inventory-service, healthy + * client of the SAME product_catalog Postgres) : flat quantiles that + * EXONERATE the database against the planted misleading client-side + * timeout logs. Load-bearing for calibration — summaries are not + * supported by the chart tools, so reading it requires raw SQL. * * Histograms use DELTA temporality (each scrape point = that window's * distribution) so the in-window shift is directly readable. @@ -41,8 +45,11 @@ * Distractors: a coincidental 1.43.1 deploy 75 min ago (the leak started * ~90 min ago — ruling it out requires comparing onsets at scrape * granularity), the correlated-but-symptomatic thread-count gauge, flat CPU, - * the healthy JVM twin, and a stable neighbor Summary that cannot be - * re-aggregated into the incident window. + * the healthy JVM twin, and — the strongest pull — misleading 'Feature-store + * query timed out' WARN logs on the subject during GC stalls that blame the + * shared product_catalog Postgres. The reporter's prompt repeats the database + * suspicion; the only exonerating evidence is the flat inventory-service + * db.client Summary on the same database. * * The prompt is blinded: the user reports slow/missing storefront * recommendations and upstream 503s at frontend-proxy. The agent must @@ -63,9 +70,14 @@ * 5. Reads the cumulative restart/GC counters as a rate to confirm the * restart cadence. * 6. Rules out the deploy (leak onset predates it), CPU (flat), thread - * growth (symptom of GC stalls, not cause), the healthy search-service - * JVM twin, and the neighbor db.client Summary (stable, unrelated, - * non-re-aggregatable). + * growth (symptom of GC stalls, not cause), and the healthy + * search-service JVM twin. + * 7. Exonerates the database: the subject's timeout logs (and the + * reporter) blame the shared product_catalog Postgres, but + * inventory-service's db.client.operation.duration Summary (same DB) + * keeps flat quantiles all window — the timeouts are client-side + * artifacts of the stalled JVM. Summaries are not chartable, so this + * step requires raw SQL against the summary table. */ import { makeLog } from '@/generators/logs'; import { @@ -588,7 +600,7 @@ function generateMetrics(rng: Rng, nowMs: number): MetricBatch { }), ); - // ── inventory-service: Summary decoy + flat CPU + non-JVM memory ───── + // ── inventory-service: exonerating Summary + flat CPU + non-JVM memory ─ const p50 = rng.range(7, 9); const p95 = rng.range(38, 44); const p99 = rng.range(85, 95); @@ -611,7 +623,12 @@ function generateMetrics(rng: Rng, nowMs: number): MetricBatch { 'service.name': NEIGHBOR_SERVICE, 'k8s.namespace.name': 'production', }, - attributes: { 'db.system': 'postgresql' }, + attributes: { + 'db.system': 'postgresql', + // Same database the subject's misleading timeout logs blame — + // the discoverable linkage for the exoneration step. + 'db.name': 'product_catalog', + }, }), ); gauge.push( @@ -692,12 +709,53 @@ function proxyUpstream503Log(rng: Rng, t: number): LogRow { }); } +/** + * Planted misleading DB-timeout WARN line on the subject during GC-stall + * pressure windows. A stalled JVM cannot service its own deadline timers, so + * its client-side feature-store reads against the shared product_catalog + * Postgres "time out" — the log trail blames the database even though it is + * healthy. The exoneration is inventory-service's db.client.operation.duration + * Summary (same database, flat quantiles all window), which is only readable + * via raw SQL because summaries are not supported by the chart tools. + */ +function featureStoreTimeoutLog(rng: Rng, t: number): LogRow { + const deadlineMs = rng.pick([250, 300, 500] as const); + const query = rng.pick([ + 'SELECT features FROM candidate_features WHERE sku = ANY($1)', + 'SELECT embedding FROM product_embeddings WHERE product_id = ANY($1)', + 'SELECT snapshot FROM catalog_snapshot WHERE category_id = $1', + ] as const); + const pod = `${SUBJECT_SERVICE}-pod-${rng.intRange(0, POD_COUNT)}`; + const body = + `Feature-store query timed out after ${deadlineMs}ms ` + + `(db=product_catalog host=postgres-primary:5432): ${query}; ` + + 'deadline exceeded, returning fallback ranking'; + return makeLog({ + timestampMs: t, + serviceName: SUBJECT_SERVICE, + severityText: 'WARN', + body, + resourceAttributes: { + 'service.name': SUBJECT_SERVICE, + 'k8s.namespace.name': 'production', + 'k8s.pod.name': pod, + }, + logAttributes: { + 'event.name': 'db.query.timeout', + 'db.system': 'postgresql', + 'db.name': 'product_catalog', + 'server.address': 'postgres-primary', + _severity_raw: 'warning', + }, + }); +} + export const metricSaturationScenario: Scenario = { name: 'metric-saturation', maxTurns: 25, agentPrompt: groundTruth.agentPrompt, description: - 'recommendation-service JVM heap leak behind a blinded prompt (storefront symptom at frontend-proxy): per-pod/per-pool memory-gauge sawtooth -> GC-pause (exp-histogram) tail -> latency-histogram shift -> staggered pod-restart (cumulative sum) cadence. Root cause is metrics-only; traces/logs show just mild latency + generic 503s. Exercises all five metric types plus discovery/group-by (healthy JVM twin on search-service, pool-split heap gauge, 3 out-of-phase pods). Distractors: coincidental deploy 15 min after leak onset, correlated-but-symptomatic thread-count gauge, flat CPU, healthy JVM twin, stable neighbor Summary.', + 'recommendation-service JVM heap leak behind a blinded prompt (storefront symptom at frontend-proxy): per-pod/per-pool memory-gauge sawtooth -> GC-pause (exp-histogram) tail -> latency-histogram shift -> staggered pod-restart (cumulative sum) cadence. Root cause is metrics-only; traces/logs show just mild latency + generic 503s. Exercises all five metric types plus discovery/group-by (healthy JVM twin on search-service, pool-split heap gauge, 3 out-of-phase pods). Distractors: coincidental deploy 15 min after leak onset, correlated-but-symptomatic thread-count gauge, flat CPU, healthy JVM twin, and misleading client-side DB-timeout WARN logs blaming the shared product_catalog Postgres — exonerated only by the flat inventory-service db.client Summary on the same database (raw SQL required; summaries are not chartable).', buildSystemPrompt: ctx => buildInvestigationSystemPrompt( 'metric-saturation', @@ -941,6 +999,23 @@ export const metricSaturationScenario: Scenario = { } } + // ── Planted misleading DB-timeout lines on the subject (fixed volume) ─ + // Client-side artifacts of GC stalls: they cluster with heap pressure + // and blame the shared product_catalog Postgres. The database is healthy + // — inventory-service's db.client Summary on the same DB stays flat — so + // these lines are the trap the summary metric exists to disarm. + for (let i = 0; i < scrapeCount; i++) { + const t = windowStart + i * SCRAPE_INTERVAL_MS; + const pressure = heapPressure(nowMs, t); + if (pressure < 0.05 || rng.next() >= pressure * 0.9) continue; + const n = rng.intRange(1, 4); + for (let k = 0; k < n; k++) { + logs.push( + featureStoreTimeoutLog(rng, t + rng.intRange(0, SCRAPE_INTERVAL_MS)), + ); + } + } + // ── Restart-adjacent hint logs (one per restart event, ambiguous) ──── for (let pod = 0; pod < POD_COUNT; pod++) { const leakStart = podLeakStartMs(nowMs, pod); diff --git a/packages/hdx-eval/src/scenarios/metric-saturation/ground-truth.json b/packages/hdx-eval/src/scenarios/metric-saturation/ground-truth.json index df51b09b42..8c83bdc6db 100644 --- a/packages/hdx-eval/src/scenarios/metric-saturation/ground-truth.json +++ b/packages/hdx-eval/src/scenarios/metric-saturation/ground-truth.json @@ -1,6 +1,6 @@ { "scenario": "metric-saturation", - "agentPrompt": "Shoppers are intermittently seeing slow or missing product recommendations on the storefront. Figure out which service is responsible and what the root cause is, and explain the mechanism end to end.", + "agentPrompt": "Shoppers are intermittently seeing slow or missing product recommendations on the storefront. An engineer who skimmed the logs earlier suspects database timeouts are to blame. Figure out which service is responsible and what the root cause is, confirming or ruling out the database on actual evidence, and explain the mechanism end to end.", "expected": { "affectedService": "recommendation-service", "rootCause": "A slow JVM heap memory leak (an unbounded candidate/embedding cache) on recommendation-service. On each of its 3 pods the G1 Old Gen heap climbs steadily from ~600MB toward the ~1950MB limit; full-GC pause time develops a heavy tail that stalls request handling, and each pod OOM-restarts on a ~22-minute cadence (staggered ~7 min apart, so the service never fully goes down — requests intermittently stall or 503). After each restart heap drops back to baseline and the cycle repeats. The leak began ~90 minutes ago — about 15 minutes BEFORE the coincidental 1.43.1 deploy.", @@ -9,7 +9,8 @@ "jvm.gc.pause (exponential histogram) develops a heavy tail (full GCs of 400-2200ms) as heap fills — the mechanism linking memory pressure to latency. Pre-incident pauses were tight and small (young-gen, ~5-25ms). The identically-named metric on search-service stays healthy throughout.", "http.server.request.duration (histogram) mass shifts from the low buckets into the 500-5000ms buckets during the incident window on recommendation-service (frontend-proxy's histogram stays healthy).", "k8s.pod.restarts (cumulative sum, per pod) steps up repeatedly on all 3 pods at staggered times; its rate/delta reveals the restart cadence. jvm.gc.collections{G1 Old Generation} accelerates over the same window.", - "jvm.threads.count (gauge) rises with each fill cycle — but it is a downstream symptom (threads pile up while requests stall in full GCs), not the cause." + "jvm.threads.count (gauge) rises with each fill cycle — but it is a downstream symptom (threads pile up while requests stall in full GCs), not the cause.", + "During GC stalls the subject's client-side reads from the shared product_catalog Postgres exceed their deadlines, emitting misleading 'Feature-store query timed out' WARN logs (the source of the reporter's database suspicion). db.client.operation.duration (summary) on inventory-service, a healthy client of the SAME database, keeps flat precomputed quantiles (p50 ~8ms / p95 ~40ms / p99 ~90ms) across the entire window — strong cross-client evidence that the database itself is healthy. On its own it cannot rule out issues specific to the subject's queries or connection pool, but the timeouts coincide exactly with the subject's GC stall windows and reset with each restart, so together the evidence points to a client-side artifact of the stalled JVM. Summary metrics are not supported by the chart tools, so reading these quantiles requires raw SQL against the summary table." ], "loadBearingSignal": "metrics", "timeline": { @@ -18,13 +19,13 @@ "restartCadenceMinutes": 22, "podStaggerMinutes": 7 }, - "whyTracesLogsInsufficient": "Traces show only mildly elevated latency and a few generic 503s ('context deadline exceeded') on recommendation-service; they carry no memory/GC/heap attributes. Proxy access logs name recommendation-service as the failing upstream (localization) but say nothing about why. Logs carry only ambiguous restart lines (liveness probe failed, SIGTERM) with no usable memory signal. The climbing-heap / GC-pause-tail / restart-cadence chain is only reconstructable from metrics — and only by filtering to the G1 Old Gen pool and grouping by pod.", + "whyTracesLogsInsufficient": "Traces show only mildly elevated latency and a few generic 503s ('context deadline exceeded') on recommendation-service; they carry no memory/GC/heap attributes. Proxy access logs name recommendation-service as the failing upstream (localization) but say nothing about why. Logs carry only ambiguous restart lines (liveness probe failed, SIGTERM) with no usable memory signal — and they actively mislead: during stall windows the subject logs 'Feature-store query timed out' WARN lines blaming the shared product_catalog Postgres, so a logs-first investigation converges on the database. The only direct database-health evidence is the inventory-service db.client.operation.duration summary (same database, flat quantiles all window), readable only via raw SQL. The climbing-heap / GC-pause-tail / restart-cadence chain is only reconstructable from metrics — and only by filtering to the G1 Old Gen pool and grouping by pod.", "metricTypesUsed": { "gauge": "process.runtime.jvm.memory.used per pod x pool (Old Gen leaks — load-bearing; other pools healthy); jvm.threads.count (correlated symptom distractor); system.cpu.utilization (flat, rules out CPU); process.memory.usage on inventory-service (flat decoy)", "sum": "k8s.pod.restarts per pod + jvm.gc.collections (cumulative; need a rate/delta); http.server.request.count on frontend-proxy (healthy)", "histogram": "http.server.request.duration (latency shift on recommendation-service, delta temporality; healthy twin series on frontend-proxy)", "exponentialHistogram": "jvm.gc.pause (heavy tail on recommendation-service, delta temporality; healthy twin series on search-service)", - "summary": "db.client.operation.duration on inventory-service (stable distractor)" + "summary": "db.client.operation.duration on inventory-service (healthy client of the same product_catalog Postgres): flat p50/p95/p99 across the window is the cross-client evidence against the misleading client-side timeout logs — load-bearing for calibration, and only readable via raw SQL because summaries are not supported by the chart tools" } }, "distractors": [ @@ -49,10 +50,10 @@ "whyNotRootCause": "search-service emits the exact same JVM metric names, so a keyword hunt for 'jvm' or 'memory' returns two services. Its heap pools are flat, its GC pauses stay small, and its counters tick at baseline all window — it is healthy. Isolating the leaker requires grouping or filtering by service rather than eyeballing metric names." }, { - "kind": "neighbor db.client latency summary", - "service": "inventory-service", - "metric": "db.client.operation.duration (summary)", - "whyNotRootCause": "A Summary metric with stable precomputed quantiles (p50/p95/p99) on a DIFFERENT service. It looks like 'a database is a bit slow', but the quantiles are flat across the whole window and uncorrelated with the recommendation-service incident. Crucially, a Summary cannot be re-aggregated or re-bucketed — you cannot zoom into the incident window to recompute it — so it is not diagnostic here. Blaming the database is a calibration failure." + "kind": "misleading client-side DB-timeout logs", + "service": "recommendation-service", + "metric": "db.client.operation.duration (summary, inventory-service, same product_catalog Postgres)", + "whyNotRootCause": "During GC stalls the subject emits 'Feature-store query timed out' WARN lines blaming the shared product_catalog Postgres, and the prompt's reporter repeats that suspicion — so the database is the most tempting wrong answer. But the database is healthy: inventory-service's db.client.operation.duration Summary against the SAME database keeps flat precomputed quantiles (~8/~40/~90ms) across the whole window, and the subject's timeouts coincide exactly with its own GC stall windows. The timeouts are client-side artifacts of the stalled JVM (a paused client cannot service its own deadline timers). The Summary is cross-service evidence — it cannot by itself rule out subject-specific queries or connection-pool issues, which is why the GC-stall timing alignment matters — and reading its quantiles requires raw SQL, because summary metrics cannot be charted by the query tools. Blaming the database without checking is a calibration failure." }, { "kind": "flat CPU", @@ -99,6 +100,11 @@ 2, "(gauge|histogram|metric|jvm[\\s._]|process[\\s._]runtime|memory[\\s._]used|pod[\\s._]restarts?|gc[\\s._]?pause|gc[\\s._]?collection|\\bheap\\b|old[\\s._-]?gen|\\bg1\\b|garbage[\\s-]?collect|restart\\s+(counter|cadence|storm|loop))" ], + [ + "exonerates_database", + 2, + "(database|\\bdb\\b|postgres\\w*|product_catalog|feature[ -]?store)[^.\\n]{0,80}?(healthy|flat|stable|normal|fine|ruled out|not (the|a) (cause|problem|issue|bottleneck)|exonerat\\w*|unaffected|red herring|client[- ]side)|(rul\\w+ out|exonerat\\w*|not caused by)[^.\\n]{0,40}?(the )?(database|\\bdb\\b|postgres\\w*)" + ], [ "false_blame_cpu", 2, @@ -114,7 +120,7 @@ [ "false_blame_database", 2, - "(root cause|caused by|due to|because of|the (issue|problem|cause) is)[^.\\n]{0,60}?(database|db\\b|inventory-service|db\\.client)", + "(root cause|caused by|due to|because of|the (issue|problem|cause) is)(?![^.\\n]{0,80}?(\\bnot\\b|n't\\b|rather than|red herring|however|\\bbut\\b|\\bhealthy|ruled out|exonerat))[^.\\n]{0,60}?(database|db\\b|inventory-service|db\\.client)", true ], [ @@ -158,6 +164,17 @@ "weight": 1, "metrics": ["process.runtime.jvm.memory.used"], "alsoPattern": "pool|pod" + }, + { + "id": "discovered_summary_metric", + "weight": 1, + "metrics": ["db.client.operation.duration"] + }, + { + "id": "queried_summary_via_sql", + "weight": 1, + "metrics": ["db.client.operation.duration"], + "alsoPattern": "\\bSELECT\\b|ValueAtQuantiles" } ], "judge": { @@ -170,12 +187,12 @@ { "id": "metric_grounding", "weight": 3, - "description": "Is the conclusion grounded in metric evidence that is NOT recoverable from traces/logs alone, at the right granularity? A 5 requires pool-level (G1 Old Gen vs other pools) and/or pod-level (per-pod sawtooth, staggered restarts) evidence — not just a service-wide average — plus at least two corroborating signals from: the GC-pause distribution tail, the latency histogram shift, the cumulative restart counter read as a rate, or Old-Gen GC-collection acceleration. An answer that only cites trace latency / 503 counts, or only an un-grouped service-level memory average, has not used the load-bearing signal." + "description": "Is the conclusion grounded in metric evidence that is NOT recoverable from traces/logs alone, at the right granularity? A 5 requires pool-level (G1 Old Gen vs other pools) and/or pod-level (per-pod sawtooth, staggered restarts) evidence — not just a service-wide average — plus at least two corroborating signals from: the GC-pause distribution tail, the latency histogram shift, the cumulative restart counter read as a rate, Old-Gen GC-collection acceleration, or the flat db.client.operation.duration summary quantiles used to exonerate the database. An answer that only cites trace latency / 503 counts, or only an un-grouped service-level memory average, has not used the load-bearing signal." }, { "id": "calibration", "weight": 2, - "description": "Did the answer rule out the distractors? Specifically: (a) the coincidental 1.43.1 deploy ~75 min ago is not the cause because the heap was already climbing ~15 min before it — a 5 REQUIRES explicitly addressing the deploy; (b) jvm.threads.count rises with the incident but is a downstream symptom of GC stalls, not a thread leak; (c) CPU is flat, so it is not CPU exhaustion; (d) search-service emits the same JVM metrics but is healthy; (e) the inventory-service db.client Summary is stable, unrelated, and cannot be re-aggregated into the incident window; (f) the 503s are a symptom, not the cause. Blaming any of these is a calibration failure; a 5 requires the deploy plus at least two others explicitly addressed." + "description": "Did the answer rule out the distractors? Specifically: (a) the coincidental 1.43.1 deploy ~75 min ago is not the cause because the heap was already climbing ~15 min before it — a 5 REQUIRES explicitly addressing the deploy; (b) jvm.threads.count rises with the incident but is a downstream symptom of GC stalls, not a thread leak; (c) CPU is flat, so it is not CPU exhaustion; (d) search-service emits the same JVM metrics but is healthy; (e) the 'Feature-store query timed out' WARN logs (and the reporter's suspicion) blame the shared product_catalog Postgres, but inventory-service's db.client.operation.duration Summary on the SAME database shows flat quantiles across the whole window and the timeouts coincide exactly with the subject's GC stalls — the timeouts are client-side artifacts, and exonerating the database requires citing this summary-quantile evidence, not merely asserting it. An answer that additionally notes the Summary is cross-service evidence (it cannot alone rule out subject-specific queries or connection pools) while grounding the exoneration in the GC-stall timing alignment is BETTER calibrated, not worse; (f) the 503s are a symptom, not the cause. Blaming any of these is a calibration failure; a 5 requires the deploy AND the database exoneration (with the summary evidence) plus at least one other explicitly addressed." }, { "id": "conciseness",