Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/mean-spoons-poke.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
"@hyperdx/hdx-eval": patch
"@hyperdx/api": patch
---

feat: expose summary metrics through the mcp
21 changes: 21 additions & 0 deletions packages/api/src/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,27 @@ export const bulkInsertMetricsHistogram = async (
);
};

export const bulkInsertMetricsSummary = async (
metrics: {
MetricName: string;
ResourceAttributes: Record<string, string>;
ScopeAttributes?: Record<string, string>;
Attributes?: Record<string, string>;
ServiceName?: string;
TimeUnix: Date;
Count?: number;
Sum?: number;
}[],
) => {
if (!config.IS_CI) {
throw new Error('ONLY execute this in CI env 😈 !!!');
}
await bulkInsertData(
`${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.SUMMARY}`,
metrics,
);
};

type ExponentialHistogramMetricPoint = {
TimeUnix: Date;
ServiceName?: string;
Expand Down
17 changes: 11 additions & 6 deletions packages/api/src/mcp/__tests__/listMetricsCursor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,16 @@ describe('listMetrics cursor', () => {
expect(decodeCursor(encodeCursor(payload))).toEqual(payload);
});

it('round-trips a summary cursor', () => {
// Summary is discoverable (listed) even though it is not queryable
// by the builder tools.
const payload = {
kind: 'summary' as const,
lastName: 'http_request_duration_seconds',
};
expect(decodeCursor(encodeCursor(payload))).toEqual(payload);
});

it('round-trips metric names with dots, dashes, and unicode', () => {
const payload = {
kind: 'gauge' as const,
Expand Down Expand Up @@ -74,12 +84,7 @@ describe('listMetrics cursor', () => {
expect(decodeCursor(malformed)).toBeNull();
});

it('returns null when kind is not a queryable metric kind', () => {
const summaryCursor = Buffer.from(
JSON.stringify({ kind: 'summary', lastName: 'x' }),
).toString('base64');
expect(decodeCursor(summaryCursor)).toBeNull();

it('returns null when kind is not a discoverable metric kind', () => {
const bogusCursor = Buffer.from(
JSON.stringify({ kind: 'bogus', lastName: 'x' }),
).toString('base64');
Expand Down
20 changes: 19 additions & 1 deletion packages/api/src/mcp/__tests__/metricKinds.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,22 @@
import { sanitizeMetricTables } from '@/mcp/tools/sources/metricKinds';
import {
DISCOVERABLE_METRIC_KINDS,
QUERYABLE_METRIC_KINDS,
sanitizeMetricTables,
} from '@/mcp/tools/sources/metricKinds';

describe('DISCOVERABLE_METRIC_KINDS', () => {
it('is the queryable kinds plus summary, with summary last', () => {
// Discovery tools list summary metrics, but the query renderer cannot
// translate them — the query/dashboard schemas must stay on the
// queryable list. Summary is last so paginated scans hit it after
// every queryable kind.
expect(DISCOVERABLE_METRIC_KINDS).toEqual([
...QUERYABLE_METRIC_KINDS,
'summary',
]);
expect(QUERYABLE_METRIC_KINDS).not.toContain('summary');
});
});

describe('sanitizeMetricTables', () => {
it('returns undefined for null / undefined input', () => {
Expand Down
96 changes: 96 additions & 0 deletions packages/api/src/mcp/__tests__/sources.int.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
bulkInsertMetricsGauge,
bulkInsertMetricsHistogram,
bulkInsertMetricsSum,
bulkInsertMetricsSummary,
DEFAULT_DATABASE,
DEFAULT_LOGS_TABLE,
DEFAULT_METRICS_TABLE,
Expand Down Expand Up @@ -429,6 +430,101 @@ describe('MCP Source Tools', () => {
);
});

it('discovers summary metrics and directs querying to clickstack_sql', async () => {
// Summary metrics are discoverable (list/describe) but not queryable:
// the query renderer has no summary translation, so every discovery
// surface must redirect the agent to clickstack_sql.
const metricSource = await Source.create({
kind: SourceKind.Metric,
team: team._id,
from: { databaseName: DEFAULT_DATABASE, tableName: '' },
metricTables: {
[MetricsDataType.Gauge.toLowerCase()]: DEFAULT_METRICS_TABLE.GAUGE,
[MetricsDataType.Summary.toLowerCase()]:
DEFAULT_METRICS_TABLE.SUMMARY,
},
timestampValueExpression: 'TimeUnix',
connection: connection._id,
name: 'Metrics With Summary',
});
const now = new Date();
await bulkInsertMetricsSummary([
{
MetricName: 'http_request_duration_seconds',
ResourceAttributes: { 'service.name': 'svc-a' },
Attributes: { endpoint: '/api/v1/users' },
ServiceName: 'svc-a',
TimeUnix: now,
Count: 10,
Sum: 1.5,
},
]);

const describeSourceResult = await callTool(
client,
'clickstack_describe_source',
{ sourceId: metricSource._id.toString() },
);
expect(describeSourceResult.isError).toBeFalsy();
const describedSource = JSON.parse(getFirstText(describeSourceResult));
// The summary table is listed and its metric names are sampled, but
// the representative discovery table stays a queryable kind.
expect(describedSource.source.metricTables.summary).toBe(
DEFAULT_METRICS_TABLE.SUMMARY,
);
expect(describedSource.source.discoveryMetricKind).toBe('gauge');
expect(describedSource.source.metricNames.summary).toEqual(
expect.arrayContaining([
expect.objectContaining({ name: 'http_request_duration_seconds' }),
]),
);
expect(describedSource.usage.metricNames).toContain('clickstack_sql');

const listResult = await callTool(client, 'clickstack_list_metrics', {
sourceId: metricSource._id.toString(),
kind: 'summary',
});
expect(listResult.isError).toBeFalsy();
const listedMetrics = JSON.parse(getFirstText(listResult));
expect(listedMetrics.metrics).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'http_request_duration_seconds',
kind: 'summary',
}),
]),
);
expect(listedMetrics.summaryNote).toContain('clickstack_sql');

const describeMetricResult = await callTool(
client,
'clickstack_describe_metric',
{
sourceId: metricSource._id.toString(),
metricName: 'http_request_duration_seconds',
kind: 'summary',
},
);
expect(describeMetricResult.isError).toBeFalsy();
const describedMetric = JSON.parse(getFirstText(describeMetricResult));
expect(describedMetric.kinds[0]).toMatchObject({ kind: 'summary' });
// Attribute discovery is table-generic and must work on the summary
// table like any other kind.
expect(describedMetric.kinds[0].attributeKeys.Attributes).toContain(
'endpoint',
);
expect(describedMetric.kinds[0].usage).toContain('clickstack_sql');
expect(describedMetric.nextSteps.query).toContain('clickstack_sql');
// The redirect must name the database-qualified table — clickstack_sql
// has no source context to resolve a bare table name against.
expect(describedMetric.nextSteps.query).toContain(
`${DEFAULT_DATABASE}.${DEFAULT_METRICS_TABLE.SUMMARY}`,
);
expect(describedMetric.nextSteps.query).not.toContain(
'clickstack_timeseries({',
);
});

it('should include usage guidance and nextSteps', async () => {
const result = await callTool(client, 'clickstack_describe_source', {
sourceId: traceSource._id.toString(),
Expand Down
2 changes: 1 addition & 1 deletion packages/api/src/mcp/prompts/dashboards/content.ts
Original file line number Diff line number Diff line change
Expand Up @@ -964,7 +964,7 @@ For configType: "sql" tiles, write ClickHouse SQL with template macros:

== METRIC SOURCES ==

Builder tiles work on metric sources. Each select item on a metric tile MUST set metricType ("gauge" | "sum" | "histogram" | "exponential histogram") and metricName (the OTel metric name, e.g. "system.cpu.utilization"). valueExpression defaults to "Value" when omitted, so a typical metric series is { aggFn: "<fn>", metricType: "<kind>", metricName: "<name>" }. summary metrics are not supported by the renderer.
Builder tiles work on metric sources. Each select item on a metric tile MUST set metricType ("gauge" | "sum" | "histogram" | "exponential histogram") and metricName (the OTel metric name, e.g. "system.cpu.utilization"). valueExpression defaults to "Value" when omitted, so a typical metric series is { aggFn: "<fn>", metricType: "<kind>", metricName: "<name>" }. summary metrics are not supported by the renderer; to chart them, use a raw SQL tile (configType: "sql") against the table named in the source's metricTables.summary.

Per-kind aggregation guidance:
gauge Use aggFn:"last_value" | "avg" | "min" | "max". Set isDelta:true for Prometheus-style delta over each bucket.
Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/mcp/tools/dashboards/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,8 @@ const mcpTileSelectItemSchema = z
.describe(
'METRIC SOURCES ONLY. OTel metric kind: gauge, sum, histogram, or exponential histogram. ' +
'Required (with metricName) when the tile sourceId is a metric source. ' +
'summary is not supported by the renderer.',
'summary is not supported by the renderer — chart summary metrics with a raw ' +
'SQL tile (configType:"sql") against the table in the source\'s metricTables.summary.',
),
metricName: z
.string()
Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/mcp/tools/query/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ const mcpMetricTypeSchema = z
' sum – cumulative or delta counters (request counts, bytes processed). Use aggFn:"increase" for counter increase, or sum/avg on the computed rate.\n' +
' histogram – bucketed distributions (request duration). Use aggFn:"quantile" with level for percentiles, or aggFn:"count" for total bucket count.\n' +
' exponential histogram – exponential-bucket distributions. Use aggFn:"quantile" with level for percentiles, or aggFn:"count" for total bucket count.\n' +
'NOTE: summary is not supported by the query renderer.',
'NOTE: summary is not supported by the query renderer — to look at summary ' +
"metrics, use clickstack_sql against the table in the source's metricTables.summary.",
);

/**
Expand Down
4 changes: 3 additions & 1 deletion packages/api/src/mcp/tools/query/sql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,9 @@ export function registerSql({ context, registerTool }: ToolRegistrar) {
description:
'Execute raw ClickHouse SQL. ' +
'ADVANCED: only use this when you need capabilities the builder tools cannot express — ' +
'JOINs, sub-queries, CTEs, or querying tables not registered as sources.\n\n' +
'JOINs, sub-queries, CTEs, querying tables not registered as sources, or looking at ' +
'summary-type metrics (the metricTables.summary table on a metric source, which the ' +
'builder tools cannot query).\n\n' +
Comment on lines +67 to +69

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just in case you haven't seen it, there is also this PR that is intended to discourage agents from using raw SQL: #2840

Does adding this here (or the clickstack_sql elsewhere) have any effect on that goal / do we need to run any evals to measure the effect? cc @brandon-pereira

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm also curious, does the agent actually generate reasonably accurate queries for summary metrics? The metrics queries tend to be very complicated.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does the agent actually generate reasonably accurate queries for summary metrics?

I think the queries are not completely accurate. Here's a couple of examples on separate runs of metric-saturation with Fable 5:

SELECT   tostartofinterval(timeunix, interval 10 minute ) AS ts,
         resourceattributes['service.name']               AS svc,
         avg (sum/count),
         max (arrayelement(valueatquantiles.value, length(valueatquantiles.value))),
         sum (count)
FROM     DEFAULT.eval_metric_saturation_otel_metrics_summary
WHERE    metricname = 'db.client.operation.duration'
AND      $__timefilter(timeunix)
GROUP BY ts,
         svc
SELECT   tostartofinterval(timeunix, interval 10 minute )                                      AS ts,
         resourceattributes['service.name']                                                    AS svc,
         max (arrayelement(valueatquantiles.value, indexof(valueatquantiles.quantile, 0.99 ))) AS p99_ms,
         max (arrayelement(valueatquantiles.value, indexof(valueatquantiles.quantile, 0.5 )))  AS p50_ms,
         sum (count)                                                                           AS ops
FROM     DEFAULT.eval_metric_saturation_otel_metrics_summary
WHERE    metricname = 'db.client.operation.duration'
AND      $__timefilter(timeunix)
GROUP BY ts,
         svc
ORDER BY ts,
         svc limit 100

I'm no expert on summary metrics but according to Claude there are two common errors in these:

  1. sum(Count) treating a cumulative counter as a per-interval value. Neither query computes per-series deltas, which is what's actually needed.
  2. Aggregating summary quantiles across series. Query 1 does max(last quantile), query 2 does max(p99) and max(p50). Both give you "worst per-series value," not a true service-wide percentile.

'Requires connectionId (not sourceId) — call clickstack_list_sources to find connections. ' +
'Call clickstack_describe_source to discover column names before writing SQL.\n\n' +
'Results are always returned as table rows — for time-series semantics, ' +
Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/mcp/tools/query/table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ export function registerTable({ context, registerTool }: ToolRegistrar) {
'Per kind: gauge uses last_value/avg/min/max; sum uses aggFn:"increase" for counter increase ' +
'(top-N capped at 20 groups when combined with groupBy), or sum/avg on the rate; ' +
'histogram and exponential histogram use aggFn:"quantile" + level for percentiles, or aggFn:"count" for total bucket count.\n' +
'summary metrics are not supported by the query renderer.',
'summary metrics are not supported by the query renderer — query them with ' +
"clickstack_sql against the table in the source's metricTables.summary.",
inputSchema: tableSchema,
},
async input => {
Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/mcp/tools/query/timeseries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ export function registerTimeseries({ context, registerTool }: ToolRegistrar) {
'histogram and exponential histogram use aggFn:"quantile" + level for percentiles, or aggFn:"count" for total bucket count.\n' +
'TOP-N CAP: aggFn:"increase" + groupBy is capped at 20 groups by the renderer ' +
'(top by max bucket sum). Narrow with where/groupBy to see other groups.\n' +
'summary metrics are not supported by the query renderer.',
'summary metrics are not supported by the query renderer — query them with ' +
"clickstack_sql against the table in the source's metricTables.summary.",
inputSchema: timeseriesSchema,
},
async input => {
Expand Down
58 changes: 43 additions & 15 deletions packages/api/src/mcp/tools/sources/describeMetric.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ import logger from '@/utils/logger';
import { trimToolResponse } from '@/utils/trimToolResponse';

import {
QUERYABLE_METRIC_KINDS,
type QueryableMetricKind,
DISCOVERABLE_METRIC_KINDS,
type DiscoverableMetricKind,
} from './metricKinds';

const DEFAULT_LOOKBACK_MS = 24 * 60 * 60 * 1000;
Expand All @@ -45,14 +45,18 @@ const MAX_ATTR_KEYS_TO_SAMPLE = 12;
// Per-kind aggregation guidance baked into the response so the agent can
// build a valid clickstack_timeseries / clickstack_table call without
// re-reading the schemas.
const KIND_USAGE: Record<QueryableMetricKind, string> = {
const KIND_USAGE: Record<DiscoverableMetricKind, string> = {
gauge:
'Gauge: use aggFn:"last_value"|"avg"|"min"|"max" on Value. Set isDelta:true for Prometheus-style delta over each bucket.',
sum: 'Sum (counter): use aggFn:"increase" for the per-bucket counter increase (reset-aware), or aggFn:"sum"/"avg" on the computed rate. increase+groupBy is capped at the top 20 groups.',
histogram:
'Histogram: use aggFn:"quantile" with level ∈ {0.5, 0.9, 0.95, 0.99} for percentiles, or aggFn:"count" for the total bucket count.',
'exponential histogram':
'Exponential histogram: use aggFn:"quantile" with level ∈ {0.5, 0.9, 0.95, 0.99} for percentiles, or aggFn:"count" for the total bucket count.',
summary:
'Summary: client-precomputed quantiles (e.g. Prometheus summaries). Not supported by ' +
'clickstack_timeseries / clickstack_table — query this table with clickstack_sql ' +
"using the source's connectionId.",
};

// ─── Schema ──────────────────────────────────────────────────────────────────
Expand All @@ -71,9 +75,9 @@ const describeMetricSchema = z.object({
'Discover via clickstack_list_metrics or clickstack_describe_source.',
),
kind: z
.enum(QUERYABLE_METRIC_KINDS)
.enum(DISCOVERABLE_METRIC_KINDS)
.describe(
'Metric kind: "gauge" | "sum" | "histogram" | "exponential histogram". Required. ' +
'Metric kind: "gauge" | "sum" | "histogram" | "exponential histogram" | "summary". Required. ' +
'Discover via clickstack_list_metrics (which returns name + kind per entry) ' +
'or clickstack_describe_source (which groups metric-name samples by kind). ' +
'A metric name can legitimately live in more than one kind (e.g. ' +
Expand Down Expand Up @@ -111,7 +115,7 @@ type AttributeValuesMeta = {
};

type KindDetail = {
kind: QueryableMetricKind;
kind: DiscoverableMetricKind;
tableName: string;
unit?: string;
description?: string;
Expand Down Expand Up @@ -641,13 +645,33 @@ async function describeMetricImpl(
!meta.unit &&
!meta.description;

const queryExample = `clickstack_timeseries({ sourceId: "${input.sourceId}", select: [{ aggFn: ${
kind === 'sum'
? '"increase"'
: kind === 'histogram' || kind === 'exponential histogram'
? '"quantile", level: 0.95'
: '"avg"'
}, metricType: "${kind}", metricName: "${input.metricName}" }] })`;
const makeNextStepsQuery = () => {
if (kind === 'summary') {
// Summary metrics have no renderer support, so the next step is
// clickstack_sql rather than a builder-tool example.
// clickstack_sql runs with no source context, so qualify the table
// with the source's database — a bare table name would resolve
// against the connection user's default database.
const qualifiedTable = databaseName
? `${databaseName}.${tableName}`
: tableName;
return (
'summary metrics cannot be queried with clickstack_timeseries / clickstack_table — ' +
`use clickstack_sql against the "${qualifiedTable}" table with this source's connectionId ` +
'(see clickstack_list_sources).'
);
}

let aggFn = '"avg"';

if (kind === 'sum') {
aggFn = '"increase"';
} else if (kind === 'histogram' || kind === 'exponential histogram') {
aggFn = '"quantile", level: 0.95';
}

return `Example: clickstack_timeseries({ sourceId: "${input.sourceId}", select: [{ aggFn: ${aggFn}, metricType: "${kind}", metricName: "${input.metricName}" }] })`;
};

const responseObj: Record<string, unknown> = {
metricName: input.metricName,
Expand All @@ -666,7 +690,7 @@ async function describeMetricImpl(
'confirm the metric name + kind combination exists.',
}),
nextSteps: {
query: `Example: ${queryExample}`,
query: makeNextStepsQuery(),
},
};

Expand Down Expand Up @@ -708,10 +732,14 @@ export function registerDescribeMetric({
'sample) to get attribute keys, sampled values, unit, and description for a ' +
'specific (metricName, kind) pair. Attribute keys vary per metric — not per source — ' +
"so always call this before clickstack_timeseries / clickstack_table for any metric you've never queried.\n\n" +
'REQUIRES `kind` — pass the gauge/sum/histogram/exponential histogram value emitted alongside the metric name by ' +
'REQUIRES `kind` — pass the gauge/sum/histogram/exponential histogram/summary value emitted alongside the metric name by ' +
'clickstack_list_metrics or clickstack_describe_source. A metric name can legitimately ' +
'live in more than one kind (e.g. "container.cpu.usage" appears in both gauge and sum); ' +
'call this tool once per kind you care about.\n\n' +
'kind:"summary" is accepted for discovery (attribute keys, sampled values, unit, ' +
'description), but summary metrics cannot be queried with clickstack_timeseries / ' +
"clickstack_table — use clickstack_sql against the table in the source's " +
'metricTables.summary.\n\n' +
'attributeValuesMeta on each kind reports sampledKeys (queried for values) and ' +
'truncatedKeys (skipped by the per-call sampling cap) — a key in truncatedKeys was ' +
'never queried, so query it directly if you need its values.\n\n' +
Expand Down
Loading
Loading