diff --git a/.changeset/describe-source-tool.md b/.changeset/describe-source-tool.md new file mode 100644 index 0000000000..36a8669b66 --- /dev/null +++ b/.changeset/describe-source-tool.md @@ -0,0 +1,17 @@ +--- +'@hyperdx/api': patch +--- + +feat(mcp): add hyperdx_describe_source tool and slim list_sources to catalog + +Add `hyperdx_describe_source` — returns full column schema, map attribute +keys, and sampled low-cardinality values (SeverityText, StatusCode, +ServiceName, etc.) for a single source. Uses existing rollup tables for +performant value sampling. + +Slim `hyperdx_list_sources` to a lightweight MongoDB-only catalog (no +ClickHouse queries). Source tools moved to a dedicated `tools/sources/` +module. + +All query tool descriptions and prompts updated to reference the two-step +`list_sources → describe_source` discovery workflow. diff --git a/MCP.md b/MCP.md index 9f702d43a6..299fd091b2 100644 --- a/MCP.md +++ b/MCP.md @@ -108,7 +108,8 @@ with: | Tool | Description | | ----------------------------- | -------------------------------------------------------------------------------------------- | -| `hyperdx_list_sources` | List all data sources and database connections, including column schemas and attribute keys | +| `hyperdx_list_sources` | List all data sources and connections as a lightweight catalog (IDs, names, kinds) | +| `hyperdx_describe_source` | Full column schema, attribute keys, and sampled low-cardinality values for a single source | | `hyperdx_timeseries` | Plot metrics over time as a line or stacked bar chart | | `hyperdx_table` | Compute aggregated metrics as a table, single number, or pie chart | | `hyperdx_search` | Browse individual log, event, or trace rows | diff --git a/packages/api/src/mcp/__tests__/dashboards.test.ts b/packages/api/src/mcp/__tests__/dashboards.test.ts index f720a039bf..6ed04e2ff9 100644 --- a/packages/api/src/mcp/__tests__/dashboards.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards.test.ts @@ -76,65 +76,6 @@ describe('MCP Dashboard Tools', () => { await server.stop(); }); - describe('hyperdx_list_sources', () => { - it('should list available sources and connections', async () => { - const result = await callTool(client, 'hyperdx_list_sources'); - - expect(result.isError).toBeFalsy(); - expect(result.content).toHaveLength(1); - - const output = JSON.parse(getFirstText(result)); - expect(output.sources).toHaveLength(1); - expect(output.sources[0]).toMatchObject({ - id: traceSource._id.toString(), - name: 'Traces', - kind: SourceKind.Trace, - }); - - expect(output.connections).toHaveLength(1); - expect(output.connections[0]).toMatchObject({ - id: connection._id.toString(), - name: 'Default', - }); - - expect(output.usage).toBeDefined(); - }); - - it('should include column schema for sources', async () => { - const result = await callTool(client, 'hyperdx_list_sources'); - const output = JSON.parse(getFirstText(result)); - const source = output.sources[0]; - - expect(source.columns).toBeDefined(); - expect(Array.isArray(source.columns)).toBe(true); - expect(source.columns.length).toBeGreaterThan(0); - // Each column should have name, type, and jsType - expect(source.columns[0]).toHaveProperty('name'); - expect(source.columns[0]).toHaveProperty('type'); - expect(source.columns[0]).toHaveProperty('jsType'); - }); - - it('should return empty sources for a team with no sources', async () => { - // Clear everything and re-register with new team - await client.close(); - await server.clearDBs(); - const result2 = await getLoggedInAgent(server); - const context2: McpContext = { - teamId: result2.team._id.toString(), - userId: result2.user._id.toString(), - }; - const client2 = await createTestClient(context2); - - const result = await callTool(client2, 'hyperdx_list_sources'); - const output = JSON.parse(getFirstText(result)); - - expect(output.sources).toHaveLength(0); - expect(output.connections).toHaveLength(0); - - await client2.close(); - }); - }); - describe('hyperdx_get_dashboard', () => { it('should list all dashboards when no id provided', async () => { await new Dashboard({ diff --git a/packages/api/src/mcp/__tests__/sources.test.ts b/packages/api/src/mcp/__tests__/sources.test.ts new file mode 100644 index 0000000000..aea5ead6d7 --- /dev/null +++ b/packages/api/src/mcp/__tests__/sources.test.ts @@ -0,0 +1,341 @@ +import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; + +import * as config from '@/config'; +import { + DEFAULT_DATABASE, + DEFAULT_LOGS_TABLE, + DEFAULT_TRACES_TABLE, + getLoggedInAgent, + getServer, +} from '@/fixtures'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; +import Team from '@/models/team'; + +import { McpContext } from '../tools/types'; +import { callTool, createTestClient, getFirstText } from './mcpTestUtils'; + +describe('MCP Source Tools', () => { + const server = getServer(); + let team: any; + let user: any; + let traceSource: any; + let logSource: any; + let connection: any; + let client: Client; + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const result = await getLoggedInAgent(server); + team = result.team; + user = result.user; + + connection = await Connection.create({ + team: team._id, + name: 'Default', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + traceSource = await Source.create({ + kind: SourceKind.Trace, + team: team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_TRACES_TABLE, + }, + timestampValueExpression: 'Timestamp', + connection: connection._id, + name: 'Traces', + spanNameExpression: 'SpanName', + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + spanKindExpression: 'SpanKind', + statusCodeExpression: 'StatusCode', + serviceNameExpression: 'ServiceName', + }); + + logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_LOGS_TABLE, + }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: + 'Timestamp, ServiceName as service, SeverityText as level, Body', + bodyExpression: 'Body', + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + traceIdExpression: 'TraceId', + connection: connection._id, + name: 'Logs', + }); + + const context: McpContext = { + teamId: team._id.toString(), + userId: user._id.toString(), + }; + client = await createTestClient(context); + }); + + afterEach(async () => { + await client?.close(); + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + // ── hyperdx_list_sources ────────────────────────────────────────────────── + + describe('hyperdx_list_sources', () => { + it('should list sources and connections as a lightweight catalog', async () => { + const result = await callTool(client, 'hyperdx_list_sources'); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + + const output = JSON.parse(getFirstText(result)); + expect(output.sources).toHaveLength(2); + + const trace = output.sources.find( + (s: any) => s.kind === SourceKind.Trace, + ); + expect(trace).toMatchObject({ + id: traceSource._id.toString(), + name: 'Traces', + kind: SourceKind.Trace, + }); + + expect(output.connections).toHaveLength(1); + expect(output.connections[0]).toMatchObject({ + id: connection._id.toString(), + name: 'Default', + }); + + // Catalog should NOT include column schema or map keys + expect(trace.columns).toBeUndefined(); + expect(trace.mapAttributeKeys).toBeUndefined(); + + // Should include nextStep guidance pointing to describe_source + expect(output.nextStep).toBeDefined(); + expect(output.nextStep).toContain('hyperdx_describe_source'); + }); + + it('should include keyColumns from source config without ClickHouse queries', async () => { + const result = await callTool(client, 'hyperdx_list_sources'); + const output = JSON.parse(getFirstText(result)); + + const trace = output.sources.find( + (s: any) => s.kind === SourceKind.Trace, + ); + expect(trace.keyColumns).toBeDefined(); + expect(trace.keyColumns).toHaveProperty('spanName'); + expect(trace.keyColumns).toHaveProperty('duration'); + + const log = output.sources.find((s: any) => s.kind === SourceKind.Log); + expect(log.keyColumns).toBeDefined(); + expect(log.keyColumns).toHaveProperty('severityText'); + expect(log.keyColumns).toHaveProperty('body'); + }); + + it('should return empty sources for a team with no sources', async () => { + await client.close(); + await server.clearDBs(); + const result2 = await getLoggedInAgent(server); + const context2: McpContext = { + teamId: result2.team._id.toString(), + userId: result2.user._id.toString(), + }; + const client2 = await createTestClient(context2); + + const result = await callTool(client2, 'hyperdx_list_sources'); + const output = JSON.parse(getFirstText(result)); + + expect(output.sources).toHaveLength(0); + expect(output.connections).toHaveLength(0); + + await client2.close(); + }); + }); + + // ── hyperdx_describe_source ─────────────────────────────────────────────── + + describe('hyperdx_describe_source', () => { + it('should return full column schema for a trace source', async () => { + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: traceSource._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + expect(result.content).toHaveLength(1); + + const output = JSON.parse(getFirstText(result)); + expect(output.source).toMatchObject({ + id: traceSource._id.toString(), + name: 'Traces', + kind: SourceKind.Trace, + }); + + // Column schema from ClickHouse DESCRIBE TABLE + expect(output.source.columns).toBeDefined(); + expect(Array.isArray(output.source.columns)).toBe(true); + expect(output.source.columns.length).toBeGreaterThan(0); + expect(output.source.columns[0]).toHaveProperty('name'); + expect(output.source.columns[0]).toHaveProperty('type'); + expect(output.source.columns[0]).toHaveProperty('jsType'); + }); + + it('should return full column schema for a log source', async () => { + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: logSource._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + + expect(output.source).toMatchObject({ + id: logSource._id.toString(), + name: 'Logs', + kind: SourceKind.Log, + }); + + // Log source should have columns + expect(output.source.columns).toBeDefined(); + expect(output.source.columns.length).toBeGreaterThan(0); + + // Should expose log-specific keyColumns + expect(output.source.keyColumns).toMatchObject({ + body: 'Body', + severityText: 'SeverityText', + serviceName: 'ServiceName', + }); + }); + + it('should include map attribute keys', async () => { + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: traceSource._id.toString(), + }); + const output = JSON.parse(getFirstText(result)); + + // Trace sources have SpanAttributes and ResourceAttributes map columns. + // Map keys may be empty when the table has no data, but the field + // should still be a plain object (or absent). + if (output.source.mapAttributeKeys) { + expect(typeof output.source.mapAttributeKeys).toBe('object'); + } + }); + + it('should include keyColumns for trace source', async () => { + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: traceSource._id.toString(), + }); + const output = JSON.parse(getFirstText(result)); + + expect(output.source.keyColumns).toBeDefined(); + expect(output.source.keyColumns).toHaveProperty('spanName'); + expect(output.source.keyColumns).toHaveProperty('duration'); + expect(output.source.keyColumns).toHaveProperty('traceId'); + expect(output.source.keyColumns).toHaveProperty('spanId'); + }); + + it('should return metricTables for a metric source (no column schema)', async () => { + const metricSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: '', + }, + metricTables: { + [MetricsDataType.Gauge.toLowerCase()]: 'otel_metrics_gauge', + [MetricsDataType.Sum.toLowerCase()]: 'otel_metrics_sum', + [MetricsDataType.Histogram.toLowerCase()]: 'otel_metrics_histogram', + }, + timestampValueExpression: 'TimeUnix', + connection: connection._id, + name: 'Metrics', + }); + + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: metricSource._id.toString(), + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + + expect(output.source).toMatchObject({ + id: metricSource._id.toString(), + name: 'Metrics', + kind: SourceKind.Metric, + }); + + // Metric sources have metricTables but no column schema or value samples + expect(output.source.metricTables).toBeDefined(); + expect(output.source.metricTables).toHaveProperty('gauge'); + expect(output.source.columns).toBeUndefined(); + expect(output.source.lowCardinalityValues).toBeUndefined(); + expect(output.source.mapAttributeKeys).toBeUndefined(); + expect(output.source.mapAttributeValues).toBeUndefined(); + }); + + it('should include usage guidance and nextSteps', async () => { + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: traceSource._id.toString(), + }); + const output = JSON.parse(getFirstText(result)); + + expect(output.usage).toBeDefined(); + expect(output.nextSteps).toBeDefined(); + expect(output.nextSteps.query).toContain(traceSource._id.toString()); + }); + + it('should return error for non-existent source', async () => { + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: '000000000000000000000000', + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + + it('should not allow access to another team source', async () => { + const otherTeam = await Team.create({ name: 'Other Team' }); + const otherConnection = await Connection.create({ + team: otherTeam._id, + name: 'Other Connection', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const otherSource = await Source.create({ + kind: SourceKind.Trace, + team: otherTeam._id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_TRACES_TABLE, + }, + timestampValueExpression: 'Timestamp', + connection: otherConnection._id, + name: 'Other Traces', + }); + + const result = await callTool(client, 'hyperdx_describe_source', { + sourceId: otherSource._id.toString(), + }); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('not found'); + }); + }); +}); diff --git a/packages/api/src/mcp/mcpServer.ts b/packages/api/src/mcp/mcpServer.ts index 4eabd05938..3a6e8abaff 100644 --- a/packages/api/src/mcp/mcpServer.ts +++ b/packages/api/src/mcp/mcpServer.ts @@ -7,6 +7,7 @@ import alertsTools from './tools/alerts/index'; import dashboardsTools from './tools/dashboards/index'; import queryTools from './tools/query/index'; import savedSearchesTools from './tools/savedSearches/index'; +import sourcesTools from './tools/sources/index'; import { McpContext } from './tools/types'; export function createServer(context: McpContext) { @@ -15,6 +16,7 @@ export function createServer(context: McpContext) { version: `${CODE_VERSION}-beta`, }); + sourcesTools(server, context); alertsTools(server, context); dashboardsTools(server, context); queryTools(server, context); diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 14439c9110..28bb45061b 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -17,16 +17,17 @@ export function buildCreateDashboardPrompt( ${userContext} ${sourceSummary} -IMPORTANT: Call hyperdx_list_sources first to get the full column schema and attribute keys for each source. The source IDs above are correct, but you need the schema details to write accurate queries. +IMPORTANT: Call hyperdx_list_sources first to get source IDs, then hyperdx_describe_source for each source you plan to query. This gives you the full column schema, attribute keys, and sampled values (e.g. SeverityText, StatusCode). The source IDs above are correct, but you need the schema details to write accurate queries. == WORKFLOW == -1. Call hyperdx_list_sources to discover source IDs, column schemas, and attribute keys. -2. Call hyperdx_get_dashboard (no id) to list existing dashboards. If any exist, fetch one or two with their id and skim them to pick up local idioms (naming style, time ranges, common groupings) before adding new ones. Skip when the workspace is empty. -3. Pick the right source kind for the question. Trace data for request volume / latency / errors. Log data for severity / messages. Metric data for system telemetry. -4. Sketch tiles, THEN group them into 2-4 containers (Overview / Trends / Errors is a sane default for a service-health pattern) before assembling the save payload. Ungrouped tiles render as a flat sprawl that fails the readability test at five or more tiles. -5. Call hyperdx_save_dashboard with the tiles, containers, and dashboard-level filters. -6. Call hyperdx_query_tile on EVERY tile (not just one) to confirm queries return data and the dashboard is not silently degraded. +1. Call hyperdx_list_sources to discover source IDs and connection IDs. +2. Call hyperdx_describe_source for each source you plan to query to get column schema, attribute keys, and sampled low-cardinality values (SeverityText, StatusCode, ServiceName, etc.). +3. Call hyperdx_get_dashboard (no id) to list existing dashboards. If any exist, fetch one or two with their id and skim them to pick up local idioms (naming style, time ranges, common groupings) before adding new ones. Skip when the workspace is empty. +4. Pick the right source kind for the question. Trace data for request volume / latency / errors. Log data for severity / messages. Metric data for system telemetry. +5. Sketch tiles, THEN group them into 2-4 containers (Overview / Trends / Errors is a sane default for a service-health pattern) before assembling the save payload. Ungrouped tiles render as a flat sprawl that fails the readability test at five or more tiles. +6. Call hyperdx_save_dashboard with the tiles, containers, and dashboard-level filters. +7. Call hyperdx_query_tile on EVERY tile (not just one) to confirm queries return data and the dashboard is not silently degraded. == UPDATING AN EXISTING DASHBOARD == @@ -71,11 +72,11 @@ Use RAW SQL tiles (with connectionId) only for queries the builder cannot expres - Top-level columns use PascalCase by default: Duration, StatusCode, SpanName, Body, SeverityText, ServiceName, SpanKind. NOTE: These are defaults for the standard HyperDX schema. Custom sources may use different names. - Always call hyperdx_list_sources to get the real column names and keyColumns for each source. + Always call hyperdx_describe_source to get the real column names, keyColumns, and sampled values for each source. - Map-type columns use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']. NEVER use dot notation for Map columns. Always use brackets. - JSON-type columns use dot notation: JsonColumn.key.subkey. - Check the jsType returned by hyperdx_list_sources to decide which a column is. + Check the jsType returned by hyperdx_describe_source to determine whether a column is Map or JSON. == DESIGN CHECKLIST == @@ -128,6 +129,11 @@ Apply these before calling hyperdx_save_dashboard. Each rule is enforced by the The dashboard_examples prompt returns concrete example shapes. Read them as patterns to adapt to the user's actual request and schema, not as literal templates to substitute names into. Specifically: the literal "ServiceName", "StatusCode:STATUS_CODE_ERROR", "Duration" come from the standard HyperDX schema. Real source schemas may use different column names and status values; always verify with hyperdx_list_sources before reusing literals. +IMPORTANT: The exact values for StatusCode and SeverityText vary by deployment. +Do NOT assume values like "STATUS_CODE_ERROR", "Ok", "error", or "fatal". +Always call hyperdx_describe_source first and inspect the lowCardinalityValues +returned for each source to discover the real values used in your data. + == DEFAULT TIME WINDOW == Dashboards open with a 15-minute default window. There is no dashboard-level field to change this default today; it is a user-side control in the header. For overview / starter dashboards on intermittent or batch-ingested data, the 15-minute window may show empty tiles even when the queries are correct. After save, tell the user the default is 15 minutes and that widening to 1h or 24h is the first thing to try if a tile looks empty. Do NOT compensate by hardcoding a wider time range into individual tiles; that desynchronizes them from the dashboard time picker. @@ -137,12 +143,12 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie - Using valueExpression with aggFn "count" (count takes no valueExpression). - Forgetting valueExpression for non-count aggFns (avg, sum, min, max, quantile all require it). - Using dot notation for Map-type attributes (always use SpanAttributes['key'] bracket syntax). -- Skipping hyperdx_list_sources (you need real source IDs and real column names). +- Skipping hyperdx_list_sources + hyperdx_describe_source (you need real source IDs, column names, and values). - Skipping hyperdx_query_tile after save (tiles can silently fail on syntax or attribute mismatches). - Setting chart-level numberFormat on a table that mixes counts and durations (counts render as 0:00:00). - Multiple select items on number / pie / heatmap tiles (each takes exactly one). - Missing level on aggFn "quantile" (must specify 0.5, 0.9, 0.95, or 0.99). -- Assuming StatusCode or SeverityText values (always inspect real values from hyperdx_list_sources). +- Assuming StatusCode or SeverityText values (always inspect lowCardinalityValues from hyperdx_describe_source). - Heatmap on a non-Trace source (heatmap is Trace-only today). - Hardcoding a focus dimension into every tile's where clause (use a dashboard-level filter instead).`; } @@ -788,7 +794,7 @@ Top-level columns (PascalCase defaults, use directly in valueExpression and grou Duration, StatusCode, SpanName, ServiceName, SpanKind, Body, SeverityText, Timestamp, TraceId, SpanId, ParentSpanId NOTE: These are defaults for the standard HyperDX schema. Custom sources may - use different column names. Always verify with hyperdx_list_sources, which returns + use different column names. Always verify with hyperdx_describe_source, which returns the real column names and keyColumns expressions for each source. Map-type columns (bracket syntax, access keys via ['key']): @@ -804,7 +810,7 @@ IMPORTANT: Always use bracket syntax for Map-type columns. Never use dot notatio JSON-type columns (dot notation, access nested keys via dot path): JsonColumn.key.subkey - NOTE: Check the jsType field returned by hyperdx_list_sources to decide + NOTE: Check the jsType field returned by hyperdx_describe_source to determine whether a column is Map (use brackets) or JSON (use dots). == LUCENE FILTER SYNTAX == @@ -1172,7 +1178,7 @@ Example: find top patterns for production services over the last 4 hours: 3. Using dot notation for Map-type attributes in valueExpression or groupBy Wrong: groupBy: "SpanAttributes.http.method" Correct: groupBy: "SpanAttributes['http.method']" - NOTE: JSON-type columns DO use dot notation. Check jsType from hyperdx_list_sources. + NOTE: JSON-type columns DO use dot notation. Check jsType from hyperdx_describe_source. 4. Multiple select items on number / pie / heatmap tiles Wrong: displayType: "number", select: [{ aggFn: "count" }, { aggFn: "avg", ... }] @@ -1202,7 +1208,7 @@ Example: find top patterns for production services over the last 4 hours: 10. Assuming StatusCode or SeverityText values Values like STATUS_CODE_ERROR, Ok, error, fatal vary by deployment. - Always call hyperdx_list_sources and inspect real keyValues from the source + Always call hyperdx_describe_source and inspect the lowCardinalityValues before writing filters that depend on these columns. 11. Heatmap tile on a non-Trace source diff --git a/packages/api/src/mcp/tools/dashboards/index.ts b/packages/api/src/mcp/tools/dashboards/index.ts index 7fc7bff4a7..a1e511e995 100644 --- a/packages/api/src/mcp/tools/dashboards/index.ts +++ b/packages/api/src/mcp/tools/dashboards/index.ts @@ -3,7 +3,6 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { McpContext, ToolDefinition } from '../types'; import { registerDeleteDashboard } from './deleteDashboard'; import { registerGetDashboard } from './getDashboard'; -import { registerListSources } from './listSources'; import { registerQueryTile } from './queryTile'; import { registerSaveDashboard } from './saveDashboard'; @@ -13,7 +12,6 @@ const dashboardsTools: ToolDefinition = ( server: McpServer, context: McpContext, ) => { - registerListSources(server, context); registerGetDashboard(server, context); registerSaveDashboard(server, context); registerDeleteDashboard(server, context); diff --git a/packages/api/src/mcp/tools/dashboards/listSources.ts b/packages/api/src/mcp/tools/dashboards/listSources.ts deleted file mode 100644 index 8b12cc6a8d..0000000000 --- a/packages/api/src/mcp/tools/dashboards/listSources.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { - convertCHDataTypeToJSType, - filterColumnMetaByType, - JSDataType, -} from '@hyperdx/common-utils/dist/clickhouse'; -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; -import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; -import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { z } from 'zod'; - -import { - getConnectionById, - getConnectionsByTeam, -} from '@/controllers/connection'; -import { getSources } from '@/controllers/sources'; -import logger from '@/utils/logger'; - -import { withToolTracing } from '../../utils/tracing'; -import type { McpContext } from '../types'; - -export function registerListSources( - server: McpServer, - context: McpContext, -): void { - const { teamId } = context; - - server.registerTool( - 'hyperdx_list_sources', - { - title: 'List Sources & Connections', - description: - 'List all data sources (logs, metrics, traces) and database connections available to this team. ' + - 'Returns source IDs (use as sourceId in hyperdx_timeseries, hyperdx_table, hyperdx_search, ' + - 'hyperdx_event_patterns, and dashboard tiles) and ' + - 'connection IDs (use as connectionId for hyperdx_sql raw SQL queries). ' + - 'Each source includes its full column schema and sampled attribute keys from map columns ' + - '(e.g. SpanAttributes, ResourceAttributes). ' + - 'Column names are PascalCase (e.g. Duration, not duration). ' + - "Map attributes must be accessed via bracket syntax: SpanAttributes['key'].\n\n" + - 'NOTE: For most queries, use source IDs with hyperdx_timeseries, hyperdx_table, ' + - 'hyperdx_search, or hyperdx_event_patterns. ' + - 'Connection IDs are only needed for hyperdx_sql (raw ClickHouse SQL).', - inputSchema: z.object({}), - }, - withToolTracing('hyperdx_list_sources', context, async () => { - const [sources, connections] = await Promise.all([ - getSources(teamId.toString()), - getConnectionsByTeam(teamId.toString()), - ]); - - const sourcesWithSchema = await Promise.all( - sources.map(async s => { - const meta: Record = { - id: s._id.toString(), - name: s.name, - kind: s.kind, - connectionId: s.connection.toString(), - timestampColumn: s.timestampValueExpression, - }; - - if ('eventAttributesExpression' in s && s.eventAttributesExpression) { - meta.eventAttributesColumn = s.eventAttributesExpression; - } - if ( - 'resourceAttributesExpression' in s && - s.resourceAttributesExpression - ) { - meta.resourceAttributesColumn = s.resourceAttributesExpression; - } - - if (s.kind === SourceKind.Trace) { - meta.keyColumns = { - spanName: s.spanNameExpression, - duration: s.durationExpression, - durationPrecision: s.durationPrecision, - statusCode: s.statusCodeExpression, - serviceName: s.serviceNameExpression, - traceId: s.traceIdExpression, - spanId: s.spanIdExpression, - }; - } else if (s.kind === SourceKind.Log) { - meta.keyColumns = { - body: s.bodyExpression, - serviceName: s.serviceNameExpression, - severityText: s.severityTextExpression, - traceId: s.traceIdExpression, - }; - } else if (s.kind === SourceKind.Metric) { - meta.metricTables = s.metricTables; - } - - // Skip column schema fetch for sources without a table (e.g. metrics - // sources store their tables in metricTables, not from.tableName). - if (s.from.tableName) { - try { - const connection = await getConnectionById( - teamId.toString(), - s.connection.toString(), - true, - ); - if (!connection) { - throw new Error(`Connection not found for source ${s._id}`); - } - - const clickhouseClient = new ClickhouseClient({ - host: connection.host, - username: connection.username, - password: connection.password, - }); - const metadata = getMetadata(clickhouseClient); - - const columns = await metadata.getColumns({ - databaseName: s.from.databaseName, - tableName: s.from.tableName, - connectionId: s.connection.toString(), - }); - - meta.columns = columns.map(c => ({ - name: c.name, - type: c.type, - jsType: convertCHDataTypeToJSType(c.type), - })); - - const mapColumns = filterColumnMetaByType(columns, [ - JSDataType.Map, - ]); - const mapKeysResults: Record = {}; - await Promise.all( - (mapColumns ?? []).map(async col => { - try { - const keys = await metadata.getMapKeys({ - databaseName: s.from.databaseName, - tableName: s.from.tableName, - column: col.name, - maxKeys: 50, - connectionId: s.connection.toString(), - }); - mapKeysResults[col.name] = keys; - } catch { - // Skip columns where key sampling fails - } - }), - ); - if (Object.keys(mapKeysResults).length > 0) { - meta.mapAttributeKeys = mapKeysResults; - } - } catch (e) { - logger.warn( - { teamId, sourceId: s._id, error: e }, - 'Failed to fetch schema for source', - ); - } - } - - return meta; - }), - ); - - const output = { - sources: sourcesWithSchema, - connections: connections.map(c => ({ - id: c._id.toString(), - name: c.name, - })), - usage: { - topLevelColumns: - 'Use directly in valueExpression/groupBy with PascalCase: Duration, StatusCode, SpanName', - mapAttributes: - "Use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']", - sourceIds: - 'Use sourceId with hyperdx_timeseries, hyperdx_table, hyperdx_search, and hyperdx_event_patterns', - connectionIds: - 'ADVANCED: Use connectionId with hyperdx_sql for raw ClickHouse SQL. ' + - 'Raw SQL is for advanced use cases like JOINs, sub-queries, or querying tables not registered as sources.', - }, - }; - return { - content: [ - { type: 'text' as const, text: JSON.stringify(output, null, 2) }, - ], - }; - }), - ); -} diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts index 8224190f90..3c65102994 100644 --- a/packages/api/src/mcp/tools/query/eventPatterns.ts +++ b/packages/api/src/mcp/tools/query/eventPatterns.ts @@ -59,7 +59,7 @@ export function registerEventPatterns(server: McpServer, context: McpContext) { '"recurring messages", or log noise analysis.\n\n' + 'Each pattern includes a "whereSnippet" — use it as the "where" parameter in ' + 'a follow-up hyperdx_search call to browse matching raw events.\n\n' + - 'Requires sourceId — call hyperdx_list_sources first.\n\n' + + 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first.\n\n' + 'When to use which tool:\n' + ' - hyperdx_event_patterns: clustering / recurring shapes / noise analysis\n' + ' - hyperdx_search: raw individual rows\n' + diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index de319ecf30..272a757a54 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -53,7 +53,7 @@ export function registerSearch(server: McpServer, context: McpContext) { 'Browse individual log/event/trace rows. ' + 'Use this when you need to see raw events, investigate specific log lines, ' + 'or drill into individual records matching a filter.\n\n' + - 'Requires sourceId — call hyperdx_list_sources first.\n\n' + + 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first.\n\n' + 'For aggregated metrics, use hyperdx_table instead. ' + 'For pattern discovery, use hyperdx_event_patterns instead.\n\n' + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts index b5cf396970..c6a096675c 100644 --- a/packages/api/src/mcp/tools/query/sql.ts +++ b/packages/api/src/mcp/tools/query/sql.ts @@ -63,7 +63,8 @@ export function registerSql(server: McpServer, context: McpContext) { 'Execute raw ClickHouse SQL. ' + 'ADVANCED: only use this when you need capabilities the builder tools cannot express — ' + 'JOINs, sub-queries, CTEs, or querying tables not registered as sources.\n\n' + - 'Requires connectionId (not sourceId) — call hyperdx_list_sources to find connections.\n\n' + + 'Requires connectionId (not sourceId) — call hyperdx_list_sources to find connections. ' + + 'Call hyperdx_describe_source to discover column names before writing SQL.\n\n' + 'Results are always returned as table rows — for time-series semantics, ' + 'include a time column and ORDER BY it in your SQL.\n\n' + 'For standard aggregations use hyperdx_table. ' + diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts index 1d826f179b..0d0bce3292 100644 --- a/packages/api/src/mcp/tools/query/table.ts +++ b/packages/api/src/mcp/tools/query/table.ts @@ -54,7 +54,7 @@ export function registerTable(server: McpServer, context: McpContext) { 'Compute aggregated metrics as a table, single number, or pie chart. ' + 'Use this for grouped aggregations, top-N queries, single-value KPIs, ' + 'or proportional breakdowns.\n\n' + - 'Requires sourceId — call hyperdx_list_sources first.\n\n' + + 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first.\n\n' + 'Each select item can have its own "where" filter, which compiles to ' + 'If(...) for multi-cohort comparison in one call ' + '(e.g. p99_before + p99_after side-by-side).\n\n' + diff --git a/packages/api/src/mcp/tools/query/timeseries.ts b/packages/api/src/mcp/tools/query/timeseries.ts index 919f39a5ec..4389d9d564 100644 --- a/packages/api/src/mcp/tools/query/timeseries.ts +++ b/packages/api/src/mcp/tools/query/timeseries.ts @@ -61,7 +61,7 @@ export function registerTimeseries(server: McpServer, context: McpContext) { 'Plot metrics over time as a line or stacked bar chart. ' + 'Use this when you need to visualize trends, compare time-series, ' + 'or monitor metric changes over a time window.\n\n' + - 'Requires sourceId — call hyperdx_list_sources first. ' + + 'Requires sourceId — call hyperdx_list_sources then hyperdx_describe_source first. ' + 'Each select item defines one plotted series.\n\n' + 'Column naming: top-level columns are PascalCase (Duration, StatusCode). ' + "Map attributes use bracket syntax: SpanAttributes['http.method'].", diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts new file mode 100644 index 0000000000..57dd26fd29 --- /dev/null +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -0,0 +1,407 @@ +import { + convertCHDataTypeToJSType, + filterColumnMetaByType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; +import logger from '@/utils/logger'; +import { trimToolResponse } from '@/utils/trimToolResponse'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; + +// How far back to look when querying the rollup tables for value samples. +const VALUE_SAMPLE_LOOKBACK_MS = 24 * 60 * 60 * 1000; // 24 hours + +// Hard timeout for the entire describe operation (ms). +const DESCRIBE_TIMEOUT_MS = 10_000; + +// Max sampled values per low-cardinality column / map attribute key. +const MAX_LC_VALUES = 20; +const MAX_MAP_KEY_VALUES = 5; +const MAX_MAP_KEYS_TO_SAMPLE = 10; + +/** + * Core schema-discovery logic. Extracted so the caller can wrap it in + * Promise.race for wall-clock timeout enforcement. + */ +async function describeSourceSchema( + teamId: string, + sourceId: string, + signal: AbortSignal, +) { + const source = await getSource(teamId, sourceId); + if (!source) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Source "${sourceId}" not found. Call hyperdx_list_sources to see available source IDs.`, + }, + ], + }; + } + + const meta: Record = { + id: source._id.toString(), + name: source.name, + kind: source.kind, + connectionId: source.connection.toString(), + timestampColumn: source.timestampValueExpression, + }; + + if ( + 'eventAttributesExpression' in source && + source.eventAttributesExpression + ) { + meta.eventAttributesColumn = source.eventAttributesExpression; + } + if ( + 'resourceAttributesExpression' in source && + source.resourceAttributesExpression + ) { + meta.resourceAttributesColumn = source.resourceAttributesExpression; + } + + // Key columns by source kind + if (source.kind === SourceKind.Trace) { + meta.keyColumns = { + spanName: source.spanNameExpression, + duration: source.durationExpression, + durationPrecision: source.durationPrecision, + statusCode: source.statusCodeExpression, + serviceName: source.serviceNameExpression, + traceId: source.traceIdExpression, + spanId: source.spanIdExpression, + }; + } else if (source.kind === SourceKind.Log) { + meta.keyColumns = { + body: source.bodyExpression, + serviceName: source.serviceNameExpression, + severityText: source.severityTextExpression, + traceId: source.traceIdExpression, + }; + } else if (source.kind === SourceKind.Metric) { + meta.metricTables = source.metricTables; + } + + // For sources without a table (e.g. metric sources), return early + if (!source.from.tableName) { + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify( + { + source: meta, + nextSteps: { + query: `Use hyperdx_timeseries, hyperdx_table, or hyperdx_search with sourceId "${sourceId}" and the metric tables above.`, + }, + }, + null, + 2, + ), + }, + ], + }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + true, + ); + if (!connection) { + return { + isError: true, + content: [ + { + type: 'text' as const, + text: `Connection not found for source "${sourceId}".`, + }, + ], + }; + } + + const clickhouseClient = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + }); + const metadata = getMetadata(clickhouseClient); + const { databaseName, tableName } = source.from; + const connectionId = source.connection.toString(); + + // Track which sampling stages were skipped due to timeout + const skippedStages: string[] = []; + + // ── 1. Column schema ────────────────────────────────────────────────── + const columns = await metadata.getColumns({ + databaseName, + tableName, + connectionId, + }); + + meta.columns = columns.map(c => ({ + name: c.name, + type: c.type, + jsType: convertCHDataTypeToJSType(c.type), + })); + + // ── 2. Map attribute keys ───────────────────────────────────────────── + const mapColumns = filterColumnMetaByType(columns, [JSDataType.Map]); + const mapKeysResults: Record = {}; + + if (!signal.aborted) { + await Promise.all( + (mapColumns ?? []).map(async col => { + try { + const keys = await metadata.getMapKeys({ + databaseName, + tableName, + column: col.name, + maxKeys: 50, + connectionId, + }); + mapKeysResults[col.name] = keys; + } catch (e) { + logger.warn( + { sourceId, column: col.name, error: e }, + 'Failed to fetch map keys for column', + ); + } + }), + ); + } + + if (signal.aborted && Object.keys(mapKeysResults).length === 0) { + skippedStages.push('mapAttributeKeys'); + } + if (Object.keys(mapKeysResults).length > 0) { + meta.mapAttributeKeys = mapKeysResults; + } + + // ── 3. Low-cardinality column value sampling ────────────────────────── + const lcColumns = columns.filter(c => { + const normalized = c.type.replace(/\s/g, ''); + return ( + normalized.startsWith('LowCardinality(') && + (normalized.includes('String') || normalized.includes('string')) + ); + }); + + const lowCardinalityValues: Record = {}; + + const metadataMVs = + 'metadataMaterializedViews' in source + ? source.metadataMaterializedViews + : undefined; + + const now = new Date(); + const dateRange: [Date, Date] = [ + new Date(now.getTime() - VALUE_SAMPLE_LOOKBACK_MS), + now, + ]; + + if (lcColumns.length > 0 && !signal.aborted) { + try { + const results = await metadata.getAllKeyValues({ + databaseName, + tableName, + keyExpressions: lcColumns.map(col => col.name), + maxValuesPerKey: MAX_LC_VALUES, + connectionId, + metadataMVs, + dateRange, + signal, + }); + for (const { key, value } of results) { + if (value.length > 0) { + lowCardinalityValues[key] = value; + } + } + } catch { + // Skip columns where value sampling fails + } + } + + if (signal.aborted && Object.keys(lowCardinalityValues).length === 0) { + skippedStages.push('lowCardinalityValues'); + } + if (Object.keys(lowCardinalityValues).length > 0) { + meta.lowCardinalityValues = lowCardinalityValues; + } + + // ── 4. Map attribute value sampling (best-effort) ───────────────────── + if (Object.keys(mapKeysResults).length > 0 && !signal.aborted) { + const mapAttributeValues: Record = {}; + + const keyExprs: string[] = []; + for (const [colName, keys] of Object.entries(mapKeysResults)) { + for (const key of keys.slice(0, MAX_MAP_KEYS_TO_SAMPLE)) { + keyExprs.push(`${colName}['${key}']`); + } + } + + try { + const results = await metadata.getAllKeyValues({ + databaseName, + tableName, + keyExpressions: keyExprs, + maxValuesPerKey: MAX_MAP_KEY_VALUES, + connectionId, + metadataMVs, + dateRange, + signal, + }); + for (const { key, value } of results) { + if (value.length > 0) { + mapAttributeValues[key] = value; + } + } + } catch { + // Best-effort; skip on failure + } + + if (signal.aborted && Object.keys(mapAttributeValues).length === 0) { + skippedStages.push('mapAttributeValues'); + } + if (Object.keys(mapAttributeValues).length > 0) { + meta.mapAttributeValues = mapAttributeValues; + } + } else if (Object.keys(mapKeysResults).length > 0) { + // Signal was already aborted before we started this stage + skippedStages.push('mapAttributeValues'); + } + + // Flag partial results so the LLM knows value samples may be incomplete + if (skippedStages.length > 0) { + meta.partial = true; + meta.skippedStages = skippedStages; + } + + const lcValuesHint = + skippedStages.length > 0 + ? 'Value sampling was partially skipped due to timeout. ' + + 'The values shown may be incomplete — verify with a hyperdx_search query if needed.' + : 'These are the REAL values in your data — use them in filters instead of guessing. ' + + 'Example: where: "SeverityText:error" (if \'error\' appears in the sampled values above).'; + + const output = trimToolResponse({ + source: meta, + usage: { + topLevelColumns: + 'Use directly in valueExpression/groupBy with PascalCase: Duration, StatusCode, SpanName', + mapAttributes: + "Use bracket syntax: SpanAttributes['http.method'], ResourceAttributes['service.name']", + lowCardinalityValues: lcValuesHint, + }, + nextSteps: { + query: `Use hyperdx_timeseries, hyperdx_table, or hyperdx_search with sourceId "${sourceId}" and the columns/attributes above.`, + mapAttributeAccess: + "Use bracket syntax for map columns: ResourceAttributes['service.name'], SpanAttributes['http.method']", + }, + }); + + return { + content: [ + { + type: 'text' as const, + text: JSON.stringify(output, null, 2), + }, + ], + }; +} + +export function registerDescribeSource( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'hyperdx_describe_source', + { + title: 'Describe Source Schema', + description: + 'CALL THIS BEFORE WRITING QUERIES — prevents unknown-column errors.\n\n' + + 'Returns the full column schema, map-attribute keys, and sampled low-cardinality ' + + 'values (e.g. SeverityText, StatusCode, ServiceName) for a single data source.\n\n' + + 'Workflow: call hyperdx_list_sources first to get source IDs, then call this tool ' + + 'for each source you plan to query.\n\n' + + 'Returns:\n' + + '- columns[]: column name, ClickHouse type, and JS type\n' + + '- mapAttributeKeys: discovered keys in Map columns (e.g. SpanAttributes, ResourceAttributes)\n' + + '- lowCardinalityValues: sampled values for LowCardinality(String) columns ' + + '(SeverityText, StatusCode, ServiceName, etc.) — use these in filters instead of guessing\n' + + '- mapAttributeValues: sampled top values for the most common map attribute keys ' + + "(e.g. ResourceAttributes['service.name'] top values) — requires rollup tables\n\n" + + 'Cost: one describe call prevents 3–5 exploratory queries against non-existent columns.', + inputSchema: z.object({ + sourceId: z + .string() + .describe( + 'The source ID to describe. Get this from hyperdx_list_sources.', + ), + }), + }, + withToolTracing( + 'hyperdx_describe_source', + context, + async ({ sourceId }) => { + const controller = new AbortController(); + + // Promise.race enforces wall-clock timeout regardless of whether + // internal ClickHouse calls honour the AbortSignal. + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + controller.abort(); + reject(new Error('DESCRIBE_TIMEOUT')); + }, DESCRIBE_TIMEOUT_MS); + }); + + try { + return await Promise.race([ + describeSourceSchema( + teamId.toString(), + sourceId, + controller.signal, + ), + timeoutPromise, + ]); + } catch (e) { + if (e instanceof Error && e.message === 'DESCRIBE_TIMEOUT') { + logger.warn( + { teamId, sourceId }, + 'hyperdx_describe_source timed out', + ); + return { + isError: true, + content: [ + { + type: 'text' as const, + text: + 'Schema discovery timed out. The ClickHouse server may be under load. ' + + 'Try again, or use hyperdx_list_sources for basic source info without schema details.', + }, + ], + }; + } + logger.warn( + { teamId, sourceId, error: e }, + 'Failed to describe source schema', + ); + throw e; + } + }, + ), + ); +} diff --git a/packages/api/src/mcp/tools/sources/index.ts b/packages/api/src/mcp/tools/sources/index.ts new file mode 100644 index 0000000000..250e4abd91 --- /dev/null +++ b/packages/api/src/mcp/tools/sources/index.ts @@ -0,0 +1,15 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + +import type { McpContext, ToolDefinition } from '../types'; +import { registerDescribeSource } from './describeSource'; +import { registerListSources } from './listSources'; + +const sourcesTools: ToolDefinition = ( + server: McpServer, + context: McpContext, +) => { + registerListSources(server, context); + registerDescribeSource(server, context); +}; + +export default sourcesTools; diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts new file mode 100644 index 0000000000..7efe719d06 --- /dev/null +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -0,0 +1,99 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { getConnectionsByTeam } from '@/controllers/connection'; +import { getSources } from '@/controllers/sources'; + +import { withToolTracing } from '../../utils/tracing'; +import type { McpContext } from '../types'; + +export function registerListSources( + server: McpServer, + context: McpContext, +): void { + const { teamId } = context; + + server.registerTool( + 'hyperdx_list_sources', + { + title: 'List Sources & Connections', + description: + 'List all data sources (logs, metrics, traces) and database connections available to this team. ' + + 'Returns source IDs, names, kinds, and connection IDs as a lightweight catalog.\n\n' + + 'NEXT STEP: After identifying the source(s) you need, call hyperdx_describe_source with the ' + + 'sourceId to get the full column schema, attribute keys, and sampled values. ' + + 'This two-step approach avoids fetching expensive schema details for sources you do not need.\n\n' + + 'NOTE: For most queries, use source IDs with hyperdx_timeseries, hyperdx_table, ' + + 'hyperdx_search, or hyperdx_event_patterns. ' + + 'Connection IDs are only needed for hyperdx_sql (raw ClickHouse SQL).', + inputSchema: z.object({}), + }, + withToolTracing('hyperdx_list_sources', context, async () => { + const [sources, connections] = await Promise.all([ + getSources(teamId.toString()), + getConnectionsByTeam(teamId.toString()), + ]); + + const sourceSummaries = sources.map(s => { + const meta: Record = { + id: s._id.toString(), + name: s.name, + kind: s.kind, + connectionId: s.connection.toString(), + timestampColumn: s.timestampValueExpression, + }; + + if ('eventAttributesExpression' in s && s.eventAttributesExpression) { + meta.eventAttributesColumn = s.eventAttributesExpression; + } + if ( + 'resourceAttributesExpression' in s && + s.resourceAttributesExpression + ) { + meta.resourceAttributesColumn = s.resourceAttributesExpression; + } + + if (s.kind === SourceKind.Trace) { + meta.keyColumns = { + spanName: s.spanNameExpression, + duration: s.durationExpression, + durationPrecision: s.durationPrecision, + statusCode: s.statusCodeExpression, + serviceName: s.serviceNameExpression, + traceId: s.traceIdExpression, + spanId: s.spanIdExpression, + }; + } else if (s.kind === SourceKind.Log) { + meta.keyColumns = { + body: s.bodyExpression, + serviceName: s.serviceNameExpression, + severityText: s.severityTextExpression, + traceId: s.traceIdExpression, + }; + } else if (s.kind === SourceKind.Metric) { + meta.metricTables = s.metricTables; + } + + return meta; + }); + + const output = { + sources: sourceSummaries, + connections: connections.map(c => ({ + id: c._id.toString(), + name: c.name, + })), + nextStep: + 'Call hyperdx_describe_source with a sourceId above to get the full column schema, ' + + 'attribute keys, and sampled low-cardinality values before writing queries. ' + + 'connectionId is only needed for hyperdx_sql.', + }; + return { + content: [ + { type: 'text' as const, text: JSON.stringify(output, null, 2) }, + ], + }; + }), + ); +}