From eb9eda4f7f0147ed683fc5a20f5378b73c1a85ee Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 8 Jun 2026 12:02:06 -0600 Subject: [PATCH 1/3] fix(mcp): reduce describe_source timeouts by using rollup tables for map key discovery Pass metadataMVs and dateRange to getMapKeys in Stage 2 so it queries the rollup table instead of scanning millions of rows in the main table. Also thread AbortSignal through getMapValues so fallback queries cancel when the 10s wall-clock timeout fires. --- .../src/mcp/tools/sources/describeSource.ts | 26 +++++++++++-------- packages/common-utils/src/core/metadata.ts | 5 ++++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index a8249ae329..c62a09ca17 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -143,6 +143,19 @@ async function describeSourceSchema( // Track which sampling stages were skipped due to timeout const skippedStages: string[] = []; + // Shared by stages 2–4 so map-key discovery can use rollup tables + // instead of falling back to expensive main-table scans. + const metadataMVs = + 'metadataMaterializedViews' in source + ? source.metadataMaterializedViews + : undefined; + + const now = new Date(); + const dateRange: [Date, Date] = [ + new Date(now.getTime() - VALUE_SAMPLE_LOOKBACK_MS), + now, + ]; + // ── 1. Column schema ────────────────────────────────────────────────── const columns = await metadata.getColumns({ databaseName, @@ -170,6 +183,8 @@ async function describeSourceSchema( column: col.name, maxKeys: 50, connectionId, + metadataMVs, + dateRange, }); mapKeysResults[col.name] = keys; } catch (e) { @@ -200,17 +215,6 @@ async function describeSourceSchema( const lowCardinalityValues: Record = {}; - const metadataMVs = - 'metadataMaterializedViews' in source - ? source.metadataMaterializedViews - : undefined; - - const now = new Date(); - const dateRange: [Date, Date] = [ - new Date(now.getTime() - VALUE_SAMPLE_LOOKBACK_MS), - now, - ]; - if (lcColumns.length > 0 && !signal.aborted) { try { const results = await metadata.getAllKeyValues({ diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index a80feb1fad..3faadfd4dd 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -640,6 +640,7 @@ export class Metadata { connectionId, dateRange, timestampValueExpression, + signal, }: { databaseName: string; tableName: string; @@ -649,6 +650,7 @@ export class Metadata { dateRange?: [Date, Date]; timestampValueExpression?: string; connectionId: string; + signal?: AbortSignal; }) { const dateRangeCacheSuffix = dateRange && timestampValueExpression @@ -719,6 +721,7 @@ export class Metadata { read_overflow_mode: 'break', ...this.getClickHouseSettings(), }, + abort_signal: signal, }) .then(res => res.json<{ value: string }>()) .then(d => d.data.map(row => row.value)); @@ -1464,6 +1467,7 @@ export class Metadata { connectionId, dateRange, timestampValueExpression, + signal, }); return { key: p.keyExpression, value: fallback }; }), @@ -1482,6 +1486,7 @@ export class Metadata { connectionId, dateRange, timestampValueExpression, + signal, }); return { key: p.keyExpression, value }; }), From f6b20ff571612c59125046acefa3f5dcc268e2dd Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 8 Jun 2026 12:04:50 -0600 Subject: [PATCH 2/3] chore: add changeset --- .changeset/fix-describe-source-timeout.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-describe-source-timeout.md diff --git a/.changeset/fix-describe-source-timeout.md b/.changeset/fix-describe-source-timeout.md new file mode 100644 index 0000000000..716b355bc0 --- /dev/null +++ b/.changeset/fix-describe-source-timeout.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +fix(mcp): reduce describe_source timeouts by using rollup tables for map key discovery From 3a170e8f3a3bc0d225d0e10964f007d4f38a2e4a Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 8 Jun 2026 12:50:30 -0600 Subject: [PATCH 3/3] fix(mcp): pass abort signal to getMapKeys ClickHouse queries --- packages/api/src/mcp/tools/sources/describeSource.ts | 1 + packages/common-utils/src/core/metadata.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index c62a09ca17..5674c7e2cf 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -185,6 +185,7 @@ async function describeSourceSchema( connectionId, metadataMVs, dateRange, + signal, }); mapKeysResults[col.name] = keys; } catch (e) { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 3faadfd4dd..d6e2b10e34 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -351,6 +351,7 @@ export class Metadata { metadataMVs, dateRange, timestampValueExpression, + signal, }: { databaseName: string; tableName: string; @@ -361,6 +362,7 @@ export class Metadata { metadataMVs?: MetadataMaterializedViews; dateRange?: [Date, Date]; timestampValueExpression?: string; + signal?: AbortSignal; }) { // Align date range to rollup granularity for consistent cache keys const alignedDateRange = @@ -419,6 +421,7 @@ export class Metadata { max_execution_time: 15, max_rows_to_read: '0', }, + abort_signal: signal, }) .then(res => res.json<{ Key: string }>()) .then(d => d.data.map(row => row.Key).filter(k => k)); @@ -524,6 +527,7 @@ export class Metadata { // Set the value to 0 (unlimited) so that the LIMIT is used instead max_rows_to_read: '0', }, + abort_signal: signal, }) .then(res => res.json<{ keysArr?: string[]; key?: string }>()) .then(d => {