From 47c003f3ac996458660481feccfa9f6f27ebd9ae Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:22:37 -0700 Subject: [PATCH 1/2] fix(clickhouse): disable per-part subcolumn size calculation on CH 26.3+ ClickHouse 26.3 flipped allow_calculating_subcolumns_sizes_for_merge_tree_reading to on by default. optimizePrewhere then calls getColumnSizes with subcolumn sizes enabled, which walks every (active part x referenced map key) to read per-part sizes. On SharedMergeTree each of those is an S3 GET, it happens during planning before any row is read, and neither max_execution_time nor client cancellation interrupts it. Measured on a 26.6 SharedMergeTree cluster with 162 active parts, via EXPLAIN PLAN so no rows are read: keys setting duration S3 GETs 1 0 5 ms 0 1 1 4,670 ms 148 10 1 36,111 ms 1,480 TIMEOUT_EXCEEDED before start Cost is linear in referenced keys at roughly one GET per active part each. The LLM dashboard reads ~64 keys per tile; on a 645-part cluster running 26.4 that came to 64k GETs and 27 minutes of planning for zero rows, and because 26.4 still held the DataParts lock across the loop it starved every other reader on the replica. Send the setting as 0 from processClickhouseSettings so every caller (charts, search, alerts, API, MCP) is covered by one change. applySettingIfAvailable only sends settings present in the server's system.settings, so pre-26.3 servers are unaffected, and caller-supplied clickhouse_settings still win. The sizes are only used to reorder PREWHERE conditions, so the pre-26.3 approximation (parent column size) is an acceptable trade. Also stop binding the ~70 KiB cost expression on the two LLM dashboard chart groups that never select it. --- .changeset/lazy-pugs-smoke.md | 15 +++++++++++ .../src/llm/dashboard/EfficiencyCharts.tsx | 3 ++- .../app/src/llm/dashboard/LatencyCharts.tsx | 3 ++- .../src/__tests__/clickhouse.test.ts | 26 +++++++++++++++++++ packages/common-utils/src/clickhouse/index.ts | 9 +++++++ 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 .changeset/lazy-pugs-smoke.md diff --git a/.changeset/lazy-pugs-smoke.md b/.changeset/lazy-pugs-smoke.md new file mode 100644 index 0000000000..0fd3d30296 --- /dev/null +++ b/.changeset/lazy-pugs-smoke.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +--- + +fix: disable per-part subcolumn size calculation on ClickHouse 26.3+ + +ClickHouse 26.3 turned on +`allow_calculating_subcolumns_sizes_for_merge_tree_reading` by default, which +makes PREWHERE planning fetch per-part sizes for every map key a query +references. On SharedMergeTree that is one S3 GET per (key × active part), it +runs before any row is read, and `max_execution_time` does not interrupt it. +Queries referencing many attribute keys — the LLM dashboard reads ~64 — could +spend minutes in planning. Queries now send the setting as `0` when the server +supports it. diff --git a/packages/app/src/llm/dashboard/EfficiencyCharts.tsx b/packages/app/src/llm/dashboard/EfficiencyCharts.tsx index 5a6fad867c..c3bf51c547 100644 --- a/packages/app/src/llm/dashboard/EfficiencyCharts.tsx +++ b/packages/app/src/llm/dashboard/EfficiencyCharts.tsx @@ -23,7 +23,8 @@ const PERCENT_NUMBER_FORMAT: NumberFormat = { */ export function EfficiencyCharts(props: LLMChartProps) { const { source, expressions } = props; - const base = baseLLMChartConfig(props); + // Neither chart selects cost, so skip the ~70 KiB cost alias binding. + const base = baseLLMChartConfig({ ...props, withCostAlias: false }); return ( <> diff --git a/packages/app/src/llm/dashboard/LatencyCharts.tsx b/packages/app/src/llm/dashboard/LatencyCharts.tsx index 7a79509d77..07737db5c6 100644 --- a/packages/app/src/llm/dashboard/LatencyCharts.tsx +++ b/packages/app/src/llm/dashboard/LatencyCharts.tsx @@ -17,7 +17,8 @@ const CHART_HEIGHT = 320; */ export function LatencyCharts(props: LLMChartProps) { const { source, expressions } = props; - const base = baseLLMChartConfig(props); + // Neither chart selects cost, so skip the ~70 KiB cost alias binding. + const base = baseLLMChartConfig({ ...props, withCostAlias: false }); return ( <> diff --git a/packages/common-utils/src/__tests__/clickhouse.test.ts b/packages/common-utils/src/__tests__/clickhouse.test.ts index 14e13606bd..2c8b85abbf 100644 --- a/packages/common-utils/src/__tests__/clickhouse.test.ts +++ b/packages/common-utils/src/__tests__/clickhouse.test.ts @@ -414,6 +414,10 @@ describe('processClickhouseSettings - optimization settings', () => { // { name: 'use_top_k_dynamic_filtering', value: '1' }, { name: 'use_skip_indexes_on_data_read', value: '1' }, { name: 'use_skip_indexes_for_disjunctions', value: '1' }, + { + name: 'allow_calculating_subcolumns_sizes_for_merge_tree_reading', + value: '1', + }, ]); await client.query({ @@ -440,9 +444,31 @@ describe('processClickhouseSettings - optimization settings', () => { // use_top_k_dynamic_filtering: '1', use_skip_indexes_on_data_read: '1', use_skip_indexes_for_disjunctions: '1', + allow_calculating_subcolumns_sizes_for_merge_tree_reading: '0', }); }); + // Pre-26.3 servers don't have the setting; sending it would fail the query. + it('should omit the subcolumn size setting when the server lacks it', async () => { + setupMockQuery([{ name: 'use_skip_indexes_for_top_k', value: '1' }]); + + await client.query({ + query: 'SELECT 1', + format: 'JSON', + connectionId: 'test-conn', + }); + + const actualQueryCall = mockQueryMethod.mock.calls.find( + (call: any) => call[0].query === 'SELECT 1', + ); + + expect(actualQueryCall).toBeDefined(); + expect( + actualQueryCall[0].clickhouse_settings + .allow_calculating_subcolumns_sizes_for_merge_tree_reading, + ).toBeUndefined(); + }); + it('should only apply available optimization settings', async () => { setupMockQuery([ { name: 'use_skip_indexes_for_top_k', value: '1' }, diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 9df6177583..6a18dc1a2e 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -546,6 +546,15 @@ export abstract class BaseClickhouseClient { // Enables full-text (inverted index) search. applySettingIfAvailable('enable_full_text_index', '1'); + // 26.3 turned this on by default. On SharedMergeTree it makes PREWHERE + // planning fetch per-part sizes for every map key referenced — one S3 GET + // each, not interruptible by max_execution_time. The sizes only reorder + // PREWHERE conditions, so the pre-26.3 approximation is fine. + applySettingIfAvailable( + 'allow_calculating_subcolumns_sizes_for_merge_tree_reading', + '0', + ); + return { ...defaultSettings, ...clickhouse_settings, From b6c879e0b991c8d3a05c1f13ede56f87bc818444 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:55:49 -0700 Subject: [PATCH 2/2] test: drop redundant subcolumn-setting omission case The exact-equality assertion in 'should only apply available optimization settings' already fails if the setting is emitted unconditionally, so the added test could not fail independently. Verified by injecting an unconditional emit: that test plus three others catch it, including the existing permissions-error case. Leaves a note on the surviving test explaining what it protects. --- .../src/__tests__/clickhouse.test.ts | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/packages/common-utils/src/__tests__/clickhouse.test.ts b/packages/common-utils/src/__tests__/clickhouse.test.ts index 2c8b85abbf..a9b4825660 100644 --- a/packages/common-utils/src/__tests__/clickhouse.test.ts +++ b/packages/common-utils/src/__tests__/clickhouse.test.ts @@ -448,27 +448,9 @@ describe('processClickhouseSettings - optimization settings', () => { }); }); - // Pre-26.3 servers don't have the setting; sending it would fail the query. - it('should omit the subcolumn size setting when the server lacks it', async () => { - setupMockQuery([{ name: 'use_skip_indexes_for_top_k', value: '1' }]); - - await client.query({ - query: 'SELECT 1', - format: 'JSON', - connectionId: 'test-conn', - }); - - const actualQueryCall = mockQueryMethod.mock.calls.find( - (call: any) => call[0].query === 'SELECT 1', - ); - - expect(actualQueryCall).toBeDefined(); - expect( - actualQueryCall[0].clickhouse_settings - .allow_calculating_subcolumns_sizes_for_merge_tree_reading, - ).toBeUndefined(); - }); - + // The exact-equality assertion below is what keeps unavailable settings off + // the wire — e.g. a pre-26.3 server that would reject + // allow_calculating_subcolumns_sizes_for_merge_tree_reading as unknown. it('should only apply available optimization settings', async () => { setupMockQuery([ { name: 'use_skip_indexes_for_top_k', value: '1' },