From a4f60f35478b419f6ebec3c433f6243ccb7c4992 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Wed, 20 May 2026 17:53:19 -0400 Subject: [PATCH 1/2] feat: only use pk + block ids to resolve a row --- packages/app/src/components/DBRowTable.tsx | 41 +++- .../components/__tests__/DBRowTable.test.tsx | 75 +++++-- .../src/hooks/__tests__/useRowWhere.test.tsx | 189 ++++++++++++++++++ packages/app/src/hooks/useRowWhere.tsx | 16 +- packages/cli/src/api/eventQuery.ts | 6 +- packages/cli/src/shared/useRowWhere.ts | 14 +- 6 files changed, 322 insertions(+), 19 deletions(-) diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index ba7e8bed41..5b4b297cbe 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -1368,9 +1368,20 @@ export function appendSelectWithAdditionalKeys( partitionKey: string, extraKeys: string[] = [], ): { select: SelectList; additionalKeysLength: number } { + // Include both the raw key expressions (e.g. toStartOfFiveMinutes(Timestamp)) + // and the extracted column references (e.g. Timestamp). The raw expressions + // are needed so the row WHERE clause can filter on PK expressions directly. const partitionKeyArr = extractColumnReferencesFromKey(partitionKey); const primaryKeyArr = extractColumnReferencesFromKey(primaryKeys); - const allKeys = new Set([...partitionKeyArr, ...primaryKeyArr, ...extraKeys]); + const rawPartitionExprs = splitAndTrimWithBracket(partitionKey); + const rawPrimaryExprs = splitAndTrimWithBracket(primaryKeys); + const allKeys = new Set([ + ...partitionKeyArr, + ...primaryKeyArr, + ...rawPartitionExprs, + ...rawPrimaryExprs, + ...extraKeys, + ]); if (typeof select === 'string') { const selectSplit = splitAndTrimWithBracket(select); const selectColumns = new Set(selectSplit); @@ -1448,7 +1459,27 @@ export function useConfigWithAdditionalSelect( extraKeys, ); - return { ...config, select, additionalKeysLength }; + // When block columns are available, the PK + partition + block columns + // uniquely identify a row. Compute the full set of key column names + // (both raw expressions like toStartOfFiveMinutes(Timestamp) and bare + // column references like Timestamp, ServiceName) so the row WHERE clause + // can use only these, avoiding expensive index loading on large columns + // like Body. + const hasBlockColumns = + extraKeys.includes('_block_number') && + extraKeys.includes('_block_offset'); + + const rowKeyColumns = hasBlockColumns + ? new Set([ + ...splitAndTrimWithBracket(partitionKey), + ...splitAndTrimWithBracket(primaryKey), + ...extractColumnReferencesFromKey(partitionKey), + ...extractColumnReferencesFromKey(primaryKey), + ...extraKeys, + ]) + : undefined; + + return { ...config, select, additionalKeysLength, rowKeyColumns }; }, [primaryKey, partitionKey, config, tableMetadata, columns, sourceId]); } @@ -1674,7 +1705,11 @@ function DBSqlRowTableComponent({ [data], ); - const getRowWhere = useRowWhere({ meta: data?.meta, aliasMap }); + const getRowWhere = useRowWhere({ + meta: data?.meta, + aliasMap, + primaryKeyColumns: mergedConfig?.rowKeyColumns, + }); const _onRowDetailsClick = useCallback( (row: Record) => { diff --git a/packages/app/src/components/__tests__/DBRowTable.test.tsx b/packages/app/src/components/__tests__/DBRowTable.test.tsx index 281ca2bdef..406148bf99 100644 --- a/packages/app/src/components/__tests__/DBRowTable.test.tsx +++ b/packages/app/src/components/__tests__/DBRowTable.test.tsx @@ -153,8 +153,9 @@ describe('appendSelectWithAdditionalKeys', () => { ' toStartOfInterval(timestamp, toIntervalDay(3))', ); expect(result).toEqual({ - additionalKeysLength: 3, - select: 'col1,col2,timestamp,id,created_at', + additionalKeysLength: 4, + select: + 'col1,col2,timestamp,id,created_at,toStartOfInterval(timestamp, toIntervalDay(3))', }); }); @@ -173,8 +174,9 @@ describe('appendSelectWithAdditionalKeys', () => { "toStartOfInterval(timestamp, toIntervalDay(3)), date_diff('DAY', col3, col4), now(), toDate(col5 + INTERVAL 1 DAY)", ); expect(result).toEqual({ - additionalKeysLength: 6, - select: 'col1,col2,timestamp,col3,col4,col5,id,timestamp2', + additionalKeysLength: 11, + select: + "col1,col2,timestamp,col3,col4,col5,id,timestamp2,toStartOfInterval(timestamp, toIntervalDay(3)),date_diff('DAY', col3, col4),now(),toDate(col5 + INTERVAL 1 DAY),toStartOfInterval(timestamp2, toIntervalDay(3))", }); }); @@ -221,8 +223,8 @@ describe('appendSelectWithAdditionalKeys', () => { `json.a, json.b.c, toStartOfDay(timestamp, json_2.d)`, ); expect(result).toEqual({ - additionalKeysLength: 5, - select: `col1,col2,json.a,json.b.c,timestamp,json_2.d,json.b`, + additionalKeysLength: 6, + select: `col1,col2,json.a,json.b.c,timestamp,json_2.d,json.b,toStartOfDay(timestamp, json_2.d)`, }); }); @@ -233,8 +235,8 @@ describe('appendSelectWithAdditionalKeys', () => { `toStartOfDay(json.a.b.:DateTime)`, ); expect(result).toEqual({ - additionalKeysLength: 2, - select: `col1,col2,json.a.b,json.b`, + additionalKeysLength: 4, + select: `col1,col2,json.a.b,json.b,toStartOfDay(json.a.b.:DateTime),json.b.:Int64`, }); }); @@ -245,8 +247,8 @@ describe('appendSelectWithAdditionalKeys', () => { ``, ); expect(result).toEqual({ - additionalKeysLength: 1, - select: `col1,col2,col3`, + additionalKeysLength: 2, + select: `col1,col2,col3,json.b.:Array(String)`, }); }); @@ -257,8 +259,8 @@ describe('appendSelectWithAdditionalKeys', () => { ``, ); expect(result).toEqual({ - additionalKeysLength: 1, - select: `col1,col2,col3`, + additionalKeysLength: 2, + select: `col1,col2,col3,map['key']['key2']`, }); }); @@ -310,4 +312,53 @@ describe('appendSelectWithAdditionalKeys', () => { ], }); }); + + // Tests matching the actual ClickHouse schemas in docker/otel-collector/schema/seed/ + + it('otel_logs schema: ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp), PARTITION BY toDate(Timestamp)', () => { + const result = appendSelectWithAdditionalKeys( + 'Timestamp, ServiceName, SeverityText, Body', + 'toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp', + 'toDate(Timestamp)', + ['_block_number', '_block_offset'], + ); + // Raw expressions (toStartOfFiveMinutes(Timestamp), toDate(Timestamp)) are + // appended alongside bare column references so the row WHERE clause can + // filter on PK expressions directly. + // Timestamp and ServiceName are already in select so they aren't added again. + expect(result).toEqual({ + additionalKeysLength: 4, + select: + 'Timestamp,ServiceName,SeverityText,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_number,_block_offset', + }); + }); + + it('otel_traces schema: ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)), PARTITION BY toDate(Timestamp)', () => { + const result = appendSelectWithAdditionalKeys( + 'Timestamp, ServiceName, SpanName, Duration', + 'ServiceName, SpanName, toDateTime(Timestamp)', + 'toDate(Timestamp)', + ['_block_number', '_block_offset'], + ); + expect(result).toEqual({ + additionalKeysLength: 4, + select: + 'Timestamp,ServiceName,SpanName,Duration,toDate(Timestamp),toDateTime(Timestamp),_block_number,_block_offset', + }); + }); + + it('otel_logs schema with __hdx_id after ServiceName in PK', () => { + // represents some potential hash + const result = appendSelectWithAdditionalKeys( + 'Timestamp, ServiceName, Body', + 'toStartOfFiveMinutes(Timestamp), ServiceName, __hdx_id, Timestamp', + 'toDate(Timestamp)', + ['_block_number', '_block_offset'], + ); + expect(result).toEqual({ + additionalKeysLength: 5, + select: + 'Timestamp,ServiceName,Body,__hdx_id,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_number,_block_offset', + }); + }); }); diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index a7209a94aa..aa9289aa87 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -584,4 +584,193 @@ describe('useRowWhere', () => { expect(() => result.current(row)).toThrow('Column type not found for id'); }); + + it('should filter to only primaryKeyColumns when provided', () => { + const meta: ColumnMetaType[] = [ + { name: 'Timestamp', type: 'DateTime64' }, + { name: 'ServiceName', type: 'String' }, + { name: 'Body', type: 'String' }, + { name: '__hdx_id', type: 'String' }, + ]; + + const primaryKeyColumns = new Set(['Timestamp', 'ServiceName', '__hdx_id']); + + const { result } = renderHook(() => + useRowWhere({ meta, primaryKeyColumns }), + ); + + const row = { + Timestamp: '2024-01-01T00:00:00Z', + ServiceName: 'my-service', + Body: 'a very long log message that should not be in the WHERE clause', + __hdx_id: 'abc123', + }; + const rowWhereResult = result.current(row); + + // Body should NOT be in the WHERE clause + expect(rowWhereResult.where).not.toContain('Body'); + expect(rowWhereResult.where).toContain('Timestamp'); + expect(rowWhereResult.where).toContain('ServiceName'); + expect(rowWhereResult.where).toContain('__hdx_id'); + }); + + it('should use all columns when primaryKeyColumns is not provided', () => { + const meta: ColumnMetaType[] = [ + { name: 'id', type: 'String' }, + { name: 'Body', type: 'String' }, + ]; + + const { result } = renderHook(() => useRowWhere({ meta })); + + const row = { id: '123', Body: 'hello' }; + const rowWhereResult = result.current(row); + + expect(rowWhereResult.where).toBe("id='123' AND Body='hello'"); + }); + + // Tests matching actual ClickHouse schemas in docker/otel-collector/schema/seed/ + // These simulate the full row data a user would see and verify that only + // PK + partition + block columns end up in the WHERE clause. + + it('otel_logs schema: only PK/partition/block columns in WHERE, Body and SeverityText excluded', () => { + // meta only contains actual columns — expression-based PK entries like + // toStartOfFiveMinutes(Timestamp) don't appear as result columns. + const meta: ColumnMetaType[] = [ + { name: 'Timestamp', type: "DateTime64(9, 'UTC')" }, + { name: 'ServiceName', type: 'String' }, + { name: 'SeverityText', type: 'String' }, + { name: 'Body', type: 'String' }, + { name: '_block_number', type: 'UInt64' }, + { name: '_block_offset', type: 'UInt64' }, + ]; + + // primaryKeyColumns includes expression names from the raw PK even though + // they won't match any row key — the filter silently skips them. + const primaryKeyColumns = new Set([ + 'Timestamp', + 'ServiceName', + 'toDate(Timestamp)', + 'toStartOfFiveMinutes(Timestamp)', + '_block_number', + '_block_offset', + ]); + + const { result } = renderHook(() => + useRowWhere({ meta, primaryKeyColumns }), + ); + + const row = { + Timestamp: '2026-05-20T21:20:00.123456789Z', + ServiceName: 'api-server', + SeverityText: 'ERROR', + Body: 'Connection refused to downstream service after 30s timeout', + _block_number: '2668', + _block_offset: '4', + }; + + const rowWhereResult = result.current(row); + + // Non-PK columns must NOT appear + expect(rowWhereResult.where).not.toContain('Body'); + expect(rowWhereResult.where).not.toContain('SeverityText'); + + // Actual PK column references and block columns must appear + expect(rowWhereResult.where).toContain('Timestamp'); + expect(rowWhereResult.where).toContain('ServiceName'); + expect(rowWhereResult.where).toContain('_block_number'); + expect(rowWhereResult.where).toContain('_block_offset'); + }); + + it('otel_traces schema: only PK/partition/block columns in WHERE, Duration and StatusCode excluded', () => { + const meta: ColumnMetaType[] = [ + { name: 'Timestamp', type: "DateTime64(9, 'UTC')" }, + { name: 'ServiceName', type: 'String' }, + { name: 'SpanName', type: 'String' }, + { name: 'Duration', type: 'Int64' }, + { name: 'StatusCode', type: 'String' }, + { name: '_block_number', type: 'UInt64' }, + { name: '_block_offset', type: 'UInt64' }, + ]; + + const primaryKeyColumns = new Set([ + 'Timestamp', + 'ServiceName', + 'SpanName', + 'toDate(Timestamp)', + 'toDateTime(Timestamp)', + '_block_number', + '_block_offset', + ]); + + const { result } = renderHook(() => + useRowWhere({ meta, primaryKeyColumns }), + ); + + const row = { + Timestamp: '2026-05-20T21:20:00.123456789Z', + ServiceName: 'api-server', + SpanName: 'GET /api/users', + Duration: '150000000', + StatusCode: 'STATUS_CODE_ERROR', + _block_number: '100', + _block_offset: '7', + }; + + const rowWhereResult = result.current(row); + + expect(rowWhereResult.where).not.toContain('Duration'); + expect(rowWhereResult.where).not.toContain('StatusCode'); + + expect(rowWhereResult.where).toContain('ServiceName'); + expect(rowWhereResult.where).toContain('SpanName'); + expect(rowWhereResult.where).toContain('_block_number'); + expect(rowWhereResult.where).toContain('_block_offset'); + }); + + it('otel_logs schema with __hdx_id in PK: hash column included, Body excluded', () => { + const meta: ColumnMetaType[] = [ + { name: 'Timestamp', type: "DateTime64(9, 'UTC')" }, + { name: 'ServiceName', type: 'String' }, + { name: 'SeverityText', type: 'String' }, + { name: 'Body', type: 'String' }, + { name: '__hdx_id', type: 'String' }, + { name: '_block_number', type: 'UInt64' }, + { name: '_block_offset', type: 'UInt64' }, + ]; + + const primaryKeyColumns = new Set([ + 'Timestamp', + 'ServiceName', + '__hdx_id', + 'toDate(Timestamp)', + 'toStartOfFiveMinutes(Timestamp)', + '_block_number', + '_block_offset', + ]); + + const { result } = renderHook(() => + useRowWhere({ meta, primaryKeyColumns }), + ); + + const row = { + Timestamp: '2026-05-20T21:20:00.123456789Z', + ServiceName: 'api-server', + SeverityText: 'INFO', + Body: 'Request completed successfully with 200 OK', + __hdx_id: 'a1b2c3d4e5f6', + _block_number: '500', + _block_offset: '12', + }; + + const rowWhereResult = result.current(row); + + expect(rowWhereResult.where).not.toContain('Body'); + expect(rowWhereResult.where).not.toContain('SeverityText'); + + expect(rowWhereResult.where).toContain('Timestamp'); + expect(rowWhereResult.where).toContain('__hdx_id'); + expect(rowWhereResult.where).toContain('ServiceName'); + expect(rowWhereResult.where).toContain('_block_number'); + expect(rowWhereResult.where).toContain('_block_offset'); + }); }); diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index 6d087b8548..25e664698f 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -132,9 +132,11 @@ export function processRowToWhereClause( export default function useRowWhere({ meta, aliasMap, + primaryKeyColumns, }: { meta?: ColumnMetaType[]; aliasMap?: Record; // map alias -> valueExpr, undefined is not supported + primaryKeyColumns?: Set; }) { const columnMap = useMemo( () => @@ -172,11 +174,21 @@ export default function useRowWhere({ [INTERNAL_ROW_FIELDS.ALIAS_WITH]: _aliasWith, ...dbRow } = row; + + // When primaryKeyColumns is provided, only use those columns in the + // WHERE clause. This avoids filtering on large columns like Body that + // trigger expensive index loading in ClickHouse. + const filteredRow = primaryKeyColumns + ? Object.fromEntries( + Object.entries(dbRow).filter(([col]) => primaryKeyColumns.has(col)), + ) + : dbRow; + return { - where: processRowToWhereClause(dbRow, columnMap), + where: processRowToWhereClause(filteredRow, columnMap), aliasWith, }; }, - [columnMap, aliasWith], + [columnMap, aliasWith, primaryKeyColumns], ); } diff --git a/packages/cli/src/api/eventQuery.ts b/packages/cli/src/api/eventQuery.ts index 5e53a7e515..d2a5650b24 100644 --- a/packages/cli/src/api/eventQuery.ts +++ b/packages/cli/src/api/eventQuery.ts @@ -337,9 +337,12 @@ export async function buildFullRowQuery( /** Column metadata from the table query response */ tableMeta: ColumnMetaType[]; metadata: Metadata; + /** When provided, only these columns are used in the row WHERE clause */ + primaryKeyColumns?: Set; }, ): Promise { - const { source, row, tableChSql, tableMeta, metadata } = opts; + const { source, row, tableChSql, tableMeta, metadata, primaryKeyColumns } = + opts; // Parse the rendered table SQL to get alias → expression mapping const aliasMap = chSqlToAliasMap(tableChSql); @@ -352,6 +355,7 @@ export async function buildFullRowQuery( row as Record, columnMap, aliasMap, + primaryKeyColumns, ); const selectList = buildRowDataSelectList(source); diff --git a/packages/cli/src/shared/useRowWhere.ts b/packages/cli/src/shared/useRowWhere.ts index 45c1a2b44e..9497c61712 100644 --- a/packages/cli/src/shared/useRowWhere.ts +++ b/packages/cli/src/shared/useRowWhere.ts @@ -168,11 +168,16 @@ function buildAliasWith( /** * Generate a RowWhereResult from a row, column map, and alias map. * Non-React equivalent of the useRowWhere hook's returned callback. + * + * When primaryKeyColumns is provided, only those columns are used in the + * WHERE clause. This avoids filtering on large columns like Body that + * trigger expensive index loading in ClickHouse. */ export function getRowWhere( row: Record, columnMap: Map, aliasMap: Record | undefined, + primaryKeyColumns?: Set, ): RowWhereResult { // Filter out synthetic columns that aren't in the database schema const { @@ -180,8 +185,15 @@ export function getRowWhere( [INTERNAL_ROW_FIELDS.ALIAS_WITH]: _aliasWith, ...dbRow } = row; + + const filteredRow = primaryKeyColumns + ? Object.fromEntries( + Object.entries(dbRow).filter(([col]) => primaryKeyColumns.has(col)), + ) + : dbRow; + return { - where: processRowToWhereClause(dbRow, columnMap), + where: processRowToWhereClause(filteredRow, columnMap), aliasWith: buildAliasWith(aliasMap), }; } From 59353ed6fce778fbe1d3d8c3b3bb84cde7a3661d Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Wed, 20 May 2026 18:00:17 -0400 Subject: [PATCH 2/2] add changeset --- .changeset/neat-pianos-cheat.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/neat-pianos-cheat.md diff --git a/.changeset/neat-pianos-cheat.md b/.changeset/neat-pianos-cheat.md new file mode 100644 index 0000000000..f3f7d03ba2 --- /dev/null +++ b/.changeset/neat-pianos-cheat.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/app": patch +"@hyperdx/cli": patch +--- + +fix: only use pk and row uniqueness to look up a row