feat: only use pk + block ids to resolve a row

.changeset/neat-pianos-cheat.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/app": patch
+"@hyperdx/cli": patch
+---
+
+fix: only use pk and row uniqueness to look up a row

packages/app/src/components/DBRowTable.tsx

@@ -1368,9 +1368,20 @@ export function appendSelectWithAdditionalKeys(
   partitionKey: string,
   extraKeys: string[] = [],
 ): { select: SelectList; additionalKeysLength: number } {
+  // Include both the raw key expressions (e.g. toStartOfFiveMinutes(Timestamp))
+  // and the extracted column references (e.g. Timestamp). The raw expressions
+  // are needed so the row WHERE clause can filter on PK expressions directly.
   const partitionKeyArr = extractColumnReferencesFromKey(partitionKey);
   const primaryKeyArr = extractColumnReferencesFromKey(primaryKeys);
-  const allKeys = new Set([...partitionKeyArr, ...primaryKeyArr, ...extraKeys]);
+  const rawPartitionExprs = splitAndTrimWithBracket(partitionKey);
+  const rawPrimaryExprs = splitAndTrimWithBracket(primaryKeys);
+  const allKeys = new Set([
+    ...partitionKeyArr,
+    ...primaryKeyArr,
+    ...rawPartitionExprs,
+    ...rawPrimaryExprs,
+    ...extraKeys,
+  ]);
   if (typeof select === 'string') {
     const selectSplit = splitAndTrimWithBracket(select);
     const selectColumns = new Set(selectSplit);
@@ -1448,7 +1459,27 @@ export function useConfigWithAdditionalSelect(
       extraKeys,
     );
 
-    return { ...config, select, additionalKeysLength };
+    // When block columns are available, the PK + partition + block columns
+    // uniquely identify a row. Compute the full set of key column names
+    // (both raw expressions like toStartOfFiveMinutes(Timestamp) and bare
+    // column references like Timestamp, ServiceName) so the row WHERE clause
+    // can use only these, avoiding expensive index loading on large columns
+    // like Body.
+    const hasBlockColumns =
+      extraKeys.includes('_block_number') &&
+      extraKeys.includes('_block_offset');
+
+    const rowKeyColumns = hasBlockColumns
+      ? new Set([
+          ...splitAndTrimWithBracket(partitionKey),
+          ...splitAndTrimWithBracket(primaryKey),
+          ...extractColumnReferencesFromKey(partitionKey),
+          ...extractColumnReferencesFromKey(primaryKey),
+          ...extraKeys,
+        ])
+      : undefined;
+
+    return { ...config, select, additionalKeysLength, rowKeyColumns };
   }, [primaryKey, partitionKey, config, tableMetadata, columns, sourceId]);
 }
 
@@ -1674,7 +1705,11 @@ function DBSqlRowTableComponent({
     [data],
   );
 
-  const getRowWhere = useRowWhere({ meta: data?.meta, aliasMap });
+  const getRowWhere = useRowWhere({
+    meta: data?.meta,
+    aliasMap,
+    primaryKeyColumns: mergedConfig?.rowKeyColumns,
+  });
 
   const _onRowDetailsClick = useCallback(
     (row: Record<string, any>) => {

packages/app/src/components/__tests__/DBRowTable.test.tsx

@@ -153,8 +153,9 @@ describe('appendSelectWithAdditionalKeys', () => {
       ' toStartOfInterval(timestamp, toIntervalDay(3))',
     );
     expect(result).toEqual({
-      additionalKeysLength: 3,
-      select: 'col1,col2,timestamp,id,created_at',
+      additionalKeysLength: 4,
+      select:
+        'col1,col2,timestamp,id,created_at,toStartOfInterval(timestamp, toIntervalDay(3))',
     });
   });
 
@@ -173,8 +174,9 @@ describe('appendSelectWithAdditionalKeys', () => {
       "toStartOfInterval(timestamp, toIntervalDay(3)), date_diff('DAY', col3, col4), now(), toDate(col5 + INTERVAL 1 DAY)",
     );
     expect(result).toEqual({
-      additionalKeysLength: 6,
-      select: 'col1,col2,timestamp,col3,col4,col5,id,timestamp2',
+      additionalKeysLength: 11,
+      select:
+        "col1,col2,timestamp,col3,col4,col5,id,timestamp2,toStartOfInterval(timestamp, toIntervalDay(3)),date_diff('DAY', col3, col4),now(),toDate(col5 + INTERVAL 1 DAY),toStartOfInterval(timestamp2, toIntervalDay(3))",
     });
   });
 
@@ -221,8 +223,8 @@ describe('appendSelectWithAdditionalKeys', () => {
       `json.a, json.b.c, toStartOfDay(timestamp, json_2.d)`,
     );
     expect(result).toEqual({
-      additionalKeysLength: 5,
-      select: `col1,col2,json.a,json.b.c,timestamp,json_2.d,json.b`,
+      additionalKeysLength: 6,
+      select: `col1,col2,json.a,json.b.c,timestamp,json_2.d,json.b,toStartOfDay(timestamp, json_2.d)`,
     });
   });
 
@@ -233,8 +235,8 @@ describe('appendSelectWithAdditionalKeys', () => {
       `toStartOfDay(json.a.b.:DateTime)`,
     );
     expect(result).toEqual({
-      additionalKeysLength: 2,
-      select: `col1,col2,json.a.b,json.b`,
+      additionalKeysLength: 4,
+      select: `col1,col2,json.a.b,json.b,toStartOfDay(json.a.b.:DateTime),json.b.:Int64`,
     });
   });
 
@@ -245,8 +247,8 @@ describe('appendSelectWithAdditionalKeys', () => {
       ``,
     );
     expect(result).toEqual({
-      additionalKeysLength: 1,
-      select: `col1,col2,col3`,
+      additionalKeysLength: 2,
+      select: `col1,col2,col3,json.b.:Array(String)`,
     });
   });
 
@@ -257,8 +259,8 @@ describe('appendSelectWithAdditionalKeys', () => {
       ``,
     );
     expect(result).toEqual({
-      additionalKeysLength: 1,
-      select: `col1,col2,col3`,
+      additionalKeysLength: 2,
+      select: `col1,col2,col3,map['key']['key2']`,
     });
   });
 
@@ -310,4 +312,53 @@ describe('appendSelectWithAdditionalKeys', () => {
       ],
     });
   });
+
+  // Tests matching the actual ClickHouse schemas in docker/otel-collector/schema/seed/
+
+  it('otel_logs schema: ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp), PARTITION BY toDate(Timestamp)', () => {
+    const result = appendSelectWithAdditionalKeys(
+      'Timestamp, ServiceName, SeverityText, Body',
+      'toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp',
+      'toDate(Timestamp)',
+      ['_block_number', '_block_offset'],
+    );
+    // Raw expressions (toStartOfFiveMinutes(Timestamp), toDate(Timestamp)) are
+    // appended alongside bare column references so the row WHERE clause can
+    // filter on PK expressions directly.
+    // Timestamp and ServiceName are already in select so they aren't added again.
+    expect(result).toEqual({
+      additionalKeysLength: 4,
+      select:
+        'Timestamp,ServiceName,SeverityText,Body,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_number,_block_offset',
+    });
+  });
+
+  it('otel_traces schema: ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)), PARTITION BY toDate(Timestamp)', () => {
+    const result = appendSelectWithAdditionalKeys(
+      'Timestamp, ServiceName, SpanName, Duration',
+      'ServiceName, SpanName, toDateTime(Timestamp)',
+      'toDate(Timestamp)',
+      ['_block_number', '_block_offset'],
+    );
+    expect(result).toEqual({
+      additionalKeysLength: 4,
+      select:
+        'Timestamp,ServiceName,SpanName,Duration,toDate(Timestamp),toDateTime(Timestamp),_block_number,_block_offset',
+    });
+  });
+
+  it('otel_logs schema with __hdx_id after ServiceName in PK', () => {
+    // represents some potential hash
+    const result = appendSelectWithAdditionalKeys(
+      'Timestamp, ServiceName, Body',
+      'toStartOfFiveMinutes(Timestamp), ServiceName, __hdx_id, Timestamp',
+      'toDate(Timestamp)',
+      ['_block_number', '_block_offset'],
+    );
+    expect(result).toEqual({
+      additionalKeysLength: 5,
+      select:
+        'Timestamp,ServiceName,Body,__hdx_id,toDate(Timestamp),toStartOfFiveMinutes(Timestamp),_block_number,_block_offset',
+    });
+  });
 });

packages/app/src/hooks/__tests__/useRowWhere.test.tsx

@@ -584,4 +584,193 @@ describe('useRowWhere', () => {
 
     expect(() => result.current(row)).toThrow('Column type not found for id');
   });
+
+  it('should filter to only primaryKeyColumns when provided', () => {
+    const meta: ColumnMetaType[] = [
+      { name: 'Timestamp', type: 'DateTime64' },
+      { name: 'ServiceName', type: 'String' },
+      { name: 'Body', type: 'String' },
+      { name: '__hdx_id', type: 'String' },
+    ];
+
+    const primaryKeyColumns = new Set(['Timestamp', 'ServiceName', '__hdx_id']);
+
+    const { result } = renderHook(() =>
+      useRowWhere({ meta, primaryKeyColumns }),
+    );
+
+    const row = {
+      Timestamp: '2024-01-01T00:00:00Z',
+      ServiceName: 'my-service',
+      Body: 'a very long log message that should not be in the WHERE clause',
+      __hdx_id: 'abc123',
+    };
+    const rowWhereResult = result.current(row);
+
+    // Body should NOT be in the WHERE clause
+    expect(rowWhereResult.where).not.toContain('Body');
+    expect(rowWhereResult.where).toContain('Timestamp');
+    expect(rowWhereResult.where).toContain('ServiceName');
+    expect(rowWhereResult.where).toContain('__hdx_id');
+  });
+
+  it('should use all columns when primaryKeyColumns is not provided', () => {
+    const meta: ColumnMetaType[] = [
+      { name: 'id', type: 'String' },
+      { name: 'Body', type: 'String' },
+    ];
+
+    const { result } = renderHook(() => useRowWhere({ meta }));
+
+    const row = { id: '123', Body: 'hello' };
+    const rowWhereResult = result.current(row);
+
+    expect(rowWhereResult.where).toBe("id='123' AND Body='hello'");
+  });
+
+  // Tests matching actual ClickHouse schemas in docker/otel-collector/schema/seed/
+  // These simulate the full row data a user would see and verify that only
+  // PK + partition + block columns end up in the WHERE clause.
+
+  it('otel_logs schema: only PK/partition/block columns in WHERE, Body and SeverityText excluded', () => {
+    // meta only contains actual columns — expression-based PK entries like
+    // toStartOfFiveMinutes(Timestamp) don't appear as result columns.
+    const meta: ColumnMetaType[] = [
+      { name: 'Timestamp', type: "DateTime64(9, 'UTC')" },
+      { name: 'ServiceName', type: 'String' },
+      { name: 'SeverityText', type: 'String' },
+      { name: 'Body', type: 'String' },
+      { name: '_block_number', type: 'UInt64' },
+      { name: '_block_offset', type: 'UInt64' },
+    ];
+
+    // primaryKeyColumns includes expression names from the raw PK even though
+    // they won't match any row key — the filter silently skips them.
+    const primaryKeyColumns = new Set([
+      'Timestamp',
+      'ServiceName',
+      'toDate(Timestamp)',
+      'toStartOfFiveMinutes(Timestamp)',
+      '_block_number',
+      '_block_offset',
+    ]);
+
+    const { result } = renderHook(() =>
+      useRowWhere({ meta, primaryKeyColumns }),
+    );
+
+    const row = {
+      Timestamp: '2026-05-20T21:20:00.123456789Z',
+      ServiceName: 'api-server',
+      SeverityText: 'ERROR',
+      Body: 'Connection refused to downstream service after 30s timeout',
+      _block_number: '2668',
+      _block_offset: '4',
+    };
+
+    const rowWhereResult = result.current(row);
+
+    // Non-PK columns must NOT appear
+    expect(rowWhereResult.where).not.toContain('Body');
+    expect(rowWhereResult.where).not.toContain('SeverityText');
+
+    // Actual PK column references and block columns must appear
+    expect(rowWhereResult.where).toContain('Timestamp');
+    expect(rowWhereResult.where).toContain('ServiceName');
+    expect(rowWhereResult.where).toContain('_block_number');
+    expect(rowWhereResult.where).toContain('_block_offset');
+  });
+
+  it('otel_traces schema: only PK/partition/block columns in WHERE, Duration and StatusCode excluded', () => {
+    const meta: ColumnMetaType[] = [
+      { name: 'Timestamp', type: "DateTime64(9, 'UTC')" },
+      { name: 'ServiceName', type: 'String' },
+      { name: 'SpanName', type: 'String' },
+      { name: 'Duration', type: 'Int64' },
+      { name: 'StatusCode', type: 'String' },
+      { name: '_block_number', type: 'UInt64' },
+      { name: '_block_offset', type: 'UInt64' },
+    ];
+
+    const primaryKeyColumns = new Set([
+      'Timestamp',
+      'ServiceName',
+      'SpanName',
+      'toDate(Timestamp)',
+      'toDateTime(Timestamp)',
+      '_block_number',
+      '_block_offset',
+    ]);
+
+    const { result } = renderHook(() =>
+      useRowWhere({ meta, primaryKeyColumns }),
+    );
+
+    const row = {
+      Timestamp: '2026-05-20T21:20:00.123456789Z',
+      ServiceName: 'api-server',
+      SpanName: 'GET /api/users',
+      Duration: '150000000',
+      StatusCode: 'STATUS_CODE_ERROR',
+      _block_number: '100',
+      _block_offset: '7',
+    };
+
+    const rowWhereResult = result.current(row);
+
+    expect(rowWhereResult.where).not.toContain('Duration');
+    expect(rowWhereResult.where).not.toContain('StatusCode');
+
+    expect(rowWhereResult.where).toContain('ServiceName');
+    expect(rowWhereResult.where).toContain('SpanName');
+    expect(rowWhereResult.where).toContain('_block_number');
+    expect(rowWhereResult.where).toContain('_block_offset');
+  });
+
+  it('otel_logs schema with __hdx_id in PK: hash column included, Body excluded', () => {
+    const meta: ColumnMetaType[] = [
+      { name: 'Timestamp', type: "DateTime64(9, 'UTC')" },
+      { name: 'ServiceName', type: 'String' },
+      { name: 'SeverityText', type: 'String' },
+      { name: 'Body', type: 'String' },
+      { name: '__hdx_id', type: 'String' },
+      { name: '_block_number', type: 'UInt64' },
+      { name: '_block_offset', type: 'UInt64' },
+    ];
+
+    const primaryKeyColumns = new Set([
+      'Timestamp',
+      'ServiceName',
+      '__hdx_id',
+      'toDate(Timestamp)',
+      'toStartOfFiveMinutes(Timestamp)',
+      '_block_number',
+      '_block_offset',
+    ]);
+
+    const { result } = renderHook(() =>
+      useRowWhere({ meta, primaryKeyColumns }),
+    );
+
+    const row = {
+      Timestamp: '2026-05-20T21:20:00.123456789Z',
+      ServiceName: 'api-server',
+      SeverityText: 'INFO',
+      Body: 'Request completed successfully with 200 OK',
+      __hdx_id: 'a1b2c3d4e5f6',
+      _block_number: '500',
+      _block_offset: '12',
+    };
+
+    const rowWhereResult = result.current(row);
+
+    expect(rowWhereResult.where).not.toContain('Body');
+    expect(rowWhereResult.where).not.toContain('SeverityText');
+
+    expect(rowWhereResult.where).toContain('Timestamp');
+    expect(rowWhereResult.where).toContain('__hdx_id');
+    expect(rowWhereResult.where).toContain('ServiceName');
+    expect(rowWhereResult.where).toContain('_block_number');
+    expect(rowWhereResult.where).toContain('_block_offset');
+  });
 });