fix: don't scan the whole table to discover Map keys

.changeset/bounded-map-key-discovery.md

@@ -0,0 +1,15 @@
+---
+'@hyperdx/common-utils': patch
+'@hyperdx/app': patch
+'@hyperdx/api': patch
+---
+
+fix: don't scan the whole table to discover Map keys
+
+`getMapKeys` only applied a time predicate when the caller passed both a date
+range and a timestamp expression; otherwise the raw `sampledKeys` scan ran with
+no `WHERE` and touched every part of the table. It now defaults a missing date
+range to the last 24 hours and skips the raw scan entirely when there is no
+timestamp expression to filter on. The chart, alert and dashboard-filter
+editors pass the source and date range they already have so Map keys keep
+autocompleting there.

packages/api/src/controllers/ai.ts

@@ -3,6 +3,7 @@ import { createOpenAI } from '@ai-sdk/openai';
 import {
   getMetadata,
   TableMetadata,
+  tcFromSource,
 } from '@hyperdx/common-utils/dist/core/metadata';
 import {
   AILineTableResponse,
@@ -128,6 +129,8 @@ export async function getAIMetadata(source: ISource) {
     databaseName,
     tableName,
     connectionId,
+    metadataMVs: tcFromSource(source).metadataMVs,
+    timestampValueExpression: source.timestampValueExpression,
   });
 
   // TODO: Dedup with DBSearchPageFilters.tsx logic

packages/app/src/DBSearchPageAlertModal.tsx

@@ -314,6 +314,7 @@ const AlertForm = ({
           </Text>
           <SQLInlineEditorControlled
             tableConnection={tcFromSource(source)}
+            sourceId={sourceId ?? undefined}
             control={control}
             name={`groupBy`}
             placeholder="SQL Columns"

packages/app/src/components/DBEditTimeChartForm/ChartActionBar.tsx

@@ -65,6 +65,8 @@ type ChartActionBarProps = {
   control: Control<ChartEditorFormState>;
   handleSubmit: UseFormHandleSubmit<ChartEditorFormState>;
   tableConnection: TableConnection;
+  sourceId?: string;
+  dateRange?: [Date, Date];
   activeTab: string;
   isRawSqlInput: boolean;
   dashboardId?: string;
@@ -98,6 +100,8 @@ export function ChartActionBar({
   control,
   handleSubmit,
   tableConnection,
+  sourceId,
+  dateRange,
   activeTab,
   isRawSqlInput,
   dashboardId,
@@ -187,6 +191,8 @@ export function ChartActionBar({
               <SQLInlineEditorControlled
                 parentRef={parentRef}
                 tableConnection={tableConnection}
+                sourceId={sourceId}
+                dateRange={dateRange}
                 // The default order by is the current group by value
                 placeholder={typeof groupBy === 'string' ? groupBy : ''}
                 control={control}

packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx

@@ -240,6 +240,8 @@ export function ChartEditorControls({
         <Flex gap="xs" direction="column">
           <SQLInlineEditorControlled
             tableConnection={tableConnection}
+            sourceId={tableSource?.id}
+            dateRange={dateRange}
             control={control}
             name="select"
             placeholder={
@@ -346,6 +348,8 @@ export function ChartEditorControls({
                 <div>
                   <SQLInlineEditorControlled
                     {...groupByConnectionProps}
+                    sourceId={tableSource?.id}
+                    dateRange={dateRange}
                     control={control}
                     name={`groupBy`}
                     placeholder="SQL Columns"
@@ -370,6 +374,8 @@ export function ChartEditorControls({
                     <div>
                       <SQLInlineEditorControlled
                         tableConnection={tableConnection}
+                        sourceId={tableSource?.id}
+                        dateRange={dateRange}
                         control={control}
                         name="having"
                         placeholder="SQL HAVING clause (ex. count() > 100)"
@@ -533,6 +539,8 @@ export function ChartEditorControls({
         <Flex gap="xs" direction="column">
           <SQLInlineEditorControlled
             tableConnection={tableConnection}
+            sourceId={tableSource?.id}
+            dateRange={dateRange}
             control={control}
             name="select"
             placeholder={

packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx

@@ -419,6 +419,8 @@ export function ChartSeriesEditor({
           >
             <SQLInlineEditorControlled
               tableConnection={tableConnection}
+              sourceId={tableSource?.id}
+              dateRange={dateRange}
               control={control}
               name={`${namePrefix}valueExpression`}
               placeholder="SQL Column"
@@ -479,6 +481,8 @@ export function ChartSeriesEditor({
                   <SQLInlineEditorControlled
                     parentRef={parentRef}
                     tableConnection={tableConnection}
+                    sourceId={tableSource?.id}
+                    dateRange={dateRange}
                     control={control}
                     name={`groupBy`}
                     placeholder="SQL Columns"
@@ -497,6 +501,8 @@ export function ChartSeriesEditor({
                     <div style={{ minWidth: 300, maxWidth: '100%' }}>
                       <SQLInlineEditorControlled
                         tableConnection={tableConnection}
+                        sourceId={tableSource?.id}
+                        dateRange={dateRange}
                         control={control}
                         name="having"
                         placeholder="SQL HAVING clause (ex. count() > 100)"

packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx

@@ -1061,6 +1061,8 @@ export default function EditTimeChartForm({
           control={control}
           handleSubmit={handleSubmit}
           tableConnection={tableConnection}
+          sourceId={tableSource?.id}
+          dateRange={dateRange}
           activeTab={activeTab}
           isRawSqlInput={isRawSqlInput}
           dashboardId={dashboardId}
@@ -1127,6 +1129,8 @@ export default function EditTimeChartForm({
         opened={heatmapSettingsOpened}
         onClose={closeHeatmapSettings}
         connection={tableConnection}
+        sourceId={tableSource?.id}
+        dateRange={dateRange}
         parentRef={parentRef}
         defaultValues={heatmapSettingsDefaults}
         onSubmit={handleUpdateHeatmapSettings}

packages/app/src/components/DashboardFiltersModal/QueryExpressionFilterEditForm.tsx

@@ -178,6 +178,7 @@ export const QueryExpressionFilterEditForm = ({
       >
         <SQLInlineEditorControlled
           tableConnection={tableConnection}
+          sourceId={sourceId}
           control={control}
           name="expression"
           placeholder="SQL column or expression"

packages/app/src/components/HeatmapSettingsDrawer.tsx

@@ -29,13 +29,17 @@ export default function HeatmapSettingsDrawer({
   opened,
   onClose,
   connection,
+  sourceId,
+  dateRange,
   parentRef,
   defaultValues,
   onSubmit,
 }: {
   opened: boolean;
   onClose: () => void;
   connection: TableConnection;
+  sourceId?: string;
+  dateRange?: [Date, Date];
   parentRef?: HTMLElement | null;
   defaultValues: HeatmapSettingsValues;
   onSubmit: (v: HeatmapSettingsValues) => void;
@@ -92,6 +96,8 @@ export default function HeatmapSettingsDrawer({
           <SQLInlineEditorControlled
             parentRef={parentRef}
             tableConnection={connection}
+            sourceId={sourceId}
+            dateRange={dateRange}
             control={form.control}
             name="value"
             size="xs"
@@ -107,6 +113,8 @@ export default function HeatmapSettingsDrawer({
           <SQLInlineEditorControlled
             parentRef={parentRef}
             tableConnection={connection}
+            sourceId={sourceId}
+            dateRange={dateRange}
             control={form.control}
             name="count"
             placeholder="SQL expression"

packages/app/src/components/Search/DBSearchHeatmapChart.tsx

@@ -189,6 +189,8 @@ export function DBSearchHeatmapChart({
         opened={settingsOpened}
         onClose={settingsHandlers.close}
         connection={tcFromSource(source)}
+        sourceId={source.id}
+        dateRange={chartConfig.dateRange}
         parentRef={container}
         defaultValues={heatmapSettingsDefaults}
         onSubmit={data => {

packages/app/src/components/alerts/EditAlertModal.tsx

@@ -427,6 +427,7 @@ export function EditAlertModal({
               </Text>
               <SQLInlineEditorControlled
                 tableConnection={tcFromSource(source)}
+                sourceId={source?.id}
                 control={control}
                 name="groupBy"
                 placeholder="SQL Columns"

packages/app/src/hooks/__tests__/useMetadata.test.tsx

@@ -336,6 +336,32 @@ describe('useMultipleAllFields', () => {
     expect(result.current.data).toEqual(fieldsA);
   });
 
+  it('falls back to each connection timestampValueExpression when none is passed', async () => {
+    const getAllFields = jest
+      .spyOn(mockMetadata, 'getAllFields')
+      .mockResolvedValue(fieldsA);
+
+    const { result } = renderHook(
+      () =>
+        useMultipleAllFields([
+          { ...tcA, timestampValueExpression: 'TimestampA' },
+          tcB,
+        ]),
+      { wrapper },
+    );
+
+    await waitFor(() => expect(result.current.isSuccess).toBe(true));
+    expect(getAllFields).toHaveBeenCalledWith(
+      expect.objectContaining({ timestampValueExpression: 'TimestampA' }),
+    );
+    expect(getAllFields).toHaveBeenCalledWith(
+      expect.objectContaining({
+        tableName: 'table_b',
+        timestampValueExpression: undefined,
+      }),
+    );
+  });
+
   it('should deduplicate fields across successful connections', async () => {
     jest
       .spyOn(mockMetadata, 'getAllFields')

packages/app/src/hooks/useMetadata.tsx

@@ -261,7 +261,12 @@ export function useMultipleAllFields(
 
       const promiseResults = await Promise.allSettled(
         tableConnections.map(tc =>
-          metadata.getAllFields({ ...tc, dateRange, timestampValueExpression }),
+          metadata.getAllFields({
+            ...tc,
+            dateRange,
+            timestampValueExpression:
+              timestampValueExpression ?? tc.timestampValueExpression,
+          }),
         ),
       );
 

packages/common-utils/src/__tests__/metadata.int.test.ts

@@ -1306,4 +1306,109 @@ describe('Metadata Integration Tests', () => {
       querySpy.mockRestore();
     });
   });
+
+  describe('getMapKeys - unbounded scan guard (#3037)', () => {
+    const tableName = 'test_map_keys_scan';
+    let metadata: Metadata;
+
+    beforeAll(async () => {
+      await client.command({
+        query: `DROP TABLE IF EXISTS default.${tableName}`,
+      });
+      // Map(LowCardinality(String), String) hits a separate, pre-existing bug on this CH version.
+      await client.command({
+        query: `CREATE TABLE default.${tableName} (
+            Timestamp DateTime64(9),
+            LogAttributes Map(String, String)
+          )
+          ENGINE = MergeTree()
+          PARTITION BY toDate(Timestamp)
+          ORDER BY Timestamp
+        `,
+      });
+
+      // One row inside the default 24h lookback, one row 5 days back in its own partition.
+      await client.insert({
+        table: `default.${tableName}`,
+        values: [
+          {
+            Timestamp: new Date(Date.now() - 60 * 60 * 1000)
+              .toISOString()
+              .replace('T', ' ')
+              .replace('Z', ''),
+            LogAttributes: { 'http.method': 'GET' },
+          },
+          {
+            Timestamp: new Date(Date.now() - 5 * 24 * 60 * 60 * 1000)
+              .toISOString()
+              .replace('T', ' ')
+              .replace('Z', ''),
+            LogAttributes: { 'legacy.key': 'value' },
+          },
+        ],
+        format: 'JSONEachRow',
+      });
+    });
+
+    afterAll(async () => {
+      await client.command({
+        query: `DROP TABLE IF EXISTS default.${tableName}`,
+      });
+    });
+
+    beforeEach(() => {
+      metadata = new Metadata(hdxClient, new MetadataCache());
+    });
+
+    it('falls back to an unbounded scan when no timestampValueExpression is given', async () => {
+      const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
+
+      const keys = await metadata.getMapKeys({
+        databaseName: 'default',
+        tableName,
+        column: 'LogAttributes',
+        connectionId: 'test_connection',
+        dateRange: [new Date(Date.now() - 60 * 60 * 1000), new Date()],
+      });
+
+      // dateRange is ignored without an expression to apply it to.
+      expect(keys).toEqual(
+        expect.arrayContaining(['http.method', 'legacy.key']),
+      );
+      expect(warnSpy).toHaveBeenCalledWith(
+        expect.stringContaining('Unbounded Map key scan'),
+      );
+
+      warnSpy.mockRestore();
+    });
+
+    it('defaults to the last 24h and excludes older partitions', async () => {
+      const keys = await metadata.getMapKeys({
+        databaseName: 'default',
+        tableName,
+        column: 'LogAttributes',
+        connectionId: 'test_connection',
+        timestampValueExpression: 'Timestamp',
+      });
+
+      expect(keys).toContain('http.method');
+      expect(keys).not.toContain('legacy.key');
+    });
+
+    it('reaches older partitions when the caller asks for that window explicitly', async () => {
+      const keys = await metadata.getMapKeys({
+        databaseName: 'default',
+        tableName,
+        column: 'LogAttributes',
+        connectionId: 'test_connection',
+        timestampValueExpression: 'Timestamp',
+        dateRange: [
+          new Date(Date.now() - 6 * 24 * 60 * 60 * 1000),
+          new Date(Date.now() - 4 * 24 * 60 * 60 * 1000),
+        ],
+      });
+
+      expect(keys).toEqual(['legacy.key']);
+    });
+  });
 });