feat: allow up to 10,000 filter keys per column in team settings

.changeset/filter-keys-fetch-limit-max.md

@@ -0,0 +1,10 @@
+---
+'@hyperdx/app': patch
+'@hyperdx/common-utils': patch
+---
+
+feat: allow up to 10,000 in the team "Filter Keys Fetch Limit" setting
+
+The setting's input in Team settings stopped at 1,000 keys per column; it now
+accepts up to 10,000. Map key discovery returns up to the setting's value per
+Map column, and never fewer than 1,000.

packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx

@@ -324,7 +324,7 @@ export default function TeamQueryConfigSection() {
             defaultValue={DEFAULT_FILTER_KEYS_FETCH_LIMIT}
             placeholder={`default = ${DEFAULT_FILTER_KEYS_FETCH_LIMIT}`}
             min={1}
-            max={1000}
+            max={10000}
             displayValue={displayValueWithUnit('keys')}
             description={`Default is ${DEFAULT_FILTER_KEYS_FETCH_LIMIT}`}
           />

packages/app/src/hooks/__tests__/useMetadata.test.tsx

@@ -734,7 +734,7 @@ describe('useMultipleAllFields', () => {
   ];
   const columnsB = [column('col_b', 'String')];
 
-  let team: { fieldMetadataDisabled?: boolean };
+  let team: { fieldMetadataDisabled?: boolean; filterKeysFetchLimit?: number };
 
   beforeEach(() => {
     jest.clearAllMocks();
@@ -811,6 +811,29 @@ describe('useMultipleAllFields', () => {
     );
   });
 
+  it.each([
+    [undefined, 1000],
+    [100, 1000],
+    [5000, 5000],
+  ])(
+    'with a filter key limit of %s, discovers up to %s keys per Map column',
+    async (filterKeysFetchLimit, maxKeys) => {
+      team.filterKeysFetchLimit = filterKeysFetchLimit;
+      const getAllFields = jest
+        .spyOn(mockMetadata, 'getAllFields')
+        .mockResolvedValue(fieldsA);
+
+      const { result } = renderHook(() => useMultipleAllFields([tcA]), {
+        wrapper,
+      });
+
+      await waitFor(() => expect(result.current.isSuccess).toBe(true));
+      expect(getAllFields).toHaveBeenCalledWith(
+        expect.objectContaining({ maxKeys }),
+      );
+    },
+  );
+
   it('should deduplicate fields across successful connections', async () => {
     jest
       .spyOn(mockMetadata, 'getAllFields')

packages/app/src/hooks/useMetadata.tsx

@@ -7,6 +7,7 @@ import {
   JSDataType,
 } from '@hyperdx/common-utils/dist/clickhouse';
 import {
+  DEFAULT_MAX_KEYS,
   Field,
   FieldsCondition,
   MetricNames,
@@ -279,6 +280,9 @@ export function useMultipleAllFields(
   const metadata = useMetadataWithSettings();
   const { data: me, isFetched } = api.useMe();
   const isFieldMetadataDisabled = !!me?.team?.fieldMetadataDisabled;
+  // The team setting only raises Map key discovery, so a low limit doesn't
+  // shrink the keys offered to autocomplete and the other field pickers.
+  const maxKeys = Math.max(useFilterKeysFetchLimit(), DEFAULT_MAX_KEYS);
   const allPopulated = tableConnections.every(
     tc => !!tc.databaseName && !!tc.tableName && !!tc.connectionId,
   );
@@ -291,6 +295,7 @@ export function useMultipleAllFields(
       timestampValueExpression,
       isFieldMetadataDisabled,
       conditions,
+      maxKeys,
     ],
     queryFn: () =>
       Promise.all(
@@ -310,6 +315,7 @@ export function useMultipleAllFields(
                   timestampValueExpression:
                     timestampValueExpression ?? tc.timestampValueExpression,
                   condition: conditions?.[i],
+                  maxKeys,
                 });
             return { fields, columns };
           } catch (error) {

packages/common-utils/src/__tests__/metadata.test.ts

@@ -3823,7 +3823,7 @@ describe('Metadata', () => {
       jest.clearAllMocks();
     });
 
-    it('threads dateRange and timestampValueExpression through to getMapKeys', async () => {
+    it('threads dateRange, timestampValueExpression, and maxKeys through to getMapKeys', async () => {
       const realCache = new (
         jest.requireActual('../core/metadata') as any
       ).MetadataCache();
@@ -3860,12 +3860,14 @@ describe('Metadata', () => {
         connectionId: 'conn-1',
         dateRange,
         timestampValueExpression: 'EventTime, EventDate',
+        maxKeys: 5000,
       });
 
       expect(getMapKeysSpy).toHaveBeenCalledWith(
         expect.objectContaining({
           dateRange,
           timestampValueExpression: 'EventTime, EventDate',
+          maxKeys: 5000,
         }),
       );
     });

packages/common-utils/src/core/metadata.ts

@@ -70,7 +70,7 @@ import {
 // If filters initially are taking too long to load, decrease this number.
 // Between 1e6 - 5e6 is a good range.
 export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6;
-const DEFAULT_MAX_KEYS = 1000;
+export const DEFAULT_MAX_KEYS = 1000;
 // shortcut: keys and values found only in parts outside the selection can be
 // missed; raise it per source (mapKeyIndexMaxParts) or pick other parts
 // (mapKeyIndexPartSelection) if they go missing.
@@ -2121,10 +2121,13 @@ export class Metadata {
     dateRange,
     timestampValueExpression,
     condition,
+    maxKeys,
   }: TableConnection & {
     dateRange?: [Date, Date];
     timestampValueExpression?: string;
     condition?: FieldsCondition;
+    /** Most keys to discover per Map or JSON column. */
+    maxKeys?: number;
   }) {
     const fields: Field[] = [];
     const columns = await this.getColumns({
@@ -2172,6 +2175,7 @@ export class Metadata {
             metricName,
             dateRange,
             timestampValueExpression,
+            maxKeys,
           });
 
           for (const path of paths) {
@@ -2196,6 +2200,7 @@ export class Metadata {
           dateRange,
           timestampValueExpression,
           conditionSql,
+          maxKeys,
         });
 
         const match = column.type.match(/Map\(.+,\s*(.+)\)/);