diff --git a/.changeset/filter-keys-fetch-limit-max.md b/.changeset/filter-keys-fetch-limit-max.md new file mode 100644 index 000000000..b05b2bd86 --- /dev/null +++ b/.changeset/filter-keys-fetch-limit-max.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/app': patch +'@hyperdx/common-utils': patch +--- + +feat: allow up to 10,000 in the team "Filter Keys Fetch Limit" setting + +The setting's input in Team settings stopped at 1,000 keys per column; it now +accepts up to 10,000. Map key discovery returns up to the setting's value per +Map column, and never fewer than 1,000. diff --git a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx index e271ec8fd..cef40fb67 100644 --- a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx +++ b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx @@ -324,7 +324,7 @@ export default function TeamQueryConfigSection() { defaultValue={DEFAULT_FILTER_KEYS_FETCH_LIMIT} placeholder={`default = ${DEFAULT_FILTER_KEYS_FETCH_LIMIT}`} min={1} - max={1000} + max={10000} displayValue={displayValueWithUnit('keys')} description={`Default is ${DEFAULT_FILTER_KEYS_FETCH_LIMIT}`} /> diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 031caa81f..46c92c95e 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -734,7 +734,7 @@ describe('useMultipleAllFields', () => { ]; const columnsB = [column('col_b', 'String')]; - let team: { fieldMetadataDisabled?: boolean }; + let team: { fieldMetadataDisabled?: boolean; filterKeysFetchLimit?: number }; beforeEach(() => { jest.clearAllMocks(); @@ -811,6 +811,29 @@ describe('useMultipleAllFields', () => { ); }); + it.each([ + [undefined, 1000], + [100, 1000], + [5000, 5000], + ])( + 'with a filter key limit of %s, discovers up to %s keys per Map column', + async (filterKeysFetchLimit, maxKeys) => { + team.filterKeysFetchLimit = filterKeysFetchLimit; + const getAllFields = jest + .spyOn(mockMetadata, 'getAllFields') + .mockResolvedValue(fieldsA); + + const { result } = renderHook(() => useMultipleAllFields([tcA]), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(getAllFields).toHaveBeenCalledWith( + expect.objectContaining({ maxKeys }), + ); + }, + ); + it('should deduplicate fields across successful connections', async () => { jest .spyOn(mockMetadata, 'getAllFields') diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 94a9fd302..fb965ed26 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -7,6 +7,7 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { + DEFAULT_MAX_KEYS, Field, FieldsCondition, MetricNames, @@ -279,6 +280,9 @@ export function useMultipleAllFields( const metadata = useMetadataWithSettings(); const { data: me, isFetched } = api.useMe(); const isFieldMetadataDisabled = !!me?.team?.fieldMetadataDisabled; + // The team setting only raises Map key discovery, so a low limit doesn't + // shrink the keys offered to autocomplete and the other field pickers. + const maxKeys = Math.max(useFilterKeysFetchLimit(), DEFAULT_MAX_KEYS); const allPopulated = tableConnections.every( tc => !!tc.databaseName && !!tc.tableName && !!tc.connectionId, ); @@ -291,6 +295,7 @@ export function useMultipleAllFields( timestampValueExpression, isFieldMetadataDisabled, conditions, + maxKeys, ], queryFn: () => Promise.all( @@ -310,6 +315,7 @@ export function useMultipleAllFields( timestampValueExpression: timestampValueExpression ?? tc.timestampValueExpression, condition: conditions?.[i], + maxKeys, }); return { fields, columns }; } catch (error) { diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 08e640d14..e12561b5e 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -3823,7 +3823,7 @@ describe('Metadata', () => { jest.clearAllMocks(); }); - it('threads dateRange and timestampValueExpression through to getMapKeys', async () => { + it('threads dateRange, timestampValueExpression, and maxKeys through to getMapKeys', async () => { const realCache = new ( jest.requireActual('../core/metadata') as any ).MetadataCache(); @@ -3860,12 +3860,14 @@ describe('Metadata', () => { connectionId: 'conn-1', dateRange, timestampValueExpression: 'EventTime, EventDate', + maxKeys: 5000, }); expect(getMapKeysSpy).toHaveBeenCalledWith( expect.objectContaining({ dateRange, timestampValueExpression: 'EventTime, EventDate', + maxKeys: 5000, }), ); }); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index b77e667d3..7dcc2e4c7 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -70,7 +70,7 @@ import { // If filters initially are taking too long to load, decrease this number. // Between 1e6 - 5e6 is a good range. export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6; -const DEFAULT_MAX_KEYS = 1000; +export const DEFAULT_MAX_KEYS = 1000; // shortcut: keys and values found only in parts outside the selection can be // missed; raise it per source (mapKeyIndexMaxParts) or pick other parts // (mapKeyIndexPartSelection) if they go missing. @@ -2121,10 +2121,13 @@ export class Metadata { dateRange, timestampValueExpression, condition, + maxKeys, }: TableConnection & { dateRange?: [Date, Date]; timestampValueExpression?: string; condition?: FieldsCondition; + /** Most keys to discover per Map or JSON column. */ + maxKeys?: number; }) { const fields: Field[] = []; const columns = await this.getColumns({ @@ -2172,6 +2175,7 @@ export class Metadata { metricName, dateRange, timestampValueExpression, + maxKeys, }); for (const path of paths) { @@ -2196,6 +2200,7 @@ export class Metadata { dateRange, timestampValueExpression, conditionSql, + maxKeys, }); const match = column.type.match(/Map\(.+,\s*(.+)\)/);