diff --git a/.changeset/map-key-index-median-parts.md b/.changeset/map-key-index-median-parts.md new file mode 100644 index 000000000..62e41e1ad --- /dev/null +++ b/.changeset/map-key-index-median-parts.md @@ -0,0 +1,15 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +perf: read only 20 median-sized parts for text index key and value lookups + +Map key discovery and "all values" lookups on text indexes read the index +dictionary of every part in the time range, which is slow on tables with many +parts. They now read the 20 parts closest to the median size. Log and trace +sources can change the count with the new "Map key index parts" setting +(`mapKeyIndexMaxParts`), and which parts are read with "Map key index part size" +(`mapKeyIndexPartSelection`: median, smallest, or largest). "Load more" on a +single filter key still reads every part in the time range. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a2c7b3fae..f199f750e 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5203,6 +5203,24 @@ "nullable": true, "example": "auto" }, + "mapKeyIndexMaxParts": { + "type": "integer", + "minimum": 1, + "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.", + "nullable": true, + "example": 20 + }, + "mapKeyIndexPartSelection": { + "type": "string", + "enum": [ + "median", + "smallest", + "largest" + ], + "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "nullable": true, + "example": "median" + }, "highlightedTraceAttributeExpressions": { "type": "array", "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.", @@ -5457,6 +5475,24 @@ "nullable": true, "example": "auto" }, + "mapKeyIndexMaxParts": { + "type": "integer", + "minimum": 1, + "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.", + "nullable": true, + "example": 20 + }, + "mapKeyIndexPartSelection": { + "type": "string", + "enum": [ + "median", + "smallest", + "largest" + ], + "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "nullable": true, + "example": "median" + }, "highlightedTraceAttributeExpressions": { "type": "array", "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.", diff --git a/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts b/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts index 75d4d99a6..f06bde491 100644 --- a/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts +++ b/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts @@ -12,6 +12,8 @@ jest.mock('@/controllers/connection', () => ({ jest.mock('@/clickhouse', () => ({ ClickhouseClient: jest.fn() })); jest.mock('@hyperdx/common-utils/dist/core/metadata', () => ({ getMetadata: () => mockMetadata, + tcFromSource: jest.requireActual('@hyperdx/common-utils/dist/core/metadata') + .tcFromSource, })); jest.mock('@/controllers/promql', () => ({ getPromqlLabelCatalog: jest.fn(), diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 881a75ed8..c096429d3 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -3,7 +3,10 @@ import { filterColumnMetaByType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; -import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + getMetadata, + tcFromSource, +} from '@hyperdx/common-utils/dist/core/metadata'; import { type MetricTable, SourceKind } from '@hyperdx/common-utils/dist/types'; import { trace } from '@opentelemetry/api'; import SqlString from 'sqlstring'; @@ -224,10 +227,8 @@ async function describeSourceSchema( // Shared by stages 2–4 so map-key discovery can use rollup tables // instead of falling back to expensive main-table scans. - const metadataMVs = - 'metadataMaterializedViews' in source - ? source.metadataMaterializedViews - : undefined; + const { metadataMVs, mapKeyIndexMaxParts, mapKeyIndexPartSelection } = + tcFromSource(source); const now = new Date(); const dateRange: [Date, Date] = [ @@ -306,6 +307,8 @@ async function describeSourceSchema( maxKeys: 50, connectionId, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -352,6 +355,8 @@ async function describeSourceSchema( maxValuesPerKey: MAX_LC_VALUES, connectionId, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -399,6 +404,8 @@ async function describeSourceSchema( maxValuesPerKey: MAX_MAP_KEY_VALUES, connectionId, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 03f293efa..5a0ef1bed 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -466,6 +466,26 @@ const mcpSaveSourceSchema = z.object({ 'Log and trace sources. Rollup tables used for fast field discovery and ' + 'value autocomplete in filters and search suggestions.', ), + mapKeyIndexMaxParts: z + .number() + .int() + .min(1) + .optional() + .describe( + 'Log and trace sources. How many data parts are read from text indexes ' + + 'to discover attribute keys and filter values, chosen by ' + + 'mapKeyIndexPartSelection. Raise it if keys or values are missing from ' + + 'filters; lower it if they load slowly. Omit for the default of 20.', + ), + mapKeyIndexPartSelection: z + .enum(['median', 'smallest', 'largest']) + .optional() + .describe( + 'Log and trace sources. Which data parts, by row count, are read from ' + + 'text indexes for attribute keys and filter values: "median" (used when unset) ' + + 'balances coverage and cost, "smallest" reads the newest unmerged ' + + 'parts, "largest" reads the most merged data.', + ), }); export type McpSaveSourceInput = z.infer; diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 6b162640d..172a69a68 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -1,6 +1,7 @@ import { BaseSourceSchema, LogSourceSchema, + MapKeyIndexPartSelection, MetricsDataType, MetricSourceSchema, PROMQL_SOURCE_BACKENDS, @@ -189,6 +190,11 @@ export const LogSource = Source.discriminator( type: MetadataMaterializedViewsMongoSchema, default: undefined, }, + mapKeyIndexMaxParts: Number, + mapKeyIndexPartSelection: { + type: String, + enum: Object.values(MapKeyIndexPartSelection), + }, orderByExpression: String, }), ); @@ -240,6 +246,11 @@ export const TraceSource = Source.discriminator( type: MetadataMaterializedViewsMongoSchema, default: undefined, }, + mapKeyIndexMaxParts: Number, + mapKeyIndexPartSelection: { + type: String, + enum: Object.values(MapKeyIndexPartSelection), + }, orderByExpression: String, }), ); diff --git a/packages/api/src/routers/api/__tests__/sources.int.test.ts b/packages/api/src/routers/api/__tests__/sources.int.test.ts index ea83a0874..9b4ca14c4 100644 --- a/packages/api/src/routers/api/__tests__/sources.int.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.int.test.ts @@ -1672,4 +1672,79 @@ describe('sources router', () => { expect(sources).toHaveLength(0); }); }); + + describe('mapKeyIndexMaxParts field', () => { + it('POST / - persists mapKeyIndexMaxParts and PUT /:id updates it', async () => { + const { agent } = await getLoggedInAgent(server); + + const created = await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexMaxParts: 50 }) + .expect(200); + expect(created.body.mapKeyIndexMaxParts).toBe(50); + + await agent + .put(`/sources/${created.body.id}`) + .send({ ...MOCK_SOURCE, id: created.body.id, mapKeyIndexMaxParts: 5 }) + .expect(200); + + const updated = await Source.findById(created.body.id).lean(); + if (updated?.kind !== SourceKind.Log) { + throw new Error(`expected Log source, got ${updated?.kind}`); + } + expect(updated.mapKeyIndexMaxParts).toBe(5); + }); + + it.each([0, 2.5, -1])( + 'POST / - rejects mapKeyIndexMaxParts %s', + async mapKeyIndexMaxParts => { + const { agent } = await getLoggedInAgent(server); + + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexMaxParts }) + .expect(400); + + expect(await Source.find({}).lean()).toHaveLength(0); + }, + ); + }); + + describe('mapKeyIndexPartSelection field', () => { + it('POST / - persists mapKeyIndexPartSelection and PUT /:id updates it', async () => { + const { agent } = await getLoggedInAgent(server); + + const created = await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexPartSelection: 'largest' }) + .expect(200); + expect(created.body.mapKeyIndexPartSelection).toBe('largest'); + + await agent + .put(`/sources/${created.body.id}`) + .send({ + ...MOCK_SOURCE, + id: created.body.id, + mapKeyIndexPartSelection: 'smallest', + }) + .expect(200); + + const updated = await Source.findById(created.body.id).lean(); + if (updated?.kind !== SourceKind.Log) { + throw new Error(`expected Log source, got ${updated?.kind}`); + } + expect(updated.mapKeyIndexPartSelection).toBe('smallest'); + }); + + it('POST / - rejects an unknown mapKeyIndexPartSelection', async () => { + const { agent } = await getLoggedInAgent(server); + + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexPartSelection: 'random' }) + .expect(400); + + expect(await Source.find({}).lean()).toHaveLength(0); + }); + }); }); diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 561625bc6..33ac3d5ce 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -412,6 +412,18 @@ function formatExternalSource(source: SourceDocument) { * description: Controls whether lucene rendering uses ClickHouse text indices via hasAllTokens() against the implicit column. "auto" detects a covering index at query time, "enabled" forces text index usage, "disabled" forces a LIKE/hasToken fallback. * nullable: true * example: auto + * mapKeyIndexMaxParts: + * type: integer + * minimum: 1 + * description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20. + * nullable: true + * example: 20 + * mapKeyIndexPartSelection: + * type: string + * enum: [median, smallest, largest] + * description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * nullable: true + * example: median * highlightedTraceAttributeExpressions: * type: array * description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace. @@ -613,6 +625,18 @@ function formatExternalSource(source: SourceDocument) { * description: Controls whether lucene rendering uses ClickHouse text indices via hasAllTokens() against the implicit column. "auto" detects a covering index at query time, "enabled" forces text index usage, "disabled" forces a LIKE/hasToken fallback. * nullable: true * example: auto + * mapKeyIndexMaxParts: + * type: integer + * minimum: 1 + * description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20. + * nullable: true + * example: 20 + * mapKeyIndexPartSelection: + * type: string + * enum: [median, smallest, largest] + * description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * nullable: true + * example: median * highlightedTraceAttributeExpressions: * type: array * description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace. diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx index 396741201..bd089bf62 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx +++ b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx @@ -807,6 +807,9 @@ describe('useFetchFacets', () => { }); expect(getAllKeyValues).toHaveBeenCalledTimes(1); + expect(getAllKeyValues).toHaveBeenCalledWith( + expect.objectContaining({ mapKeyIndexMaxParts: null }), + ); expect(getKeyValuesWithMVs).not.toHaveBeenCalled(); expect(result.current.data.keyValues).toEqual([ { key: 'ServiceName', value: ['api', 'web'] }, diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 54267aeaf..467f93800 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -255,6 +255,8 @@ function useFacets({ tableName: tableConnection.tableName, connectionId: tableConnection.connectionId, metadataMVs: tableConnection.metadataMVs, + // "Load more" wants every value in the time range, not a sample of parts. + mapKeyIndexMaxParts: null, keyExpressions: [sqlKey], maxValuesPerKey: SINGLE_KEY_VALUES_LIMIT, dateRange, diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index b936c352f..9a41cf3dd 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -20,6 +20,10 @@ import { import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; import { HighlightedAttributeExpressionsFormRow } from './HighlightedAttributes'; +import { + MapKeyIndexMaxPartsFormRow, + MapKeyIndexPartSelectionFormRow, +} from './MapKeyIndexMaxPartsFormRow'; import { MaterializedViewsFormSection, MetadataMaterializedViewsFormSection, @@ -286,6 +290,8 @@ export function LogTableModelForm(props: TableModelProps) { /> + + ; +}) { + return ( + + ( + field.onChange(v === '' ? undefined : v)} + min={1} + allowDecimal={false} + allowNegative={false} + placeholder={String(DEFAULT_MAP_KEY_INDEX_MAX_PARTS)} + error={fieldState.error?.message} + /> + )} + /> + + ); +} + +export function MapKeyIndexPartSelectionFormRow({ + control, +}: { + control: Control; +}) { + return ( + + + + ); +} diff --git a/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx index 47265e50e..01b523c19 100644 --- a/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx @@ -16,6 +16,10 @@ import { import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; import { HighlightedAttributeExpressionsFormRow } from './HighlightedAttributes'; +import { + MapKeyIndexMaxPartsFormRow, + MapKeyIndexPartSelectionFormRow, +} from './MapKeyIndexMaxPartsFormRow'; import { MaterializedViewsFormSection, MetadataMaterializedViewsFormSection, @@ -380,6 +384,8 @@ export function TraceTableModelForm(props: TableModelProps) { /> + + { ).toEqual([{ kvRollupTable: 'logs_kv' }, { kvRollupTable: 'traces_kv' }]); }); + // Sources here only carry the fields the hook reads. + const mockSources = (data: Record[]) => + jest.mocked(useSources).mockReturnValue({ + data, + isLoading: false, + } as unknown as ReturnType); + + it("passes log and trace sources' part settings to 'all' mode lookups, and no others", async () => { + const defaultSources = jest.mocked(useSources)(); + mockSources([ + { + id: 'logs', + kind: 'log', + mapKeyIndexMaxParts: 7, + mapKeyIndexPartSelection: 'largest', + }, + // Not a log or trace source, so its fields don't apply. + { + id: 'metrics', + kind: 'metric', + mapKeyIndexMaxParts: 9, + mapKeyIndexPartSelection: 'smallest', + }, + ]); + const streamAllKeyValues = mockLookups.all([], []); + + try { + const { result } = renderValues({ + mode: 'all', + chartConfigs: [ + logsConfig, + createMockChartConfig({ + source: 'metrics', + from: { databaseName: 'db', tableName: 'metrics' }, + }), + ], + }); + await waitForSettled(result); + + expect( + streamAllKeyValues.mock.calls.map(([args]) => [ + args.tableName, + args.mapKeyIndexMaxParts, + args.mapKeyIndexPartSelection, + ]), + ).toEqual([ + ['logs', 7, 'largest'], + ['metrics', undefined, undefined], + ]); + } finally { + jest.mocked(useSources).mockReturnValue(defaultSources); + } + }); + + it("refetches 'all' mode values when the source's part settings change", async () => { + const defaultSources = jest.mocked(useSources)(); + const withParts = (mapKeyIndexMaxParts: number) => + mockSources([{ id: 'logs', kind: 'log', mapKeyIndexMaxParts }]); + const streamAllKeyValues = mockLookups.all([], []); + + try { + withParts(20); + const { result, rerender } = renderValues({ + mode: 'all', + chartConfigs: [logsConfig], + }); + await waitForSettled(result); + + withParts(5); + rerender(); + await waitFor(() => expect(streamAllKeyValues).toHaveBeenCalledTimes(2)); + expect( + streamAllKeyValues.mock.calls.map(([args]) => args.mapKeyIndexMaxParts), + ).toEqual([20, 5]); + } finally { + jest.mocked(useSources).mockReturnValue(defaultSources); + } + }); + it("passes limit to 'all' mode lookups as maxValuesPerKey", async () => { const streamAllKeyValues = mockLookups.all([]); diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 9a61524c3..94a9fd302 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -13,6 +13,7 @@ import { parseKeyPath, TableConnection, TableMetadata, + tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, @@ -530,12 +531,24 @@ export function useMultipleGetKeyValues( ? { retry, structuralSharing, refetchOnWindowFocus: true } : {}; + // Resolved here, not in the stream, so editing a source's part settings + // changes the query key instead of serving cached facets. + const partSettings = chartConfigsArr.map(chartConfig => { + const { mapKeyIndexMaxParts, mapKeyIndexPartSelection } = tcFromSource( + chartConfig.source + ? sources?.find(s => s.id === chartConfig.source) + : undefined, + ); + return { mapKeyIndexMaxParts, mapKeyIndexPartSelection }; + }); + const query = useStreamingQuery({ queryKey: [ 'useMetadata.useGetKeyValues', mode, metadataMVsOverride, ...chartConfigsArr.map(cc => ({ ...cc })), + partSettings, ...keys, // Serialized: react-query hashes keys with JSON.stringify, which would // flatten every distinct Set selection to `{}`. @@ -573,7 +586,7 @@ export function useMultipleGetKeyValues( }, ); - const lookups = chartConfigsArr.map(chartConfig => { + const lookups = chartConfigsArr.map((chartConfig, i) => { const source = chartConfig.source ? sources?.find(s => s.id === chartConfig.source) : undefined; @@ -582,7 +595,7 @@ export function useMultipleGetKeyValues( (source && (isLogSource(source) || isTraceSource(source)) ? source.metadataMaterializedViews : undefined); - return { chartConfig, source, metadataMVs }; + return { chartConfig, source, metadataMVs, ...partSettings[i] }; }); if (mode === 'exact') { @@ -606,18 +619,26 @@ export function useMultipleGetKeyValues( } // mode === 'all' - const streams = lookups.map(({ chartConfig, metadataMVs }) => - metadata.streamAllKeyValues({ - databaseName: chartConfig.from.databaseName, - tableName: chartConfig.from.tableName, - keyExpressions: limitedKeys, - maxValuesPerKey: limit, - connectionId: chartConfig.connection, + const streams = lookups.map( + ({ + chartConfig, metadataMVs, - dateRange: chartConfig.dateRange, - timestampValueExpression: chartConfig.timestampValueExpression, - signal, - }), + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, + }) => + metadata.streamAllKeyValues({ + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + keyExpressions: limitedKeys, + maxValuesPerKey: limit, + connectionId: chartConfig.connection, + metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, + dateRange: chartConfig.dateRange, + timestampValueExpression: chartConfig.timestampValueExpression, + signal, + }), ); for await (const keyValues of mergeStreams(streams)) { yield keyValues.map(({ key, value }) => ({ diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index d3cefe697..8ea619374 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -15,7 +15,12 @@ import { } from '@/core/renderChartConfig'; import { isBuilderChartConfig } from '@/guards'; import { TextIndexInfoLookup } from '@/queryParser'; -import { BuilderChartConfigWithDateRange, SourceKind, TSource } from '@/types'; +import { + BuilderChartConfigWithDateRange, + MapKeyIndexPartSelection, + SourceKind, + TSource, +} from '@/types'; // Mock ClickhouseClient const mockClickhouseClient = { @@ -1398,6 +1403,88 @@ describe('Metadata', () => { expect(sql).toContain('GROUP BY column, key'); }); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads only the parts nearest the median size from the %s text index', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + }); + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexMaxParts: 5, + }); + + const partsLimits = (mockClickhouseClient.query as jest.Mock).mock.calls + .filter(([opts]) => opts.query.includes('mergeTreeTextIndex(')) + .map(([opts]) => { + const match = opts.query.match( + /ORDER BY abs\(rn - \(cnt \+ 1\) \/ 2\), name\s+LIMIT \{(\w+):Int32\}/, + ); + expect(match).not.toBeNull(); + return opts.query_params[match[1]]; + }); + expect(partsLimits).toEqual([20, 5]); + }, + ); + + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads every part in the time range from the %s text index when the part limit is null', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexMaxParts: null, + }); + + const textIndexCall = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].query).toMatch( + /part_name IN \(SELECT name\s+FROM system\.parts/, + ); + expect(textIndexCall[0].query).not.toContain('row_number()'); + }, + ); + + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'orders the %s text index parts by the configured part selection', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexPartSelection: MapKeyIndexPartSelection.Largest, + }); + + const textIndexCall = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].query).toMatch( + /\)\s+ORDER BY rn DESC\s+LIMIT \{\w+:Int32\}/, + ); + }, + ); + it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { setupDefaultLogsSchema(); @@ -3290,6 +3377,90 @@ describe('Metadata', () => { expect(mockClickhouseClient.query).toHaveBeenCalledTimes(1); }, ); + + it.each(textIndexLookups)( + 'reads only the parts nearest the median size on the %s text index path', + async (_, textIndexLookup) => { + const md = new Metadata(mockClickhouseClient, new MetadataCache()); + jest.spyOn(md, 'getServerVersion').mockResolvedValue([26, 3, 0, 0]); + jest + .spyOn(md, 'getMapColumnTextIndexes') + .mockResolvedValue(textIndexLookup); + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + const args = { + databaseName: 'default', + tableName: 'otel_logs', + column: 'LogAttributes', + connectionId: 'test_connection', + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:00:00Z'), + ] as [Date, Date], + timestampValueExpression: 'Timestamp', + }; + + await md.getMapKeys(args); + await md.getMapKeys({ ...args, mapKeyIndexMaxParts: 5 }); + + // A different limit is a different result, not a cache hit. + expect(mockClickhouseClient.query).toHaveBeenCalledTimes(2); + const calls = (mockClickhouseClient.query as jest.Mock).mock.calls; + const partsLimits = calls.map(([opts]) => { + const match = opts.query.match( + /row_number\(\) OVER \(ORDER BY rows, name\)[\s\S]+ORDER BY abs\(rn - \(cnt \+ 1\) \/ 2\), name\s+LIMIT \{(\w+):Int32\}/, + ); + expect(match).not.toBeNull(); + return opts.query_params[match[1]]; + }); + expect(partsLimits).toEqual([20, 5]); + }, + ); + + it.each(textIndexLookups)( + 'orders the parts by the configured part selection on the %s text index path', + async (_, textIndexLookup) => { + const md = new Metadata(mockClickhouseClient, new MetadataCache()); + jest.spyOn(md, 'getServerVersion').mockResolvedValue([26, 3, 0, 0]); + jest + .spyOn(md, 'getMapColumnTextIndexes') + .mockResolvedValue(textIndexLookup); + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + const args = { + databaseName: 'default', + tableName: 'otel_logs', + column: 'LogAttributes', + connectionId: 'test_connection', + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:00:00Z'), + ] as [Date, Date], + timestampValueExpression: 'Timestamp', + }; + + for (const mapKeyIndexPartSelection of [ + MapKeyIndexPartSelection.Smallest, + MapKeyIndexPartSelection.Largest, + MapKeyIndexPartSelection.Median, + ]) { + await md.getMapKeys({ ...args, mapKeyIndexPartSelection }); + } + + // Each selection is a different result, not a cache hit. + const orders = (mockClickhouseClient.query as jest.Mock).mock.calls.map( + ([opts]) => + opts.query.match(/\)\s+ORDER BY (.+)\s+LIMIT \{\w+:Int32\}/)?.[1], + ); + expect(orders).toEqual([ + 'rn', + 'rn DESC', + 'abs(rn - (cnt + 1) / 2), name', + ]); + }, + ); }); describe('getMapValues', () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 6eb9ac386..81a93eeda 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -38,7 +38,12 @@ import type { MetadataMaterializedViews, TSource, } from '@/types'; -import { isLogSource, isTraceSource, SourceKind } from '@/types'; +import { + isLogSource, + isTraceSource, + MapKeyIndexPartSelection, + SourceKind, +} from '@/types'; import { ClickHouseVersion, @@ -65,6 +70,10 @@ import { // Between 1e6 - 5e6 is a good range. export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6; const DEFAULT_MAX_KEYS = 1000; +// shortcut: keys and values found only in parts outside the selection can be +// missed; raise it per source (mapKeyIndexMaxParts) or pick other parts +// (mapKeyIndexPartSelection) if they go missing. +export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; @@ -103,6 +112,12 @@ type AllKeyValuesQuery = { maxValuesPerKey?: number; connectionId: string; metadataMVs?: MetadataMaterializedViews; + /** + * Parts the text index paths read; see `partsOverlapFilter`. Unset uses the + * default; null reads every part in the time range. + */ + mapKeyIndexMaxParts?: number | null; + mapKeyIndexPartSelection?: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; @@ -896,21 +911,51 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts, + partSelection = MapKeyIndexPartSelection.Median, }: { databaseName: string; tableName: string; dateRange?: [Date, Date]; timestampValueExpression?: string; + /** Keep only `maxParts` overlapping parts, chosen by `partSelection`. */ + maxParts?: number; + /** Which parts by row count; only applies with `maxParts`. */ + partSelection?: MapKeyIndexPartSelection; }): Promise { if (!dateRange || !timestampValueExpression) return chSql`1`; const startTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})`; const endTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }})`; - return chSql`part_name IN ( - SELECT name + const overlappingParts = chSql` FROM system.parts WHERE database = ${{ String: databaseName }} AND table = ${{ String: tableName }} AND active=1 - AND ((min_time >= ${startTime} AND min_time <= ${endTime}) OR (max_time <= ${endTime} AND max_time >= ${startTime}) OR (min_time <= ${startTime} AND max_time >= ${endTime})) + AND ((min_time >= ${startTime} AND min_time <= ${endTime}) OR (max_time <= ${endTime} AND max_time >= ${startTime}) OR (min_time <= ${startTime} AND max_time >= ${endTime}))`; + if (maxParts == null) { + return chSql`part_name IN (SELECT name ${overlappingParts})`; + } + // Rank by size. Median picks from the middle out: tiny unmerged parts + // hold few keys and the largest merged parts have the biggest dictionaries. + let order: string; + switch (partSelection) { + case MapKeyIndexPartSelection.Smallest: + order = 'rn'; + break; + case MapKeyIndexPartSelection.Largest: + order = 'rn DESC'; + break; + default: + order = 'abs(rn - (cnt + 1) / 2), name'; + } + return chSql`part_name IN ( + SELECT name FROM ( + SELECT name, + row_number() OVER (ORDER BY rows, name) AS rn, + count() OVER () AS cnt + ${overlappingParts} + ) + ORDER BY ${{ UNSAFE_RAW_SQL: order }} + LIMIT ${{ Int32: maxParts }} )`; } @@ -1075,6 +1120,8 @@ export class Metadata { dateRange: callerDateRange, timestampValueExpression, conditionSql, + mapKeyIndexMaxParts = DEFAULT_MAP_KEY_INDEX_MAX_PARTS, + mapKeyIndexPartSelection = MapKeyIndexPartSelection.Median, signal, }: { databaseName: string; @@ -1087,6 +1134,9 @@ export class Metadata { dateRange?: [Date, Date]; timestampValueExpression?: string; conditionSql?: string; + /** Parts the text index paths read; see `partsOverlapFilter`. */ + mapKeyIndexMaxParts?: number; + mapKeyIndexPartSelection?: MapKeyIndexPartSelection; signal?: AbortSignal; }) { inlineNonNegativeInt(maxKeys, 'maxKeys'); @@ -1110,10 +1160,10 @@ export class Metadata { ? `${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.${timestampValueExpression}` : ''; const unscopedCacheKey = metricName - ? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.${dateRangeCacheSuffix}.${maxKeys}.keys` + ? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.${mapKeyIndexPartSelection}.keys` : alignedDateRange - ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedCacheSuffix}.${maxKeys}.keys` - : `${connectionId}.${databaseName}.${tableName}.${column}.${dateRangeCacheSuffix}.${maxKeys}.keys`; + ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.${mapKeyIndexPartSelection}.keys` + : `${connectionId}.${databaseName}.${tableName}.${column}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.${mapKeyIndexPartSelection}.keys`; const cacheKey = conditionSql ? `${unscopedCacheKey}.${objectHash(conditionSql)}.condition` : unscopedCacheKey; @@ -1137,6 +1187,8 @@ export class Metadata { alignedDateRange, timestampValueExpression, conditionSql, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, cacheKey, signal, }), @@ -1156,6 +1208,8 @@ export class Metadata { alignedDateRange, timestampValueExpression, conditionSql, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, cacheKey, signal, }: { @@ -1170,6 +1224,8 @@ export class Metadata { alignedDateRange?: [Date, Date]; timestampValueExpression?: string; conditionSql?: string; + mapKeyIndexMaxParts: number; + mapKeyIndexPartSelection: MapKeyIndexPartSelection; cacheKey: string; signal?: AbortSignal; }): Promise { @@ -1205,6 +1261,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, }); const index = textIndexInfo.key.indexName; // `cardinality` is the token's row count in the part, not a distinct count. @@ -1250,6 +1308,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, }); const index = textIndexInfo.kv.indexName; const separator = textIndexInfo.kv.separator; @@ -1696,17 +1756,22 @@ export class Metadata { tableName, connectionId, queryOptions, + maxParts, + partSelection, dateRange, timestampValueExpression, signal, }: TableConnection & { queryOptions: TextIndexMapColumnQueryOptions; + /** null reads every part in the time range. */ + maxParts: number | null; + partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; }): Promise { const queryOptionsHash = objectHash(queryOptions); - const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${timestampValueExpression}.getMapTextIndexKeyValues`; + const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getMapTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { const sqlBranches: Array = []; @@ -1729,6 +1794,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts: maxParts ?? undefined, + partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; const sql = chSql` @@ -1788,17 +1855,22 @@ export class Metadata { tableName, connectionId, queryOptions, + maxParts, + partSelection, dateRange, timestampValueExpression, signal, }: TableConnection & { queryOptions: TextIndexColumnQueryOptions; + /** null reads every part in the time range. */ + maxParts: number | null; + partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; }): Promise { const queryOptionsHash = objectHash(queryOptions); - const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${timestampValueExpression}.getTextIndexKeyValues`; + const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { const sqlBranches: Array = []; @@ -1808,6 +1880,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts: maxParts ?? undefined, + partSelection, }); const sql = chSql` SELECT * FROM ( @@ -1965,6 +2039,8 @@ export class Metadata { connectionId, metricName, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, condition, @@ -2038,6 +2114,8 @@ export class Metadata { connectionId, metricName, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, conditionSql, @@ -2868,6 +2946,8 @@ export class Metadata { maxValuesPerKey, connectionId, metadataMVs, + mapKeyIndexMaxParts = DEFAULT_MAP_KEY_INDEX_MAX_PARTS, + mapKeyIndexPartSelection = MapKeyIndexPartSelection.Median, dateRange, timestampValueExpression, signal, @@ -2987,6 +3067,8 @@ export class Metadata { tableName, connectionId, queryOptions: mapTextIndexQueryOptions, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -3000,6 +3082,8 @@ export class Metadata { tableName, connectionId, queryOptions: nativeTextIndexQueryOptions, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -3468,6 +3552,8 @@ export type TableConnection = { connectionId: string; metricName?: string; metadataMVs?: MetadataMaterializedViews; + mapKeyIndexMaxParts?: number; + mapKeyIndexPartSelection?: MapKeyIndexPartSelection; // Bounds Map key discovery; without it Map keys are skipped. #3037 timestampValueExpression?: string; }; @@ -3511,6 +3597,14 @@ export function tcFromSource(source?: TSource): TableConnection { source && (isLogSource(source) || isTraceSource(source)) ? source.metadataMaterializedViews : undefined, + mapKeyIndexMaxParts: + source && (isLogSource(source) || isTraceSource(source)) + ? source.mapKeyIndexMaxParts + : undefined, + mapKeyIndexPartSelection: + source && (isLogSource(source) || isTraceSource(source)) + ? source.mapKeyIndexPartSelection + : undefined, timestampValueExpression: source?.timestampValueExpression, }; } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dba9c7854..4c7b27d3c 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -28,6 +28,17 @@ export enum UseTextIndex { export const UseTextIndexSchema = z.nativeEnum(UseTextIndex); +/** Which data parts, by row count, text index key and value lookups read. */ +export enum MapKeyIndexPartSelection { + Median = 'median', + Smallest = 'smallest', + Largest = 'largest', +} + +export const MapKeyIndexPartSelectionSchema = z.nativeEnum( + MapKeyIndexPartSelection, +); + // -------------------------- // UI // -------------------------- @@ -2701,6 +2712,15 @@ export type MetadataMaterializedViews = z.infer< typeof MetadataMaterializedViewsSchema >; +/** + * How many parts key and value discovery reads from a text index. Unset uses + * DEFAULT_MAP_KEY_INDEX_MAX_PARTS. A cleared form NumberInput yields `''`. + */ +const MapKeyIndexMaxPartsSchema = z.preprocess( + v => (v === '' || v === null ? undefined : v), + z.number().int().min(1).optional(), +); + // Log source form schema export const LogSourceSchema = BaseSourceSchema.extend({ kind: z.literal(SourceKind.Log), @@ -2753,6 +2773,8 @@ export const LogSourceSchema = BaseSourceSchema.extend({ HighlightedAttributeExpressionsSchema.optional(), materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), + mapKeyIndexMaxParts: MapKeyIndexMaxPartsSchema, + mapKeyIndexPartSelection: MapKeyIndexPartSelectionSchema.optional(), orderByExpression: z.string().optional(), useTextIndexForImplicitColumn: UseTextIndexSchema.optional(), }); @@ -2798,6 +2820,8 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ HighlightedAttributeExpressionsSchema.optional(), materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), + mapKeyIndexMaxParts: MapKeyIndexMaxPartsSchema, + mapKeyIndexPartSelection: MapKeyIndexPartSelectionSchema.optional(), orderByExpression: z.string().optional(), useTextIndexForImplicitColumn: UseTextIndexSchema.optional(), });