diff --git a/.changeset/filters-direct-read.md b/.changeset/filters-direct-read.md new file mode 100644 index 0000000000..26182d6853 --- /dev/null +++ b/.changeset/filters-direct-read.md @@ -0,0 +1,9 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/app": minor +--- + +feat: emit Lucene conditions from sidebar/dashboard filters to enable KV items direct_read optimization on Map columns + +Legacy `type: 'sql'` filters in URLs are automatically migrated to Lucene +on page load. The persisted `DashboardFilter.expression` in MongoDB is unchanged. diff --git a/AGENTS.md b/AGENTS.md index 90fafcbdb0..c1e02f3ee5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,19 @@ and lint issues across all packages. Pre-commit hooks handle this when committing, but if you finish edits without committing, run `yarn lint:fix` before stopping. +## Before Writing New Utility Functions + +Before implementing a new parsing, formatting, or transformation function, +search for existing functions that already do the same thing or something close. +Common places to check: + +- `packages/common-utils/src/core/metadata.ts` — key/column parsing utilities + (e.g. `parseKeyPath`) +- `packages/common-utils/src/queryParser.ts` — Lucene parsing (`parse`, + `decodeSpecialTokens`), AST type guards, serialization helpers +- `packages/common-utils/src/filters.ts` — filter state serialization and + parsing (`filtersToQuery`, `parseLuceneFilter`) + ## Key Principles 1. **Multi-tenancy**: All data is scoped to `Team` - ensure proper filtering diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index b2d3e53ebb..bb6906e674 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -146,7 +146,7 @@ export type ExternalDashboardFilter = z.infer< >; export const externalDashboardSavedFilterValueSchema = z.object({ - type: z.literal('sql').optional().default('sql'), + type: z.enum(['sql', 'lucene']).optional().default('sql'), condition: z.string().max(10000), }); diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index 544c6762d5..a2a9a05c4b 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -17,6 +17,10 @@ import { getAlignedDateRange, Granularity, } from '@hyperdx/common-utils/dist/core/utils'; +import { + type FilterState, + filtersToQuery, +} from '@hyperdx/common-utils/dist/filters'; import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { AggregateFunction as AggFnV2, @@ -923,13 +927,19 @@ export function buildEventsSearchUrl({ // Add group-by column filters if (groupFilters && groupFilters.length > 0) { - groupFilters.forEach(({ column, value }) => { + const filterState: FilterState = {}; + for (const { column, value } of groupFilters) { if (column && value != null) { - // Can't use SQLString.escape here because the search endpoint relies on exist match for UI - const condition = `${column} IN (${SqlString.escape(value)})`; - additionalFilters.push({ type: 'sql', condition }); + if (!filterState[column]) { + filterState[column] = { + included: new Set(), + excluded: new Set(), + }; + } + filterState[column].included.add(String(value)); } - }); + } + additionalFilters.push(...filtersToQuery(filterState)); } // Add Y-axis value range filter (±threshold) for charts diff --git a/packages/app/src/__tests__/searchFilters.test.ts b/packages/app/src/__tests__/searchFilters.test.ts index f838f49710..b108556838 100644 --- a/packages/app/src/__tests__/searchFilters.test.ts +++ b/packages/app/src/__tests__/searchFilters.test.ts @@ -1,5 +1,11 @@ import { enableMapSet } from 'immer'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { filtersToQuery } from '@hyperdx/common-utils/dist/filters'; +import { + CustomSchemaSQLSerializerV2, + SearchQueryBuilder, +} from '@hyperdx/common-utils/dist/queryParser'; import { act, renderHook } from '@testing-library/react'; import { @@ -630,6 +636,222 @@ describe('searchFilters', () => { }); }); + describe('round-trip: Lucene filters', () => { + it('round-trips Map filter with single included value', () => { + const filters = { + "LogAttributes['service.name']": { + included: new Set(['my-app']), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + expect(query).toEqual([ + { + type: 'lucene', + condition: 'LogAttributes.service.name:"my-app"', + }, + ]); + // After round-trip, bracket-notation key becomes dot notation + const parsed = parseQuery(query); + expect(parsed.filters).toEqual({ + 'LogAttributes.service.name': { + included: new Set(['my-app']), + excluded: new Set(), + }, + }); + }); + + it('round-trips Map filter with multiple included values', () => { + const filters = { + "ResourceAttributes['env']": { + included: new Set(['prod', 'staging']), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters).toEqual({ + 'ResourceAttributes.env': { + included: new Set(['prod', 'staging']), + excluded: new Set(), + }, + }); + }); + + it('round-trips Map filter with excluded values', () => { + const filters = { + "LogAttributes['level']": { + included: new Set(), + excluded: new Set(['debug', 'trace']), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters).toEqual({ + 'LogAttributes.level': { + included: new Set(), + excluded: new Set(['debug', 'trace']), + }, + }); + }); + + it('round-trips values containing double quotes', () => { + const filters = { + "LogAttributes['msg']": { + included: new Set(['say "hello"']), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters).toEqual({ + 'LogAttributes.msg': { + included: new Set(['say "hello"']), + excluded: new Set(), + }, + }); + }); + + it('round-trips values containing backslashes', () => { + const filters = { + "LogAttributes['path']": { + included: new Set(['C:\\path\\to\\file']), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters).toEqual({ + 'LogAttributes.path': { + included: new Set(['C:\\path\\to\\file']), + excluded: new Set(), + }, + }); + }); + + it('round-trips plain key filters', () => { + const filters = { + service: { + included: new Set(['app']), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters).toEqual({ + service: { + included: new Set(['app']), + excluded: new Set(), + }, + }); + }); + + it('round-trips boolean filter values', () => { + const filters = { + isRootSpan: { + included: new Set([true]), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters).toEqual({ + isRootSpan: { + included: new Set([true]), + excluded: new Set(), + }, + }); + // Verify .has(true) works (boolean, not string) + expect(parsed.filters.isRootSpan.included.has(true)).toBe(true); + expect(parsed.filters.isRootSpan.included.has('true')).toBe(false); + }); + + it('round-trips empty string values', () => { + const filters = { + tag: { + included: new Set(['']), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters.tag.included).toEqual(new Set([''])); + }); + + it('round-trips values with Lucene reserved characters', () => { + const filters = { + query: { + included: new Set(['(foo) AND [bar]']), + excluded: new Set(), + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters.query.included).toEqual( + new Set(['(foo) AND [bar]']), + ); + }); + + it('round-trips range filters through Lucene', () => { + const filters = { + duration: { + included: new Set(), + excluded: new Set(), + range: { min: 10, max: 500 }, + }, + }; + const query = filtersToQuery(filters); + const parsed = parseQuery(query); + expect(parsed.filters.duration.range).toEqual({ min: 10, max: 500 }); + }); + + it('merges mixed sql and lucene filters for the same key', () => { + const parsed = parseQuery([ + { type: 'sql', condition: `service IN ('app')` }, + { type: 'lucene', condition: 'service:"web"' }, + ]); + expect(parsed.filters.service.included).toEqual(new Set(['app', 'web'])); + }); + + it('parses existing lucene filter from URL/API', () => { + const parsed = parseQuery([ + { + type: 'lucene', + condition: + 'LogAttributes.service:"app-1" OR LogAttributes.service:"app-2"', + }, + ]); + expect(parsed.filters).toEqual({ + 'LogAttributes.service': { + included: new Set(['app-1', 'app-2']), + excluded: new Set(), + }, + }); + }); + + it('preserves unparseable Lucene filters as passthrough', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const parsed = parseQuery([ + { type: 'lucene', condition: '((((' }, + { type: 'lucene', condition: 'service:"app"' }, + ]); + // Valid filter is parsed + expect(parsed.filters.service.included).toEqual(new Set(['app'])); + // Invalid filter is preserved in passthroughFilters + expect(parsed.passthroughFilters).toEqual([ + { type: 'lucene', condition: '((((' }, + ]); + warnSpy.mockRestore(); + }); + + it('returns empty array for valid but non-quoted Lucene (no passthrough)', () => { + const parsed = parseQuery([{ type: 'lucene', condition: 'service:foo' }]); + // Unquoted term — valid Lucene but not a filter, nothing extracted + expect(Object.keys(parsed.filters)).toHaveLength(0); + expect(parsed.passthroughFilters).toHaveLength(0); + }); + }); + describe('useSearchPageFilterState', () => { const onFilterChange = jest.fn(); @@ -647,20 +869,15 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenLastCalledWith([ - { - type: 'sql', - condition: "service IN ('app')", - }, - { - type: 'sql', - condition: "level IN ('error')", - }, + { type: 'lucene', condition: 'service:"app"' }, + { type: 'lucene', condition: 'level:"error"' }, ]); }); - it('updating filter query', () => { + it('updating filter query from legacy sql input', () => { const { result } = renderHook(() => useSearchPageFilterState({ + // Legacy sql filters are still parsed correctly searchQuery: [ { type: 'sql', condition: `service IN ('hdx-oss-dev-app')` }, { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, @@ -675,10 +892,14 @@ describe('searchFilters', () => { result.current.setFilterValue('another_facet', 'some_value'); }); + // Output is now lucene after round-trip expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, - { type: 'sql', condition: `level IN ('info', 'ok')` }, - { type: 'sql', condition: `another_facet IN ('some_value')` }, + { type: 'lucene', condition: 'hyperdx_event_type:"span"' }, + { + type: 'lucene', + condition: '(level:"info" OR level:"ok")', + }, + { type: 'lucene', condition: 'another_facet:"some_value"' }, ]); }); @@ -699,12 +920,12 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'sql', condition: `service IN ('hdx-oss-dev-app')` }, - { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, + { type: 'lucene', condition: 'service:"hdx-oss-dev-app"' }, + { type: 'lucene', condition: 'hyperdx_event_type:"span"' }, ]); }); - it('correctly hydrates filter state from query', () => { + it('correctly hydrates filter state from sql query', () => { const { result } = renderHook(() => useSearchPageFilterState({ searchQuery: [ @@ -745,9 +966,152 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenCalledWith([ - { type: 'sql', condition: `service IN ('app')` }, - { type: 'sql', condition: `level IN ('info')` }, // Should only have the included value, no excluded values + { type: 'lucene', condition: 'service:"app"' }, + { type: 'lucene', condition: 'level:"info"' }, + ]); + }); + + it('migrates legacy SQL filters to Lucene on load', () => { + const onFilterChangeMigrate = jest.fn(); + renderHook(() => + useSearchPageFilterState({ + searchQuery: [ + { type: 'sql', condition: `service IN ('app')` }, + { type: 'sql', condition: `level IN ('info', 'ok')` }, + ], + onFilterChange: onFilterChangeMigrate, + }), + ); + + // Migration fires on mount, converting SQL to Lucene + expect(onFilterChangeMigrate).toHaveBeenCalledWith([ + { type: 'lucene', condition: 'service:"app"' }, + { type: 'lucene', condition: '(level:"info" OR level:"ok")' }, ]); }); + + it('does not migrate when filters are already Lucene', () => { + const onFilterChangeNoMigrate = jest.fn(); + renderHook(() => + useSearchPageFilterState({ + searchQuery: [{ type: 'lucene', condition: 'service:"app"' }], + onFilterChange: onFilterChangeNoMigrate, + }), + ); + + // No migration needed — onFilterChange should not be called + expect(onFilterChangeNoMigrate).not.toHaveBeenCalled(); + }); + }); + + describe('filters use direct_read optimization', () => { + const metadata = getMetadata( + new ClickhouseClient({ host: 'http://localhost:8123' }), + ); + + metadata.getColumn = jest.fn().mockImplementation(async ({ column }) => { + if (column === 'LogAttributes') { + return { name: 'LogAttributes', type: 'Map(String, String)' }; + } + return undefined; + }); + metadata.getMaterializedColumnsLookupTable = jest + .fn() + .mockResolvedValue(new Map()); + metadata.getColumns = jest.fn().mockResolvedValue([ + { + name: 'LogAttributes', + type: 'Map(String, String)', + default_type: '', + default_expression: '', + }, + { + name: 'LogAttributeItems', + type: 'Array(String)', + default_type: 'ALIAS', + default_expression: + "arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String)))", + }, + ]); + metadata.getSkipIndices = jest.fn().mockResolvedValue([ + { + name: 'idx_log_attr_items', + type: 'text', + typeFull: 'text(tokenizer=array)', + expression: 'LogAttributeItems', + granularity: 1, + }, + ]); + metadata.getSetting = jest.fn().mockResolvedValue('0'); + + const serializer = new CustomSchemaSQLSerializerV2({ + metadata, + databaseName: 'default', + tableName: 'otel_logs', + connectionId: 'test', + }); + + it('single included filter value uses has() direct_read', async () => { + const filters = filtersToQuery({ + "LogAttributes['service.name']": { + included: new Set(['my-app']), + excluded: new Set(), + }, + }); + + const condition = filters[0]; + expect(condition.type).toBe('lucene'); + const builder = new SearchQueryBuilder( + (condition as { condition: string }).condition, + serializer, + ); + const sql = await builder.build(); + expect(sql).toContain( + "has(`LogAttributeItems`, concat('service.name', '=', 'my-app'))", + ); + }); + + it('multiple included filter values each use has() direct_read', async () => { + const filters = filtersToQuery({ + "LogAttributes['env']": { + included: new Set(['prod', 'staging']), + excluded: new Set(), + }, + }); + + const condition = filters[0]; + expect(condition.type).toBe('lucene'); + const builder = new SearchQueryBuilder( + (condition as { condition: string }).condition, + serializer, + ); + const sql = await builder.build(); + expect(sql).toContain( + "has(`LogAttributeItems`, concat('env', '=', 'prod'))", + ); + expect(sql).toContain( + "has(`LogAttributeItems`, concat('env', '=', 'staging'))", + ); + }); + + it('excluded filter value uses NOT has() direct_read', async () => { + const filters = filtersToQuery({ + "LogAttributes['level']": { + included: new Set(), + excluded: new Set(['debug']), + }, + }); + + const condition = filters[0]; + expect(condition.type).toBe('lucene'); + const builder = new SearchQueryBuilder( + (condition as { condition: string }).condition, + serializer, + ); + const sql = await builder.build(); + expect(sql).toContain( + "NOT has(`LogAttributeItems`, concat('level', '=', 'debug'))", + ); + }); }); }); diff --git a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx index baa8904b3b..2aa490a135 100644 --- a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx @@ -90,7 +90,7 @@ describe('useDashboardFilters', () => { ); }); - it('should generate IN clause for multi-select values', () => { + it('should generate lucene condition for multi-select values', () => { const { result } = renderHook(() => useDashboardFilters(mockFilters)); act(() => { @@ -105,7 +105,7 @@ describe('useDashboardFilters', () => { const query = result2.current.filterQueries[0]; const condition = 'condition' in query ? query.condition : ''; expect(condition).toEqual( - "toString(environment) IN ('production', 'staging')", + '(environment:"production" OR environment:"staging")', ); }); @@ -272,4 +272,81 @@ describe('useDashboardFilters', () => { expect(result.current.ignoredFilterExpressions).toEqual([]); }); }); + + it('should match bracket-notation expressions after Lucene round-trip', () => { + const bracketFilters: DashboardFilter[] = [ + { + id: 'filter-bracket', + type: 'QUERY_EXPRESSION', + name: 'Pod', + expression: "SpanAttributes['k8s.pod.name']", + source: 'traces', + }, + ]; + + const { result } = renderHook(() => useDashboardFilters(bracketFilters)); + + act(() => { + result.current.setFilterValue("SpanAttributes['k8s.pod.name']", [ + 'pod-1', + ]); + }); + + const { result: result2 } = renderHook(() => + useDashboardFilters(bracketFilters), + ); + + // The bracket-notation expression should still match after the Lucene + // round-trip converts the key to dot notation internally. + expect( + result2.current.filterValues["SpanAttributes['k8s.pod.name']"]?.included, + ).toEqual(new Set(['pod-1'])); + expect(result2.current.ignoredFilterExpressions).toEqual([]); + }); + + it('should match dot-notation URL key to bracket-notation expression', () => { + const bracketFilters: DashboardFilter[] = [ + { + id: 'filter-bracket', + type: 'QUERY_EXPRESSION', + name: 'Pod', + expression: "SpanAttributes['k8s.pod.name']", + source: 'traces', + }, + ]; + + // Pre-seed URL state with a dot-notation Lucene filter (as would be + // stored after a round-trip through filtersToQuery → parseQuery). + mockState = [ + { + type: 'lucene', + condition: 'SpanAttributes.k8s.pod.name:"pod-1"', + }, + ]; + + const { result } = renderHook(() => useDashboardFilters(bracketFilters)); + + expect( + result.current.filterValues["SpanAttributes['k8s.pod.name']"]?.included, + ).toEqual(new Set(['pod-1'])); + expect(result.current.ignoredFilterExpressions).toEqual([]); + }); + + it('should migrate legacy SQL filters to Lucene on load', () => { + // Pre-seed URL with old-format SQL filters + mockState = [{ type: 'sql', condition: "environment IN ('production')" }]; + + renderHook(() => useDashboardFilters(mockFilters)); + + // Migration should have called setFilterQueries with Lucene format + expect(mockSetState).toHaveBeenCalled(); + const lastCall = + mockSetState.mock.calls[mockSetState.mock.calls.length - 1]; + // setFilterQueries receives an updater or a value; resolve it + const result = + typeof lastCall[0] === 'function' ? lastCall[0](mockState) : lastCall[0]; + expect(result).toEqual([ + { type: 'lucene', condition: 'environment:"production"' }, + ]); + }); }); diff --git a/packages/app/src/hooks/useDashboardFilters.tsx b/packages/app/src/hooks/useDashboardFilters.tsx index e1b03c6c40..7aa439465e 100644 --- a/packages/app/src/hooks/useDashboardFilters.tsx +++ b/packages/app/src/hooks/useDashboardFilters.tsx @@ -1,5 +1,6 @@ -import { useCallback, useMemo } from 'react'; +import { useCallback, useEffect, useMemo, useRef } from 'react'; import { useQueryState } from 'nuqs'; +import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, filtersToQuery, @@ -21,19 +22,19 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { (expression: string, values: string[]) => { setFilterQueries(prev => { const { filters: filterValues } = parseQuery(prev ?? []); + // Normalize the expression to dot notation so it matches the keys + // returned by parseQuery (which converts bracket notation to dots). + const key = parseKeyPath(expression).join('.'); if (values.length === 0) { - delete filterValues[expression]; + delete filterValues[key]; } else { - filterValues[expression] = { + filterValues[key] = { included: new Set(values), excluded: new Set(), }; } - return filtersToQuery( - filterValues, - { stringifyKeys: false }, // Don't wrap keys with toString(), to preserve exact key names in URL query parameters - ); + return filtersToQuery(filterValues); }); }, [setFilterQueries], @@ -46,32 +47,53 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { } = useMemo(() => { const { filters: parsedFilters } = parseQuery(filterQueries ?? []); const valuesForExistingFilters: FilterState = {}; - const knownExpressions = new Set(filters.map(f => f.expression)); const ignored: string[] = []; + // Build a normalized lookup so bracket-notation expressions + // (e.g. SpanAttributes['k8s.pod.name']) match the dot-notation keys + // returned by parseLuceneFilter (e.g. SpanAttributes.k8s.pod.name). + const normalizeKey = (k: string) => parseKeyPath(k).join('.'); + const normalizedParsed = new Map( + Object.entries(parsedFilters).map(([k, v]) => [normalizeKey(k), v]), + ); + const knownNormalized = new Set( + filters.map(f => normalizeKey(f.expression)), + ); + for (const { expression } of filters) { - if (expression in parsedFilters) { - valuesForExistingFilters[expression] = parsedFilters[expression]; + const norm = normalizeKey(expression); + const match = normalizedParsed.get(norm); + if (match) { + valuesForExistingFilters[expression] = match; } } for (const key of Object.keys(parsedFilters)) { - if (!knownExpressions.has(key)) { + if (!knownNormalized.has(normalizeKey(key))) { ignored.push(key); } } return { valuesForExistingFilters, - queriesForExistingFilters: filtersToQuery( - valuesForExistingFilters, - // Wrap keys in `toString()` to support JSON/Dynamic-type columns. - // All keys can be stringified, since filter select values are stringified as well. - { stringifyKeys: true }, - ), + queriesForExistingFilters: filtersToQuery(valuesForExistingFilters), ignoredExpressions: ignored, }; }, [filterQueries, filters]); + // Migrate legacy SQL filters in the URL to Lucene on load + const hasMigratedRef = useRef(false); + useEffect(() => { + if (hasMigratedRef.current || !filterQueries) return; + const hasSqlFilters = filterQueries.some( + f => 'condition' in f && f.type === 'sql', + ); + if (hasSqlFilters) { + hasMigratedRef.current = true; + const { filters: parsed, passthroughFilters } = parseQuery(filterQueries); + setFilterQueries([...filtersToQuery(parsed), ...passthroughFilters]); + } + }, [filterQueries, setFilterQueries]); + return { filterValues: valuesForExistingFilters, filterQueries: queriesForExistingFilters, diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index fbb6292857..cc35e484fd 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -1,8 +1,11 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import produce from 'immer'; +import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { + coerceBooleanValue, type FilterState, filtersToQuery, + parseLuceneFilter, } from '@hyperdx/common-utils/dist/filters'; import type { Filter } from '@hyperdx/common-utils/dist/types'; @@ -297,6 +300,8 @@ export const parseQuery = ( q: Filter[], ): { filters: FilterState; + /** Filters that could not be parsed into FilterState (preserved for re-emission) */ + passthroughFilters: Filter[]; } => { const state = new Map< string, @@ -306,7 +311,29 @@ export const parseQuery = ( range?: { min: number; max: number }; } >(); + const passthroughFilters: Filter[] = []; for (const filter of q) { + if (filter.type === 'lucene') { + const parsedFields = parseLuceneFilter(filter.condition); + if (parsedFields === undefined) { + // Parse failure — preserve so it isn't silently lost + if (filter.condition.trim()) { + passthroughFilters.push(filter); + } + } else { + for (const { key, included, excluded, range } of parsedFields) { + if (!state.has(key)) { + state.set(key, { included: new Set(), excluded: new Set() }); + } + const sets = state.get(key)!; + for (const v of included) sets.included.add(v); + for (const v of excluded) sets.excluded.add(v); + if (range) sets.range = range; + } + } + continue; + } + if (filter.type !== 'sql') continue; // Check for BETWEEN condition (only when BETWEEN appears outside quotes) @@ -352,7 +379,7 @@ export const parseQuery = ( }); } } - return { filters: Object.fromEntries(state) }; + return { filters: Object.fromEntries(state), passthroughFilters }; }; export const useSearchPageFilterState = ({ @@ -367,7 +394,7 @@ export const useSearchPageFilterState = ({ return parseQuery(searchQuery); } catch (e) { console.error(e); - return { filters: {} }; + return { filters: {}, passthroughFilters: [] }; } }, [searchQuery]); @@ -381,11 +408,32 @@ export const useSearchPageFilterState = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [parsedQuery.filters]); + // Migrate legacy SQL filters to Lucene on load so URLs become canonical + const hasMigratedRef = useRef(false); + useEffect(() => { + if (hasMigratedRef.current) return; + const hasSqlFilters = searchQuery?.some( + f => 'condition' in f && f.type === 'sql', + ); + if (hasSqlFilters && Object.keys(parsedQuery.filters).length > 0) { + hasMigratedRef.current = true; + onFilterChange([ + ...filtersToQuery(parsedQuery.filters), + ...parsedQuery.passthroughFilters, + ]); + } + }, [searchQuery, parsedQuery, onFilterChange]); + const updateFilterQuery = useCallback( (newFilters: FilterState) => { - onFilterChange(filtersToQuery(newFilters)); + // Preserve unparseable filters so they aren't silently lost when + // the user toggles a sidebar facet. + onFilterChange([ + ...filtersToQuery(newFilters), + ...parsedQuery.passthroughFilters, + ]); }, - [onFilterChange], + [onFilterChange, parsedQuery.passthroughFilters], ); const setFilterValue = useCallback( @@ -394,15 +442,21 @@ export const useSearchPageFilterState = ({ value: string | boolean, action?: 'only' | 'exclude' | 'include', ) => { + // Normalize bracket notation to dot notation so the key matches + // what parseQuery returns after a Lucene round-trip. + const key = parseKeyPath(property).join('.'); + // Normalize "true"/"false" strings to booleans so the value matches + // what parseLuceneFilter returns after a Lucene round-trip. + const normalizedValue = coerceBooleanValue(value); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - if (!draft[property]) { - draft[property] = { included: new Set(), excluded: new Set() }; + if (!draft[key]) { + draft[key] = { included: new Set(), excluded: new Set() }; } if (action === 'only') { - draft[property] = { - included: new Set([value]), + draft[key] = { + included: new Set([normalizedValue]), excluded: new Set(), }; return; @@ -410,22 +464,22 @@ export const useSearchPageFilterState = ({ if (action === 'exclude') { // Remove from included if it was there - draft[property].included.delete(value); + draft[key].included.delete(normalizedValue); // Toggle in excluded - if (draft[property].excluded.has(value)) { - draft[property].excluded.delete(value); + if (draft[key].excluded.has(normalizedValue)) { + draft[key].excluded.delete(normalizedValue); } else { - draft[property].excluded.add(value); + draft[key].excluded.add(normalizedValue); } return; } // Regular toggle (include) - draft[property].excluded.delete(value); - if (draft[property].included.has(value)) { - draft[property].included.delete(value); + draft[key].excluded.delete(normalizedValue); + if (draft[key].included.has(normalizedValue)) { + draft[key].included.delete(normalizedValue); } else { - draft[property].included.add(value); + draft[key].included.add(normalizedValue); } }); updateFilterQuery(newFilters); @@ -437,12 +491,13 @@ export const useSearchPageFilterState = ({ const setFilterRange = useCallback( (property: string, range: { min: number; max: number }) => { + const key = parseKeyPath(property).join('.'); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - if (!draft[property]) { - draft[property] = { included: new Set(), excluded: new Set() }; + if (!draft[key]) { + draft[key] = { included: new Set(), excluded: new Set() }; } - draft[property].range = range; + draft[key].range = range; }); updateFilterQuery(newFilters); return newFilters; @@ -453,9 +508,10 @@ export const useSearchPageFilterState = ({ const clearFilter = useCallback( (property: string) => { + const key = parseKeyPath(property).join('.'); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - delete draft[property]; + delete draft[key]; }); updateFilterQuery(newFilters); return newFilters; diff --git a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts index 88c4d43d0e..d4f127b2aa 100644 --- a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts @@ -205,10 +205,10 @@ test.describe( const url = new URL(page.url()); expect(url.searchParams.get('source')).toBe(logsSourceId); expect(url.searchParams.get('isLive')).toBe('false'); - // Default row click encodes the group-by filter as a SQL IN clause in - // the JSON-encoded `filters` param. + // Default row click encodes the group-by filter as a Lucene condition + // in the JSON-encoded `filters` param. const filters = url.searchParams.get('filters') ?? '[]'; - expect(filters).toContain(`ServiceName IN ('${serviceName}')`); + expect(filters).toContain(`ServiceName:\\"${serviceName}\\"`); const from = Number(url.searchParams.get('from')); const to = Number(url.searchParams.get('to')); expect(from).toBeGreaterThan(0); @@ -563,8 +563,8 @@ test.describe( const filters = JSON.parse(decodeURIComponent(filtersRaw!)); expect(filters).toEqual([ { - type: 'sql', - condition: `ServiceName IN ('${serviceName}')`, + type: 'lucene', + condition: `ServiceName:"${serviceName}"`, }, ]); }); @@ -649,8 +649,8 @@ test.describe( const filters = JSON.parse(decodeURIComponent(filtersRaw!)); expect(filters).toEqual([ { - type: 'sql', - condition: `ServiceName IN ('${serviceName}')`, + type: 'lucene', + condition: `ServiceName:"${serviceName}"`, }, ]); }); diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts index 20253b4c8b..d2ba8ebb2f 100644 --- a/packages/common-utils/src/__tests__/filters.test.ts +++ b/packages/common-utils/src/__tests__/filters.test.ts @@ -1,36 +1,38 @@ -import { filtersToQuery } from '@/filters'; +import { type FilterState, filtersToQuery, parseLuceneFilter } from '@/filters'; +import type { Filter } from '@/types'; + +/** Extract condition string from a Filter (filtersToQuery never emits sql_ast) */ +const getCondition = (f: Filter): string => + 'condition' in f ? f.condition : ''; describe('filters', () => { describe('filtersToQuery', () => { - it('should return empty string when no filters', () => { - const filters = {}; - expect(filtersToQuery(filters)).toEqual([]); + it('should return empty array when no filters', () => { + expect(filtersToQuery({})).toEqual([]); }); - it('should return query for one filter', () => { + it('should emit lucene for a single included value', () => { const filters = { a: { included: new Set(['b']), excluded: new Set() }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'sql', condition: "a IN ('b')" }, + { type: 'lucene', condition: 'a:"b"' }, ]); }); - it('should return query for multiple filters', () => { + it('should emit lucene with parens for multiple included values', () => { const filters = { - a: { included: new Set(['b']), excluded: new Set() }, c: { included: new Set(['d', 'x']), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'sql', condition: "a IN ('b')" }, - { type: 'sql', condition: "c IN ('d', 'x')" }, + { type: 'lucene', condition: '(c:"d" OR c:"x")' }, ]); }); - it('should handle excluded values', () => { + it('should handle excluded values without parens', () => { const filters = { a: { included: new Set(['b']), @@ -38,129 +40,442 @@ describe('filters', () => { }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'sql', condition: "a IN ('b')" }, - { type: 'sql', condition: "a NOT IN ('c')" }, + { type: 'lucene', condition: 'a:"b"' }, + { type: 'lucene', condition: '-a:"c"' }, ]); }); - it('should wrap keys with toString() when specified', () => { + it('should handle multiple excluded values without parens', () => { const filters = { - 'json.key': { - included: new Set(['value']), - excluded: new Set(['other value']), + a: { + included: new Set(), + excluded: new Set([true, false]), }, }; - expect(filtersToQuery(filters, { stringifyKeys: true })).toEqual([ - { type: 'sql', condition: "toString(json.key) IN ('value')" }, - { type: 'sql', condition: "toString(json.key) NOT IN ('other value')" }, + expect(filtersToQuery(filters)).toEqual([ + { + type: 'lucene', + condition: '-a:"true" AND -a:"false"', + }, ]); }); - it('should should handle boolean filter values', () => { + it('should handle boolean filter values', () => { const filters = { isRootSpan: { included: new Set([true]), excluded: new Set([]), }, - another_column: { - included: new Set([]), - excluded: new Set([true, false]), - }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'sql', condition: 'isRootSpan IN (true)' }, - { type: 'sql', condition: 'another_column NOT IN (true, false)' }, + { type: 'lucene', condition: 'isRootSpan:"true"' }, ]); }); - it('should escape single quotes in filter values', () => { + it('should escape double quotes in values', () => { const filters = { message: { - included: new Set(["my 'filter' key"]), + included: new Set(['say "hello"']), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ - { - type: 'sql', - condition: "message IN ('my ''filter'' key')", - }, + { type: 'lucene', condition: 'message:"say \\"hello\\""' }, ]); }); - it('should escape single quotes in excluded filter values', () => { + it('should pass through single quotes in values', () => { const filters = { message: { - included: new Set(), - excluded: new Set(["it's a test"]), + included: new Set(["it's a test"]), + excluded: new Set(), + }, + }; + expect(filtersToQuery(filters)).toEqual([ + { type: 'lucene', condition: 'message:"it\'s a test"' }, + ]); + }); + + it('should escape backslashes in values', () => { + const filters = { + FilePath: { + included: new Set(['C:\\path\\to\\file']), + excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ { - type: 'sql', - condition: "message NOT IN ('it''s a test')", + type: 'lucene', + condition: 'FilePath:"C:\\\\path\\\\to\\\\file"', }, ]); }); - it('should escape single quotes with stringifyKeys', () => { + it('should convert Map bracket-notation keys to dot notation', () => { const filters = { - 'json.key': { - included: new Set(["value with 'quotes'"]), + "LogAttributes['service.name']": { + included: new Set(['my-app']), excluded: new Set(), }, }; - expect(filtersToQuery(filters, { stringifyKeys: true })).toEqual([ + expect(filtersToQuery(filters)).toEqual([ { - type: 'sql', - condition: "toString(json.key) IN ('value with ''quotes''')", + type: 'lucene', + condition: 'LogAttributes.service.name:"my-app"', }, ]); }); - it('should escape backslashes in filter values', () => { + it('should handle Map bracket-notation with multiple values', () => { const filters = { - FilePath: { - included: new Set(['C:\\path\\to\\file']), + "LogAttributes['env']": { + included: new Set(['prod', 'staging']), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ { - type: 'sql', - condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')", + type: 'lucene', + condition: + '(LogAttributes.env:"prod" OR LogAttributes.env:"staging")', }, ]); }); - it('should escape backslashes in excluded filter values', () => { + it('should handle Map bracket-notation excluded values', () => { const filters = { - FilePath: { + "LogAttributes['level']": { included: new Set(), - excluded: new Set(['C:\\path\\to\\file']), + excluded: new Set(['debug']), }, }; expect(filtersToQuery(filters)).toEqual([ - { - type: 'sql', - condition: "FilePath NOT IN ('C:\\\\path\\\\to\\\\file')", + { type: 'lucene', condition: '-LogAttributes.level:"debug"' }, + ]); + }); + + it('should handle Map bracket-notation with both included and excluded', () => { + const filters = { + "ResourceAttributes['host']": { + included: new Set(['web-1']), + excluded: new Set(['web-2']), }, + }; + expect(filtersToQuery(filters)).toEqual([ + { type: 'lucene', condition: 'ResourceAttributes.host:"web-1"' }, + { type: 'lucene', condition: '-ResourceAttributes.host:"web-2"' }, ]); }); - it('should escape backslashes before single quotes so quotes stay escaped', () => { + it('should emit lucene range syntax', () => { const filters = { - message: { - included: new Set(["a\\'b"]), + duration: { + included: new Set(), excluded: new Set(), + range: { min: 10, max: 500 }, }, }; expect(filtersToQuery(filters)).toEqual([ + { type: 'lucene', condition: 'duration:[10 TO 500]' }, + ]); + }); + + it('should emit lucene range with dot notation for Map bracket-notation keys', () => { + const filters = { + "LogAttributes['latency']": { + included: new Set(), + excluded: new Set(), + range: { min: 0, max: 100 }, + }, + }; + expect(filtersToQuery(filters)).toEqual([ + { type: 'lucene', condition: 'LogAttributes.latency:[0 TO 100]' }, + ]); + }); + }); + + describe('parseLuceneFilter', () => { + it('should return empty array for empty string', () => { + expect(parseLuceneFilter('')).toEqual([]); + }); + + it('should return empty array for unquoted terms', () => { + expect(parseLuceneFilter('service:foo')).toEqual([]); + }); + + it('should parse a single included term', () => { + expect(parseLuceneFilter('service:"app"')).toEqual([ + { key: 'service', included: ['app'], excluded: [] }, + ]); + }); + + it('should parse a single excluded term', () => { + expect(parseLuceneFilter('-service:"app"')).toEqual([ + { key: 'service', included: [], excluded: ['app'] }, + ]); + }); + + it('should parse multiple included values with OR', () => { + expect(parseLuceneFilter('(service:"app" OR service:"web")')).toEqual([ + { key: 'service', included: ['app', 'web'], excluded: [] }, + ]); + }); + + it('should parse multiple excluded values with AND', () => { + expect( + parseLuceneFilter('-service:"debug" AND -service:"trace"'), + ).toEqual([ + { key: 'service', included: [], excluded: ['debug', 'trace'] }, + ]); + }); + + it('should parse mixed included and excluded for the same field', () => { + expect( + parseLuceneFilter('-service:"bingo" (service:"foo" OR service:"bar")'), + ).toEqual([ + { key: 'service', included: ['foo', 'bar'], excluded: ['bingo'] }, + ]); + }); + + it('should parse multiple distinct fields', () => { + expect(parseLuceneFilter('service:"app" level:"info"')).toEqual([ + { key: 'service', included: ['app'], excluded: [] }, + { key: 'level', included: ['info'], excluded: [] }, + ]); + }); + + it('should parse multiple fields with mixed negation', () => { + expect( + parseLuceneFilter('service:"app" -level:"debug" level:"info"'), + ).toEqual([ + { key: 'service', included: ['app'], excluded: [] }, + { key: 'level', included: ['info'], excluded: ['debug'] }, + ]); + }); + + it('should parse dot-notation fields (Map columns)', () => { + expect(parseLuceneFilter('LogAttributes.env:"prod"')).toEqual([ + { key: 'LogAttributes.env', included: ['prod'], excluded: [] }, + ]); + }); + + it('should handle values with escaped double quotes', () => { + expect(parseLuceneFilter('message:"say \\"hello\\""')).toEqual([ + { key: 'message', included: ['say "hello"'], excluded: [] }, + ]); + }); + + it('should handle values with backslashes', () => { + expect(parseLuceneFilter('path:"C:\\\\dir\\\\file"')).toEqual([ + { key: 'path', included: ['C:\\dir\\file'], excluded: [] }, + ]); + }); + + it('should coerce "true"/"false" back to booleans', () => { + expect(parseLuceneFilter('isRootSpan:"true"')).toEqual([ + { key: 'isRootSpan', included: [true], excluded: [] }, + ]); + expect(parseLuceneFilter('-col:"true" AND -col:"false"')).toEqual([ + { key: 'col', included: [], excluded: [true, false] }, + ]); + }); + + it('should parse range terms', () => { + expect(parseLuceneFilter('duration:[10 TO 500]')).toEqual([ + { + key: 'duration', + included: [], + excluded: [], + range: { min: 10, max: 500 }, + }, + ]); + }); + + it('should parse range with dot-notation field', () => { + expect(parseLuceneFilter('LogAttributes.latency:[0 TO 100]')).toEqual([ { - type: 'sql', - condition: "message IN ('a\\\\''b')", + key: 'LogAttributes.latency', + included: [], + excluded: [], + range: { min: 0, max: 100 }, + }, + ]); + }); + + it('should return undefined for invalid lucene syntax', () => { + expect(parseLuceneFilter('((((')).toBeUndefined(); + }); + }); + + describe('filtersToQuery -> parseLuceneFilter round-trip', () => { + it('round-trips a single included value', () => { + const state: FilterState = { + service: { + included: new Set(['app']), + excluded: new Set(), }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed).toEqual([ + { key: 'service', included: ['app'], excluded: [] }, ]); }); + + it('round-trips multiple included values', () => { + const state: FilterState = { + env: { + included: new Set(['prod', 'staging']), + excluded: new Set(), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed?.[0].key).toBe('env'); + expect(parsed?.[0].included).toEqual( + expect.arrayContaining(['prod', 'staging']), + ); + }); + + it('round-trips excluded values', () => { + const state: FilterState = { + level: { + included: new Set(), + excluded: new Set(['debug', 'trace']), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed?.[0].key).toBe('level'); + expect(parsed?.[0].excluded).toEqual( + expect.arrayContaining(['debug', 'trace']), + ); + }); + + it('round-trips boolean values preserving type', () => { + const state: FilterState = { + isRootSpan: { + included: new Set([true]), + excluded: new Set([false]), + }, + }; + const filters = filtersToQuery(state); + const includedParsed = parseLuceneFilter(getCondition(filters[0])); + const excludedParsed = parseLuceneFilter(getCondition(filters[1])); + expect(includedParsed?.[0].included).toEqual([true]); + expect(excludedParsed?.[0].excluded).toEqual([false]); + }); + + it('round-trips the literal string "true" without boolean coercion', () => { + // The string "true" is indistinguishable from boolean true after + // Lucene round-trip due to coerceBooleanValue. This test documents + // that known limitation. + const state: FilterState = { + status: { + included: new Set(['true']), + excluded: new Set(), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + // Coerced to boolean — this is the expected (lossy) behavior + expect(parsed?.[0].included).toEqual([true]); + }); + + it('round-trips values with double quotes', () => { + const state: FilterState = { + msg: { + included: new Set(['say "hello"']), + excluded: new Set(), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed?.[0].included).toEqual(['say "hello"']); + }); + + it('round-trips values with backslashes', () => { + const state: FilterState = { + path: { + included: new Set(['C:\\dir\\file']), + excluded: new Set(), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed?.[0].included).toEqual(['C:\\dir\\file']); + }); + + it('round-trips empty string values', () => { + const state: FilterState = { + tag: { + included: new Set(['']), + excluded: new Set(), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed?.[0].included).toEqual(['']); + }); + + it('round-trips values containing Lucene reserved characters', () => { + const state: FilterState = { + query: { + included: new Set(['(foo) AND [bar]']), + excluded: new Set(), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed?.[0].included).toEqual(['(foo) AND [bar]']); + }); + + it('round-trips range filters', () => { + const state: FilterState = { + duration: { + included: new Set(), + excluded: new Set(), + range: { min: 10, max: 500 }, + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + expect(parsed?.[0].range).toEqual({ min: 10, max: 500 }); + }); + + it('round-trips Map bracket-notation key', () => { + const state: FilterState = { + "LogAttributes['service.name']": { + included: new Set(['my-app']), + excluded: new Set(), + }, + }; + const filters = filtersToQuery(state); + const parsed = parseLuceneFilter(getCondition(filters[0])); + // Bracket notation normalizes to dot notation + expect(parsed?.[0].key).toBe('LogAttributes.service.name'); + expect(parsed?.[0].included).toEqual(['my-app']); + }); + + it('round-trips mixed included and excluded for same field', () => { + const state: FilterState = { + env: { + included: new Set(['prod', 'staging']), + excluded: new Set(['dev']), + }, + }; + const filters = filtersToQuery(state); + // filtersToQuery emits separate Filter entries for included/excluded + expect(filters).toHaveLength(2); + // Parse both and merge + const allParsed = filters.flatMap( + f => parseLuceneFilter(getCondition(f)) ?? [], + ); + const envEntries = allParsed.filter(p => p.key === 'env'); + const included = envEntries.flatMap(e => e.included); + const excluded = envEntries.flatMap(e => e.excluded); + expect(included).toEqual(expect.arrayContaining(['prod', 'staging'])); + expect(excluded).toEqual(['dev']); + }); }); }); diff --git a/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts b/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts index 2dd7ec5fd5..9305dac1c5 100644 --- a/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts +++ b/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts @@ -243,7 +243,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'sql', condition: "ServiceName IN ('MyService')" }]); + ).toEqual([{ type: 'lucene', condition: 'ServiceName:"MyService"' }]); } }); @@ -277,7 +277,9 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'sql', condition: "ServiceName IN ('A', 'B')" }]); + ).toEqual([ + { type: 'lucene', condition: '(ServiceName:"A" OR ServiceName:"B")' }, + ]); } }); @@ -312,8 +314,8 @@ describe('renderOnClickSearch', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'sql', condition: "ServiceName IN ('MyService')" }, - { type: 'sql', condition: "SeverityText IN ('error')" }, + { type: 'lucene', condition: 'ServiceName:"MyService"' }, + { type: 'lucene', condition: 'SeverityText:"error"' }, ]); } }); @@ -343,7 +345,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'sql', condition: "ServiceName IN ('O''Malley')" }]); + ).toEqual([{ type: 'lucene', condition: 'ServiceName:"O\'Malley"' }]); } }); @@ -373,7 +375,7 @@ describe('renderOnClickSearch', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'sql', condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')" }, + { type: 'lucene', condition: 'FilePath:"C:\\\\path\\\\to\\\\file"' }, ]); } }); @@ -405,8 +407,8 @@ describe('renderOnClickSearch', () => { JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ { - type: 'sql', - condition: "SpanAttributes['url'] IN ('/users%2F42')", + type: 'lucene', + condition: 'SpanAttributes.url:"/users%2F42"', }, ]); } @@ -675,7 +677,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'sql', condition: "ServiceName IN ('MyService')" }]); + ).toEqual([{ type: 'lucene', condition: 'ServiceName:"MyService"' }]); } }); @@ -709,7 +711,9 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'sql', condition: "ServiceName IN ('A', 'B')" }]); + ).toEqual([ + { type: 'lucene', condition: '(ServiceName:"A" OR ServiceName:"B")' }, + ]); } }); @@ -744,8 +748,8 @@ describe('renderOnClickDashboard', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'sql', condition: "ServiceName IN ('MyService')" }, - { type: 'sql', condition: "SeverityText IN ('error')" }, + { type: 'lucene', condition: 'ServiceName:"MyService"' }, + { type: 'lucene', condition: 'SeverityText:"error"' }, ]); } }); @@ -775,7 +779,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'sql', condition: "ServiceName IN ('O''Malley')" }]); + ).toEqual([{ type: 'lucene', condition: 'ServiceName:"O\'Malley"' }]); } }); @@ -805,7 +809,7 @@ describe('renderOnClickDashboard', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'sql', condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')" }, + { type: 'lucene', condition: 'FilePath:"C:\\\\path\\\\to\\\\file"' }, ]); } }); @@ -837,8 +841,8 @@ describe('renderOnClickDashboard', () => { JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ { - type: 'sql', - condition: "SpanAttributes['url'] IN ('/users%2F42')", + type: 'lucene', + condition: 'SpanAttributes.url:"/users%2F42"', }, ]); } diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 9a7ac150b1..d53d64a23a 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -1,3 +1,14 @@ +import lucene from '@hyperdx/lucene'; + +import { parseKeyPath } from '@/core/metadata'; +import { + decodeSpecialTokens, + isBinaryAST, + isLeftOnlyAST, + isNodeRangedTerm, + isNodeTerm, + parse, +} from '@/queryParser'; import { Filter } from '@/types'; export type FilterState = { @@ -8,14 +19,12 @@ export type FilterState = { }; }; -const escapeString = (s: string) => { - return s.replace(/\\/g, '\\\\').replace(/'/g, "''"); +/** Escape a value for use inside a Lucene quoted term ("...") */ +const escapeLuceneQuotedTerm = (s: string) => { + return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); }; -export const filtersToQuery = ( - filters: FilterState, - { stringifyKeys = false }: { stringifyKeys?: boolean } = {}, -): Filter[] => { +export const filtersToQuery = (filters: FilterState): Filter[] => { return Object.entries(filters) .filter( ([_, values]) => @@ -25,30 +34,137 @@ export const filtersToQuery = ( ) .flatMap(([key, values]) => { const conditions: Filter[] = []; - const actualKey = stringifyKeys ? `toString(${key})` : key; + const luceneField = parseKeyPath(key).join('.'); if (values.included.size > 0) { + const terms = Array.from(values.included).map( + v => `${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`, + ); conditions.push({ - type: 'sql' as const, - condition: `${actualKey} IN (${Array.from(values.included) - .map(v => (typeof v === 'string' ? `'${escapeString(v)}'` : v)) - .join(', ')})`, + type: 'lucene' as const, + condition: terms.length > 1 ? `(${terms.join(' OR ')})` : terms[0], }); } if (values.excluded.size > 0) { conditions.push({ - type: 'sql' as const, - condition: `${actualKey} NOT IN (${Array.from(values.excluded) - .map(v => (typeof v === 'string' ? `'${escapeString(v)}'` : v)) - .join(', ')})`, + type: 'lucene' as const, + condition: Array.from(values.excluded) + .map(v => `-${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`) + .join(' AND '), }); } + if (values.range != null) { + // Lucene range syntax: field:[min TO max] conditions.push({ - type: 'sql' as const, - condition: `${actualKey} BETWEEN ${values.range.min} AND ${values.range.max}`, + type: 'lucene' as const, + condition: `${luceneField}:[${values.range.min} TO ${values.range.max}]`, }); } return conditions; }); }; + +type CollectedTerm = { field: string; value: string; negated: boolean }; +type CollectedRange = { field: string; min: number; max: number }; + +/** + * Collect quoted terms and range terms from a Lucene AST. + */ +function collectFromAst( + ast: lucene.AST | lucene.Node, + terms: CollectedTerm[], + ranges: CollectedRange[], +): void { + if (isNodeTerm(ast)) { + if (!ast.quoted) return; + const negated = ast.field.startsWith('-'); + const field = negated ? ast.field.slice(1) : ast.field; + terms.push({ field, value: decodeSpecialTokens(ast.term), negated }); + } else if (isNodeRangedTerm(ast)) { + const field = ast.field; + const min = parseFloat(ast.term_min); + const max = parseFloat(ast.term_max); + if (!isNaN(min) && !isNaN(max)) { + ranges.push({ field, min, max }); + } + } else if (isBinaryAST(ast)) { + collectFromAst(ast.left, terms, ranges); + collectFromAst(ast.right, terms, ranges); + } else if (isLeftOnlyAST(ast)) { + collectFromAst(ast.left, terms, ranges); + } +} + +/** Coerce "true"/"false" strings back to booleans, pass through otherwise */ +export function coerceBooleanValue(v: string | boolean): string | boolean { + if (typeof v === 'boolean') return v; + if (v === 'true') return true; + if (v === 'false') return false; + return v; +} + +export type ParsedLuceneFilter = { + key: string; + included: (string | boolean)[]; + excluded: (string | boolean)[]; + range?: { min: number; max: number }; +}; + +/** + * Parse a Lucene filter condition back into per-field included/excluded values + * and optional ranges. + * + * Returns an array with one entry per distinct field (empty array if the input + * is valid Lucene but contains no quoted terms or ranges), or undefined if + * parsing fails entirely. + */ +export function parseLuceneFilter( + condition: string, +): ParsedLuceneFilter[] | undefined { + try { + const ast = parse(condition); + const terms: CollectedTerm[] = []; + const ranges: CollectedRange[] = []; + collectFromAst(ast, terms, ranges); + + // Group by field, coercing "true"/"false" back to booleans + const byField = new Map< + string, + { + included: (string | boolean)[]; + excluded: (string | boolean)[]; + range?: { min: number; max: number }; + } + >(); + + const getEntry = (field: string) => { + if (!byField.has(field)) { + byField.set(field, { included: [], excluded: [] }); + } + return byField.get(field)!; + }; + + for (const t of terms) { + const entry = getEntry(t.field); + const value = coerceBooleanValue(t.value); + if (t.negated) { + entry.excluded.push(value); + } else { + entry.included.push(value); + } + } + + for (const r of ranges) { + const entry = getEntry(r.field); + entry.range = { min: r.min, max: r.max }; + } + + return Array.from(byField.entries()).map(([key, vals]) => ({ + key, + ...vals, + })); + } catch { + return undefined; + } +} diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 0781019705..10bf9ac59e 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -31,7 +31,7 @@ function encodeSpecialTokens(query: string): string { .replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1') .replace(/\\:/g, 'HDX_COLON'); } -function decodeSpecialTokens(query: string): string { +export function decodeSpecialTokens(query: string): string { return query .replace(/\\"/g, '"') .replace(/HDX_BACKSLASH_LITERAL/g, '\\') @@ -59,17 +59,21 @@ function normalizeChExpression(expr: string): string { const IMPLICIT_FIELD = ''; // Type guards for lucene AST types -function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { +export function isNodeTerm( + node: lucene.Node | lucene.AST, +): node is lucene.NodeTerm { return 'term' in node && node.term != null; } -function isNodeRangedTerm( +export function isNodeRangedTerm( node: lucene.Node | lucene.AST, ): node is lucene.NodeRangedTerm { return 'inclusive' in node && node.inclusive != null; } -function isBinaryAST(ast: lucene.AST | lucene.Node): ast is lucene.BinaryAST { +export function isBinaryAST( + ast: lucene.AST | lucene.Node, +): ast is lucene.BinaryAST { return 'right' in ast && ast.right != null; } @@ -79,7 +83,7 @@ function hasStart( return 'start' in ast && !!ast.start; } -function isLeftOnlyAST( +export function isLeftOnlyAST( ast: lucene.AST | lucene.Node, ): ast is lucene.LeftOnlyAST { return (