diff --git a/.changeset/thin-pets-relate.md b/.changeset/thin-pets-relate.md deleted file mode 100644 index 71fe2ca5b1..0000000000 --- a/.changeset/thin-pets-relate.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/app": patch ---- - -fix: make filters aware of map column return arrayElement diff --git a/AGENTS.md b/AGENTS.md index 6455078d16..70dbe826a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,19 +71,6 @@ and lint issues across all packages. Pre-commit hooks handle this when committing, but if you finish edits without committing, run `yarn lint:fix` before stopping. -## Before Writing New Utility Functions - -Before implementing a new parsing, formatting, or transformation function, -search for existing functions that already do the same thing or something close. -Common places to check: - -- `packages/common-utils/src/core/metadata.ts` — key/column parsing utilities - (e.g. `parseKeyPath`) -- `packages/common-utils/src/queryParser.ts` — Lucene parsing (`parse`, - `decodeSpecialTokens`), AST type guards, serialization helpers -- `packages/common-utils/src/filters.ts` — filter state serialization and - parsing (`filtersToQuery`, `parseLuceneFilter`) - ## Key Principles 1. **Multi-tenancy**: All data is scoped to `Team` - ensure proper filtering diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index bb6906e674..b2d3e53ebb 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -146,7 +146,7 @@ export type ExternalDashboardFilter = z.infer< >; export const externalDashboardSavedFilterValueSchema = z.object({ - type: z.enum(['sql', 'lucene']).optional().default('sql'), + type: z.literal('sql').optional().default('sql'), condition: z.string().max(10000), }); diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index a2a9a05c4b..544c6762d5 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -17,10 +17,6 @@ import { getAlignedDateRange, Granularity, } from '@hyperdx/common-utils/dist/core/utils'; -import { - type FilterState, - filtersToQuery, -} from '@hyperdx/common-utils/dist/filters'; import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { AggregateFunction as AggFnV2, @@ -927,19 +923,13 @@ export function buildEventsSearchUrl({ // Add group-by column filters if (groupFilters && groupFilters.length > 0) { - const filterState: FilterState = {}; - for (const { column, value } of groupFilters) { + groupFilters.forEach(({ column, value }) => { if (column && value != null) { - if (!filterState[column]) { - filterState[column] = { - included: new Set(), - excluded: new Set(), - }; - } - filterState[column].included.add(String(value)); + // Can't use SQLString.escape here because the search endpoint relies on exist match for UI + const condition = `${column} IN (${SqlString.escape(value)})`; + additionalFilters.push({ type: 'sql', condition }); } - } - additionalFilters.push(...filtersToQuery(filterState)); + }); } // Add Y-axis value range filter (±threshold) for charts diff --git a/packages/app/src/__tests__/searchFilters.test.ts b/packages/app/src/__tests__/searchFilters.test.ts index 1d30fc7314..ec91631c03 100644 --- a/packages/app/src/__tests__/searchFilters.test.ts +++ b/packages/app/src/__tests__/searchFilters.test.ts @@ -1,11 +1,5 @@ import { enableMapSet } from 'immer'; -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; -import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { filtersToQuery } from '@hyperdx/common-utils/dist/filters'; -import { - CustomSchemaSQLSerializerV2, - SearchQueryBuilder, -} from '@hyperdx/common-utils/dist/queryParser'; import { act, renderHook } from '@testing-library/react'; import { @@ -636,222 +630,6 @@ describe('searchFilters', () => { }); }); - describe('round-trip: Lucene filters', () => { - it('round-trips Map filter with single included value', () => { - const filters = { - "LogAttributes['service.name']": { - included: new Set(['my-app']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - expect(query).toEqual([ - { - type: 'lucene', - condition: 'LogAttributes.service.name:"my-app"', - }, - ]); - // After round-trip, bracket-notation key becomes dot notation - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.service.name': { - included: new Set(['my-app']), - excluded: new Set(), - }, - }); - }); - - it('round-trips Map filter with multiple included values', () => { - const filters = { - "ResourceAttributes['env']": { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'ResourceAttributes.env': { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, - }); - }); - - it('round-trips Map filter with excluded values', () => { - const filters = { - "LogAttributes['level']": { - included: new Set(), - excluded: new Set(['debug', 'trace']), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.level': { - included: new Set(), - excluded: new Set(['debug', 'trace']), - }, - }); - }); - - it('round-trips values containing double quotes', () => { - const filters = { - "LogAttributes['msg']": { - included: new Set(['say "hello"']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.msg': { - included: new Set(['say "hello"']), - excluded: new Set(), - }, - }); - }); - - it('round-trips values containing backslashes', () => { - const filters = { - "LogAttributes['path']": { - included: new Set(['C:\\path\\to\\file']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - 'LogAttributes.path': { - included: new Set(['C:\\path\\to\\file']), - excluded: new Set(), - }, - }); - }); - - it('round-trips plain key filters', () => { - const filters = { - service: { - included: new Set(['app']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - service: { - included: new Set(['app']), - excluded: new Set(), - }, - }); - }); - - it('round-trips boolean filter values', () => { - const filters = { - isRootSpan: { - included: new Set([true]), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters).toEqual({ - isRootSpan: { - included: new Set([true]), - excluded: new Set(), - }, - }); - // Verify .has(true) works (boolean, not string) - expect(parsed.filters.isRootSpan.included.has(true)).toBe(true); - expect(parsed.filters.isRootSpan.included.has('true')).toBe(false); - }); - - it('round-trips empty string values', () => { - const filters = { - tag: { - included: new Set(['']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters.tag.included).toEqual(new Set([''])); - }); - - it('round-trips values with Lucene reserved characters', () => { - const filters = { - query: { - included: new Set(['(foo) AND [bar]']), - excluded: new Set(), - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters.query.included).toEqual( - new Set(['(foo) AND [bar]']), - ); - }); - - it('round-trips range filters through Lucene', () => { - const filters = { - duration: { - included: new Set(), - excluded: new Set(), - range: { min: 10, max: 500 }, - }, - }; - const query = filtersToQuery(filters); - const parsed = parseQuery(query); - expect(parsed.filters.duration.range).toEqual({ min: 10, max: 500 }); - }); - - it('merges mixed sql and lucene filters for the same key', () => { - const parsed = parseQuery([ - { type: 'sql', condition: `service IN ('app')` }, - { type: 'lucene', condition: 'service:"web"' }, - ]); - expect(parsed.filters.service.included).toEqual(new Set(['app', 'web'])); - }); - - it('parses existing lucene filter from URL/API', () => { - const parsed = parseQuery([ - { - type: 'lucene', - condition: - 'LogAttributes.service:"app-1" OR LogAttributes.service:"app-2"', - }, - ]); - expect(parsed.filters).toEqual({ - 'LogAttributes.service': { - included: new Set(['app-1', 'app-2']), - excluded: new Set(), - }, - }); - }); - - it('preserves unparseable Lucene filters as passthrough', () => { - const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); - const parsed = parseQuery([ - { type: 'lucene', condition: '((((' }, - { type: 'lucene', condition: 'service:"app"' }, - ]); - // Valid filter is parsed - expect(parsed.filters.service.included).toEqual(new Set(['app'])); - // Invalid filter is preserved in passthroughFilters - expect(parsed.passthroughFilters).toEqual([ - { type: 'lucene', condition: '((((' }, - ]); - warnSpy.mockRestore(); - }); - - it('returns empty array for valid but non-quoted Lucene (no passthrough)', () => { - const parsed = parseQuery([{ type: 'lucene', condition: 'service:foo' }]); - // Unquoted term — valid Lucene but not a filter, nothing extracted - expect(Object.keys(parsed.filters)).toHaveLength(0); - expect(parsed.passthroughFilters).toHaveLength(0); - }); - }); - describe('useSearchPageFilterState', () => { const onFilterChange = jest.fn(); @@ -869,15 +647,20 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'service:"app"' }, - { type: 'lucene', condition: 'level:"error"' }, + { + type: 'sql', + condition: "service IN ('app')", + }, + { + type: 'sql', + condition: "level IN ('error')", + }, ]); }); - it('updating filter query from legacy sql input', () => { + it('updating filter query', () => { const { result } = renderHook(() => useSearchPageFilterState({ - // Legacy sql filters are still parsed correctly searchQuery: [ { type: 'sql', condition: `service IN ('hdx-oss-dev-app')` }, { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, @@ -892,14 +675,10 @@ describe('searchFilters', () => { result.current.setFilterValue('another_facet', 'some_value'); }); - // Output is now lucene after round-trip expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'hyperdx_event_type:"span"' }, - { - type: 'lucene', - condition: '(level:"info" OR level:"ok")', - }, - { type: 'lucene', condition: 'another_facet:"some_value"' }, + { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, + { type: 'sql', condition: `level IN ('info', 'ok')` }, + { type: 'sql', condition: `another_facet IN ('some_value')` }, ]); }); @@ -920,12 +699,12 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'service:"hdx-oss-dev-app"' }, - { type: 'lucene', condition: 'hyperdx_event_type:"span"' }, + { type: 'sql', condition: `service IN ('hdx-oss-dev-app')` }, + { type: 'sql', condition: `hyperdx_event_type IN ('span')` }, ]); }); - it('correctly hydrates filter state from sql query', () => { + it('correctly hydrates filter state from query', () => { const { result } = renderHook(() => useSearchPageFilterState({ searchQuery: [ @@ -966,43 +745,11 @@ describe('searchFilters', () => { }); expect(onFilterChange).toHaveBeenCalledWith([ - { type: 'lucene', condition: 'service:"app"' }, - { type: 'lucene', condition: 'level:"info"' }, - ]); - }); - - it('migrates legacy SQL filters to Lucene on load', () => { - const onFilterChangeMigrate = jest.fn(); - renderHook(() => - useSearchPageFilterState({ - searchQuery: [ - { type: 'sql', condition: `service IN ('app')` }, - { type: 'sql', condition: `level IN ('info', 'ok')` }, - ], - onFilterChange: onFilterChangeMigrate, - }), - ); - - // Migration fires on mount, converting SQL to Lucene - expect(onFilterChangeMigrate).toHaveBeenCalledWith([ - { type: 'lucene', condition: 'service:"app"' }, - { type: 'lucene', condition: '(level:"info" OR level:"ok")' }, + { type: 'sql', condition: `service IN ('app')` }, + { type: 'sql', condition: `level IN ('info')` }, // Should only have the included value, no excluded values ]); }); - it('does not migrate when filters are already Lucene', () => { - const onFilterChangeNoMigrate = jest.fn(); - renderHook(() => - useSearchPageFilterState({ - searchQuery: [{ type: 'lucene', condition: 'service:"app"' }], - onFilterChange: onFilterChangeNoMigrate, - }), - ); - - // No migration needed — onFilterChange should not be called - expect(onFilterChangeNoMigrate).not.toHaveBeenCalled(); - }); - describe('retainFiltersByColumns', () => { it('returns [] and does not touch URL when filter state is empty', () => { const onFilterChangeLocal = jest.fn(); @@ -1078,8 +825,8 @@ describe('searchFilters', () => { const { result } = renderHook(() => useSearchPageFilterState({ searchQuery: [ - { type: 'lucene', condition: 'OldColumn:"x"' }, - { type: 'lucene', condition: 'AnotherGone:"y"' }, + { type: 'sql', condition: `OldColumn IN ('x')` }, + { type: 'sql', condition: `AnotherGone IN ('y')` }, ], onFilterChange: onFilterChangeLocal, }), @@ -1101,8 +848,8 @@ describe('searchFilters', () => { const { result } = renderHook(() => useSearchPageFilterState({ searchQuery: [ - { type: 'lucene', condition: 'ServiceName:"app"' }, - { type: 'lucene', condition: 'Body:"oops"' }, + { type: 'sql', condition: `ServiceName IN ('app')` }, + { type: 'sql', condition: `Body IN ('oops')` }, ], onFilterChange: onFilterChangeLocal, }), @@ -1117,157 +864,9 @@ describe('searchFilters', () => { expect(dropped).toEqual(['Body']); expect(onFilterChangeLocal).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'ServiceName:"app"' }, - ]); - }); - - it('preserves passthrough filters when reconciling', () => { - const onFilterChangeLocal = jest.fn(); - const warnSpy = jest - .spyOn(console, 'warn') - .mockImplementation(() => {}); - // Unparseable Lucene (mismatched parens) is preserved by parseQuery - // as a passthrough filter. - const { result } = renderHook(() => - useSearchPageFilterState({ - searchQuery: [ - { type: 'lucene', condition: 'ServiceName:"app"' }, - { type: 'lucene', condition: 'Body:"oops"' }, - { type: 'lucene', condition: '((((' }, - ], - onFilterChange: onFilterChangeLocal, - }), - ); - - let dropped: string[] = []; - act(() => { - dropped = result.current.retainFiltersByColumns( - new Set(['ServiceName']), - ); - }); - - expect(dropped).toEqual(['Body']); - // Passthrough filter rides along after the kept filters. - expect(onFilterChangeLocal).toHaveBeenLastCalledWith([ - { type: 'lucene', condition: 'ServiceName:"app"' }, - { type: 'lucene', condition: '((((' }, + { type: 'sql', condition: `ServiceName IN ('app')` }, ]); - warnSpy.mockRestore(); - }); - }); - }); - - describe('filters use direct_read optimization', () => { - const metadata = getMetadata( - new ClickhouseClient({ host: 'http://localhost:8123' }), - ); - - metadata.getColumn = jest.fn().mockImplementation(async ({ column }) => { - if (column === 'LogAttributes') { - return { name: 'LogAttributes', type: 'Map(String, String)' }; - } - return undefined; - }); - metadata.getMaterializedColumnsLookupTable = jest - .fn() - .mockResolvedValue(new Map()); - metadata.getColumns = jest.fn().mockResolvedValue([ - { - name: 'LogAttributes', - type: 'Map(String, String)', - default_type: '', - default_expression: '', - }, - { - name: 'LogAttributeItems', - type: 'Array(String)', - default_type: 'ALIAS', - default_expression: - "arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String)))", - }, - ]); - metadata.getSkipIndices = jest.fn().mockResolvedValue([ - { - name: 'idx_log_attr_items', - type: 'text', - typeFull: 'text(tokenizer=array)', - expression: 'LogAttributeItems', - granularity: 1, - }, - ]); - metadata.getSetting = jest.fn().mockResolvedValue('0'); - metadata.getServerVersion = jest - .fn() - .mockResolvedValue([26, 5, 0, 0] as const); - - const serializer = new CustomSchemaSQLSerializerV2({ - metadata, - databaseName: 'default', - tableName: 'otel_logs', - connectionId: 'test', - }); - - it('single included filter value uses has() direct_read', async () => { - const filters = filtersToQuery({ - "LogAttributes['service.name']": { - included: new Set(['my-app']), - excluded: new Set(), - }, - }); - - const condition = filters[0]; - expect(condition.type).toBe('lucene'); - const builder = new SearchQueryBuilder( - (condition as { condition: string }).condition, - serializer, - ); - const sql = await builder.build(); - expect(sql).toContain( - "has(`LogAttributeItems`, concat('service.name', '=', 'my-app'))", - ); - }); - - it('multiple included filter values each use has() direct_read', async () => { - const filters = filtersToQuery({ - "LogAttributes['env']": { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, }); - - const condition = filters[0]; - expect(condition.type).toBe('lucene'); - const builder = new SearchQueryBuilder( - (condition as { condition: string }).condition, - serializer, - ); - const sql = await builder.build(); - expect(sql).toContain( - "has(`LogAttributeItems`, concat('env', '=', 'prod'))", - ); - expect(sql).toContain( - "has(`LogAttributeItems`, concat('env', '=', 'staging'))", - ); - }); - - it('excluded filter value uses NOT has() direct_read', async () => { - const filters = filtersToQuery({ - "LogAttributes['level']": { - included: new Set(), - excluded: new Set(['debug']), - }, - }); - - const condition = filters[0]; - expect(condition.type).toBe('lucene'); - const builder = new SearchQueryBuilder( - (condition as { condition: string }).condition, - serializer, - ); - const sql = await builder.build(); - expect(sql).toContain( - "NOT has(`LogAttributeItems`, concat('level', '=', 'debug'))", - ); }); }); }); diff --git a/packages/app/src/components/DBRowJsonViewer.test.tsx b/packages/app/src/components/DBRowJsonViewer.test.tsx index e889a6df69..161a5abfe6 100644 --- a/packages/app/src/components/DBRowJsonViewer.test.tsx +++ b/packages/app/src/components/DBRowJsonViewer.test.tsx @@ -53,10 +53,10 @@ describe('DBRowJsonViewer', () => { }); // Helper to render component - const renderComponent = (data: any, jsonColumns?: string[]) => { + const renderComponent = (data: any) => { return renderWithMantine( - + , ); }; @@ -133,21 +133,6 @@ describe('DBRowJsonViewer', () => { ); }); - it('adds filters on JSON-typed column as plain dot notation (no toString, no backticks)', () => { - const jsonData = { - Metadata: { - endpoint: 'grpc://thingy:8888/Verify', - }, - }; - renderComponent(jsonData, ['Metadata']); - clickLineButton('endpoint', 'Add to Filters'); - - expect(mockOnPropertyAddClick).toHaveBeenCalledWith( - 'Metadata.endpoint', - 'grpc://thingy:8888/Verify', - ); - }); - it('toggles columns with correct path formatting', () => { renderComponent(logData); clickLineButton('field1', 'Column'); diff --git a/packages/app/src/components/DBRowJsonViewer.tsx b/packages/app/src/components/DBRowJsonViewer.tsx index e6aea39c5f..65abd56115 100644 --- a/packages/app/src/components/DBRowJsonViewer.tsx +++ b/packages/app/src/components/DBRowJsonViewer.tsx @@ -27,7 +27,6 @@ import { IconTextWrap, } from '@tabler/icons-react'; -import { cleanClickHouseExpression } from '@/components/DBSearchPageFilters/utils'; import HyperJson, { GetLineActions, LineAction } from '@/components/HyperJson'; import { mergePath } from '@/utils'; import { @@ -371,6 +370,8 @@ export function DBRowJsonViewer({ ({ keyPath, value, isInParsedJson, parsedJsonRootPath }) => { const actions: LineAction[] = []; const fieldPath = mergePath(keyPath, jsonColumns); + const isJsonColumn = + keyPath.length > 0 && jsonColumns?.includes(keyPath[0]); // Add to Filters action (strings only) // FIXME: TOTAL HACK To disallow adding timestamp to filters @@ -393,6 +394,7 @@ export function DBRowJsonViewer({ onClick: () => { let filterFieldPath = fieldPath; + // Handle parsed JSON from string columns using JSONExtractString if (isInParsedJson && parsedJsonRootPath) { const jsonQuery = buildJSONExtractQuery( keyPath, @@ -401,14 +403,23 @@ export function DBRowJsonViewer({ ); if (jsonQuery) { filterFieldPath = jsonQuery; + } else { + // We're at the root of the parsed JSON, treat as string + filterFieldPath = isJsonColumn + ? `toString(${fieldPath})` + : fieldPath; } + } else { + // Regular JSON column or non-JSON field + filterFieldPath = isJsonColumn + ? `toString(${fieldPath})` + : fieldPath; } - const cleanedPath = cleanClickHouseExpression(filterFieldPath); - onPropertyAddClick(cleanedPath, value); + onPropertyAddClick(filterFieldPath, value); notifications.show({ color: 'green', - message: `Added "${cleanedPath} = ${value}" to filters`, + message: `Added "${fieldPath} = ${value}" to filters`, }); }, }); diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 53779e8d1d..9d2c8c31e6 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -29,7 +29,6 @@ import { isJSDataTypeJSONStringifiable, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { BuilderChartConfigWithDateRange, @@ -1509,18 +1508,6 @@ function selectColumnMapWithoutAdditionalKeys( ); } -export function addMapAliasesToSelect(select: string): string { - const selects = splitAndTrimWithBracket(select); - for (let i = 0; i < selects.length; i++) { - const key = selects[i]; - const path = parseKeyPath(key); - if (path.length === 2) { - selects[i] = `${key} as "${path[0]}['${path[1]}']"`; - } - } - return selects.join(','); -} - export type DBRowTableVariant = 'default' | 'muted'; function DBSqlRowTableComponent({ @@ -1615,9 +1602,6 @@ function DBSqlRowTableComponent({ ...searchChartConfigDefaults(me?.team), ...config, }; - if (typeof base.select === 'string') { - base.select = addMapAliasesToSelect(base.select); - } if (orderByArray.length) { base.orderBy = orderByArray.map(o => { return { diff --git a/packages/app/src/components/EventTag.tsx b/packages/app/src/components/EventTag.tsx index d2ae465604..9cb6177fb0 100644 --- a/packages/app/src/components/EventTag.tsx +++ b/packages/app/src/components/EventTag.tsx @@ -5,7 +5,6 @@ import { SearchConditionLanguage } from '@hyperdx/common-utils/dist/types'; import { Button, Group, Popover, Stack, Text, Tooltip } from '@mantine/core'; import { IconCirclePlus, IconLink, IconSearch } from '@tabler/icons-react'; -import { cleanClickHouseExpression } from '@/components/DBSearchPageFilters/utils'; import { isLinkableUrl } from '@/utils/highlightedAttributes'; export default function EventTag({ @@ -106,10 +105,7 @@ export default function EventTag({ size="xs" rightSection={} onClick={() => { - onPropertyAddClick( - cleanClickHouseExpression(sqlExpression), - value, - ); + onPropertyAddClick(sqlExpression, value); setOpened(false); }} > diff --git a/packages/app/src/components/__tests__/DBRowTable.test.tsx b/packages/app/src/components/__tests__/DBRowTable.test.tsx index 03e8a80db6..ffb459b2f5 100644 --- a/packages/app/src/components/__tests__/DBRowTable.test.tsx +++ b/packages/app/src/components/__tests__/DBRowTable.test.tsx @@ -2,7 +2,6 @@ import { screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { - addMapAliasesToSelect, appendSelectWithAdditionalKeys, RawLogTable, } from '@/components/DBRowTable'; @@ -417,90 +416,3 @@ describe('appendSelectWithAdditionalKeys', () => { }); }); }); - -describe('addMapAliasesToSelect', () => { - it('leaves plain columns unchanged and collapses ", " to ","', () => { - expect(addMapAliasesToSelect('Timestamp, ServiceName, SeverityText')).toBe( - 'Timestamp,ServiceName,SeverityText', - ); - }); - - it('aliases a single-quoted bracket-form map key', () => { - expect(addMapAliasesToSelect("ResourceAttributes['service.name']")).toBe( - `ResourceAttributes['service.name'] as "ResourceAttributes['service.name']"`, - ); - }); - - it('aliases only the map keys in a mixed multi-column input', () => { - expect( - addMapAliasesToSelect( - "Timestamp, ResourceAttributes['service.name'], SeverityText", - ), - ).toBe( - `Timestamp,ResourceAttributes['service.name'] as "ResourceAttributes['service.name']",SeverityText`, - ); - }); - - it('preserves function-call expressions whose arguments contain commas', () => { - expect( - addMapAliasesToSelect( - "Timestamp, toDateTime(now64(3, 'UTC')), ServiceName", - ), - ).toBe(`Timestamp,toDateTime(now64(3, 'UTC')),ServiceName`); - }); - - it('aliases an arrayElement(Col, key) Map subscript', () => { - expect( - addMapAliasesToSelect("arrayElement(ResourceAttributes, 'service.name')"), - ).toBe( - `arrayElement(ResourceAttributes, 'service.name') as "ResourceAttributes['service.name']"`, - ); - }); - - it('passes already-aliased plain columns through unchanged', () => { - expect(addMapAliasesToSelect('Timestamp as ts, ServiceName as svc')).toBe( - 'Timestamp as ts,ServiceName as svc', - ); - }); - - it('does NOT re-alias a bracket-form map subscript that already carries an "as " clause', () => { - expect( - addMapAliasesToSelect("ResourceAttributes['service.name'] as svc"), - ).toBe("ResourceAttributes['service.name'] as svc"); - }); - - it('does NOT re-alias an already-aliased map subscript within a multi-column input', () => { - expect( - addMapAliasesToSelect( - "Timestamp, ResourceAttributes['service.name'] as svc, SeverityText", - ), - ).toBe("Timestamp,ResourceAttributes['service.name'] as svc,SeverityText"); - }); - - it('does NOT alias unbracketed dot-notation tokens', () => { - expect(addMapAliasesToSelect('user_data.name')).toBe('user_data.name'); - expect(addMapAliasesToSelect('user_data.address.city')).toBe( - 'user_data.address.city', - ); - }); - - it('does NOT alias multi-segment dotted native columns', () => { - expect(addMapAliasesToSelect('SpanAttributes.k8s.resource.name')).toBe( - 'SpanAttributes.k8s.resource.name', - ); - }); - - it('does NOT alias toString(x).y attribute-access expressions', () => { - expect(addMapAliasesToSelect('toString(SpanAttributes).k8s')).toBe( - 'toString(SpanAttributes).k8s', - ); - }); - - it('does NOT alias backtick-quoted dotted JSON paths', () => { - expect(addMapAliasesToSelect('`host`.`arch`')).toBe('`host`.`arch`'); - }); - - it('returns an empty string for empty input', () => { - expect(addMapAliasesToSelect('')).toBe(''); - }); -}); diff --git a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx index 2aa490a135..baa8904b3b 100644 --- a/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardFilters.test.tsx @@ -90,7 +90,7 @@ describe('useDashboardFilters', () => { ); }); - it('should generate lucene condition for multi-select values', () => { + it('should generate IN clause for multi-select values', () => { const { result } = renderHook(() => useDashboardFilters(mockFilters)); act(() => { @@ -105,7 +105,7 @@ describe('useDashboardFilters', () => { const query = result2.current.filterQueries[0]; const condition = 'condition' in query ? query.condition : ''; expect(condition).toEqual( - '(environment:"production" OR environment:"staging")', + "toString(environment) IN ('production', 'staging')", ); }); @@ -272,81 +272,4 @@ describe('useDashboardFilters', () => { expect(result.current.ignoredFilterExpressions).toEqual([]); }); }); - - it('should match bracket-notation expressions after Lucene round-trip', () => { - const bracketFilters: DashboardFilter[] = [ - { - id: 'filter-bracket', - type: 'QUERY_EXPRESSION', - name: 'Pod', - expression: "SpanAttributes['k8s.pod.name']", - source: 'traces', - }, - ]; - - const { result } = renderHook(() => useDashboardFilters(bracketFilters)); - - act(() => { - result.current.setFilterValue("SpanAttributes['k8s.pod.name']", [ - 'pod-1', - ]); - }); - - const { result: result2 } = renderHook(() => - useDashboardFilters(bracketFilters), - ); - - // The bracket-notation expression should still match after the Lucene - // round-trip converts the key to dot notation internally. - expect( - result2.current.filterValues["SpanAttributes['k8s.pod.name']"]?.included, - ).toEqual(new Set(['pod-1'])); - expect(result2.current.ignoredFilterExpressions).toEqual([]); - }); - - it('should match dot-notation URL key to bracket-notation expression', () => { - const bracketFilters: DashboardFilter[] = [ - { - id: 'filter-bracket', - type: 'QUERY_EXPRESSION', - name: 'Pod', - expression: "SpanAttributes['k8s.pod.name']", - source: 'traces', - }, - ]; - - // Pre-seed URL state with a dot-notation Lucene filter (as would be - // stored after a round-trip through filtersToQuery → parseQuery). - mockState = [ - { - type: 'lucene', - condition: 'SpanAttributes.k8s.pod.name:"pod-1"', - }, - ]; - - const { result } = renderHook(() => useDashboardFilters(bracketFilters)); - - expect( - result.current.filterValues["SpanAttributes['k8s.pod.name']"]?.included, - ).toEqual(new Set(['pod-1'])); - expect(result.current.ignoredFilterExpressions).toEqual([]); - }); - - it('should migrate legacy SQL filters to Lucene on load', () => { - // Pre-seed URL with old-format SQL filters - mockState = [{ type: 'sql', condition: "environment IN ('production')" }]; - - renderHook(() => useDashboardFilters(mockFilters)); - - // Migration should have called setFilterQueries with Lucene format - expect(mockSetState).toHaveBeenCalled(); - const lastCall = - mockSetState.mock.calls[mockSetState.mock.calls.length - 1]; - // setFilterQueries receives an updater or a value; resolve it - const result = - typeof lastCall[0] === 'function' ? lastCall[0](mockState) : lastCall[0]; - expect(result).toEqual([ - { type: 'lucene', condition: 'environment:"production"' }, - ]); - }); }); diff --git a/packages/app/src/hooks/useDashboardFilters.tsx b/packages/app/src/hooks/useDashboardFilters.tsx index 093cf84254..53211b0000 100644 --- a/packages/app/src/hooks/useDashboardFilters.tsx +++ b/packages/app/src/hooks/useDashboardFilters.tsx @@ -1,6 +1,5 @@ -import { useCallback, useEffect, useMemo, useRef } from 'react'; +import { useCallback, useMemo } from 'react'; import { useQueryState } from 'nuqs'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, filtersToQuery, @@ -22,19 +21,19 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { (expression: string, values: string[]) => { setFilterQueries(prev => { const { filters: filterValues } = parseQuery(prev ?? []); - // Normalize the expression to dot notation so it matches the keys - // returned by parseQuery (which converts bracket notation to dots). - const key = parseKeyPath(expression).join('.'); if (values.length === 0) { - delete filterValues[key]; + delete filterValues[expression]; } else { - filterValues[key] = { + filterValues[expression] = { included: new Set(values), excluded: new Set(), }; } - return filtersToQuery(filterValues); + return filtersToQuery( + filterValues, + { stringifyKeys: false }, // Don't wrap keys with toString(), to preserve exact key names in URL query parameters + ); }); }, [setFilterQueries], @@ -48,28 +47,16 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { } = useMemo(() => { const { filters: parsedFilters } = parseQuery(filterQueries ?? []); const valuesForExistingFilters: FilterState = {}; + const knownExpressions = new Set(filters.map(f => f.expression)); const ignored: string[] = []; - // Build a normalized lookup so bracket-notation expressions - // (e.g. SpanAttributes['k8s.pod.name']) match the dot-notation keys - // returned by parseLuceneFilter (e.g. SpanAttributes.k8s.pod.name). - const normalizeKey = (k: string) => parseKeyPath(k).join('.'); - const normalizedParsed = new Map( - Object.entries(parsedFilters).map(([k, v]) => [normalizeKey(k), v]), - ); - const knownNormalized = new Set( - filters.map(f => normalizeKey(f.expression)), - ); - for (const { expression } of filters) { - const norm = normalizeKey(expression); - const match = normalizedParsed.get(norm); - if (match) { - valuesForExistingFilters[expression] = match; + if (expression in parsedFilters) { + valuesForExistingFilters[expression] = parsedFilters[expression]; } } for (const key of Object.keys(parsedFilters)) { - if (!knownNormalized.has(normalizeKey(key))) { + if (!knownExpressions.has(key)) { ignored.push(key); } } @@ -88,7 +75,12 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { return { valuesForExistingFilters, - queriesForExistingFilters: filtersToQuery(valuesForExistingFilters), + queriesForExistingFilters: filtersToQuery( + valuesForExistingFilters, + // Wrap keys in `toString()` to support JSON/Dynamic-type columns. + // All keys can be stringified, since filter select values are stringified as well. + { stringifyKeys: true }, + ), ignoredExpressions: ignored, filtersByExpression, }; @@ -123,20 +115,6 @@ const useDashboardFilters = (filters: DashboardFilter[]) => { [valuesForExistingFilters, filtersByExpression], ); - // Migrate legacy SQL filters in the URL to Lucene on load - const hasMigratedRef = useRef(false); - useEffect(() => { - if (hasMigratedRef.current || !filterQueries) return; - const hasSqlFilters = filterQueries.some( - f => 'condition' in f && f.type === 'sql', - ); - if (hasSqlFilters) { - hasMigratedRef.current = true; - const { filters: parsed, passthroughFilters } = parseQuery(filterQueries); - setFilterQueries([...filtersToQuery(parsed), ...passthroughFilters]); - } - }, [filterQueries, setFilterQueries]); - return { filterValues: valuesForExistingFilters, filterQueries: queriesForExistingFilters, diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index 23ec18ba42..28d8fa50ae 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -1,11 +1,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import produce from 'immer'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; import { - coerceBooleanValue, type FilterState, filtersToQuery, - parseLuceneFilter, } from '@hyperdx/common-utils/dist/filters'; import type { Filter } from '@hyperdx/common-utils/dist/types'; @@ -300,8 +297,6 @@ export const parseQuery = ( q: Filter[], ): { filters: FilterState; - /** Filters that could not be parsed into FilterState (preserved for re-emission) */ - passthroughFilters: Filter[]; } => { const state = new Map< string, @@ -311,29 +306,7 @@ export const parseQuery = ( range?: { min: number; max: number }; } >(); - const passthroughFilters: Filter[] = []; for (const filter of q) { - if (filter.type === 'lucene') { - const parsedFields = parseLuceneFilter(filter.condition); - if (parsedFields === undefined) { - // Parse failure — preserve so it isn't silently lost - if (filter.condition.trim()) { - passthroughFilters.push(filter); - } - } else { - for (const { key, included, excluded, range } of parsedFields) { - if (!state.has(key)) { - state.set(key, { included: new Set(), excluded: new Set() }); - } - const sets = state.get(key)!; - for (const v of included) sets.included.add(v); - for (const v of excluded) sets.excluded.add(v); - if (range) sets.range = range; - } - } - continue; - } - if (filter.type !== 'sql') continue; // Check for BETWEEN condition (only when BETWEEN appears outside quotes) @@ -379,7 +352,7 @@ export const parseQuery = ( }); } } - return { filters: Object.fromEntries(state), passthroughFilters }; + return { filters: Object.fromEntries(state) }; }; export const useSearchPageFilterState = ({ @@ -394,7 +367,7 @@ export const useSearchPageFilterState = ({ return parseQuery(searchQuery); } catch (e) { console.error(e); - return { filters: {}, passthroughFilters: [] }; + return { filters: {} }; } }, [searchQuery]); @@ -408,32 +381,11 @@ export const useSearchPageFilterState = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [parsedQuery.filters]); - // Migrate legacy SQL filters to Lucene on load so URLs become canonical - const hasMigratedRef = useRef(false); - useEffect(() => { - if (hasMigratedRef.current) return; - const hasSqlFilters = searchQuery?.some( - f => 'condition' in f && f.type === 'sql', - ); - if (hasSqlFilters && Object.keys(parsedQuery.filters).length > 0) { - hasMigratedRef.current = true; - onFilterChange([ - ...filtersToQuery(parsedQuery.filters), - ...parsedQuery.passthroughFilters, - ]); - } - }, [searchQuery, parsedQuery, onFilterChange]); - const updateFilterQuery = useCallback( (newFilters: FilterState) => { - // Preserve unparseable filters so they aren't silently lost when - // the user toggles a sidebar facet. - onFilterChange([ - ...filtersToQuery(newFilters), - ...parsedQuery.passthroughFilters, - ]); + onFilterChange(filtersToQuery(newFilters)); }, - [onFilterChange, parsedQuery.passthroughFilters], + [onFilterChange], ); const setFilterValue = useCallback( @@ -442,21 +394,15 @@ export const useSearchPageFilterState = ({ value: string | boolean, action?: 'only' | 'exclude' | 'include', ) => { - // Normalize bracket notation to dot notation so the key matches - // what parseQuery returns after a Lucene round-trip. - const key = parseKeyPath(property).join('.'); - // Normalize "true"/"false" strings to booleans so the value matches - // what parseLuceneFilter returns after a Lucene round-trip. - const normalizedValue = coerceBooleanValue(value); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - if (!draft[key]) { - draft[key] = { included: new Set(), excluded: new Set() }; + if (!draft[property]) { + draft[property] = { included: new Set(), excluded: new Set() }; } if (action === 'only') { - draft[key] = { - included: new Set([normalizedValue]), + draft[property] = { + included: new Set([value]), excluded: new Set(), }; return; @@ -464,22 +410,22 @@ export const useSearchPageFilterState = ({ if (action === 'exclude') { // Remove from included if it was there - draft[key].included.delete(normalizedValue); + draft[property].included.delete(value); // Toggle in excluded - if (draft[key].excluded.has(normalizedValue)) { - draft[key].excluded.delete(normalizedValue); + if (draft[property].excluded.has(value)) { + draft[property].excluded.delete(value); } else { - draft[key].excluded.add(normalizedValue); + draft[property].excluded.add(value); } return; } // Regular toggle (include) - draft[key].excluded.delete(normalizedValue); - if (draft[key].included.has(normalizedValue)) { - draft[key].included.delete(normalizedValue); + draft[property].excluded.delete(value); + if (draft[property].included.has(value)) { + draft[property].included.delete(value); } else { - draft[key].included.add(normalizedValue); + draft[property].included.add(value); } }); updateFilterQuery(newFilters); @@ -491,13 +437,12 @@ export const useSearchPageFilterState = ({ const setFilterRange = useCallback( (property: string, range: { min: number; max: number }) => { - const key = parseKeyPath(property).join('.'); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - if (!draft[key]) { - draft[key] = { included: new Set(), excluded: new Set() }; + if (!draft[property]) { + draft[property] = { included: new Set(), excluded: new Set() }; } - draft[key].range = range; + draft[property].range = range; }); updateFilterQuery(newFilters); return newFilters; @@ -508,10 +453,9 @@ export const useSearchPageFilterState = ({ const clearFilter = useCallback( (property: string) => { - const key = parseKeyPath(property).join('.'); setFilters(prevFilters => { const newFilters = produce(prevFilters, draft => { - delete draft[key]; + delete draft[property]; }); updateFilterQuery(newFilters); return newFilters; diff --git a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts index d4f127b2aa..88c4d43d0e 100644 --- a/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts +++ b/packages/app/tests/e2e/features/dashboard-table-linking.spec.ts @@ -205,10 +205,10 @@ test.describe( const url = new URL(page.url()); expect(url.searchParams.get('source')).toBe(logsSourceId); expect(url.searchParams.get('isLive')).toBe('false'); - // Default row click encodes the group-by filter as a Lucene condition - // in the JSON-encoded `filters` param. + // Default row click encodes the group-by filter as a SQL IN clause in + // the JSON-encoded `filters` param. const filters = url.searchParams.get('filters') ?? '[]'; - expect(filters).toContain(`ServiceName:\\"${serviceName}\\"`); + expect(filters).toContain(`ServiceName IN ('${serviceName}')`); const from = Number(url.searchParams.get('from')); const to = Number(url.searchParams.get('to')); expect(from).toBeGreaterThan(0); @@ -563,8 +563,8 @@ test.describe( const filters = JSON.parse(decodeURIComponent(filtersRaw!)); expect(filters).toEqual([ { - type: 'lucene', - condition: `ServiceName:"${serviceName}"`, + type: 'sql', + condition: `ServiceName IN ('${serviceName}')`, }, ]); }); @@ -649,8 +649,8 @@ test.describe( const filters = JSON.parse(decodeURIComponent(filtersRaw!)); expect(filters).toEqual([ { - type: 'lucene', - condition: `ServiceName:"${serviceName}"`, + type: 'sql', + condition: `ServiceName IN ('${serviceName}')`, }, ]); }); diff --git a/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts b/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts deleted file mode 100644 index 59cc207949..0000000000 --- a/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { filtersToQuery, parseLuceneFilter } from '@/filters'; -import { parse } from '@/queryParser'; - -describe('Colon-in-MAP-KEY scenario (LogAttributes with key containing colon)', () => { - it('parses Lucene when the colon in a field name is properly escaped', () => { - expect(() => - parse(String.raw`LogAttributes.foo\:bar:"value"`), - ).not.toThrow(); - }); - - it('filtersToQuery emits parseable Lucene for bracket-form Map key containing colon', () => { - const filters = { - "LogAttributes['foo:bar']": { - included: new Set(['value1']), - excluded: new Set(), - }, - }; - const result = filtersToQuery(filters); - const condition = (result[0] as { condition: string }).condition; - expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`); - expect(() => parse(condition)).not.toThrow(); - }); - - it('filtersToQuery emits parseable Lucene for dot-form Map key containing colon', () => { - const filters = { - 'LogAttributes.foo:bar': { - included: new Set(['value1']), - excluded: new Set(), - }, - }; - const result = filtersToQuery(filters); - const condition = (result[0] as { condition: string }).condition; - expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`); - expect(() => parse(condition)).not.toThrow(); - }); - - it('round-trips Map keys containing colons through filtersToQuery + parseLuceneFilter', () => { - const filters = { - "LogAttributes['foo:bar']": { - included: new Set(['value1']), - excluded: new Set(), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - const parsed = parseLuceneFilter(cond); - expect(parsed).toEqual([ - { - key: 'LogAttributes.foo:bar', - included: ['value1'], - excluded: [], - }, - ]); - }); - - it('round-trips Map keys with colons and excluded values', () => { - const filters = { - 'LogAttributes.foo:bar': { - included: new Set(), - excluded: new Set(['unwanted:value']), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - expect(cond).toBe(String.raw`-LogAttributes.foo\:bar:"unwanted:value"`); - expect(parseLuceneFilter(cond)).toEqual([ - { - key: 'LogAttributes.foo:bar', - included: [], - excluded: ['unwanted:value'], - }, - ]); - }); - - it('round-trips Map keys with multiple colons in the key segment', () => { - const filters = { - "LogAttributes['k8s.io:annotation:foo']": { - included: new Set(['v1']), - excluded: new Set(), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - expect(() => parse(cond)).not.toThrow(); - expect(parseLuceneFilter(cond)).toEqual([ - { - key: 'LogAttributes.k8s.io:annotation:foo', - included: ['v1'], - excluded: [], - }, - ]); - }); - - it('round-trips Map key colon-in-key + colon-in-value simultaneously', () => { - const filters = { - "LogAttributes['foo:bar']": { - included: new Set(['https://example.com:8080']), - excluded: new Set(), - }, - }; - const [emitted] = filtersToQuery(filters); - const cond = (emitted as { condition: string }).condition; - expect(() => parse(cond)).not.toThrow(); - expect(parseLuceneFilter(cond)).toEqual([ - { - key: 'LogAttributes.foo:bar', - included: ['https://example.com:8080'], - excluded: [], - }, - ]); - }); -}); diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts index 9ba03e7c30..20253b4c8b 100644 --- a/packages/common-utils/src/__tests__/filters.test.ts +++ b/packages/common-utils/src/__tests__/filters.test.ts @@ -1,38 +1,36 @@ -import { type FilterState, filtersToQuery, parseLuceneFilter } from '@/filters'; -import type { Filter } from '@/types'; - -/** Extract condition string from a Filter (filtersToQuery never emits sql_ast) */ -const getCondition = (f: Filter): string => - 'condition' in f ? f.condition : ''; +import { filtersToQuery } from '@/filters'; describe('filters', () => { describe('filtersToQuery', () => { - it('should return empty array when no filters', () => { - expect(filtersToQuery({})).toEqual([]); + it('should return empty string when no filters', () => { + const filters = {}; + expect(filtersToQuery(filters)).toEqual([]); }); - it('should emit lucene for a single included value', () => { + it('should return query for one filter', () => { const filters = { a: { included: new Set(['b']), excluded: new Set() }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'a:"b"' }, + { type: 'sql', condition: "a IN ('b')" }, ]); }); - it('should emit lucene with parens for multiple included values', () => { + it('should return query for multiple filters', () => { const filters = { + a: { included: new Set(['b']), excluded: new Set() }, c: { included: new Set(['d', 'x']), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: '(c:"d" OR c:"x")' }, + { type: 'sql', condition: "a IN ('b')" }, + { type: 'sql', condition: "c IN ('d', 'x')" }, ]); }); - it('should handle excluded values without parens', () => { + it('should handle excluded values', () => { const filters = { a: { included: new Set(['b']), @@ -40,458 +38,129 @@ describe('filters', () => { }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'a:"b"' }, - { type: 'lucene', condition: '-a:"c"' }, + { type: 'sql', condition: "a IN ('b')" }, + { type: 'sql', condition: "a NOT IN ('c')" }, ]); }); - it('should handle multiple excluded values without parens', () => { + it('should wrap keys with toString() when specified', () => { const filters = { - a: { - included: new Set(), - excluded: new Set([true, false]), + 'json.key': { + included: new Set(['value']), + excluded: new Set(['other value']), }, }; - expect(filtersToQuery(filters)).toEqual([ - { - type: 'lucene', - condition: '-a:"true" AND -a:"false"', - }, + expect(filtersToQuery(filters, { stringifyKeys: true })).toEqual([ + { type: 'sql', condition: "toString(json.key) IN ('value')" }, + { type: 'sql', condition: "toString(json.key) NOT IN ('other value')" }, ]); }); - it('should handle boolean filter values', () => { + it('should should handle boolean filter values', () => { const filters = { isRootSpan: { included: new Set([true]), excluded: new Set([]), }, - }; - expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'isRootSpan:"true"' }, - ]); - }); - - it('should escape double quotes in values', () => { - const filters = { - message: { - included: new Set(['say "hello"']), - excluded: new Set(), + another_column: { + included: new Set([]), + excluded: new Set([true, false]), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'message:"say \\"hello\\""' }, + { type: 'sql', condition: 'isRootSpan IN (true)' }, + { type: 'sql', condition: 'another_column NOT IN (true, false)' }, ]); }); - it('should pass through single quotes in values', () => { + it('should escape single quotes in filter values', () => { const filters = { message: { - included: new Set(["it's a test"]), + included: new Set(["my 'filter' key"]), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'message:"it\'s a test"' }, + { + type: 'sql', + condition: "message IN ('my ''filter'' key')", + }, ]); }); - it('should escape backslashes in values', () => { + it('should escape single quotes in excluded filter values', () => { const filters = { - FilePath: { - included: new Set(['C:\\path\\to\\file']), - excluded: new Set(), + message: { + included: new Set(), + excluded: new Set(["it's a test"]), }, }; expect(filtersToQuery(filters)).toEqual([ { - type: 'lucene', - condition: 'FilePath:"C:\\\\path\\\\to\\\\file"', + type: 'sql', + condition: "message NOT IN ('it''s a test')", }, ]); }); - it('should convert Map bracket-notation keys to dot notation', () => { + it('should escape single quotes with stringifyKeys', () => { const filters = { - "LogAttributes['service.name']": { - included: new Set(['my-app']), + 'json.key': { + included: new Set(["value with 'quotes'"]), excluded: new Set(), }, }; - expect(filtersToQuery(filters)).toEqual([ + expect(filtersToQuery(filters, { stringifyKeys: true })).toEqual([ { - type: 'lucene', - condition: 'LogAttributes.service.name:"my-app"', + type: 'sql', + condition: "toString(json.key) IN ('value with ''quotes''')", }, ]); }); - it('should handle Map bracket-notation with multiple values', () => { + it('should escape backslashes in filter values', () => { const filters = { - "LogAttributes['env']": { - included: new Set(['prod', 'staging']), + FilePath: { + included: new Set(['C:\\path\\to\\file']), excluded: new Set(), }, }; expect(filtersToQuery(filters)).toEqual([ { - type: 'lucene', - condition: - '(LogAttributes.env:"prod" OR LogAttributes.env:"staging")', + type: 'sql', + condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')", }, ]); }); - it('should handle Map bracket-notation excluded values', () => { + it('should escape backslashes in excluded filter values', () => { const filters = { - "LogAttributes['level']": { + FilePath: { included: new Set(), - excluded: new Set(['debug']), - }, - }; - expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: '-LogAttributes.level:"debug"' }, - ]); - }); - - it('should handle Map bracket-notation with both included and excluded', () => { - const filters = { - "ResourceAttributes['host']": { - included: new Set(['web-1']), - excluded: new Set(['web-2']), + excluded: new Set(['C:\\path\\to\\file']), }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'ResourceAttributes.host:"web-1"' }, - { type: 'lucene', condition: '-ResourceAttributes.host:"web-2"' }, - ]); - }); - - it('should emit lucene range syntax', () => { - const filters = { - duration: { - included: new Set(), - excluded: new Set(), - range: { min: 10, max: 500 }, + { + type: 'sql', + condition: "FilePath NOT IN ('C:\\\\path\\\\to\\\\file')", }, - }; - expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'duration:[10 TO 500]' }, ]); }); - it('should emit lucene range with dot notation for Map bracket-notation keys', () => { + it('should escape backslashes before single quotes so quotes stay escaped', () => { const filters = { - "LogAttributes['latency']": { - included: new Set(), + message: { + included: new Set(["a\\'b"]), excluded: new Set(), - range: { min: 0, max: 100 }, }, }; expect(filtersToQuery(filters)).toEqual([ - { type: 'lucene', condition: 'LogAttributes.latency:[0 TO 100]' }, - ]); - }); - }); - - describe('parseLuceneFilter', () => { - it('should return empty array for empty string', () => { - expect(parseLuceneFilter('')).toEqual([]); - }); - - it('should return empty array for unquoted terms', () => { - expect(parseLuceneFilter('service:foo')).toEqual([]); - }); - - it('should parse a single included term', () => { - expect(parseLuceneFilter('service:"app"')).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - ]); - }); - - it('should parse a single excluded term', () => { - expect(parseLuceneFilter('-service:"app"')).toEqual([ - { key: 'service', included: [], excluded: ['app'] }, - ]); - }); - - it('should parse multiple included values with OR', () => { - expect(parseLuceneFilter('(service:"app" OR service:"web")')).toEqual([ - { key: 'service', included: ['app', 'web'], excluded: [] }, - ]); - }); - - it('should parse multiple excluded values with AND', () => { - expect( - parseLuceneFilter('-service:"debug" AND -service:"trace"'), - ).toEqual([ - { key: 'service', included: [], excluded: ['debug', 'trace'] }, - ]); - }); - - it('should parse mixed included and excluded for the same field', () => { - expect( - parseLuceneFilter('-service:"bingo" (service:"foo" OR service:"bar")'), - ).toEqual([ - { key: 'service', included: ['foo', 'bar'], excluded: ['bingo'] }, - ]); - }); - - it('should parse multiple distinct fields', () => { - expect(parseLuceneFilter('service:"app" level:"info"')).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - { key: 'level', included: ['info'], excluded: [] }, - ]); - }); - - it('should parse multiple fields with mixed negation', () => { - expect( - parseLuceneFilter('service:"app" -level:"debug" level:"info"'), - ).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - { key: 'level', included: ['info'], excluded: ['debug'] }, - ]); - }); - - it('should parse dot-notation fields (Map columns)', () => { - expect(parseLuceneFilter('LogAttributes.env:"prod"')).toEqual([ - { key: 'LogAttributes.env', included: ['prod'], excluded: [] }, - ]); - }); - - it('should handle values with escaped double quotes', () => { - expect(parseLuceneFilter('message:"say \\"hello\\""')).toEqual([ - { key: 'message', included: ['say "hello"'], excluded: [] }, - ]); - }); - - it('should handle values with backslashes', () => { - expect(parseLuceneFilter('path:"C:\\\\dir\\\\file"')).toEqual([ - { key: 'path', included: ['C:\\dir\\file'], excluded: [] }, - ]); - }); - - it('should coerce "true"/"false" back to booleans', () => { - expect(parseLuceneFilter('isRootSpan:"true"')).toEqual([ - { key: 'isRootSpan', included: [true], excluded: [] }, - ]); - expect(parseLuceneFilter('-col:"true" AND -col:"false"')).toEqual([ - { key: 'col', included: [], excluded: [true, false] }, - ]); - }); - - it('should parse range terms', () => { - expect(parseLuceneFilter('duration:[10 TO 500]')).toEqual([ - { - key: 'duration', - included: [], - excluded: [], - range: { min: 10, max: 500 }, - }, - ]); - }); - - it('should parse range with dot-notation field', () => { - expect(parseLuceneFilter('LogAttributes.latency:[0 TO 100]')).toEqual([ { - key: 'LogAttributes.latency', - included: [], - excluded: [], - range: { min: 0, max: 100 }, + type: 'sql', + condition: "message IN ('a\\\\''b')", }, ]); }); - - it('should return undefined for invalid lucene syntax', () => { - expect(parseLuceneFilter('((((')).toBeUndefined(); - }); - }); - - describe('filtersToQuery -> parseLuceneFilter round-trip', () => { - it('round-trips a single included value', () => { - const state: FilterState = { - service: { - included: new Set(['app']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed).toEqual([ - { key: 'service', included: ['app'], excluded: [] }, - ]); - }); - - it('round-trips multiple included values', () => { - const state: FilterState = { - env: { - included: new Set(['prod', 'staging']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].key).toBe('env'); - expect(parsed?.[0].included).toEqual( - expect.arrayContaining(['prod', 'staging']), - ); - }); - - it('round-trips excluded values', () => { - const state: FilterState = { - level: { - included: new Set(), - excluded: new Set(['debug', 'trace']), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].key).toBe('level'); - expect(parsed?.[0].excluded).toEqual( - expect.arrayContaining(['debug', 'trace']), - ); - }); - - it('round-trips boolean values preserving type', () => { - const state: FilterState = { - isRootSpan: { - included: new Set([true]), - excluded: new Set([false]), - }, - }; - const filters = filtersToQuery(state); - const includedParsed = parseLuceneFilter(getCondition(filters[0])); - const excludedParsed = parseLuceneFilter(getCondition(filters[1])); - expect(includedParsed?.[0].included).toEqual([true]); - expect(excludedParsed?.[0].excluded).toEqual([false]); - }); - - it('round-trips the literal string "true" without boolean coercion', () => { - // The string "true" is indistinguishable from boolean true after - // Lucene round-trip due to coerceBooleanValue. This test documents - // that known limitation. - const state: FilterState = { - status: { - included: new Set(['true']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - // Coerced to boolean — this is the expected (lossy) behavior - expect(parsed?.[0].included).toEqual([true]); - }); - - it('round-trips values with double quotes', () => { - const state: FilterState = { - msg: { - included: new Set(['say "hello"']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['say "hello"']); - }); - - it('round-trips values with backslashes', () => { - const state: FilterState = { - path: { - included: new Set(['C:\\dir\\file']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['C:\\dir\\file']); - }); - - it('round-trips empty string values', () => { - const state: FilterState = { - tag: { - included: new Set(['']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['']); - }); - - it('round-trips values containing Lucene reserved characters', () => { - const state: FilterState = { - query: { - included: new Set(['(foo) AND [bar]']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].included).toEqual(['(foo) AND [bar]']); - }); - - it('round-trips range filters', () => { - const state: FilterState = { - duration: { - included: new Set(), - excluded: new Set(), - range: { min: 10, max: 500 }, - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - expect(parsed?.[0].range).toEqual({ min: 10, max: 500 }); - }); - - it('round-trips Map bracket-notation key', () => { - const state: FilterState = { - "LogAttributes['service.name']": { - included: new Set(['my-app']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const parsed = parseLuceneFilter(getCondition(filters[0])); - // Bracket notation normalizes to dot notation - expect(parsed?.[0].key).toBe('LogAttributes.service.name'); - expect(parsed?.[0].included).toEqual(['my-app']); - }); - - it('round-trips a JSON column dot-path with colons and slashes in value', () => { - const state: FilterState = { - 'Metadata.endpoint': { - included: new Set(['grpc://thingy:8888/Verify']), - excluded: new Set(), - }, - }; - const filters = filtersToQuery(state); - const condition = getCondition(filters[0]); - expect(condition).not.toMatch(/^toString\(/); - expect(condition).not.toMatch(/`/); - const parsed = parseLuceneFilter(condition); - expect(parsed?.[0].key).toBe('Metadata.endpoint'); - expect(parsed?.[0].included).toEqual(['grpc://thingy:8888/Verify']); - }); - - it('round-trips mixed included and excluded for same field', () => { - const state: FilterState = { - env: { - included: new Set(['prod', 'staging']), - excluded: new Set(['dev']), - }, - }; - const filters = filtersToQuery(state); - // filtersToQuery emits separate Filter entries for included/excluded - expect(filters).toHaveLength(2); - // Parse both and merge - const allParsed = filters.flatMap( - f => parseLuceneFilter(getCondition(f)) ?? [], - ); - const envEntries = allParsed.filter(p => p.key === 'env'); - const included = envEntries.flatMap(e => e.included); - const excluded = envEntries.flatMap(e => e.excluded); - expect(included).toEqual(expect.arrayContaining(['prod', 'staging'])); - expect(excluded).toEqual(['dev']); - }); }); }); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 083c9005a2..bbd40781ef 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1471,28 +1471,4 @@ describe('parseKeyPath', () => { "ResourceAttributes['service.name", ]); }); - - it('parses single-quoted arrayElement function-call form', () => { - expect( - parseKeyPath("arrayElement(ResourceAttributes, 'service.name')"), - ).toEqual(['ResourceAttributes', 'service.name']); - }); - - it('parses arrayElement keys with dots in the map key', () => { - expect( - parseKeyPath("arrayElement(SpanAttributes, 'http.request.method')"), - ).toEqual(['SpanAttributes', 'http.request.method']); - }); - - it('tolerates whitespace inside arrayElement', () => { - expect( - parseKeyPath("arrayElement( ResourceAttributes , 'service.name' )"), - ).toEqual(['ResourceAttributes', 'service.name']); - }); - - it('does not parse arrayElement with a missing closing paren', () => { - expect( - parseKeyPath("arrayElement(ResourceAttributes, 'service.name'"), - ).toEqual(["arrayElement(ResourceAttributes, 'service.name'"]); - }); }); diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index 343b9b9f2e..a2afc2118f 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -494,19 +494,6 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { expect(actualSql).toBe(expectedSql); }); - it('"Add to Filters" repro: JSON column path + value with colons and slashes emits valid SQL', async () => { - const lucene = - 'ResourceAttributesJSON.endpoint:"grpc://auth-svc:50051/Verify"'; - const builder = new SearchQueryBuilder(lucene, serializer); - const actualSql = await builder.build(); - expect(actualSql).toContain( - 'toString(`ResourceAttributesJSON`.`endpoint`)', - ); - expect(actualSql).toContain('grpc://auth-svc:50051/Verify'); - expect(actualSql).not.toContain('http_COLON_'); - expect(actualSql).not.toContain('HDX_COLON'); - }); - describe('CustomSchemaSQLSerializerV2: implicit column falls back to bodyExpression', () => { // Symmetric counterpart to the one-way fallback in // `getEventBody` (packages/app/src/source.ts): a log source admin who diff --git a/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts b/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts index ee7b64b7f5..067e3dbbce 100644 --- a/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts +++ b/packages/common-utils/src/core/__tests__/linkUrlBuilder.test.ts @@ -244,7 +244,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"MyService"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('MyService')" }]); } }); @@ -278,9 +278,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([ - { type: 'lucene', condition: '(ServiceName:"A" OR ServiceName:"B")' }, - ]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('A', 'B')" }]); } }); @@ -315,8 +313,8 @@ describe('renderOnClickSearch', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'ServiceName:"MyService"' }, - { type: 'lucene', condition: 'SeverityText:"error"' }, + { type: 'sql', condition: "ServiceName IN ('MyService')" }, + { type: 'sql', condition: "SeverityText IN ('error')" }, ]); } }); @@ -346,7 +344,7 @@ describe('renderOnClickSearch', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"O\'Malley"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('O''Malley')" }]); } }); @@ -376,7 +374,7 @@ describe('renderOnClickSearch', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'FilePath:"C:\\\\path\\\\to\\\\file"' }, + { type: 'sql', condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')" }, ]); } }); @@ -408,8 +406,8 @@ describe('renderOnClickSearch', () => { JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ { - type: 'lucene', - condition: 'SpanAttributes.url:"/users%2F42"', + type: 'sql', + condition: "SpanAttributes['url'] IN ('/users%2F42')", }, ]); } @@ -678,7 +676,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"MyService"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('MyService')" }]); } }); @@ -712,9 +710,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([ - { type: 'lucene', condition: '(ServiceName:"A" OR ServiceName:"B")' }, - ]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('A', 'B')" }]); } }); @@ -749,8 +745,8 @@ describe('renderOnClickDashboard', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'ServiceName:"MyService"' }, - { type: 'lucene', condition: 'SeverityText:"error"' }, + { type: 'sql', condition: "ServiceName IN ('MyService')" }, + { type: 'sql', condition: "SeverityText IN ('error')" }, ]); } }); @@ -780,7 +776,7 @@ describe('renderOnClickDashboard', () => { const params = new URLSearchParams(result.url.split('?')[1]); expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), - ).toEqual([{ type: 'lucene', condition: 'ServiceName:"O\'Malley"' }]); + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('O''Malley')" }]); } }); @@ -810,7 +806,7 @@ describe('renderOnClickDashboard', () => { expect( JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ - { type: 'lucene', condition: 'FilePath:"C:\\\\path\\\\to\\\\file"' }, + { type: 'sql', condition: "FilePath IN ('C:\\\\path\\\\to\\\\file')" }, ]); } }); @@ -842,8 +838,8 @@ describe('renderOnClickDashboard', () => { JSON.parse(decodeURIComponent(params.get('filters') ?? '')), ).toEqual([ { - type: 'lucene', - condition: 'SpanAttributes.url:"/users%2F42"', + type: 'sql', + condition: "SpanAttributes['url'] IN ('/users%2F42')", }, ]); } diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 59688cef15..a80feb1fad 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -1768,17 +1768,9 @@ export type Field = { }; /** - * Parses a Map-access key string into a path array. - * - * Handles both ClickHouse Map access forms: - * - Bracket form (canonical input): - * `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` - * `ResourceAttributes["service.name"]` → `['ResourceAttributes', 'service.name']` - * - Function-call form (what ClickHouse renders in result column names): - * `arrayElement(ResourceAttributes, 'service.name')` → `['ResourceAttributes', 'service.name']` - * - * Native columns return a single-element path: - * `ServiceName` → `['ServiceName']` + * Parses a bracket-notation key string into a path array. + * e.g. `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` + * `ServiceName` → `['ServiceName']` */ export function parseKeyPath(key: string): string[] { const singleIdx = key.indexOf("['"); @@ -1789,15 +1781,6 @@ export function parseKeyPath(key: string): string[] { if (doubleIdx !== -1 && key.endsWith('"]')) { return [key.slice(0, doubleIdx), key.slice(doubleIdx + 2, -2)]; } - // ClickHouse Map access function-call form. ClickHouse renders Map - // subscript expressions (e.g. `SpanAttributes['http.route']`) as - // `arrayElement(SpanAttributes, 'http.route')` in result column names. - const ARRAY_ELEMENT_PREFIX = 'arrayElement('; - if (key.startsWith(ARRAY_ELEMENT_PREFIX) && key.endsWith(')')) { - const inner = key.slice(ARRAY_ELEMENT_PREFIX.length, -1); - const parts = inner.split(',').map(e => e.trim()); - return [parts[0], parts[1].replaceAll("'", '')]; - } return [key]; } diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index e5ded218a4..9a7ac150b1 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -1,14 +1,3 @@ -import lucene from '@hyperdx/lucene'; - -import { parseKeyPath } from '@/core/metadata'; -import { - decodeSpecialTokens, - isBinaryAST, - isLeftOnlyAST, - isNodeRangedTerm, - isNodeTerm, - parse, -} from '@/queryParser'; import { Filter } from '@/types'; export type FilterState = { @@ -19,25 +8,14 @@ export type FilterState = { }; }; -/** Escape a value for use inside a Lucene quoted term ("...") */ -const escapeLuceneQuotedTerm = (s: string) => { - return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +const escapeString = (s: string) => { + return s.replace(/\\/g, '\\\\').replace(/'/g, "''"); }; -/** - * Escape backslashes and colons so the field name survives Lucene parsing. - * Map sub-keys can legitimately contain `:` (e.g. `LogAttributes['foo:bar']` - * normalizes to `LogAttributes.foo:bar` via parseKeyPath().join('.')), and - * `:` is the Lucene field/value separator. Backslashes are escaped first so - * the inserted `\:` survives `encodeSpecialTokens`' `\\` → backslash-literal - * substitution; the encoder's matching `\:` → HDX_COLON rule then makes the - * colon opaque to the parser, and `decodeSpecialTokens` restores the - * original key on the consumer side. - */ -const escapeLuceneFieldName = (key: string): string => - key.replace(/\\/g, '\\\\').replace(/:/g, '\\:'); - -export const filtersToQuery = (filters: FilterState): Filter[] => { +export const filtersToQuery = ( + filters: FilterState, + { stringifyKeys = false }: { stringifyKeys?: boolean } = {}, +): Filter[] => { return Object.entries(filters) .filter( ([_, values]) => @@ -47,137 +25,30 @@ export const filtersToQuery = (filters: FilterState): Filter[] => { ) .flatMap(([key, values]) => { const conditions: Filter[] = []; - const luceneField = escapeLuceneFieldName(parseKeyPath(key).join('.')); + const actualKey = stringifyKeys ? `toString(${key})` : key; if (values.included.size > 0) { - const terms = Array.from(values.included).map( - v => `${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`, - ); conditions.push({ - type: 'lucene' as const, - condition: terms.length > 1 ? `(${terms.join(' OR ')})` : terms[0], + type: 'sql' as const, + condition: `${actualKey} IN (${Array.from(values.included) + .map(v => (typeof v === 'string' ? `'${escapeString(v)}'` : v)) + .join(', ')})`, }); } if (values.excluded.size > 0) { conditions.push({ - type: 'lucene' as const, - condition: Array.from(values.excluded) - .map(v => `-${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`) - .join(' AND '), + type: 'sql' as const, + condition: `${actualKey} NOT IN (${Array.from(values.excluded) + .map(v => (typeof v === 'string' ? `'${escapeString(v)}'` : v)) + .join(', ')})`, }); } - if (values.range != null) { - // Lucene range syntax: field:[min TO max] conditions.push({ - type: 'lucene' as const, - condition: `${luceneField}:[${values.range.min} TO ${values.range.max}]`, + type: 'sql' as const, + condition: `${actualKey} BETWEEN ${values.range.min} AND ${values.range.max}`, }); } return conditions; }); }; - -type CollectedTerm = { field: string; value: string; negated: boolean }; -type CollectedRange = { field: string; min: number; max: number }; - -/** - * Collect quoted terms and range terms from a Lucene AST. - */ -function collectFromAst( - ast: lucene.AST | lucene.Node, - terms: CollectedTerm[], - ranges: CollectedRange[], -): void { - if (isNodeTerm(ast)) { - if (!ast.quoted) return; - const negated = ast.field.startsWith('-'); - const field = decodeSpecialTokens(negated ? ast.field.slice(1) : ast.field); - terms.push({ field, value: decodeSpecialTokens(ast.term), negated }); - } else if (isNodeRangedTerm(ast)) { - const field = decodeSpecialTokens(ast.field); - const min = parseFloat(ast.term_min); - const max = parseFloat(ast.term_max); - if (!isNaN(min) && !isNaN(max)) { - ranges.push({ field, min, max }); - } - } else if (isBinaryAST(ast)) { - collectFromAst(ast.left, terms, ranges); - collectFromAst(ast.right, terms, ranges); - } else if (isLeftOnlyAST(ast)) { - collectFromAst(ast.left, terms, ranges); - } -} - -/** Coerce "true"/"false" strings back to booleans, pass through otherwise */ -export function coerceBooleanValue(v: string | boolean): string | boolean { - if (typeof v === 'boolean') return v; - if (v === 'true') return true; - if (v === 'false') return false; - return v; -} - -export type ParsedLuceneFilter = { - key: string; - included: (string | boolean)[]; - excluded: (string | boolean)[]; - range?: { min: number; max: number }; -}; - -/** - * Parse a Lucene filter condition back into per-field included/excluded values - * and optional ranges. - * - * Returns an array with one entry per distinct field (empty array if the input - * is valid Lucene but contains no quoted terms or ranges), or undefined if - * parsing fails entirely. - */ -export function parseLuceneFilter( - condition: string, -): ParsedLuceneFilter[] | undefined { - try { - const ast = parse(condition); - const terms: CollectedTerm[] = []; - const ranges: CollectedRange[] = []; - collectFromAst(ast, terms, ranges); - - // Group by field, coercing "true"/"false" back to booleans - const byField = new Map< - string, - { - included: (string | boolean)[]; - excluded: (string | boolean)[]; - range?: { min: number; max: number }; - } - >(); - - const getEntry = (field: string) => { - if (!byField.has(field)) { - byField.set(field, { included: [], excluded: [] }); - } - return byField.get(field)!; - }; - - for (const t of terms) { - const entry = getEntry(t.field); - const value = coerceBooleanValue(t.value); - if (t.negated) { - entry.excluded.push(value); - } else { - entry.included.push(value); - } - } - - for (const r of ranges) { - const entry = getEntry(r.field); - entry.range = { min: r.min, max: r.max }; - } - - return Array.from(byField.entries()).map(([key, vals]) => ({ - key, - ...vals, - })); - } catch { - return undefined; - } -} diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index fb195d016b..f96ffb635b 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -33,7 +33,7 @@ function encodeSpecialTokens(query: string): string { .replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1') .replace(/\\:/g, 'HDX_COLON'); } -export function decodeSpecialTokens(query: string): string { +function decodeSpecialTokens(query: string): string { return query .replace(/\\"/g, '"') .replace(/HDX_BACKSLASH_LITERAL/g, '\\') @@ -61,21 +61,17 @@ function normalizeChExpression(expr: string): string { const IMPLICIT_FIELD = ''; // Type guards for lucene AST types -export function isNodeTerm( - node: lucene.Node | lucene.AST, -): node is lucene.NodeTerm { +function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { return 'term' in node && node.term != null; } -export function isNodeRangedTerm( +function isNodeRangedTerm( node: lucene.Node | lucene.AST, ): node is lucene.NodeRangedTerm { return 'inclusive' in node && node.inclusive != null; } -export function isBinaryAST( - ast: lucene.AST | lucene.Node, -): ast is lucene.BinaryAST { +function isBinaryAST(ast: lucene.AST | lucene.Node): ast is lucene.BinaryAST { return 'right' in ast && ast.right != null; } @@ -85,7 +81,7 @@ function hasStart( return 'start' in ast && !!ast.start; } -export function isLeftOnlyAST( +function isLeftOnlyAST( ast: lucene.AST | lucene.Node, ): ast is lucene.LeftOnlyAST { return ( @@ -1766,14 +1762,9 @@ async function nodeTerm( serializer: Serializer, context: SerializerContext, ): Promise { - const isImplicitField = node.field === IMPLICIT_FIELD; - const rawField = node.field[0] === '-' ? node.field.slice(1) : node.field; - // Decode special-token placeholders the emitter inserted in the field name - // (e.g. HDX_COLON for an escaped `:` in a Map sub-key). Leave the implicit - // field sentinel untouched so downstream comparisons against IMPLICIT_FIELD - // still match. - const field = isImplicitField ? rawField : decodeSpecialTokens(rawField); + const field = node.field[0] === '-' ? node.field.slice(1) : node.field; let isNegatedField = node.field[0] === '-'; + const isImplicitField = node.field === IMPLICIT_FIELD; // NodeTerm if (isNodeTerm(node)) { @@ -1875,7 +1866,7 @@ function createSerializerContext( return { ...currentContext, - implicitColumnExpression: decodeSpecialTokens(fieldWithoutNegation), + implicitColumnExpression: fieldWithoutNegation, ...(isNegatedAndParenthesized(ast) ? { isNegatedAndParenthesized: true } : {}),