diff --git a/.changeset/pattern-filter-actions.md b/.changeset/pattern-filter-actions.md new file mode 100644 index 0000000000..d2a5e3a51f --- /dev/null +++ b/.changeset/pattern-filter-actions.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/app': minor +--- + +feat: filter for or exclude a pattern from Event Patterns + +Each row in Event Patterns now has "Filter for this pattern" and "Exclude this +pattern" buttons on hover. Each adds a filter for every event that fits the +pattern in the time range, not only the sampled rows, so the results table, +histogram and counts all update. Exclusions stack, show as pills next to the +other filters, survive sidebar filter changes, and are kept in the URL and in +saved searches. A new include replaces the previous one, and switching source +removes pattern filters. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 2a90cae9c6..814e6d36da 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -40,6 +40,7 @@ import { import { BuilderChartConfigWithDateRange, ChartConfigWithDateRange, + ConditionFilter, DisplayType, Filter, isPersistableUserId, @@ -102,6 +103,13 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover' import { InlineNameInput } from '@/components/InlineNameInput/InlineNameInput'; import { InputControlled } from '@/components/InputControlled'; import OnboardingModal from '@/components/OnboardingModal'; +import { + isPatternFilter, + parsePatternFilter, + PatternFilter, + PatternFilterAction, + withPatternFilter, +} from '@/components/Patterns/patternFilter'; import { isCustomSearchGroupBy, resolveSearchGroupBy, @@ -1468,8 +1476,37 @@ function DBSearchPageContent() { onFilterChange: handleSetFilters, dateTimeColumns, knownColumns, + isPreservedFilter: isPatternFilter, }); + const patternFilters = useMemo( + () => + (filters ?? []).flatMap(filter => { + const parsed = parsePatternFilter(filter); + return parsed ? [parsed] : []; + }), + [filters], + ); + const removePatternFilter = useCallback( + (removed: PatternFilter) => + handleSetFilters( + (filters ?? []).filter( + f => f.type !== 'sql' || f.condition !== removed.condition, + ), + ), + [filters, handleSetFilters], + ); + // Showing matches switches to the results view. + const addPatternFilter = useCallback( + (added: ConditionFilter, action: PatternFilterAction) => { + handleSetFilters(withPatternFilter(filters ?? [], added)); + if (action === 'include') { + setAnalysisMode('results'); + } + }, + [filters, handleSetFilters, setAnalysisMode], + ); + useEffect(() => { // If the user changes the source dropdown, reset the select and orderby fields // to match the new source selected @@ -2659,6 +2696,8 @@ function DBSearchPageContent() { searchFilters={searchFilters} chartConfig={filtersChartConfig} dateTimeColumns={dateTimeColumns} + patternFilters={patternFilters} + onRemovePatternFilter={removePatternFilter} mt={6} /> @@ -2800,6 +2839,7 @@ function DBSearchPageContent() { draftPatternColumn={draftPatternColumn} onDraftPatternColumnChange={setDraftPatternColumn} onSubmit={onSubmit} + onPatternFilter={addPatternFilter} totalCountConfig={totalCountConfig} totalCountQueryKeyPrefix={QUERY_KEY_PREFIX} /> diff --git a/packages/app/src/__tests__/searchFilters.test.ts b/packages/app/src/__tests__/searchFilters.test.ts index a6eec0c225..81e5b9b903 100644 --- a/packages/app/src/__tests__/searchFilters.test.ts +++ b/packages/app/src/__tests__/searchFilters.test.ts @@ -990,6 +990,83 @@ describe('searchFilters', () => { ]); }); + it('keeps preserved filters through updates until clear all', () => { + onFilterChange.mockClear(); + const pattern = { type: 'sql' as const, condition: "match(Body, '^a$')" }; + const other = { type: 'sql' as const, condition: "match(Body, '^b$')" }; + const { result } = renderHook(() => + useSearchPageFilterState({ + searchQuery: [pattern, other], + onFilterChange, + knownColumns: new Set(), + isPreservedFilter: f => f === pattern, + }), + ); + + act(() => { + result.current.setFilterValue('service', 'app'); + }); + expect(onFilterChange).toHaveBeenLastCalledWith([ + { type: 'sql', condition: "service IN ('app')" }, + pattern, + ]); + + act(() => { + result.current.clearAllFilters(); + }); + expect(onFilterChange).toHaveBeenLastCalledWith([]); + }); + + it('without isPreservedFilter, still drops other SQL filters on update', () => { + onFilterChange.mockClear(); + const service = { type: 'sql' as const, condition: "service IN ('app')" }; + const raw = { type: 'sql' as const, condition: "match(Body, '^a$')" }; + const { result } = renderHook(() => + useSearchPageFilterState({ + searchQuery: [service, raw], + onFilterChange, + knownColumns: new Set(), + }), + ); + + act(() => { + result.current.setFilterValue('level', 'error'); + }); + expect(onFilterChange).toHaveBeenLastCalledWith([ + service, + { type: 'sql', condition: "level IN ('error')" }, + ]); + + let dropped: string[] = []; + act(() => { + dropped = result.current.retainFiltersByColumns( + new Set(['service', 'level']), + ); + }); + expect(dropped).toEqual([]); + }); + + it('drops preserved filters when reconciling to another source', () => { + onFilterChange.mockClear(); + const service = { type: 'sql' as const, condition: "service IN ('app')" }; + const pattern = { type: 'sql' as const, condition: "match(Body, '^a$')" }; + const { result } = renderHook(() => + useSearchPageFilterState({ + searchQuery: [service, pattern], + onFilterChange, + knownColumns: new Set(), + isPreservedFilter: f => f === pattern, + }), + ); + + let dropped: string[] = []; + act(() => { + dropped = result.current.retainFiltersByColumns(new Set(['service'])); + }); + expect(dropped).toEqual([pattern.condition]); + expect(onFilterChange).toHaveBeenLastCalledWith([service]); + }); + it('setOnlyFilters applies all filters in a single onFilterChange emit', () => { onFilterChange.mockClear(); const { result } = renderHook(() => diff --git a/packages/app/src/components/ActiveFilterPills.tsx b/packages/app/src/components/ActiveFilterPills.tsx index 86d8234951..9d5906fc76 100644 --- a/packages/app/src/components/ActiveFilterPills.tsx +++ b/packages/app/src/components/ActiveFilterPills.tsx @@ -3,6 +3,8 @@ import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist import { Flex, FlexProps, Text } from '@mantine/core'; import { FilterPill, FilterValueEditor } from '@/components/FilterPill'; +import type { PatternFilter } from '@/components/Patterns/patternFilter'; +import { simplifyPattern } from '@/components/Patterns/simplifyPattern'; import { useGetKeyValues } from '@/hooks/useMetadata'; import type { FilterStateHook } from '@/searchFilters'; import { useFormatTime } from '@/useFormatTime'; @@ -14,6 +16,7 @@ const VALUE_EDIT_LIMIT = 50; // Stable identity so a caller that omits the prop doesn't invalidate the // flattenFilters useMemo on every render. const EMPTY_DATE_TIME_COLUMNS: ReadonlyMap = new Map(); +const NO_PATTERN_FILTERS: PatternFilter[] = []; type FormatTime = ReturnType; @@ -153,9 +156,14 @@ export const ActiveFilterPills = memo(function ActiveFilterPills({ invalidFieldReason, chartConfig, dateTimeColumns = EMPTY_DATE_TIME_COLUMNS, + patternFilters = NO_PATTERN_FILTERS, + onRemovePatternFilter, ...flexProps }: { searchFilters: FilterStateHook; + /** Filters added from the Event Patterns view, shown after the others. */ + patternFilters?: PatternFilter[]; + onRemovePatternFilter?: (filter: PatternFilter) => void; /** * Map of DateTime/Date column name → ClickHouse type. Their pill values are * formatted to the user's locale/timezone for display, matching the results @@ -262,12 +270,19 @@ export const ActiveFilterPills = memo(function ActiveFilterPills({ clearTimeout(confirmTimerRef.current); }, [confirmClear, clearAllFilters]); - if (pills.length === 0) { + const totalCount = pills.length + patternFilters.length; + if (totalCount === 0) { return null; } - const visiblePills = expanded ? pills : pills.slice(0, MAX_VISIBLE_PILLS); - const hiddenCount = pills.length - MAX_VISIBLE_PILLS; + // The cap covers facet and pattern pills together, facet pills first. + const visibleCount = expanded ? totalCount : MAX_VISIBLE_PILLS; + const visiblePills = pills.slice(0, visibleCount); + const visiblePatternFilters = patternFilters.slice( + 0, + Math.max(visibleCount - pills.length, 0), + ); + const hiddenCount = totalCount - MAX_VISIBLE_PILLS; return ( @@ -288,6 +303,17 @@ export const ActiveFilterPills = memo(function ActiveFilterPills({ /> ); })} + {visiblePatternFilters.map(filter => ( + onRemovePatternFilter?.(filter)} + data-testid="active-filter-pill-pattern" + /> + ))} {!expanded && hiddenCount > 0 && ( )} - {pills.length >= 2 && ( + {totalCount >= 2 && ( void; getRowWhere?: (row: Record) => RowWhereResult; + /** Hover buttons for a row, shown instead of the row copy buttons. */ + rowActions?: (row: Record) => React.ReactNode; variant?: DBRowTableVariant; onRemoveColumn?: (column: string) => void; }) => { @@ -1336,7 +1339,12 @@ export const RawLogTable = memo( ); })} {/* Row-level copy buttons */} - {getRowWhere && ( + {rowActions && ( +
+ {rowActions(row.original)} +
+ )} + {getRowWhere && !rowActions && ( void; onSubmit?: () => void; + onPatternFilter?: ( + filter: ConditionFilter, + action: PatternFilterAction, + ) => void; totalCountQueryKeyPrefix: string; source?: TSource; }) { @@ -94,6 +102,22 @@ export default function PatternTable({ ) as Pattern[]; }, [groupedResults]); + // The rows are the patterns. One without a template gets no buttons. + const renderRowActions = useMemo( + () => + onPatternFilter == null + ? undefined + : (row: Record) => + typeof row.template === 'string' ? ( + + ) : null, + [effectiveBodyValueExpression, onPatternFilter], + ); + return ( <> {hasExternalPatternColumn && ( @@ -164,6 +188,7 @@ export default function PatternTable({ }} config={patternQueryConfig} showExpandButton={false} + rowActions={renderRowActions} /> {selectedPattern && source && ( void; +}) { + const filters = useMemo( + () => ({ + include: buildPatternFilter(valueExpression, template, 'include'), + exclude: buildPatternFilter(valueExpression, template, 'exclude'), + }), + [valueExpression, template], + ); + const { include, exclude } = filters; + if (include == null || exclude == null) return null; + + return ( + <> + onFilter(include, 'include')} + > + + + onFilter(exclude, 'exclude')} + > + + + + ); +} diff --git a/packages/app/src/components/Patterns/__tests__/PatternRowActions.test.tsx b/packages/app/src/components/Patterns/__tests__/PatternRowActions.test.tsx new file mode 100644 index 0000000000..0801b3b45b --- /dev/null +++ b/packages/app/src/components/Patterns/__tests__/PatternRowActions.test.tsx @@ -0,0 +1,59 @@ +import { fireEvent, screen } from '@testing-library/react'; + +import { parsePatternFilter } from '@/components/Patterns/patternFilter'; +import { PatternRowActions } from '@/components/Patterns/PatternRowActions'; + +describe('PatternRowActions', () => { + it('filters for or excludes the row pattern', () => { + const onFilter = jest.fn(); + renderWithMantine( + , + ); + + fireEvent.click( + screen.getByRole('button', { name: 'Filter for this pattern' }), + ); + fireEvent.click( + screen.getByRole('button', { name: 'Exclude this pattern' }), + ); + + expect( + onFilter.mock.calls.map(([filter, action]) => [ + parsePatternFilter(filter), + action, + ]), + ).toEqual([ + [ + expect.objectContaining({ + valueExpression: 'Body', + template: 'GetCartAsync called with userId <*>', + action: 'include', + }), + 'include', + ], + [ + expect.objectContaining({ + valueExpression: 'Body', + template: 'GetCartAsync called with userId <*>', + action: 'exclude', + }), + 'exclude', + ], + ]); + }); + + it('renders nothing for a template too long to filter on', () => { + renderWithMantine( + , + ); + expect(screen.queryAllByRole('button')).toHaveLength(0); + }); +}); diff --git a/packages/app/src/components/Patterns/__tests__/patternFilter.test.ts b/packages/app/src/components/Patterns/__tests__/patternFilter.test.ts new file mode 100644 index 0000000000..b98f30b678 --- /dev/null +++ b/packages/app/src/components/Patterns/__tests__/patternFilter.test.ts @@ -0,0 +1,120 @@ +import type { ConditionFilter, Filter } from '@hyperdx/common-utils/dist/types'; + +import { + buildPatternFilter, + isPatternFilter, + parsePatternFilter, + withPatternFilter, +} from '@/components/Patterns/patternFilter'; + +const include = (template: string): ConditionFilter => + buildPatternFilter('Body', template, 'include')!; +const exclude = (template: string): ConditionFilter => + buildPatternFilter('Body', template, 'exclude')!; +const facet: Filter = { type: 'sql', condition: "ServiceName IN ('cart')" }; + +// Runs a pattern filter on one body the way ClickHouse would: strip ANSI +// codes, turn separator runs into single spaces, then match. +function fits(filter: ConditionFilter, body: string): boolean { + const re2ToJs = (re2: string) => + re2 + .replace(/(? m[1].replace(/''/g, "'").replace(/\\\\/g, '\\')); + const normalized = body + .replace(RegExp(re2ToJs(ansi), 'gu'), '') + .replace(RegExp(re2ToJs(separators), 'gu'), ' '); + const match = RegExp(re2ToJs(regex), 'u').test(normalized); + return filter.condition.startsWith('NOT ') ? !match : match; +} + +describe('pattern filters', () => { + it('builds include and exclude filters that read back', () => { + expect( + parsePatternFilter(include('GetCartAsync called with userId <*>')), + ).toMatchObject({ + valueExpression: 'Body', + template: 'GetCartAsync called with userId <*>', + action: 'include', + }); + expect(exclude('a <*>').condition).toMatch( + /^NOT match\(replaceRegexpAll\(replaceRegexpAll\(Body, /, + ); + expect(parsePatternFilter(exclude('a <*>'))?.action).toBe('exclude'); + expect(isPatternFilter(facet)).toBe(false); + }); + + it('matches bodies the way the miner saw them', () => { + const filter = include('info request done'); + expect(fits(filter, 'info: request done')).toBe(true); + // Color codes, which the miner strips before mining. + expect( + fits(filter, '\x1b[32minfo\x1b[39m: request \x1b[1;31mdone\x1b[0m'), + ).toBe(true); + // Separators that Python's str.split() has and JS `\s` lacks. + expect(fits(filter, 'info\x1erequest\x85done')).toBe(true); + expect(fits(filter, 'warn: request done')).toBe(false); + expect( + fits(exclude('info request done'), '\x1b[32minfo\x1b[39m request done'), + ).toBe(false); + }); + + it('only reads back filters it built', () => { + // The same predicate without the ANSI step is not a pattern filter. + const condition = include('a <*>').condition.replace( + /^match\(replaceRegexpAll\(replaceRegexpAll\(Body, '(?:[^'\\]|\\\\|'')*', ''\)/, + 'match(replaceRegexpAll(Body', + ); + expect(condition).toMatch(/^match\(replaceRegexpAll\(Body, /); + expect(isPatternFilter({ type: 'sql', condition })).toBe(false); + }); + + it('reads back a value expression with quotes', () => { + const expr = "JSONExtractString(Body, 'message')"; + const filter = buildPatternFilter(expr, 'user <*> logged in', 'exclude')!; + expect(parsePatternFilter(filter)).toMatchObject({ + valueExpression: expr, + template: 'user <*> logged in', + action: 'exclude', + }); + }); + + it('builds no filter for a template with punctuation inside a token', () => { + // The miner splits on punctuation, so its templates never have any. + expect(buildPatternFilter('Body', "O'Malley C:\\x (a|b)", 'include')).toBe( + null, + ); + }); + + it('flips a pattern between include and exclude instead of keeping both', () => { + expect( + withPatternFilter([facet, include('a <*>')], exclude('a <*>')), + ).toEqual([facet, exclude('a <*>')]); + expect( + withPatternFilter([facet, exclude('a <*>')], include('a <*>')), + ).toEqual([facet, include('a <*>')]); + }); + + it('replaces an include on another column too', () => { + const other = buildPatternFilter('Message', 'b <*>', 'include')!; + expect(withPatternFilter([include('a <*>')], other)).toEqual([other]); + }); + + it('keeps one include and stacks excludes', () => { + let filters: Filter[] = [facet]; + filters = withPatternFilter(filters, exclude('a <*>')); + filters = withPatternFilter(filters, exclude('b <*>')); + filters = withPatternFilter(filters, include('c <*>')); + filters = withPatternFilter(filters, include('d <*>')); + expect(filters).toEqual([ + facet, + exclude('a <*>'), + exclude('b <*>'), + include('d <*>'), + ]); + }); +}); diff --git a/packages/app/src/components/Patterns/drainDelimiters.ts b/packages/app/src/components/Patterns/drainDelimiters.ts new file mode 100644 index 0000000000..69b421edf7 --- /dev/null +++ b/packages/app/src/components/Patterns/drainDelimiters.ts @@ -0,0 +1,6 @@ +import { ASCII_PUNCTUATION } from '@hyperdx/common-utils/dist/drain'; + +// Python's string.punctuation, the extra delimiters of the app's miner. +// Anything that splits bodies or templates the way the miner did (the denoise +// matcher, pattern filters) must use the same list. +export const DRAIN_EXTRA_DELIMITERS = ASCII_PUNCTUATION.split(''); diff --git a/packages/app/src/components/Patterns/patternFilter.ts b/packages/app/src/components/Patterns/patternFilter.ts new file mode 100644 index 0000000000..bc34453e45 --- /dev/null +++ b/packages/app/src/components/Patterns/patternFilter.ts @@ -0,0 +1,112 @@ +import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; +import { + buildCompactPatternPredicate, + parsePatternPredicate, + type PatternRegexOptions, +} from '@hyperdx/common-utils/dist/drain'; +import type { ConditionFilter, Filter } from '@hyperdx/common-utils/dist/types'; + +import { DRAIN_EXTRA_DELIMITERS } from '@/components/Patterns/drainDelimiters'; + +export type PatternFilterAction = 'include' | 'exclude'; + +export type PatternFilter = { + condition: string; + valueExpression: string; + template: string; + action: PatternFilterAction; +}; + +// The templates come from the app's miner (Python Drain3), so the regex must +// split where it did. +const options: PatternRegexOptions = { + extraDelimiters: DRAIN_EXTRA_DELIMITERS, + whitespace: 'python', +}; +const NOT = 'NOT '; + +// The miner strips ANSI escape codes before mining (usePatterns), so the +// filter strips the ones logs carry first: CSI sequences such as colors. +const ANSI_CSI = '\\x{1B}\\[[0-?]*[ -/]*[@-~]'; +const STRIP_ANSI_START = 'replaceRegexpAll('; +const STRIP_ANSI_END = `, '${escapeSqlString(ANSI_CSI)}', '')`; + +/** + * A search filter that keeps (include) or drops (exclude) every event whose + * value fits the template, or null when the template is too long to filter on. + */ +export function buildPatternFilter( + valueExpression: string, + template: string, + action: PatternFilterAction, +): ConditionFilter | null { + // The compact form: filters live in the URL, and people stack many. + const predicate = buildCompactPatternPredicate( + `${STRIP_ANSI_START}${valueExpression}${STRIP_ANSI_END}`, + template, + options, + ); + if (predicate == null) return null; + return { + type: 'sql', + condition: action === 'exclude' ? `${NOT}${predicate}` : predicate, + }; +} + +/** The pattern behind a filter built by buildPatternFilter, else null. */ +export function parsePatternFilter(filter: Filter): PatternFilter | null { + if (filter.type !== 'sql') return null; + const excluded = filter.condition.startsWith(NOT); + const parsed = parsePatternPredicate( + excluded ? filter.condition.slice(NOT.length) : filter.condition, + options, + ); + if (parsed == null) return null; + const { valueExpression, template } = parsed; + if ( + !valueExpression.startsWith(STRIP_ANSI_START) || + !valueExpression.endsWith(STRIP_ANSI_END) + ) { + return null; + } + return { + condition: filter.condition, + valueExpression: valueExpression.slice( + STRIP_ANSI_START.length, + -STRIP_ANSI_END.length, + ), + template, + action: excluded ? 'exclude' : 'include', + }; +} + +export function isPatternFilter(filter: Filter): boolean { + return parsePatternFilter(filter) != null; +} + +/** + * `filters` with `added` appended. Any filter on the same pattern goes, so + * flipping include and exclude doesn't leave both. An include also replaces + * the other includes: filters are ANDed, so two would match nothing. + * Excludes stack. + */ +export function withPatternFilter( + filters: Filter[], + added: ConditionFilter, +): Filter[] { + const pattern = parsePatternFilter(added); + if (pattern == null) return [...filters, added]; + return [ + ...filters.filter(filter => { + const existing = parsePatternFilter(filter); + if (existing == null) return true; + const samePattern = + existing.template === pattern.template && + existing.valueExpression === pattern.valueExpression; + const otherInclude = + pattern.action === 'include' && existing.action === 'include'; + return !samePattern && !otherInclude; + }), + added, + ]; +} diff --git a/packages/app/src/components/__tests__/ActiveFilterPills.test.tsx b/packages/app/src/components/__tests__/ActiveFilterPills.test.tsx index 8829c9ec1f..7437030818 100644 --- a/packages/app/src/components/__tests__/ActiveFilterPills.test.tsx +++ b/packages/app/src/components/__tests__/ActiveFilterPills.test.tsx @@ -151,6 +151,56 @@ describe('ActiveFilterPills', () => { expect(screen.getByText('100 – 500')).toBeInTheDocument(); }); + it('renders pattern filters and removes them through their handler', () => { + const patternFilter = { + condition: "NOT match(Body, '^x$')", + valueExpression: 'Body', + template: 'Connection to db <*> <*> timed out', + action: 'exclude' as const, + }; + const onRemovePatternFilter = jest.fn(); + renderWithMantine( + , + ); + expect(screen.getByText('Pattern')).toBeInTheDocument(); + expect( + screen.getByText('Connection to db … timed out'), + ).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button')); + expect(onRemovePatternFilter).toHaveBeenCalledWith(patternFilter); + }); + + it('counts pattern pills toward MAX_VISIBLE_PILLS after the facet pills', () => { + const values = new Set( + Array.from({ length: 7 }, (_, i) => `val${i}`), + ); + const patternFilters = ['first <*>', 'second <*>'].map(template => ({ + condition: `NOT match(Body, '${template}')`, + valueExpression: 'Body', + template, + action: 'exclude' as const, + })); + renderWithMantine( + () }, + })} + chartConfig={CHART_CONFIG} + patternFilters={patternFilters} + />, + ); + // Seven facet pills and the first pattern pill fill the eight slots. + expect(screen.getAllByTestId('active-filter-pill-pattern')).toHaveLength(1); + fireEvent.click(screen.getByText('+1 more')); + expect(screen.getAllByTestId('active-filter-pill-pattern')).toHaveLength(2); + expect(screen.getByText('Show less')).toBeInTheDocument(); + }); + it('calls setFilterValue when removing an included pill', () => { const searchFilters = makeSearchFilters({ status: { diff --git a/packages/app/src/hooks/__tests__/useDenoisedRows.test.tsx b/packages/app/src/hooks/__tests__/useDenoisedRows.test.tsx index 252fe5f714..73cfd4f4a9 100644 --- a/packages/app/src/hooks/__tests__/useDenoisedRows.test.tsx +++ b/packages/app/src/hooks/__tests__/useDenoisedRows.test.tsx @@ -10,7 +10,6 @@ import { useGroupedPatterns } from '@/hooks/usePatterns'; jest.mock('@/hooks/usePatterns', () => ({ LEVEL_COLUMN_ALIAS: '__hdx_level', - DRAIN_EXTRA_DELIMITERS: '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''), useGroupedPatterns: jest.fn(), })); diff --git a/packages/app/src/hooks/useDenoisedRows.ts b/packages/app/src/hooks/useDenoisedRows.ts index 7ba421cc59..4552b58937 100644 --- a/packages/app/src/hooks/useDenoisedRows.ts +++ b/packages/app/src/hooks/useDenoisedRows.ts @@ -11,11 +11,8 @@ import { TSource, } from '@hyperdx/common-utils/dist/types'; -import { - DRAIN_EXTRA_DELIMITERS, - LEVEL_COLUMN_ALIAS, - useGroupedPatterns, -} from '@/hooks/usePatterns'; +import { DRAIN_EXTRA_DELIMITERS } from '@/components/Patterns/drainDelimiters'; +import { LEVEL_COLUMN_ALIAS, useGroupedPatterns } from '@/hooks/usePatterns'; import { getEventBody, getLevelExpression } from '@/source'; import { getLogLevelClass, selectItemExpression } from '@/utils'; diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index 57a42e7267..baa34a6cc1 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -6,6 +6,7 @@ import { useQuery } from '@tanstack/react-query'; import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils'; import { useConfigWithAdditionalSelect } from '@/components/DBRowTable'; +import { DRAIN_EXTRA_DELIMITERS } from '@/components/Patterns/drainDelimiters'; import { reconstructTemplate } from '@/components/Patterns/reconstructTemplate'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { getFirstTimestampValueExpression } from '@/source'; @@ -43,11 +44,6 @@ function usePyodide(options: { enabled: boolean }) { }); } -// Python's string.punctuation. Shared with the denoise matcher, which must -// split bodies into the same tokens the miner built its templates from. -export const DRAIN_EXTRA_DELIMITERS = - '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''); - class Miner { private minerVariableName; private id; @@ -120,6 +116,11 @@ type SampleLog = { export type Pattern = { id: string; pattern: string; + /** + * The cluster's Drain template, which `pattern` reconstructs for display. + * Missing when the miner returned none for the cluster. + */ + template?: string; count: number; samples: SampleLog[]; }; @@ -306,6 +307,8 @@ export function useGroupedPatterns({ fullPatternGroups[patternId] = { id: patternId, pattern: reconstructedPattern, // last pattern is usually the most up to date templated pattern + template: + results?.clusterTemplates?.[patternId] ?? lastRow?.__hdx_pattern, count, countStr: `~${count}`, level: lastRow?.[LEVEL_COLUMN_ALIAS], // last level is usually representative of the entire pattern set diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index c84c1079fc..ebd7ce9bc6 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -96,6 +96,7 @@ export const useSearchPageFilterState = ({ onFilterChange, dateTimeColumns, knownColumns, + isPreservedFilter, }: { searchQuery?: Filter[]; onFilterChange: (filters: Filter[]) => void; @@ -106,6 +107,15 @@ export const useSearchPageFilterState = ({ * (eg. service-name --> `service-name`). **/ knownColumns: Set; + /** + * Filters this state can't represent (e.g. pattern filters) that should + * survive its updates instead of being dropped. "Clear all" and + * retainFiltersByColumns (a source change) still drop them, since they + * can't be checked against the new source's columns. Match only filters + * the facet state drops (not IN, NOT IN or BETWEEN), or they'd be emitted + * twice. + */ + isPreservedFilter?: (filter: Filter) => boolean; }) => { // Access knownColumns through a ref so the returned mutators (which depend on // updateFilterQuery) keep stable identities across knownColumns reference @@ -116,6 +126,14 @@ export const useSearchPageFilterState = ({ knownColumnsRef.current = knownColumns; }, [knownColumns]); + // Same reason as knownColumnsRef. + const preservedFiltersRef = useRef([]); + useEffect(() => { + preservedFiltersRef.current = isPreservedFilter + ? searchQuery.filter(isPreservedFilter) + : []; + }, [searchQuery, isPreservedFilter]); + // Persisted filters carry canonical (escaped) keys; convert back to the clean // keys the sidebar/comparisons use as they enter in-memory FilterState. const parsedQuery = useMemo(() => { @@ -140,14 +158,17 @@ export const useSearchPageFilterState = ({ }, [parsedQuery.filters]); const updateFilterQuery = useCallback( - (newFilters: FilterState) => { + (newFilters: FilterState, { keepPreserved = true } = {}) => { // Escape filter keys here prior to saving so the URL / saved // search holds valid-SQL keys while FilterState stays clean. const escapedFilters = escapeFilterStateKeys( newFilters, knownColumnsRef.current, ); - onFilterChange(filtersToQuery(escapedFilters, { dateTimeColumns })); + onFilterChange([ + ...filtersToQuery(escapedFilters, { dateTimeColumns }), + ...(keepPreserved ? preservedFiltersRef.current : []), + ]); }, [onFilterChange, dateTimeColumns], ); @@ -285,12 +306,13 @@ export const useSearchPageFilterState = ({ const clearAllFilters = useCallback(() => { setFilters(() => ({})); - updateFilterQuery({}); + updateFilterQuery({}, { keepPreserved: false }); }, [updateFilterQuery]); - // Keep only filters whose root column appears in `allowedColumnNames`. - // Returns the keys of the filters that were dropped so callers can surface - // a notice when filter state was thrown away (e.g. on source change). + // Keep only filters whose root column appears in `allowedColumnNames`, and + // drop preserved filters. Returns the keys (or conditions) of the filters + // that were dropped so callers can surface a notice when filter state was + // thrown away (e.g. on source change). const retainFiltersByColumns = useCallback( (allowedColumnNames: Set): string[] => { const dropped: string[] = []; @@ -302,9 +324,14 @@ export const useSearchPageFilterState = ({ dropped.push(key); } } + for (const filter of preservedFiltersRef.current) { + dropped.push( + filter.type === 'sql_ast' ? filter.left : filter.condition, + ); + } if (dropped.length > 0) { setFilters(kept); - updateFilterQuery(kept); + updateFilterQuery(kept, { keepPreserved: false }); } return dropped; }, diff --git a/packages/app/tests/e2e/features/search/event-patterns.spec.ts b/packages/app/tests/e2e/features/search/event-patterns.spec.ts index 5b6d2f1357..09adf879f0 100644 --- a/packages/app/tests/e2e/features/search/event-patterns.spec.ts +++ b/packages/app/tests/e2e/features/search/event-patterns.spec.ts @@ -3,6 +3,7 @@ import { CUSTOM_SERVICE_LOGS_APP_NAMES } from '../../seed-clickhouse'; import { expect, test } from '../../utils/base-test'; import { CUSTOM_SERVICE_LOGS_SOURCE_NAME, + DEFAULT_LOGS_SOURCE_NAME, DEFAULT_TRACES_SOURCE_NAME, } from '../../utils/constants'; @@ -56,4 +57,57 @@ test.describe('Event Patterns', { tag: ['@full-stack', '@search'] }, () => { await expect(panel.levelColumnHeader).toBeVisible(); await expect(panel.levelCell(0)).toHaveText(TRACE_STATUS_CODE_PATTERN); }); + + test.describe('pattern filters', () => { + let searchPage: SearchPage; + + test.beforeEach(async ({ page }) => { + searchPage = new SearchPage(page); + await searchPage.goto(); + await searchPage.selectSource(DEFAULT_LOGS_SOURCE_NAME); + await searchPage.timePicker.selectRelativeTime('Last 1 hour'); + await searchPage.table.waitForRowsToPopulate(); + await searchPage.switchToEventPatterns(); + }); + + test('excludes a pattern from its row and keeps it excluded when a facet changes', async () => { + const excluded = await searchPage.patternListPatternCell(0).innerText(); + await searchPage.clickPatternRowAction('Exclude this pattern'); + + await expect(searchPage.patternFilterPills).toHaveCount(1); + await expect(searchPage.patternFilterPills).toContainText('!='); + // The list is mined again without the excluded pattern. + await expect(searchPage.patternListRows.first()).toBeVisible(); + await expect( + searchPage.patternListTable.getByText(excluded, { exact: true }), + ).toHaveCount(0); + + await searchPage.filters.openFilterGroup('SeverityText'); + await searchPage.filters.applyFilter('SeverityText', 'info'); + await expect(searchPage.patternFilterPills).toHaveCount(1); + }); + + test('filters for a pattern from its row and shows the results table', async ({ + page, + }) => { + await searchPage.clickPatternRowAction('Filter for this pattern'); + + await expect( + page.getByRole('tab', { name: 'Results Table' }), + ).toHaveAttribute('aria-selected', 'true'); + await expect(searchPage.patternFilterPills).toHaveCount(1); + await expect(searchPage.patternFilterPills).not.toContainText('!='); + await searchPage.table.waitForRowsToPopulate(); + await expect(searchPage.getTableError()).toHaveCount(0); + }); + + test('drops pattern filters when the source changes', async () => { + await searchPage.clickPatternRowAction('Exclude this pattern'); + await expect(searchPage.patternFilterPills).toHaveCount(1); + + await searchPage.selectSource(DEFAULT_TRACES_SOURCE_NAME); + await expect(searchPage.getDroppedFiltersToast()).toBeVisible(); + await expect(searchPage.patternFilterPills).toHaveCount(0); + }); + }); }); diff --git a/packages/app/tests/e2e/page-objects/SearchPage.ts b/packages/app/tests/e2e/page-objects/SearchPage.ts index 6467474f5d..9bb4af3030 100644 --- a/packages/app/tests/e2e/page-objects/SearchPage.ts +++ b/packages/app/tests/e2e/page-objects/SearchPage.ts @@ -209,6 +209,30 @@ export class SearchPage { .nth(PATTERN_LIST_LEVEL_COLUMN_INDEX); } + /** The Pattern-column cell of a pattern-list row (see patternListLevelCell). */ + patternListPatternCell(rowIndex = 0) { + const PATTERN_LIST_PATTERN_COLUMN_INDEX = 3; + return this.patternListRows + .nth(rowIndex) + .locator('td > button > div') + .nth(PATTERN_LIST_PATTERN_COLUMN_INDEX); + } + + /** Active-filter pills for filters added from the Event Patterns view. */ + get patternFilterPills() { + return this.page.getByTestId('active-filter-pill-pattern'); + } + + /** Hover a pattern-list row and click one of its filter buttons. */ + async clickPatternRowAction( + name: 'Filter for this pattern' | 'Exclude this pattern', + rowIndex = 0, + ) { + const row = this.patternListRows.nth(rowIndex); + await row.hover(); + await row.getByRole('button', { name }).click(); + } + /** Open the first pattern's sample flyout and wait for it to render. */ async openFirstPattern(timeout = 30_000) { await this.patternListRows.first().click();