diff --git a/.changeset/shared-pattern-predicate.md b/.changeset/shared-pattern-predicate.md new file mode 100644 index 000000000..99784c662 --- /dev/null +++ b/.changeset/shared-pattern-predicate.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/common-utils': minor +--- + +feat: share the Event Patterns drill-down predicate + +`templateToRegex` and `buildPatternPredicate`, which the MCP server uses to +build the `clickstack_event_patterns` drill-down filter, are now in +`@hyperdx/common-utils/dist/drain`. New options match templates mined with extra +delimiters (`extraDelimiters`) or by Python Drain3 (`whitespace: 'python'`), and +`parsePatternPredicate` reads a predicate back into its value expression and +template. Without options the output is the same as the MCP server's. diff --git a/packages/common-utils/src/__tests__/pattern-predicate.test.ts b/packages/common-utils/src/__tests__/pattern-predicate.test.ts new file mode 100644 index 000000000..5a0f0eeb9 --- /dev/null +++ b/packages/common-utils/src/__tests__/pattern-predicate.test.ts @@ -0,0 +1,182 @@ +import { TemplateMinerConfig } from '@/drain/config'; +import { + buildPatternPredicate, + MAX_PREDICATE_TEMPLATE_LENGTH, + parsePatternPredicate, + PatternRegexOptions, + templateToRegex, +} from '@/drain/pattern-predicate'; +import { TemplateMiner } from '@/drain/template-miner'; + +// Python's string.punctuation, the extra delimiters the app's miner uses. +const PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''); +const withPunctuation = { extraDelimiters: PUNCTUATION }; + +// RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag) +// spells them \u{HHHH} and \p{Z}, and has no POSIX classes. RE2's \s is +// [\t\n\f\r ]; JS's also covers Unicode spaces and BOM. +function toJsRegex(re2: string): RegExp { + return RegExp( + re2 + .replace(/(? { + it('splits on whitespace only without extra delimiters', () => { + const ws = '[\\s\\x{0B}\\pZ\\x{FEFF}]'; + expect(templateToRegex('Sending <*> rows')).toBe( + `^${ws}*Sending${ws}+[^\\s\\x{0B}\\pZ\\x{FEFF}]+${ws}+rows${ws}*$`, + ); + }); + + it('spells all ASCII punctuation as [:punct:] and lists other delimiters', () => { + expect(templateToRegex('a', withPunctuation)).toBe( + '^[\\s\\x{0B}\\pZ\\x{FEFF}[:punct:]]*a[\\s\\x{0B}\\pZ\\x{FEFF}[:punct:]]*$', + ); + const partial = { extraDelimiters: ['/', ':', '-', ']', '\\', '^'] }; + const regex = toJsRegex(templateToRegex('a <*> c d e f g', partial)); + expect(regex.test('a/x:c-d]e\\f^g')).toBe(true); + expect(regex.test('a/x:c-d]e\\f^g.h')).toBe(false); + }); + + it('splits on Python whitespace for templates mined by Drain3', () => { + const python: PatternRegexOptions = { + extraDelimiters: PUNCTUATION, + whitespace: 'python', + }; + // Python's str.split() splits on the \x1C-\x1F separators and NEL but not + // on BOM; JS `\s` is the other way round. + const drain3 = toJsRegex(templateToRegex('a b c', python)); + expect(drain3.test('a\x1eb\x85c')).toBe(true); + expect(drain3.test('a\ufeffb c')).toBe(false); + const ts = toJsRegex(templateToRegex('a b c', withPunctuation)); + expect(ts.test('a\x1eb\x85c')).toBe(false); + expect(ts.test('a\ufeffb c')).toBe(true); + // So a Drain3 token can hold a BOM, as in a body that starts with one. + const bomTemplate = '\ufeffGET api <*>'; + expect( + toJsRegex(templateToRegex(bomTemplate, python)).test('\ufeffGET /api/x'), + ).toBe(true); + expect( + parsePatternPredicate( + buildPatternPredicate('Body', bomTemplate, python)!, + python, + )?.template, + ).toBe(bomTemplate); + + const predicate = buildPatternPredicate('Body', 'a b c', python)!; + expect(parsePatternPredicate(predicate, python)?.template).toBe('a b c'); + expect(parsePatternPredicate(predicate, withPunctuation)).toBeNull(); + }); + + it('rejects delimiters longer than one character', () => { + expect(() => templateToRegex('a', { extraDelimiters: ['::'] })).toThrow( + 'single characters', + ); + }); + + it('treats extra delimiters as separators', () => { + const regex = toJsRegex( + templateToRegex('GET api v1 users <*> <*> <*>', withPunctuation), + ); + expect(regex.test('GET /api/v1/users/123 200 15ms')).toBe(true); + expect(regex.test('GET /api/v1/users/9 404 3ms')).toBe(true); + expect(regex.test(' GET /api/v1/users/9 404 3ms.')).toBe(true); + + expect(regex.test('POST /api/v1/users/9 404 3ms')).toBe(false); + expect(regex.test('GET /api/v1/users/9/orders 404 3ms')).toBe(false); + expect(regex.test('GET /api/v1/users/9 404')).toBe(false); + expect(regex.test('GET /api/v2/users/9 404 3ms')).toBe(false); + }); + + it('matches every member of every cluster mined with the same delimiters', () => { + const config = new TemplateMinerConfig(); + config.drainExtraDelimiters = PUNCTUATION; + const miner = new TemplateMiner(config); + const messages = [ + 'Connection to db-1:5432 timed out after 30s (retry 1/3)', + 'Connection to db-2:5432 timed out after 30s (retry 2/3)', + 'user=alice action=login status=ok', + 'user=bob action=login status=ok', + '[2026-04-17T08:35:57.725Z] "GET /api/products/2ZYF HTTP/1.1" 200 -', + '[2026-04-17T08:36:02.101Z] "GET /api/products/L9EC HTTP/1.1" 200 -', + 'path=C:\\temp\\a.txt mode=[r^w] note={a|b} cost=$5 ~ok', + 'path=C:\\temp\\b.txt mode=[r^w] note={a|b} cost=$7 ~ok', + ]; + const members = new Map(); + for (const message of messages) { + const { clusterId } = miner.addLogMessage(message); + members.set(clusterId, [...(members.get(clusterId) ?? []), message]); + } + for (const [clusterId, clusterMembers] of members) { + const template = miner.drain.getCluster(clusterId)!.getTemplate(); + const regex = toJsRegex(templateToRegex(template, withPunctuation)); + for (const message of messages) { + expect(regex.test(message)).toBe(clusterMembers.includes(message)); + } + } + }); +}); + +describe('buildPatternPredicate', () => { + it('builds no predicate for a template over the length limit', () => { + const atLimit = 'x'.repeat(MAX_PREDICATE_TEMPLATE_LENGTH); + expect(buildPatternPredicate('Body', atLimit)).not.toBeNull(); + expect(buildPatternPredicate('Body', `${atLimit}x`)).toBeNull(); + }); +}); + +describe('parsePatternPredicate', () => { + it.each([ + ['Body', 'GET api v1 users <*> <*> <*>'], + ["JSONExtractString(Body, 'message')", 'user <*> action login status ok'], + ['Body', '<*>'], + ])('reads back what buildPatternPredicate built (%s)', (expr, template) => { + const predicate = buildPatternPredicate(expr, template, withPunctuation)!; + expect(parsePatternPredicate(predicate, withPunctuation)).toEqual({ + valueExpression: expr, + template, + }); + }); + + it('reads back what buildPatternPredicate built without options', () => { + const predicate = buildPatternPredicate('Body', 'Sending <*> rows')!; + expect(parsePatternPredicate(predicate)).toEqual({ + valueExpression: 'Body', + template: 'Sending <*> rows', + }); + }); + + it('reads back tokens that hold quotes, backslashes and regex syntax', () => { + // Without delimiters a token keeps its punctuation. + const expr = "JSONExtractString(Body, 'msg')"; + const template = "it's C:\\tmp\\a.txt (ok) <*>"; + const predicate = buildPatternPredicate(expr, template)!; + expect(parsePatternPredicate(predicate)).toEqual({ + valueExpression: expr, + template, + }); + }); + + it('rejects other conditions', () => { + expect(parsePatternPredicate("ServiceName IN ('api')")).toBeNull(); + expect(parsePatternPredicate("match(Body, 'timed out')")).toBeNull(); + // A predicate only reads back with the options it was built with. + expect( + parsePatternPredicate( + buildPatternPredicate('Body', 'Sending <*> rows')!, + withPunctuation, + ), + ).toBeNull(); + expect( + parsePatternPredicate( + buildPatternPredicate('Body', 'Sending <*> rows', withPunctuation)!, + ), + ).toBeNull(); + }); +}); diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index 7615f1109..65c433ea1 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -18,5 +18,13 @@ export { } from './mine-patterns'; export { Node } from './node'; export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher'; +export type { PatternRegexOptions } from './pattern-predicate'; +export { + ASCII_PUNCTUATION, + buildPatternPredicate, + MAX_PREDICATE_TEMPLATE_LENGTH, + parsePatternPredicate, + templateToRegex, +} from './pattern-predicate'; export type { AddLogMessageResult, ExtractedParameter } from './template-miner'; export { TemplateMiner } from './template-miner'; diff --git a/packages/common-utils/src/drain/pattern-predicate.ts b/packages/common-utils/src/drain/pattern-predicate.ts new file mode 100644 index 000000000..d7de608bf --- /dev/null +++ b/packages/common-utils/src/drain/pattern-predicate.ts @@ -0,0 +1,136 @@ +import { escapeSqlString } from '@/core/utils'; + +const PATTERN_WILDCARD = '<*>'; + +/** + * Templates longer than this get no predicate: the regex (and the SQL that + * embeds it) would be unwieldy, and very long templates are usually a + * single unparameterized payload rather than a reusable shape. + */ +export const MAX_PREDICATE_TEMPLATE_LENGTH = 4000; + +export type PatternRegexOptions = { + /** + * The extra delimiters the miner was configured with, one character each. + * Drain turns each one into a space before splitting, so they separate + * tokens like whitespace. + */ + extraDelimiters?: string[]; + /** + * The whitespace the miner split on: JavaScript's `\s` for the TypeScript + * Drain port (the default), or Python's `str.split()` for Drain3. + */ + whitespace?: 'js' | 'python'; +}; + +// The TypeScript Drain port tokenizes with JS `\s`, which includes Unicode +// spaces and BOM; RE2's `\s` is ASCII-only. JS `\s` = RE2 `\s` + \v + \pZ +// (Zs/Zl/Zp) + BOM. `\pZ` keeps the snippet short; spelling out the code +// points repeats ~100 chars between every token. +const JS_WHITESPACE = '\\s\\x{0B}\\pZ\\x{FEFF}'; +// Python's `str.split()`, which Drain3 uses: RE2 `\s` + \v + the separators +// \x1C-\x1F + NEL + \pZ, and no BOM. +const PYTHON_WHITESPACE = '\\s\\x{0B}\\x{1C}-\\x{1F}\\x{85}\\pZ'; + +function escapeRe2Literal(token: string): string { + return token.replace(/[\\^$.|?*+()[\]{}]/g, '\\$&'); +} + +// The characters that are special inside a character class. `[` can open a +// POSIX class such as `[:alpha:]` in RE2. +function escapeRe2ClassChars(chars: string): string { + return chars.replace(/[\\\]^[-]/g, '\\$&'); +} + +/** The ASCII punctuation: exactly RE2's `[:punct:]`, and Python's `string.punctuation`. */ +export const ASCII_PUNCTUATION = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'; + +function separatorClass(options?: PatternRegexOptions): string { + const delimiters = options?.extraDelimiters ?? []; + // Drain splits on each delimiter as a whole string, which a character class + // can't express for one longer than a character. + if (delimiters.some(d => [...d].length !== 1)) { + throw new Error('extraDelimiters must be single characters'); + } + const whitespace = + options?.whitespace === 'python' ? PYTHON_WHITESPACE : JS_WHITESPACE; + // The class repeats between every token, so spelling out all 32 ASCII + // punctuation characters (most of which need URL encoding) makes the + // filter too long to keep in a URL. + const allPunctuation = [...ASCII_PUNCTUATION].every(c => + delimiters.includes(c), + ); + const rest = allPunctuation + ? delimiters.filter(c => !ASCII_PUNCTUATION.includes(c)) + : delimiters; + return ( + whitespace + + (allPunctuation ? '[:punct:]' : '') + + escapeRe2ClassChars(rest.join('')) + ); +} + +/** Anchored RE2 regex that a value matches when it fits the Drain template. */ +export function templateToRegex( + template: string, + options?: PatternRegexOptions, +): string { + const separators = separatorClass(options); + const separator = `[${separators}]`; + // Drain3 keeps a BOM inside a token, where JS `\s` would split. + const split = options?.whitespace === 'python' ? /[^\S\u{FEFF}]+/u : /\s+/; + const tokens = template.split(split).filter(t => t.length > 0); + const body = tokens + .map(t => + t === PATTERN_WILDCARD ? `[^${separators}]+` : escapeRe2Literal(t), + ) + .join(`${separator}+`); + return `^${separator}*${body}${separator}*$`; +} + +/** + * SQL predicate selecting events whose `valueExpression` fits the template, + * or null when the template is too long to turn into a predicate. + */ +export function buildPatternPredicate( + valueExpression: string, + template: string, + options?: PatternRegexOptions, +): string | null { + if (template.length > MAX_PREDICATE_TEMPLATE_LENGTH) return null; + return `match(${valueExpression}, '${escapeSqlString(templateToRegex(template, options))}')`; +} + +/** + * Reverse of buildPatternPredicate: the value expression and template of a + * predicate it built with the same options, or null for any other condition. + */ +export function parsePatternPredicate( + condition: string, + options?: PatternRegexOptions, +): { valueExpression: string; template: string } | null { + const call = /^match\((.+), '((?:[^'\\]|\\\\|'')*)'\)$/s.exec(condition); + if (call == null) return null; + const [, valueExpression, literal] = call; + const regex = literal.replace(/''/g, "'").replace(/\\\\/g, '\\'); + + const separators = separatorClass(options); + const separator = `[${separators}]`; + const prefix = `^${separator}*`; + const suffix = `${separator}*$`; + if (!regex.startsWith(prefix) || !regex.endsWith(suffix)) return null; + const template = regex + .slice(prefix.length, regex.length - suffix.length) + .split(`${separator}+`) + .map(part => + part === `[^${separators}]+` + ? PATTERN_WILDCARD + : part.replace(/\\(.)/g, '$1'), + ) + .join(' '); + // Only what this module builds counts, so a hand-written match() isn't + // mistaken for a pattern. + return templateToRegex(template, options) === regex + ? { valueExpression, template } + : null; +}