From 96615caad507940f855b06721f706aa42674c9b3 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 26 May 2026 12:35:30 -0400 Subject: [PATCH] fix: escape colons in Lucene field names for Map sub-key filters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Map columns can contain keys with `:` (e.g. `LogAttributes['foo:bar']`). After `parseKeyPath().join('.')` the FilterState key becomes `LogAttributes.foo:bar` and `filtersToQuery` previously emitted the unescaped form `LogAttributes.foo:bar:"value"`, which the Lucene parser cannot parse — `:` is the field/value separator and `LogAttributes.foo` ends up as the field with `bar` as an unquoted value, then the next `:` is a syntax error. This change reuses the existing `\:` → HDX_COLON → `:` pipeline in `encodeSpecialTokens`/`decodeSpecialTokens`: - `filtersToQuery` backslash-escapes `\\` and `:` in the field name before emitting (`escapeLuceneFieldName`). - `parseLuceneFilter` (`collectFromAst`) and the SQL serializer (`nodeTerm`, `createSerializerContext`) call `decodeSpecialTokens` on `ast.field` so consumers see the original key. The implicit-field sentinel is preserved unmodified so downstream `field === IMPLICIT_FIELD` comparisons keep working. Adds `colon-in-mapkey.test.ts` covering: - bracket-form and dot-form Map keys with `:` - excluded values + colon-in-key - multiple colons in the key segment (e.g. `k8s.io:annotation:foo`) - compound case: colon in BOTH key and value simultaneously --- .changeset/lucene-colon-in-map-key.md | 10 ++ .../src/__tests__/colon-in-mapkey.test.ts | 112 ++++++++++++++++++ packages/common-utils/src/filters.ts | 19 ++- packages/common-utils/src/queryParser.ts | 11 +- 4 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 .changeset/lucene-colon-in-map-key.md create mode 100644 packages/common-utils/src/__tests__/colon-in-mapkey.test.ts diff --git a/.changeset/lucene-colon-in-map-key.md b/.changeset/lucene-colon-in-map-key.md new file mode 100644 index 0000000000..c74b10241a --- /dev/null +++ b/.changeset/lucene-colon-in-map-key.md @@ -0,0 +1,10 @@ +--- +"@hyperdx/common-utils": patch +--- + +fix: escape colons in Lucene field names so filters on Map sub-keys containing +`:` (e.g. `LogAttributes['foo:bar']`) parse correctly + +`filtersToQuery` now backslash-escapes `:` and `\` in the emitted Lucene field +name, and `parseLuceneFilter` + the SQL serializer decode those placeholders +when consuming the AST so the original key is restored end-to-end. diff --git a/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts b/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts new file mode 100644 index 0000000000..59cc207949 --- /dev/null +++ b/packages/common-utils/src/__tests__/colon-in-mapkey.test.ts @@ -0,0 +1,112 @@ +import { filtersToQuery, parseLuceneFilter } from '@/filters'; +import { parse } from '@/queryParser'; + +describe('Colon-in-MAP-KEY scenario (LogAttributes with key containing colon)', () => { + it('parses Lucene when the colon in a field name is properly escaped', () => { + expect(() => + parse(String.raw`LogAttributes.foo\:bar:"value"`), + ).not.toThrow(); + }); + + it('filtersToQuery emits parseable Lucene for bracket-form Map key containing colon', () => { + const filters = { + "LogAttributes['foo:bar']": { + included: new Set(['value1']), + excluded: new Set(), + }, + }; + const result = filtersToQuery(filters); + const condition = (result[0] as { condition: string }).condition; + expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`); + expect(() => parse(condition)).not.toThrow(); + }); + + it('filtersToQuery emits parseable Lucene for dot-form Map key containing colon', () => { + const filters = { + 'LogAttributes.foo:bar': { + included: new Set(['value1']), + excluded: new Set(), + }, + }; + const result = filtersToQuery(filters); + const condition = (result[0] as { condition: string }).condition; + expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`); + expect(() => parse(condition)).not.toThrow(); + }); + + it('round-trips Map keys containing colons through filtersToQuery + parseLuceneFilter', () => { + const filters = { + "LogAttributes['foo:bar']": { + included: new Set(['value1']), + excluded: new Set(), + }, + }; + const [emitted] = filtersToQuery(filters); + const cond = (emitted as { condition: string }).condition; + const parsed = parseLuceneFilter(cond); + expect(parsed).toEqual([ + { + key: 'LogAttributes.foo:bar', + included: ['value1'], + excluded: [], + }, + ]); + }); + + it('round-trips Map keys with colons and excluded values', () => { + const filters = { + 'LogAttributes.foo:bar': { + included: new Set(), + excluded: new Set(['unwanted:value']), + }, + }; + const [emitted] = filtersToQuery(filters); + const cond = (emitted as { condition: string }).condition; + expect(cond).toBe(String.raw`-LogAttributes.foo\:bar:"unwanted:value"`); + expect(parseLuceneFilter(cond)).toEqual([ + { + key: 'LogAttributes.foo:bar', + included: [], + excluded: ['unwanted:value'], + }, + ]); + }); + + it('round-trips Map keys with multiple colons in the key segment', () => { + const filters = { + "LogAttributes['k8s.io:annotation:foo']": { + included: new Set(['v1']), + excluded: new Set(), + }, + }; + const [emitted] = filtersToQuery(filters); + const cond = (emitted as { condition: string }).condition; + expect(() => parse(cond)).not.toThrow(); + expect(parseLuceneFilter(cond)).toEqual([ + { + key: 'LogAttributes.k8s.io:annotation:foo', + included: ['v1'], + excluded: [], + }, + ]); + }); + + it('round-trips Map key colon-in-key + colon-in-value simultaneously', () => { + const filters = { + "LogAttributes['foo:bar']": { + included: new Set(['https://example.com:8080']), + excluded: new Set(), + }, + }; + const [emitted] = filtersToQuery(filters); + const cond = (emitted as { condition: string }).condition; + expect(() => parse(cond)).not.toThrow(); + expect(parseLuceneFilter(cond)).toEqual([ + { + key: 'LogAttributes.foo:bar', + included: ['https://example.com:8080'], + excluded: [], + }, + ]); + }); +}); diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index d53d64a23a..e5ded218a4 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -24,6 +24,19 @@ const escapeLuceneQuotedTerm = (s: string) => { return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); }; +/** + * Escape backslashes and colons so the field name survives Lucene parsing. + * Map sub-keys can legitimately contain `:` (e.g. `LogAttributes['foo:bar']` + * normalizes to `LogAttributes.foo:bar` via parseKeyPath().join('.')), and + * `:` is the Lucene field/value separator. Backslashes are escaped first so + * the inserted `\:` survives `encodeSpecialTokens`' `\\` → backslash-literal + * substitution; the encoder's matching `\:` → HDX_COLON rule then makes the + * colon opaque to the parser, and `decodeSpecialTokens` restores the + * original key on the consumer side. + */ +const escapeLuceneFieldName = (key: string): string => + key.replace(/\\/g, '\\\\').replace(/:/g, '\\:'); + export const filtersToQuery = (filters: FilterState): Filter[] => { return Object.entries(filters) .filter( @@ -34,7 +47,7 @@ export const filtersToQuery = (filters: FilterState): Filter[] => { ) .flatMap(([key, values]) => { const conditions: Filter[] = []; - const luceneField = parseKeyPath(key).join('.'); + const luceneField = escapeLuceneFieldName(parseKeyPath(key).join('.')); if (values.included.size > 0) { const terms = Array.from(values.included).map( @@ -79,10 +92,10 @@ function collectFromAst( if (isNodeTerm(ast)) { if (!ast.quoted) return; const negated = ast.field.startsWith('-'); - const field = negated ? ast.field.slice(1) : ast.field; + const field = decodeSpecialTokens(negated ? ast.field.slice(1) : ast.field); terms.push({ field, value: decodeSpecialTokens(ast.term), negated }); } else if (isNodeRangedTerm(ast)) { - const field = ast.field; + const field = decodeSpecialTokens(ast.field); const min = parseFloat(ast.term_min); const max = parseFloat(ast.term_max); if (!isNaN(min) && !isNaN(max)) { diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index de783790a0..2cabcba99d 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -1720,9 +1720,14 @@ async function nodeTerm( serializer: Serializer, context: SerializerContext, ): Promise { - const field = node.field[0] === '-' ? node.field.slice(1) : node.field; - let isNegatedField = node.field[0] === '-'; const isImplicitField = node.field === IMPLICIT_FIELD; + const rawField = node.field[0] === '-' ? node.field.slice(1) : node.field; + // Decode special-token placeholders the emitter inserted in the field name + // (e.g. HDX_COLON for an escaped `:` in a Map sub-key). Leave the implicit + // field sentinel untouched so downstream comparisons against IMPLICIT_FIELD + // still match. + const field = isImplicitField ? rawField : decodeSpecialTokens(rawField); + let isNegatedField = node.field[0] === '-'; // NodeTerm if (isNodeTerm(node)) { @@ -1824,7 +1829,7 @@ function createSerializerContext( return { ...currentContext, - implicitColumnExpression: fieldWithoutNegation, + implicitColumnExpression: decodeSpecialTokens(fieldWithoutNegation), ...(isNegatedAndParenthesized(ast) ? { isNegatedAndParenthesized: true } : {}),