fix: escape colons in Lucene field names for Map sub-key filters

.changeset/lucene-colon-in-map-key.md

@@ -0,0 +1,10 @@
+---
+"@hyperdx/common-utils": patch
+---
+
+fix: escape colons in Lucene field names so filters on Map sub-keys containing
+`:` (e.g. `LogAttributes['foo:bar']`) parse correctly
+
+`filtersToQuery` now backslash-escapes `:` and `\` in the emitted Lucene field
+name, and `parseLuceneFilter` + the SQL serializer decode those placeholders
+when consuming the AST so the original key is restored end-to-end.

packages/common-utils/src/__tests__/colon-in-mapkey.test.ts

@@ -0,0 +1,112 @@
+import { filtersToQuery, parseLuceneFilter } from '@/filters';
+import { parse } from '@/queryParser';
+
+describe('Colon-in-MAP-KEY scenario (LogAttributes with key containing colon)', () => {
+  it('parses Lucene when the colon in a field name is properly escaped', () => {
+    expect(() =>
+      parse(String.raw`LogAttributes.foo\:bar:"value"`),
+    ).not.toThrow();
+  });
+
+  it('filtersToQuery emits parseable Lucene for bracket-form Map key containing colon', () => {
+    const filters = {
+      "LogAttributes['foo:bar']": {
+        included: new Set<string | boolean>(['value1']),
+        excluded: new Set<string | boolean>(),
+      },
+    };
+    const result = filtersToQuery(filters);
+    const condition = (result[0] as { condition: string }).condition;
+    expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`);
+    expect(() => parse(condition)).not.toThrow();
+  });
+
+  it('filtersToQuery emits parseable Lucene for dot-form Map key containing colon', () => {
+    const filters = {
+      'LogAttributes.foo:bar': {
+        included: new Set<string | boolean>(['value1']),
+        excluded: new Set<string | boolean>(),
+      },
+    };
+    const result = filtersToQuery(filters);
+    const condition = (result[0] as { condition: string }).condition;
+    expect(condition).toBe(String.raw`LogAttributes.foo\:bar:"value1"`);
+    expect(() => parse(condition)).not.toThrow();
+  });
+
+  it('round-trips Map keys containing colons through filtersToQuery + parseLuceneFilter', () => {
+    const filters = {
+      "LogAttributes['foo:bar']": {
+        included: new Set<string | boolean>(['value1']),
+        excluded: new Set<string | boolean>(),
+      },
+    };
+    const [emitted] = filtersToQuery(filters);
+    const cond = (emitted as { condition: string }).condition;
+    const parsed = parseLuceneFilter(cond);
+    expect(parsed).toEqual([
+      {
+        key: 'LogAttributes.foo:bar',
+        included: ['value1'],
+        excluded: [],
+      },
+    ]);
+  });
+
+  it('round-trips Map keys with colons and excluded values', () => {
+    const filters = {
+      'LogAttributes.foo:bar': {
+        included: new Set<string | boolean>(),
+        excluded: new Set<string | boolean>(['unwanted:value']),
+      },
+    };
+    const [emitted] = filtersToQuery(filters);
+    const cond = (emitted as { condition: string }).condition;
+    expect(cond).toBe(String.raw`-LogAttributes.foo\:bar:"unwanted:value"`);
+    expect(parseLuceneFilter(cond)).toEqual([
+      {
+        key: 'LogAttributes.foo:bar',
+        included: [],
+        excluded: ['unwanted:value'],
+      },
+    ]);
+  });
+
+  it('round-trips Map keys with multiple colons in the key segment', () => {
+    const filters = {
+      "LogAttributes['k8s.io:annotation:foo']": {
+        included: new Set<string | boolean>(['v1']),
+        excluded: new Set<string | boolean>(),
+      },
+    };
+    const [emitted] = filtersToQuery(filters);
+    const cond = (emitted as { condition: string }).condition;
+    expect(() => parse(cond)).not.toThrow();
+    expect(parseLuceneFilter(cond)).toEqual([
+      {
+        key: 'LogAttributes.k8s.io:annotation:foo',
+        included: ['v1'],
+        excluded: [],
+      },
+    ]);
+  });
+
+  it('round-trips Map key colon-in-key + colon-in-value simultaneously', () => {
+    const filters = {
+      "LogAttributes['foo:bar']": {
+        included: new Set<string | boolean>(['https://example.com:8080']),
+        excluded: new Set<string | boolean>(),
+      },
+    };
+    const [emitted] = filtersToQuery(filters);
+    const cond = (emitted as { condition: string }).condition;
+    expect(() => parse(cond)).not.toThrow();
+    expect(parseLuceneFilter(cond)).toEqual([
+      {
+        key: 'LogAttributes.foo:bar',
+        included: ['https://example.com:8080'],
+        excluded: [],
+      },
+    ]);
+  });
+});

packages/common-utils/src/filters.ts

@@ -24,6 +24,19 @@ const escapeLuceneQuotedTerm = (s: string) => {
   return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
 };
 
+/**
+ * Escape backslashes and colons so the field name survives Lucene parsing.
+ * Map sub-keys can legitimately contain `:` (e.g. `LogAttributes['foo:bar']`
+ * normalizes to `LogAttributes.foo:bar` via parseKeyPath().join('.')), and
+ * `:` is the Lucene field/value separator. Backslashes are escaped first so
+ * the inserted `\:` survives `encodeSpecialTokens`' `\\` → backslash-literal
+ * substitution; the encoder's matching `\:` → HDX_COLON rule then makes the
+ * colon opaque to the parser, and `decodeSpecialTokens` restores the
+ * original key on the consumer side.
+ */
+const escapeLuceneFieldName = (key: string): string =>
+  key.replace(/\\/g, '\\\\').replace(/:/g, '\\:');
+
 export const filtersToQuery = (filters: FilterState): Filter[] => {
   return Object.entries(filters)
     .filter(
@@ -34,7 +47,7 @@ export const filtersToQuery = (filters: FilterState): Filter[] => {
     )
     .flatMap(([key, values]) => {
       const conditions: Filter[] = [];
-      const luceneField = parseKeyPath(key).join('.');
+      const luceneField = escapeLuceneFieldName(parseKeyPath(key).join('.'));
 
       if (values.included.size > 0) {
         const terms = Array.from(values.included).map(
@@ -79,10 +92,10 @@ function collectFromAst(
   if (isNodeTerm(ast)) {
     if (!ast.quoted) return;
     const negated = ast.field.startsWith('-');
-    const field = negated ? ast.field.slice(1) : ast.field;
+    const field = decodeSpecialTokens(negated ? ast.field.slice(1) : ast.field);
     terms.push({ field, value: decodeSpecialTokens(ast.term), negated });
   } else if (isNodeRangedTerm(ast)) {
-    const field = ast.field;
+    const field = decodeSpecialTokens(ast.field);
     const min = parseFloat(ast.term_min);
     const max = parseFloat(ast.term_max);
     if (!isNaN(min) && !isNaN(max)) {

packages/common-utils/src/queryParser.ts

@@ -1720,9 +1720,14 @@ async function nodeTerm(
   serializer: Serializer,
   context: SerializerContext,
 ): Promise<string> {
-  const field = node.field[0] === '-' ? node.field.slice(1) : node.field;
-  let isNegatedField = node.field[0] === '-';
   const isImplicitField = node.field === IMPLICIT_FIELD;
+  const rawField = node.field[0] === '-' ? node.field.slice(1) : node.field;
+  // Decode special-token placeholders the emitter inserted in the field name
+  // (e.g. HDX_COLON for an escaped `:` in a Map sub-key). Leave the implicit
+  // field sentinel untouched so downstream comparisons against IMPLICIT_FIELD
+  // still match.
+  const field = isImplicitField ? rawField : decodeSpecialTokens(rawField);
+  let isNegatedField = node.field[0] === '-';
 
   // NodeTerm
   if (isNodeTerm(node)) {
@@ -1824,7 +1829,7 @@ function createSerializerContext(
 
     return {
       ...currentContext,
-      implicitColumnExpression: fieldWithoutNegation,
+      implicitColumnExpression: decodeSpecialTokens(fieldWithoutNegation),
       ...(isNegatedAndParenthesized(ast)
         ? { isNegatedAndParenthesized: true }
         : {}),