feat: emit Lucene filters for KV items direct_read optimization

.changeset/filters-direct-read.md

@@ -0,0 +1,9 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/app": minor
+---
+
+feat: emit Lucene conditions from sidebar/dashboard filters to enable KV items direct_read optimization on Map columns
+
+Legacy `type: 'sql'` filters in URLs are automatically migrated to Lucene
+on page load. The persisted `DashboardFilter.expression` in MongoDB is unchanged.

AGENTS.md

@@ -55,6 +55,19 @@ and lint issues across all packages. Pre-commit hooks handle this when
 committing, but if you finish edits without committing, run `yarn lint:fix`
 before stopping.
 
+## Before Writing New Utility Functions
+
+Before implementing a new parsing, formatting, or transformation function,
+search for existing functions that already do the same thing or something close.
+Common places to check:
+
+- `packages/common-utils/src/core/metadata.ts` — key/column parsing utilities
+  (e.g. `parseKeyPath`)
+- `packages/common-utils/src/queryParser.ts` — Lucene parsing (`parse`,
+  `decodeSpecialTokens`), AST type guards, serialization helpers
+- `packages/common-utils/src/filters.ts` — filter state serialization and
+  parsing (`filtersToQuery`, `parseLuceneFilter`)
+
 ## Key Principles
 
 1. **Multi-tenancy**: All data is scoped to `Team` - ensure proper filtering

packages/api/src/utils/zod.ts

@@ -146,7 +146,7 @@ export type ExternalDashboardFilter = z.infer<
 >;
 
 export const externalDashboardSavedFilterValueSchema = z.object({
-  type: z.literal('sql').optional().default('sql'),
+  type: z.enum(['sql', 'lucene']).optional().default('sql'),
   condition: z.string().max(10000),
 });
 

packages/app/src/ChartUtils.tsx

@@ -17,6 +17,10 @@ import {
   getAlignedDateRange,
   Granularity,
 } from '@hyperdx/common-utils/dist/core/utils';
+import {
+  type FilterState,
+  filtersToQuery,
+} from '@hyperdx/common-utils/dist/filters';
 import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards';
 import {
   AggregateFunction as AggFnV2,
@@ -923,13 +927,19 @@ export function buildEventsSearchUrl({
 
   // Add group-by column filters
   if (groupFilters && groupFilters.length > 0) {
-    groupFilters.forEach(({ column, value }) => {
+    const filterState: FilterState = {};
+    for (const { column, value } of groupFilters) {
       if (column && value != null) {
-        // Can't use SQLString.escape here because the search endpoint relies on exist match for UI
-        const condition = `${column} IN (${SqlString.escape(value)})`;
-        additionalFilters.push({ type: 'sql', condition });
+        if (!filterState[column]) {
+          filterState[column] = {
+            included: new Set(),
+            excluded: new Set(),
+          };
+        }
+        filterState[column].included.add(String(value));
       }
-    });
+    }
+    additionalFilters.push(...filtersToQuery(filterState));
   }
 
   // Add Y-axis value range filter (±threshold) for charts