fix(app): serialize JSON column filters with dot access (HDX-5085)

.changeset/fix-json-column-filter-bracket-notation.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/app': patch
+---
+
+Fix search page filters on JSON columns generating invalid bracket access. Adding or excluding a value on a sub-key of a JSON-typed column (e.g. `ResourceAttributes.region`) previously serialized to `ResourceAttributes['region']`, which ClickHouse rejects on a JSON column ("First argument for function 'arrayElement' must be array, got 'JSON'"). Sub-keys of JSON columns now serialize with dot access (`ResourceAttributes.\`region\``).

packages/app/src/DBSearchPage.tsx

@@ -29,6 +29,8 @@ import HyperDX from '@hyperdx/browser';
 import {
   ClickHouseQueryError,
   ColumnMeta,
+  filterColumnMetaByType,
+  JSDataType,
 } from '@hyperdx/common-utils/dist/clickhouse';
 import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
 import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig';
@@ -1318,6 +1320,20 @@ export function DBSearchPage() {
         : new Set<string>(),
     [inputSourceColumns],
   );
+  // JSON-typed columns for the active source. Sub-key filters on these must
+  // serialize as dot access (`ResourceAttributes.`region``) rather than the
+  // bracket access ClickHouse rejects on a JSON column. HDX-5085.
+  const jsonColumns = useMemo(
+    () =>
+      inputSourceColumns
+        ? new Set(
+            filterColumnMetaByType(inputSourceColumns, [JSDataType.JSON])?.map(
+              c => c.name,
+            ) ?? [],
+          )
+        : new Set<string>(),
+    [inputSourceColumns],
+  );
 
   const watchedSource = useWatch({
     control,
@@ -1353,6 +1369,7 @@ export function DBSearchPage() {
     onFilterChange: handleSetFilters,
     dateTimeColumns,
     knownColumns,
+    jsonColumns,
   });
 
   useEffect(() => {

packages/app/src/components/DBSearchPageFilters.tsx

@@ -1138,6 +1138,12 @@ const DBSearchPageFiltersComponent = ({
   const { data: source } = useSource({ id: sourceId });
   const sourceTableConnection = tcFromSource(source);
   const { data: jsonColumns } = useJsonColumns(sourceTableConnection);
+  // Set form for O(1) membership checks when serializing sub-key filters on
+  // JSON columns to dot access. HDX-5085.
+  const jsonColumnsSet = useMemo(
+    () => new Set(jsonColumns ?? []),
+    [jsonColumns],
+  );
 
   // Special case for live tail
   const [dateRange, setDateRange] = useState<[Date, Date]>(
@@ -1458,6 +1464,7 @@ const DBSearchPageFiltersComponent = ({
                 sqlKey: toQuotedClickHouseKeyExpression(
                   child.key,
                   knownColumns,
+                  jsonColumnsSet,
                 ),
               }))}
               selectedValues={group.children.reduce((acc, child) => {
@@ -1524,6 +1531,7 @@ const DBSearchPageFiltersComponent = ({
             const facetSqlKey = toQuotedClickHouseKeyExpression(
               facet.key,
               knownColumns,
+              jsonColumnsSet,
             );
             return (
               <FilterGroup
@@ -1603,6 +1611,7 @@ const DBSearchPageFiltersComponent = ({
       setFilterRange,
       tableMetadata,
       knownColumns,
+      jsonColumnsSet,
       extraFacetKeys,
     ],
   );

packages/app/src/components/DBSearchPageFilters/hooks.ts

@@ -60,6 +60,13 @@ function useFacets({
   );
   const { data: jsonColumns } = useJsonColumns(tableConnection);
   const { data: mapColumns } = useMapColumns(tableConnection);
+  // Set form for JSON-aware key serialization: sub-keys of JSON columns must
+  // render as dot access rather than the bracket access ClickHouse rejects on a
+  // JSON column. HDX-5085.
+  const jsonColumnsSet = useMemo(
+    () => new Set(jsonColumns ?? []),
+    [jsonColumns],
+  );
 
   const {
     data: allFields,
@@ -135,12 +142,16 @@ function useFacets({
 
     const sqlKeyToUiKey = new Map<string, string>();
     const escapedKeysToFetch = keysToFetch.map(key => {
-      const sqlKey = toQuotedClickHouseKeyExpression(key, knownColumns);
+      const sqlKey = toQuotedClickHouseKeyExpression(
+        key,
+        knownColumns,
+        jsonColumnsSet,
+      );
       sqlKeyToUiKey.set(sqlKey, key);
       return sqlKey;
     });
     return { escapedKeysToFetch, sqlKeyToUiKey };
-  }, [isColumnsLoading, keysToFetch, knownColumns]);
+  }, [isColumnsLoading, keysToFetch, knownColumns, jsonColumnsSet]);
 
   const facetsChartConfig = useMemo(
     () =>
@@ -174,7 +185,11 @@ function useFacets({
   const loadMoreFacetsForKey = useCallback(
     async (key: string): Promise<Facet | undefined> => {
       try {
-        const sqlKey = toQuotedClickHouseKeyExpression(key, knownColumns);
+        const sqlKey = toQuotedClickHouseKeyExpression(
+          key,
+          knownColumns,
+          jsonColumnsSet,
+        );
         if (mode === 'exact') {
           const strippedFilterState: FilterState = { ...filterState };
           delete strippedFilterState[key];
@@ -184,7 +199,11 @@ function useFacets({
               ...chartConfig,
               dateRange,
               filters: filtersToQuery(
-                escapeFilterStateKeys(strippedFilterState, knownColumns),
+                escapeFilterStateKeys(
+                  strippedFilterState,
+                  knownColumns,
+                  jsonColumnsSet,
+                ),
                 { dateTimeColumns },
               ),
             },
@@ -231,6 +250,7 @@ function useFacets({
       chartConfig,
       dateRange,
       knownColumns,
+      jsonColumnsSet,
       source,
       filterState,
       dateTimeColumns,

packages/app/src/components/DBSearchPageFilters/jsonAddToFilter.pipeline.test.ts

@@ -165,3 +165,79 @@ describe('parsed-JSON "Add to Filters" -> valid SQL pipeline (HDX-4427)', () =>
     expect(entry.included).toEqual(new Set(["O'Brien"]));
   });
 });
+
+// HDX-5085: filtering a sub-key of a *native* JSON column (e.g. from clicking
+// "Add to Filters" then "Exclude" in the sidebar) must serialize as dot access,
+// not the bracket access ClickHouse rejects on a JSON column with "First
+// argument for function 'arrayElement' must be array, got 'JSON'".
+describe('JSON-column sub-key filter -> valid dot-access SQL pipeline (HDX-5085)', () => {
+  // ResourceAttributes is a native JSON column here (unlike the Map columns
+  // above), so its sub-keys must render with dot access.
+  const jsonKnownColumns = new Set(['ResourceAttributes', 'ServiceName']);
+  const jsonColumns = new Set(['ResourceAttributes']);
+
+  const runJsonFilter = (
+    key: string,
+    values: { included?: string[]; excluded?: string[] },
+  ) => {
+    const state: FilterState = {
+      [key]: {
+        included: new Set<string | boolean>(values.included ?? []),
+        excluded: new Set<string | boolean>(values.excluded ?? []),
+      },
+    };
+    return filtersToQuery(
+      escapeFilterStateKeys(state, jsonKnownColumns, jsonColumns),
+    );
+  };
+
+  it('excludes a JSON sub-key value with dot access, not bracket access', () => {
+    expect(
+      runJsonFilter('ResourceAttributes.region', {
+        excluded: ['eu-central-1'],
+      }),
+    ).toEqual([
+      {
+        type: 'sql',
+        condition: "ResourceAttributes.`region` NOT IN ('eu-central-1')",
+      },
+    ]);
+  });
+
+  it('never emits the illegal bracket subscript on a JSON column', () => {
+    const query = runJsonFilter('ResourceAttributes.region', {
+      included: ['us-east-1'],
+      excluded: ['eu-central-1'],
+    });
+    expect(query.length).toBeGreaterThan(0);
+    for (const filter of query) {
+      expect(filter.type).toBe('sql');
+      if (filter.type === 'sql') {
+        expect(filter.condition).not.toContain("ResourceAttributes['");
+        expect(isValidFilterCondition(filter.condition, 'sql')).toBe(true);
+      }
+    }
+  });
+
+  it('renders a nested JSON path with backtick-quoted segments', () => {
+    expect(
+      runJsonFilter('ResourceAttributes.host.name', { included: ['web-1'] }),
+    ).toEqual([
+      {
+        type: 'sql',
+        condition: "ResourceAttributes.`host`.`name` IN ('web-1')",
+      },
+    ]);
+  });
+
+  it('round-trips the JSON sub-key back to its clean dot form', () => {
+    const query = runJsonFilter('ResourceAttributes.region', {
+      excluded: ['eu-central-1'],
+    });
+    const back = parseQuery(query).filters;
+    const restoredKeys = Object.keys(back).map(cleanClickHouseExpression);
+    expect(restoredKeys).toEqual(['ResourceAttributes.region']);
+    const entry = back[Object.keys(back)[0]];
+    expect(entry.excluded).toEqual(new Set(['eu-central-1']));
+  });
+});

packages/app/src/components/DBSearchPageFilters/utils.test.ts

@@ -181,6 +181,45 @@ describe('toClickHouseKeyExpression', () => {
     );
   });
 
+  // HDX-5085: a sub-key on a JSON-typed column must serialize as dot access,
+  // not bracket access. ClickHouse rejects `ResourceAttributes['region']` on a
+  // JSON column with "First argument for function 'arrayElement' must be array,
+  // got 'JSON'". When the base column is known to be JSON, emit dot notation
+  // with backtick-quoted segments instead.
+  describe('JSON columns (HDX-5085)', () => {
+    const jsonColumns = new Set(['ResourceAttributes']);
+
+    it('rewrites a dot-form JSON sub-key to dot access, not bracket access', () => {
+      expect(
+        toClickHouseKeyExpression('ResourceAttributes.region', jsonColumns),
+      ).toBe('ResourceAttributes.`region`');
+    });
+
+    it('renders a nested JSON path with each segment backtick-quoted', () => {
+      expect(
+        toClickHouseKeyExpression('ResourceAttributes.host.name', jsonColumns),
+      ).toBe('ResourceAttributes.`host`.`name`');
+    });
+
+    it('still uses bracket form for a Map column not listed as JSON', () => {
+      expect(
+        toClickHouseKeyExpression('LogAttributes.region', jsonColumns),
+      ).toBe("LogAttributes['region']");
+    });
+
+    it('falls back to bracket form when no JSON columns are provided', () => {
+      expect(toClickHouseKeyExpression('ResourceAttributes.region')).toBe(
+        "ResourceAttributes['region']",
+      );
+    });
+
+    it('leaves an already-backtick JSON path unchanged', () => {
+      expect(
+        toClickHouseKeyExpression('ResourceAttributes.`region`', jsonColumns),
+      ).toBe('ResourceAttributes.`region`');
+    });
+  });
+
   // HDX-4427: "Add to Filters" on a value inside parsed JSON from a String
   // column builds a JSONExtract* function call as the filter key. These are
   // already valid ClickHouse expressions and must pass through untouched.
@@ -320,4 +359,41 @@ describe('toQuotedClickHouseKeyExpression', () => {
       ).toBe("LogAttributes['host.name']");
     });
   });
+
+  // HDX-5085: a sub-key on a JSON column must serialize as dot access.
+  describe('with jsonColumns (JSON-aware)', () => {
+    const cols = new Set(['ResourceAttributes']);
+    const jsonCols = new Set(['ResourceAttributes']);
+
+    it('renders a JSON sub-key as dot access instead of bracket access', () => {
+      expect(
+        toQuotedClickHouseKeyExpression(
+          'ResourceAttributes.region',
+          cols,
+          jsonCols,
+        ),
+      ).toBe('ResourceAttributes.`region`');
+    });
+
+    it('renders a nested JSON path with backtick-quoted segments', () => {
+      expect(
+        toQuotedClickHouseKeyExpression(
+          'ResourceAttributes.host.name',
+          cols,
+          jsonCols,
+        ),
+      ).toBe('ResourceAttributes.`host`.`name`');
+    });
+
+    it('uses bracket access for a Map column absent from jsonColumns', () => {
+      const mapCols = new Set(['LogAttributes']);
+      expect(
+        toQuotedClickHouseKeyExpression(
+          'LogAttributes.region',
+          mapCols,
+          jsonCols,
+        ),
+      ).toBe("LogAttributes['region']");
+    });
+  });
 });