feat: field filtering and priority classification

.changeset/bright-queens-shop.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+feat: field filtering and priority classification for Event Deltas

packages/app/src/components/DBDeltaChart.tsx

@@ -5,7 +5,15 @@ import {
   ChartConfigWithOptDateRange,
   Filter,
 } from '@hyperdx/common-utils/dist/types';
-import { Box, Code, Container, Flex, Pagination, Text } from '@mantine/core';
+import {
+  Box,
+  Code,
+  Container,
+  Divider,
+  Flex,
+  Pagination,
+  Text,
+} from '@mantine/core';
 import { useElementSize } from '@mantine/hooks';
 
 import { isAggregateFunction } from '@/ChartUtils';
@@ -15,6 +23,8 @@ import { getFirstTimestampValueExpression } from '@/source';
 import { SQLPreview } from './ChartSQLPreview';
 import {
   getPropertyStatistics,
+  isDenylisted,
+  isHighCardinality,
   mergeValueStatisticsMaps,
 } from './deltaChartUtils';
 import {
@@ -194,52 +204,87 @@ export default function DBDeltaChart({
     limit: { limit: 1000 },
   });
 
+  // Column metadata for field classification (from ClickHouse response)
+  const columnMeta = useMemo<{ name: string; type: string }[]>(
+    () => outlierData?.meta ?? inlierData?.meta ?? [],
+    [outlierData?.meta, inlierData?.meta],
+  );
+
   // TODO: Is loading state
-  const { sortedProperties, outlierValueOccurences, inlierValueOccurences } =
-    useMemo(() => {
-      const { percentageOccurences: outlierValueOccurences } =
-        getPropertyStatistics(outlierData?.data ?? []);
-
-      const { percentageOccurences: inlierValueOccurences } =
-        getPropertyStatistics(inlierData?.data ?? []);
-
-      // Get all the unique keys from the outliers
-      let uniqueKeys = new Set([...outlierValueOccurences.keys()]);
-      // If there's no outliers, use inliers as the unique keys
-      if (uniqueKeys.size === 0) {
-        uniqueKeys = new Set([...inlierValueOccurences.keys()]);
+  const {
+    visibleProperties,
+    hiddenProperties,
+    outlierValueOccurences,
+    inlierValueOccurences,
+  } = useMemo(() => {
+    const {
+      percentageOccurences: outlierValueOccurences,
+      propertyOccurences: outlierPropertyOccurences,
+      valueOccurences: outlierRawValueOccurences,
+    } = getPropertyStatistics(outlierData?.data ?? []);
+
+    const {
+      percentageOccurences: inlierValueOccurences,
+      propertyOccurences: inlierPropertyOccurences,
+      valueOccurences: inlierRawValueOccurences,
+    } = getPropertyStatistics(inlierData?.data ?? []);
+
+    // Get all the unique keys from the outliers
+    let uniqueKeys = new Set([...outlierValueOccurences.keys()]);
+    // If there's no outliers, use inliers as the unique keys
+    if (uniqueKeys.size === 0) {
+      uniqueKeys = new Set([...inlierValueOccurences.keys()]);
+    }
+    // Now process the keys to find the ones with the highest delta between outlier and inlier percentages
+    const sortedProperties = Array.from(uniqueKeys)
+      .map(key => {
+        const inlierCount =
+          inlierValueOccurences.get(key) ?? new Map<string, number>();
+        const outlierCount =
+          outlierValueOccurences.get(key) ?? new Map<string, number>();
+
+        const mergedArray = mergeValueStatisticsMaps(outlierCount, inlierCount);
+        let maxValueDelta = 0;
+        mergedArray.forEach(item => {
+          const delta = Math.abs(item.outlierCount - item.inlierCount);
+          if (delta > maxValueDelta) {
+            maxValueDelta = delta;
+          }
+        });
+
+        return [key, maxValueDelta] as const;
+      })
+      .sort((a, b) => b[1] - a[1])
+      .map(a => a[0]);
+
+    // Split into visible (primary) and hidden (denylisted + high cardinality)
+    const visibleProperties: string[] = [];
+    const hiddenProperties: string[] = [];
+    sortedProperties.forEach(key => {
+      if (isDenylisted(key, columnMeta)) {
+        hiddenProperties.push(key);
+      } else if (
+        isHighCardinality(
+          key,
+          outlierRawValueOccurences,
+          inlierRawValueOccurences,
+          outlierPropertyOccurences,
+          inlierPropertyOccurences,
+        )
+      ) {
+        hiddenProperties.push(key);
+      } else {
+        visibleProperties.push(key);
       }
-      // Now process the keys to find the ones with the highest delta between outlier and inlier percentages
-      const sortedProperties = Array.from(uniqueKeys)
-        .map(key => {
-          const inlierCount =
-            inlierValueOccurences.get(key) ?? new Map<string, number>();
-          const outlierCount =
-            outlierValueOccurences.get(key) ?? new Map<string, number>();
-
-          const mergedArray = mergeValueStatisticsMaps(
-            outlierCount,
-            inlierCount,
-          );
-          let maxValueDelta = 0;
-          mergedArray.forEach(item => {
-            const delta = Math.abs(item.outlierCount - item.inlierCount);
-            if (delta > maxValueDelta) {
-              maxValueDelta = delta;
-            }
-          });
-
-          return [key, maxValueDelta] as const;
-        })
-        .sort((a, b) => b[1] - a[1])
-        .map(a => a[0]);
-
-      return {
-        sortedProperties,
-        outlierValueOccurences,
-        inlierValueOccurences,
-      };
-    }, [outlierData?.data, inlierData?.data]);
+    });
+
+    return {
+      visibleProperties,
+      hiddenProperties,
+      outlierValueOccurences,
+      inlierValueOccurences,
+    };
+  }, [outlierData?.data, inlierData?.data, columnMeta]);
 
   const [activePage, setPage] = useState(1);
 
@@ -260,11 +305,9 @@ export default function DBDeltaChart({
         (CHART_HEIGHT + CHART_GAP),
     ),
   );
-  const PAGE_SIZE = columns * rows;
-
   useEffect(() => {
     setPage(1);
-  }, [PAGE_SIZE, xMin, xMax, yMin, yMax]);
+  }, [columns, rows, xMin, xMax, yMin, yMax]);
 
   if (error) {
     return (
@@ -301,7 +344,37 @@ export default function DBDeltaChart({
     );
   }
 
-  const totalPages = Math.ceil(sortedProperties.length / PAGE_SIZE);
+  // Row-based pagination: visible and hidden sections are separate grids,
+  // so an incomplete last row of visible fields does not get "filled" with hidden items.
+  const visibleRows = Math.ceil(visibleProperties.length / columns);
+  const hiddenRows = Math.ceil(hiddenProperties.length / columns);
+  const totalRows = visibleRows + hiddenRows;
+  const totalPages = Math.ceil(totalRows / rows);
+
+  const pageRowStart = (activePage - 1) * rows;
+  const pageRowEnd = activePage * rows;
+
+  // Rows occupied by the visible section on this page
+  const visRowStart = Math.min(pageRowStart, visibleRows);
+  const visRowEnd = Math.min(pageRowEnd, visibleRows);
+  const visibleOnPage = visibleProperties.slice(
+    visRowStart * columns,
+    Math.min(visRowEnd * columns, visibleProperties.length),
+  );
+
+  // Rows occupied by the hidden section on this page
+  const hidRowStart = Math.max(0, pageRowStart - visibleRows);
+  const hidRowEnd = Math.min(hiddenRows, Math.max(0, pageRowEnd - visibleRows));
+  const hiddenOnPage = hiddenProperties.slice(
+    hidRowStart * columns,
+    Math.min(hidRowEnd * columns, hiddenProperties.length),
+  );
+
+  // Show a divider when both sections appear on the same page
+  const showDivider = visibleOnPage.length > 0 && hiddenOnPage.length > 0;
+  // Show a header when ONLY hidden fields appear on this page (no divider above)
+  const showHiddenHeader =
+    hiddenOnPage.length > 0 && visibleOnPage.length === 0;
 
   return (
     <Box
@@ -314,16 +387,16 @@ export default function DBDeltaChart({
         flexDirection: 'column',
       }}
     >
-      <div
-        style={{
-          display: 'grid',
-          gridTemplateColumns: `repeat(${columns}, 1fr)`,
-          gap: CHART_GAP,
-        }}
-      >
-        {sortedProperties
-          .slice((activePage - 1) * PAGE_SIZE, activePage * PAGE_SIZE)
-          .map(property => (
+      {/* Primary fields */}
+      {visibleOnPage.length > 0 && (
+        <div
+          style={{
+            display: 'grid',
+            gridTemplateColumns: `repeat(${columns}, 1fr)`,
+            gap: CHART_GAP,
+          }}
+        >
+          {visibleOnPage.map(property => (
             <PropertyComparisonChart
               name={property}
               outlierValueOccurences={
@@ -335,7 +408,44 @@ export default function DBDeltaChart({
               key={property}
             />
           ))}
-      </div>
+        </div>
+      )}
+      {/* Divider between primary and lower-priority fields */}
+      {(showDivider || showHiddenHeader) && (
+        <Divider
+          mt="lg"
+          mb="xs"
+          label={
+            <Text size="xs" c="dimmed">
+              Lower-priority fields ({hiddenProperties.length})
+            </Text>
+          }
+          labelPosition="left"
+        />
+      )}
+      {/* Lower-priority fields — separate grid so rows align independently */}
+      {hiddenOnPage.length > 0 && (
+        <div
+          style={{
+            display: 'grid',
+            gridTemplateColumns: `repeat(${columns}, 1fr)`,
+            gap: CHART_GAP,
+          }}
+        >
+          {hiddenOnPage.map(key => (
+            <PropertyComparisonChart
+              name={key}
+              outlierValueOccurences={
+                outlierValueOccurences.get(key) ?? new Map()
+              }
+              inlierValueOccurences={
+                inlierValueOccurences.get(key) ?? new Map()
+              }
+              key={key}
+            />
+          ))}
+        </div>
+      )}
       <Flex
         justify="flex-end"
         style={{