From ddf937782f2e848331604324437a6531f9c6c824 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev Date: Mon, 2 Mar 2026 09:21:32 -0800 Subject: [PATCH 1/2] feat: field filtering and priority classification (#1825) - deltaChartUtils: add stripTypeWrappers, isIdField, isTimestampArrayField, isDenylisted, isHighCardinality classification functions - getPropertyStatistics now also returns propertyOccurences and valueOccurences (needed for cardinality detection) - DBDeltaChart: split properties into visible (primary) and hidden (denylisted + high cardinality) with two separate CSS grids - Row-based pagination: hidden fields always start from column 1 - "Lower-priority fields (N)" divider between sections - Tests: 29 unit tests covering all classification functions Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/app/src/components/DBDeltaChart.tsx | 230 ++++++++---- .../deltaChartFieldClassification.test.ts | 334 ++++++++++++++++++ .../app/src/components/deltaChartUtils.ts | 115 ++++++ 3 files changed, 619 insertions(+), 60 deletions(-) create mode 100644 packages/app/src/components/__tests__/deltaChartFieldClassification.test.ts diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index 9c5e486134..fd5bd68de5 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -5,7 +5,15 @@ import { ChartConfigWithOptDateRange, Filter, } from '@hyperdx/common-utils/dist/types'; -import { Box, Code, Container, Flex, Pagination, Text } from '@mantine/core'; +import { + Box, + Code, + Container, + Divider, + Flex, + Pagination, + Text, +} from '@mantine/core'; import { useElementSize } from '@mantine/hooks'; import { isAggregateFunction } from '@/ChartUtils'; @@ -15,6 +23,8 @@ import { getFirstTimestampValueExpression } from '@/source'; import { SQLPreview } from './ChartSQLPreview'; import { getPropertyStatistics, + isDenylisted, + isHighCardinality, mergeValueStatisticsMaps, } from './deltaChartUtils'; import { @@ -194,52 +204,87 @@ export default function DBDeltaChart({ limit: { limit: 1000 }, }); + // Column metadata for field classification (from ClickHouse response) + const columnMeta = useMemo<{ name: string; type: string }[]>( + () => outlierData?.meta ?? inlierData?.meta ?? [], + [outlierData?.meta, inlierData?.meta], + ); + // TODO: Is loading state - const { sortedProperties, outlierValueOccurences, inlierValueOccurences } = - useMemo(() => { - const { percentageOccurences: outlierValueOccurences } = - getPropertyStatistics(outlierData?.data ?? []); - - const { percentageOccurences: inlierValueOccurences } = - getPropertyStatistics(inlierData?.data ?? []); - - // Get all the unique keys from the outliers - let uniqueKeys = new Set([...outlierValueOccurences.keys()]); - // If there's no outliers, use inliers as the unique keys - if (uniqueKeys.size === 0) { - uniqueKeys = new Set([...inlierValueOccurences.keys()]); + const { + visibleProperties, + hiddenProperties, + outlierValueOccurences, + inlierValueOccurences, + } = useMemo(() => { + const { + percentageOccurences: outlierValueOccurences, + propertyOccurences: outlierPropertyOccurences, + valueOccurences: outlierRawValueOccurences, + } = getPropertyStatistics(outlierData?.data ?? []); + + const { + percentageOccurences: inlierValueOccurences, + propertyOccurences: inlierPropertyOccurences, + valueOccurences: inlierRawValueOccurences, + } = getPropertyStatistics(inlierData?.data ?? []); + + // Get all the unique keys from the outliers + let uniqueKeys = new Set([...outlierValueOccurences.keys()]); + // If there's no outliers, use inliers as the unique keys + if (uniqueKeys.size === 0) { + uniqueKeys = new Set([...inlierValueOccurences.keys()]); + } + // Now process the keys to find the ones with the highest delta between outlier and inlier percentages + const sortedProperties = Array.from(uniqueKeys) + .map(key => { + const inlierCount = + inlierValueOccurences.get(key) ?? new Map(); + const outlierCount = + outlierValueOccurences.get(key) ?? new Map(); + + const mergedArray = mergeValueStatisticsMaps(outlierCount, inlierCount); + let maxValueDelta = 0; + mergedArray.forEach(item => { + const delta = Math.abs(item.outlierCount - item.inlierCount); + if (delta > maxValueDelta) { + maxValueDelta = delta; + } + }); + + return [key, maxValueDelta] as const; + }) + .sort((a, b) => b[1] - a[1]) + .map(a => a[0]); + + // Split into visible (primary) and hidden (denylisted + high cardinality) + const visibleProperties: string[] = []; + const hiddenProperties: string[] = []; + sortedProperties.forEach(key => { + if (isDenylisted(key, columnMeta)) { + hiddenProperties.push(key); + } else if ( + isHighCardinality( + key, + outlierRawValueOccurences, + inlierRawValueOccurences, + outlierPropertyOccurences, + inlierPropertyOccurences, + ) + ) { + hiddenProperties.push(key); + } else { + visibleProperties.push(key); } - // Now process the keys to find the ones with the highest delta between outlier and inlier percentages - const sortedProperties = Array.from(uniqueKeys) - .map(key => { - const inlierCount = - inlierValueOccurences.get(key) ?? new Map(); - const outlierCount = - outlierValueOccurences.get(key) ?? new Map(); - - const mergedArray = mergeValueStatisticsMaps( - outlierCount, - inlierCount, - ); - let maxValueDelta = 0; - mergedArray.forEach(item => { - const delta = Math.abs(item.outlierCount - item.inlierCount); - if (delta > maxValueDelta) { - maxValueDelta = delta; - } - }); - - return [key, maxValueDelta] as const; - }) - .sort((a, b) => b[1] - a[1]) - .map(a => a[0]); - - return { - sortedProperties, - outlierValueOccurences, - inlierValueOccurences, - }; - }, [outlierData?.data, inlierData?.data]); + }); + + return { + visibleProperties, + hiddenProperties, + outlierValueOccurences, + inlierValueOccurences, + }; + }, [outlierData?.data, inlierData?.data, columnMeta]); const [activePage, setPage] = useState(1); @@ -260,11 +305,9 @@ export default function DBDeltaChart({ (CHART_HEIGHT + CHART_GAP), ), ); - const PAGE_SIZE = columns * rows; - useEffect(() => { setPage(1); - }, [PAGE_SIZE, xMin, xMax, yMin, yMax]); + }, [columns, rows, xMin, xMax, yMin, yMax]); if (error) { return ( @@ -301,7 +344,37 @@ export default function DBDeltaChart({ ); } - const totalPages = Math.ceil(sortedProperties.length / PAGE_SIZE); + // Row-based pagination: visible and hidden sections are separate grids, + // so an incomplete last row of visible fields does not get "filled" with hidden items. + const visibleRows = Math.ceil(visibleProperties.length / columns); + const hiddenRows = Math.ceil(hiddenProperties.length / columns); + const totalRows = visibleRows + hiddenRows; + const totalPages = Math.ceil(totalRows / rows); + + const pageRowStart = (activePage - 1) * rows; + const pageRowEnd = activePage * rows; + + // Rows occupied by the visible section on this page + const visRowStart = Math.min(pageRowStart, visibleRows); + const visRowEnd = Math.min(pageRowEnd, visibleRows); + const visibleOnPage = visibleProperties.slice( + visRowStart * columns, + Math.min(visRowEnd * columns, visibleProperties.length), + ); + + // Rows occupied by the hidden section on this page + const hidRowStart = Math.max(0, pageRowStart - visibleRows); + const hidRowEnd = Math.min(hiddenRows, Math.max(0, pageRowEnd - visibleRows)); + const hiddenOnPage = hiddenProperties.slice( + hidRowStart * columns, + Math.min(hidRowEnd * columns, hiddenProperties.length), + ); + + // Show a divider when both sections appear on the same page + const showDivider = visibleOnPage.length > 0 && hiddenOnPage.length > 0; + // Show a header when ONLY hidden fields appear on this page (no divider above) + const showHiddenHeader = + hiddenOnPage.length > 0 && visibleOnPage.length === 0; return ( -
- {sortedProperties - .slice((activePage - 1) * PAGE_SIZE, activePage * PAGE_SIZE) - .map(property => ( + {/* Primary fields */} + {visibleOnPage.length > 0 && ( +
+ {visibleOnPage.map(property => ( ))} -
+
+ )} + {/* Divider between primary and lower-priority fields */} + {(showDivider || showHiddenHeader) && ( + + Lower-priority fields ({hiddenProperties.length}) + + } + labelPosition="left" + /> + )} + {/* Lower-priority fields — separate grid so rows align independently */} + {hiddenOnPage.length > 0 && ( +
+ {hiddenOnPage.map(key => ( + + ))} +
+ )} { + it('removes LowCardinality wrapper', () => { + expect(stripTypeWrappers('LowCardinality(String)')).toBe('String'); + }); + + it('removes Nullable wrapper', () => { + expect(stripTypeWrappers('Nullable(String)')).toBe('String'); + }); + + it('removes nested wrappers', () => { + expect(stripTypeWrappers('LowCardinality(Nullable(String))')).toBe( + 'String', + ); + }); + + it('returns unwrapped type as-is', () => { + expect(stripTypeWrappers('String')).toBe('String'); + expect(stripTypeWrappers('Array(String)')).toBe('Array(String)'); + }); +}); + +describe('isIdField', () => { + it('identifies top-level String columns ending in Id', () => { + expect(isIdField('TraceId', traceColumnMeta)).toBe(true); + expect(isIdField('SpanId', traceColumnMeta)).toBe(true); + expect(isIdField('ParentSpanId', traceColumnMeta)).toBe(true); + }); + + it('identifies Array(String) column elements ending in Id', () => { + expect(isIdField('Links.TraceId[0]', traceColumnMeta)).toBe(true); + expect(isIdField('Links.SpanId[0]', traceColumnMeta)).toBe(true); + expect(isIdField('Links.TraceId[5]', traceColumnMeta)).toBe(true); + }); + + it('identifies plain Array(String) column reference ending in Id', () => { + expect(isIdField('Links.TraceId', traceColumnMeta)).toBe(true); + expect(isIdField('Links.SpanId', traceColumnMeta)).toBe(true); + }); + + it('does not match non-ID String columns', () => { + expect(isIdField('Timestamp', traceColumnMeta)).toBe(false); + expect(isIdField('Events.Name[0]', traceColumnMeta)).toBe(false); + }); + + it('does not match Map or Array(Map) columns even if name ends in Id', () => { + const meta = [{ name: 'MyMapId', type: 'Map(String, String)' }]; + expect(isIdField('MyMapId', meta)).toBe(false); + }); + + it('does not match keys with sub-keys after array index', () => { + expect(isIdField('Events.Attributes[0].spanId', traceColumnMeta)).toBe( + false, + ); + }); + + it('returns false for unknown columns', () => { + expect(isIdField('UnknownId', traceColumnMeta)).toBe(false); + }); + + it('returns false for empty columnMeta', () => { + expect(isIdField('TraceId', [])).toBe(false); + }); +}); + +describe('isTimestampArrayField', () => { + it('identifies Array(DateTime64) column elements by index', () => { + expect(isTimestampArrayField('Events.Timestamp[0]', traceColumnMeta)).toBe( + true, + ); + expect(isTimestampArrayField('Events.Timestamp[23]', traceColumnMeta)).toBe( + true, + ); + expect(isTimestampArrayField('Links.Timestamp[0]', traceColumnMeta)).toBe( + true, + ); + }); + + it('identifies plain Array(DateTime64) column reference', () => { + expect(isTimestampArrayField('Events.Timestamp', traceColumnMeta)).toBe( + true, + ); + expect(isTimestampArrayField('Links.Timestamp', traceColumnMeta)).toBe( + true, + ); + }); + + it('does not match non-DateTime64 array columns', () => { + expect(isTimestampArrayField('Events.Name[0]', traceColumnMeta)).toBe( + false, + ); + expect(isTimestampArrayField('Links.TraceId[0]', traceColumnMeta)).toBe( + false, + ); + }); + + it('does not match non-array DateTime64 columns', () => { + expect(isTimestampArrayField('Timestamp', traceColumnMeta)).toBe(false); + }); + + it('returns false for unknown columns', () => { + expect(isTimestampArrayField('Unknown.Timestamp[0]', traceColumnMeta)).toBe( + false, + ); + }); + + it('handles Array(DateTime64) with timezone parameter', () => { + const meta = [ + { name: 'MyTimestamps', type: "Array(DateTime64(9, 'UTC'))" }, + ]; + expect(isTimestampArrayField('MyTimestamps[0]', meta)).toBe(true); + }); +}); + +describe('isDenylisted', () => { + it('denylists ID fields', () => { + expect(isDenylisted('TraceId', traceColumnMeta)).toBe(true); + expect(isDenylisted('SpanId', traceColumnMeta)).toBe(true); + expect(isDenylisted('ParentSpanId', traceColumnMeta)).toBe(true); + expect(isDenylisted('Links.TraceId[0]', traceColumnMeta)).toBe(true); + }); + + it('denylists timestamp array fields', () => { + expect(isDenylisted('Events.Timestamp[0]', traceColumnMeta)).toBe(true); + expect(isDenylisted('Links.Timestamp[3]', traceColumnMeta)).toBe(true); + }); + + it('does not denylist useful fields', () => { + expect( + isDenylisted('ResourceAttributes.service.name', traceColumnMeta), + ).toBe(false); + expect(isDenylisted('SpanAttributes.http.method', traceColumnMeta)).toBe( + false, + ); + expect(isDenylisted('Events.Name[0]', traceColumnMeta)).toBe(false); + }); +}); + +describe('isHighCardinality', () => { + it('identifies high cardinality fields (all unique values)', () => { + const outlierValues = new Map(); + for (let i = 0; i < 1000; i++) { + outlierValues.set(`value-${i}`, 0.1); + } + const outlierValueOccurences = new Map([['TraceId', outlierValues]]); + const outlierPropertyOccurences = new Map([['TraceId', 1000]]); + + expect( + isHighCardinality( + 'TraceId', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + new Map(), + ), + ).toBe(true); + }); + + it('keeps low cardinality fields visible', () => { + const outlierValues = new Map([ + ['GET', 80], + ['POST', 20], + ]); + const outlierValueOccurences = new Map([['http.method', outlierValues]]); + const outlierPropertyOccurences = new Map([['http.method', 1000]]); + + expect( + isHighCardinality( + 'http.method', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + new Map(), + ), + ).toBe(false); + }); + + it('uses min of both groups — keeps visible if either group has low cardinality', () => { + const outlierValues = new Map([ + ['GET', 80], + ['POST', 20], + ]); + const outlierValueOccurences = new Map([['method', outlierValues]]); + const outlierPropertyOccurences = new Map([['method', 1000]]); + + const inlierValues = new Map(); + for (let i = 0; i < 500; i++) inlierValues.set(`v${i}`, 0.2); + const inlierValueOccurences = new Map([['method', inlierValues]]); + const inlierPropertyOccurences = new Map([['method', 500]]); + + expect( + isHighCardinality( + 'method', + outlierValueOccurences, + inlierValueOccurences, + outlierPropertyOccurences, + inlierPropertyOccurences, + ), + ).toBe(false); + }); + + it('hides field when BOTH groups have high cardinality', () => { + const makeHighCardinalityMap = (n: number) => { + const m = new Map(); + for (let i = 0; i < n; i++) m.set(`v${i}`, 100 / n); + return m; + }; + + const outlierValues = makeHighCardinalityMap(500); + const inlierValues = makeHighCardinalityMap(400); + const outlierValueOccurences = new Map([['url', outlierValues]]); + const inlierValueOccurences = new Map([['url', inlierValues]]); + const outlierPropertyOccurences = new Map([['url', 500]]); + const inlierPropertyOccurences = new Map([['url', 400]]); + + expect( + isHighCardinality( + 'url', + outlierValueOccurences, + inlierValueOccurences, + outlierPropertyOccurences, + inlierPropertyOccurences, + ), + ).toBe(true); + }); + + it('keeps visible when combined sample size is <= 20', () => { + const outlierValues = new Map(); + for (let i = 0; i < 10; i++) outlierValues.set(`v${i}`, 10); + const outlierValueOccurences = new Map([['field', outlierValues]]); + const outlierPropertyOccurences = new Map([['field', 10]]); + const inlierPropertyOccurences = new Map([['field', 10]]); + + expect( + isHighCardinality( + 'field', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + inlierPropertyOccurences, + ), + ).toBe(false); + }); + + it('uses single group uniqueness when other group has no data', () => { + const outlierValues = new Map(); + for (let i = 0; i < 100; i++) outlierValues.set(`v${i}`, 1); + const outlierValueOccurences = new Map([['id', outlierValues]]); + const outlierPropertyOccurences = new Map([['id', 100]]); + + expect( + isHighCardinality( + 'id', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + new Map(), + ), + ).toBe(true); + }); + + it('returns false for field not present in either group', () => { + expect( + isHighCardinality( + 'unknownField', + new Map(), + new Map(), + new Map(), + new Map(), + ), + ).toBe(false); + }); +}); + +describe('field split logic (visible vs hidden)', () => { + it('correctly classifies a mix of ID, timestamp, cardinality, and useful fields', () => { + expect(isDenylisted('TraceId', traceColumnMeta)).toBe(true); + expect(isDenylisted('Events.Timestamp[0]', traceColumnMeta)).toBe(true); + expect( + isDenylisted('ResourceAttributes.service.name', traceColumnMeta), + ).toBe(false); + + // High cardinality field → hidden + const hcValues = new Map(); + for (let i = 0; i < 1000; i++) hcValues.set(`trace-${i}`, 0.1); + expect( + isHighCardinality( + 'trace.id', + new Map([['trace.id', hcValues]]), + new Map(), + new Map([['trace.id', 1000]]), + new Map(), + ), + ).toBe(true); + + // Low cardinality field → visible + const lcValues = new Map([ + ['production', 70], + ['staging', 30], + ]); + expect( + isHighCardinality( + 'deployment.env', + new Map([['deployment.env', lcValues]]), + new Map(), + new Map([['deployment.env', 1000]]), + new Map(), + ), + ).toBe(false); + }); +}); diff --git a/packages/app/src/components/deltaChartUtils.ts b/packages/app/src/components/deltaChartUtils.ts index f22ec17175..8562b7bf49 100644 --- a/packages/app/src/components/deltaChartUtils.ts +++ b/packages/app/src/components/deltaChartUtils.ts @@ -78,6 +78,8 @@ export function getPropertyStatistics(data: Record[]) { return { percentageOccurences, + propertyOccurences, + valueOccurences, }; } @@ -108,3 +110,116 @@ export function mergeValueStatisticsMaps( return mergedArray; } + +// --------------------------------------------------------------------------- +// Field classification helpers +// --------------------------------------------------------------------------- + +/** Removes `LowCardinality(...)` and `Nullable(...)` wrappers from ClickHouse type strings. */ +export function stripTypeWrappers(type: string): string { + let t = type.trim(); + let changed = true; + while (changed) { + changed = false; + if (t.startsWith('LowCardinality(') && t.endsWith(')')) { + t = t.slice('LowCardinality('.length, -1).trim(); + changed = true; + } else if (t.startsWith('Nullable(') && t.endsWith(')')) { + t = t.slice('Nullable('.length, -1).trim(); + changed = true; + } + } + return t; +} + +/** + * Returns true if the field is a structural ID field that should always be hidden. + * Matches top-level String columns or Array(String) elements ending in "Id"/"ID". + */ +export function isIdField( + key: string, + columnMeta: { name: string; type: string }[], +): boolean { + const arrMatch = key.match(/^([^[]+)\[(\d+)\]$/); + const colName = arrMatch ? arrMatch[1] : key.includes('[') ? null : key; + if (!colName) return false; + if (!/(Id|ID)$/.test(colName)) return false; + + const col = columnMeta.find(c => c.name === colName); + if (!col) return false; + const baseType = stripTypeWrappers(col.type); + if (baseType === 'String') return true; + if (baseType.startsWith('Array(')) { + const innerType = stripTypeWrappers(baseType.slice('Array('.length, -1)); + return innerType === 'String'; + } + return false; +} + +/** + * Returns true if the field is a per-index timestamp array element + * (e.g., Events.Timestamp[0]) from a column of type Array(DateTime64(...)). + */ +export function isTimestampArrayField( + key: string, + columnMeta: { name: string; type: string }[], +): boolean { + const arrMatch = key.match(/^([^[]+)\[(\d+)\]$/); + const colName = arrMatch ? arrMatch[1] : key.includes('[') ? null : key; + if (!colName) return false; + + const col = columnMeta.find(c => c.name === colName); + if (!col) return false; + const baseType = stripTypeWrappers(col.type); + if (!baseType.startsWith('Array(')) return false; + const innerType = stripTypeWrappers(baseType.slice('Array('.length, -1)); + return innerType.startsWith('DateTime64('); +} + +/** + * Returns true if the field should always be hidden (ID fields + timestamp arrays). + */ +export function isDenylisted( + key: string, + columnMeta: { name: string; type: string }[], +): boolean { + return isIdField(key, columnMeta) || isTimestampArrayField(key, columnMeta); +} + +/** + * Returns true if the field has high cardinality (most values unique). + * Uses min(outlierUniqueness, inlierUniqueness) > 0.9 with combined sample > 20. + */ +export function isHighCardinality( + key: string, + outlierValueOccurences: Map>, + inlierValueOccurences: Map>, + outlierPropertyOccurences: Map, + inlierPropertyOccurences: Map, +): boolean { + const outlierTotal = outlierPropertyOccurences.get(key) ?? 0; + const inlierTotal = inlierPropertyOccurences.get(key) ?? 0; + const combinedSampleSize = outlierTotal + inlierTotal; + if (combinedSampleSize <= 20) return false; + + const outlierUniqueValues = outlierValueOccurences.get(key)?.size ?? 0; + const inlierUniqueValues = inlierValueOccurences.get(key)?.size ?? 0; + + const outlierUniqueness = + outlierTotal > 0 ? outlierUniqueValues / outlierTotal : null; + const inlierUniqueness = + inlierTotal > 0 ? inlierUniqueValues / inlierTotal : null; + + let effectiveUniqueness: number; + if (outlierUniqueness !== null && inlierUniqueness !== null) { + effectiveUniqueness = Math.min(outlierUniqueness, inlierUniqueness); + } else if (outlierUniqueness !== null) { + effectiveUniqueness = outlierUniqueness; + } else if (inlierUniqueness !== null) { + effectiveUniqueness = inlierUniqueness; + } else { + return false; + } + + return effectiveUniqueness > 0.9; +} From 37f56c2409fa7f2785958a43e40cd64314eb98d9 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev Date: Mon, 2 Mar 2026 19:01:10 -0800 Subject: [PATCH 2/2] refactor: extract getBaseColumnName() helper and add changeset Address review feedback: - Extract duplicated colName parsing into getBaseColumnName() helper - Add unit tests for the new helper (4 tests) - Add missing changeset for the PR Co-Authored-By: Claude Opus 4.6 (1M context) --- .changeset/bright-queens-shop.md | 5 ++++ .../deltaChartFieldClassification.test.ts | 23 +++++++++++++++++++ .../app/src/components/deltaChartUtils.ts | 16 +++++++++---- 3 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 .changeset/bright-queens-shop.md diff --git a/.changeset/bright-queens-shop.md b/.changeset/bright-queens-shop.md new file mode 100644 index 0000000000..38f31ed095 --- /dev/null +++ b/.changeset/bright-queens-shop.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat: field filtering and priority classification for Event Deltas diff --git a/packages/app/src/components/__tests__/deltaChartFieldClassification.test.ts b/packages/app/src/components/__tests__/deltaChartFieldClassification.test.ts index e6c9a60756..9829975f86 100644 --- a/packages/app/src/components/__tests__/deltaChartFieldClassification.test.ts +++ b/packages/app/src/components/__tests__/deltaChartFieldClassification.test.ts @@ -1,4 +1,5 @@ import { + getBaseColumnName, isDenylisted, isHighCardinality, isIdField, @@ -22,6 +23,28 @@ const traceColumnMeta = [ { name: 'Links.Attributes', type: 'Array(Map(String, String))' }, ]; +describe('getBaseColumnName', () => { + it('returns plain key as-is', () => { + expect(getBaseColumnName('TraceId')).toBe('TraceId'); + expect(getBaseColumnName('SpanAttributes.http.method')).toBe( + 'SpanAttributes.http.method', + ); + }); + + it('strips array index from key', () => { + expect(getBaseColumnName('Events.Name[0]')).toBe('Events.Name'); + expect(getBaseColumnName('Links.TraceId[5]')).toBe('Links.TraceId'); + }); + + it('returns null for keys with sub-keys after array index', () => { + expect(getBaseColumnName('Events.Attributes[0].spanId')).toBeNull(); + }); + + it('returns null for malformed array keys', () => { + expect(getBaseColumnName('Events[0][1]')).toBeNull(); + }); +}); + describe('stripTypeWrappers', () => { it('removes LowCardinality wrapper', () => { expect(stripTypeWrappers('LowCardinality(String)')).toBe('String'); diff --git a/packages/app/src/components/deltaChartUtils.ts b/packages/app/src/components/deltaChartUtils.ts index 8562b7bf49..2fa48451d4 100644 --- a/packages/app/src/components/deltaChartUtils.ts +++ b/packages/app/src/components/deltaChartUtils.ts @@ -115,6 +115,16 @@ export function mergeValueStatisticsMaps( // Field classification helpers // --------------------------------------------------------------------------- +/** + * Extracts the base column name from a flattened key. + * Strips array indices (e.g., "Events.Name[0]" → "Events.Name"). + * Returns null for keys with sub-keys after array indices (e.g., "Events.Attributes[0].spanId"). + */ +export function getBaseColumnName(key: string): string | null { + const arrMatch = key.match(/^([^[]+)\[(\d+)\]$/); + return arrMatch ? arrMatch[1] : key.includes('[') ? null : key; +} + /** Removes `LowCardinality(...)` and `Nullable(...)` wrappers from ClickHouse type strings. */ export function stripTypeWrappers(type: string): string { let t = type.trim(); @@ -140,8 +150,7 @@ export function isIdField( key: string, columnMeta: { name: string; type: string }[], ): boolean { - const arrMatch = key.match(/^([^[]+)\[(\d+)\]$/); - const colName = arrMatch ? arrMatch[1] : key.includes('[') ? null : key; + const colName = getBaseColumnName(key); if (!colName) return false; if (!/(Id|ID)$/.test(colName)) return false; @@ -164,8 +173,7 @@ export function isTimestampArrayField( key: string, columnMeta: { name: string; type: string }[], ): boolean { - const arrMatch = key.match(/^([^[]+)\[(\d+)\]$/); - const colName = arrMatch ? arrMatch[1] : key.includes('[') ? null : key; + const colName = getBaseColumnName(key); if (!colName) return false; const col = columnMeta.find(c => c.name === colName);