diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 63bbe5b62a..806430e08f 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -1900,6 +1900,7 @@ function DBSearchPage() { }} isReady={isReady} source={searchedSource} + onAddFilter={searchFilters.setFilterValue} /> )} {analysisMode === 'results' && ( diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index e1cd43368b..7e1125a0eb 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -1,290 +1,78 @@ -import { memo, useEffect, useMemo, useState } from 'react'; -import { withErrorBoundary } from 'react-error-boundary'; -import { - Bar, - BarChart, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from 'recharts'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { ChartConfigWithDateRange, ChartConfigWithOptDateRange, Filter, } from '@hyperdx/common-utils/dist/types'; -import { Box, Code, Container, Flex, Pagination, Text } from '@mantine/core'; +import { Box, Code, Divider, Flex, Pagination, Text } from '@mantine/core'; import { useElementSize } from '@mantine/hooks'; import { isAggregateFunction } from '@/ChartUtils'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; import { getFirstTimestampValueExpression } from '@/source'; -import { - getChartColorError, - getChartColorSuccess, - truncateMiddle, -} from '@/utils'; +import { getChartColorError, getChartColorSuccess } from '@/utils'; import { SQLPreview } from './ChartSQLPreview'; +import type { AddFilterFn, HighlightPoint } from './deltaChartUtils'; +import { + ALL_SPANS_COLOR, + computeComparisonScore, + computeDistributionScore, + computeEffectiveSampleSize, + computeEntropyScore, + computeYValue, + DISTRIBUTION_SCORING, + flattenData, + flattenedKeyToFilterKey, + getPropertyStatistics, + isDenylisted, + isHighCardinality, + mergeValueStatisticsMaps, + SAMPLE_SIZE, + semanticBoost, + STABLE_SAMPLE_EXPR, + stripTypeWrappers, +} from './deltaChartUtils'; +import { + CHART_GAP, + CHART_HEIGHT, + CHART_WIDTH, + PAGINATION_HEIGHT, + PropertyComparisonChart, +} from './PropertyComparisonChart'; -import styles from '../../styles/HDXLineChart.module.scss'; - -/* - * Response Data is like... -{ - Timestamp: "", - Map: { - "property": value, - } -} - -- Flatten -- Count Property Occurences -- Pick most common properties -- Count values for most common properties - -- Merge both sets of properties? one property? - */ - -// TODO: doesn't work for empty objects? -// https://stackoverflow.com/a/19101235 -function flattenData(data: Record) { - const result: Record = {}; - function recurse(cur: Record, prop: string) { - if (Object(cur) !== cur) { - result[prop] = cur; - } else if (Array.isArray(cur)) { - let l; - for (let i = 0, l = cur.length; i < l; i++) - recurse(cur[i], prop + '[' + i + ']'); - if (l == 0) result[prop] = []; - } else { - let isEmpty = true; - for (const p in cur) { - isEmpty = false; - recurse(cur[p], prop ? prop + '.' + p : p); - } - if (isEmpty && prop) result[prop] = {}; - } - } - recurse(data, ''); - return result; -} - -function getPropertyStatistics(data: Record[]) { - const flattened = data.map(flattenData); - const propertyOccurences = new Map(); - - const MIN_PROPERTY_OCCURENCES = 5; - const commonProperties = new Set(); - - flattened.forEach(item => { - Object.entries(item).forEach(([key, value]) => { - const count = propertyOccurences.get(key) || 0; - propertyOccurences.set(key, count + 1); - - if (count + 1 >= MIN_PROPERTY_OCCURENCES) { - commonProperties.add(key); - } - }); - }); - - // property -> (value -> count) - const valueOccurences = new Map>(); - flattened.forEach(item => { - Object.entries(item).forEach(([key, value]) => { - if (commonProperties.has(key)) { - let valuesMap = valueOccurences.get(key); - if (!valuesMap) { - valuesMap = new Map(); - valueOccurences.set(key, valuesMap); - } - - const valueCount = valuesMap.get(value) || 0; - valuesMap.set(value, valueCount + 1); - } - }); - }); - - const percentageOccurences = new Map>(); - valueOccurences.forEach((valuesMap, property) => { - const percentageMap = new Map(); - valuesMap.forEach((valueCount, value) => { - percentageMap.set( - value, - (valueCount / (propertyOccurences.get(property) ?? 0)) * 100, - ); - }); - percentageOccurences.set(property, percentageMap); - }); - - return { - // valueOccurences, - percentageOccurences, - // commonProperties, - // propertyOccurences, - }; -} - -function mergeValueStatisticsMaps( - outlierValues: Map, // value -> count - inlierValues: Map, -) { - const mergedArray: { - name: string; - outlierCount: number; - inlierCount: number; - }[] = []; - // Collect all value names for this property - // we sort them so timestamps are ordered - const allValues = Array.from( - new Set([...outlierValues.keys(), ...inlierValues.keys()]), - ).sort(); - - allValues.forEach(value => { - const count1 = outlierValues.get(value) || 0; - const count2 = inlierValues.get(value) || 0; - mergedArray.push({ - name: value, - outlierCount: count1, - inlierCount: count2, - }); - }); - - return mergedArray; -} - -const HDXBarChartTooltip = withErrorBoundary( - memo((props: any) => { - const { active, payload, label, title } = props; - if (active && payload && payload.length) { - return ( -
-
- {title && ( - - {title} - - )} - - {label.length === 0 ? Empty String : label} - - {payload - .sort((a: any, b: any) => b.value - a.value) - .map((p: any) => ( -
- {p.name}: {p.value.toFixed(2)}% -
- ))} -
-
- ); - } - return null; - }), - { - onError: console.error, - fallback: ( -
- An error occurred while rendering the tooltip. -
- ), - }, -); - -function PropertyComparisonChart({ - name, - outlierValueOccurences, - inlierValueOccurences, -}: { - name: string; - outlierValueOccurences: Map; - inlierValueOccurences: Map; -}) { - const mergedValueStatistics = mergeValueStatisticsMaps( - outlierValueOccurences, - inlierValueOccurences, - ); - - return ( -
- - {truncateMiddle(name, 32)} - - - - {/* */} - - - } - allowEscapeViewBox={{ y: true }} - /> - - - - -
- ); -} - -// Layout constants for dynamic grid calculation. -// CHART_WIDTH is the minimum chart width used to determine how many columns fit; actual rendered -// width expands to fill the container (charts use width: '100%' inside a CSS grid). -// CHART_HEIGHT must match PropertyComparisonChart's outer div height. -// CHART_GAP is used both in the column/row formula and as the CSS grid gap. -const CHART_WIDTH = 340; // minimum column width threshold (px) -const CHART_HEIGHT = 120; // must match PropertyComparisonChart outer div height (px) -const CHART_GAP = 16; // px; used in grid gap and layout math -// Space reserved for the pagination row: Pagination control (~32px) + top padding (16px). -// Always reserved (even when pagination is hidden via visibility:hidden) so rows count is stable. -const PAGINATION_HEIGHT = 48; +// Re-export types so callers importing from DBDeltaChart don't need to change. +export type { AddFilterFn, HighlightPoint } from './deltaChartUtils'; export default function DBDeltaChart({ config, valueExpr, - xMin, - xMax, - yMin, - yMax, + xMin: rawXMin, + xMax: rawXMax, + yMin: rawYMin, + yMax: rawYMax, + onAddFilter, + onHighlightPoints, }: { config: ChartConfigWithDateRange; valueExpr: string; - xMin: number; - xMax: number; - yMin: number; - yMax: number; + xMin?: number | null; + xMax?: number | null; + yMin?: number | null; + yMax?: number | null; + onAddFilter?: AddFilterFn; + onHighlightPoints?: (points: HighlightPoint[] | null) => void; }) { + const hasSelection = + rawXMin != null && rawXMax != null && rawYMin != null && rawYMax != null; + // Safe numeric defaults so query builders always get valid values + // (outlier/inlier queries are gated by enabled:hasSelection) + const xMin = rawXMin ?? 0; + const xMax = rawXMax ?? 0; + const yMin = rawYMin ?? 0; + const yMax = rawYMax ?? 0; + // Determine if the value expression uses aggregate functions const isAggregate = isAggregateFunction(valueExpr); @@ -371,8 +159,8 @@ export default function DBDeltaChart({ ] : []), ], - orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], - limit: { limit: 1000 }, + orderBy: [{ ordering: 'DESC', valueExpression: STABLE_SAMPLE_EXPR }], + limit: { limit: SAMPLE_SIZE }, }, }, ]; @@ -421,70 +209,267 @@ export default function DBDeltaChart({ ]; }; - const { data: outlierData, error } = useQueriedChartConfig({ + // Lightweight count query to support adaptive sample sizing. + // ClickHouse resolves count() from MergeTree metadata, so this is near-instant. + // useQueriedChartConfig internally gates on source/MV metadata readiness. + const { data: countData } = useQueriedChartConfig({ ...config, - with: buildWithClauses(true), - select: '*', - filters: buildFilters(true), - orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], - limit: { limit: 1000 }, + select: 'count() as total', }); + const totalCount = Number( + (countData?.data as Record[])?.[0]?.total ?? 0, + ); + const effectiveSampleSize = computeEffectiveSampleSize(totalCount); - const { data: inlierData } = useQueriedChartConfig({ - ...config, - with: buildWithClauses(false), - select: '*', - filters: buildFilters(false), - orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }], - limit: { limit: 1000 }, - }); + const { + data: outlierData, + error: outlierError, + isLoading: isOutlierLoading, + } = useQueriedChartConfig( + { + ...config, + with: buildWithClauses(true), + select: '*', + filters: buildFilters(true), + orderBy: [{ ordering: 'DESC', valueExpression: STABLE_SAMPLE_EXPR }], + limit: { limit: effectiveSampleSize }, + }, + { enabled: hasSelection }, + ); + + const { data: inlierData, isLoading: isInlierLoading } = + useQueriedChartConfig( + { + ...config, + with: buildWithClauses(false), + select: '*', + filters: buildFilters(false), + orderBy: [{ ordering: 'DESC', valueExpression: STABLE_SAMPLE_EXPR }], + limit: { limit: effectiveSampleSize }, + }, + { enabled: hasSelection }, + ); + + // When no selection exists, fetch all spans without any range filter + const { + data: allSpansData, + error: allSpansError, + isLoading: isAllSpansLoading, + } = useQueriedChartConfig( + { + ...config, + select: '*', + orderBy: [{ ordering: 'DESC', valueExpression: STABLE_SAMPLE_EXPR }], + limit: { limit: effectiveSampleSize }, + }, + { enabled: !hasSelection }, + ); - // TODO: Is loading state - const { sortedProperties, outlierValueOccurences, inlierValueOccurences } = - useMemo(() => { - const { percentageOccurences: outlierValueOccurences } = - getPropertyStatistics(outlierData?.data ?? []); + const isLoading = hasSelection + ? isOutlierLoading || isInlierLoading + : isAllSpansLoading; - const { percentageOccurences: inlierValueOccurences } = - getPropertyStatistics(inlierData?.data ?? []); + const error = outlierError ?? allSpansError; - // Get all the unique keys from the outliers - let uniqueKeys = new Set([...outlierValueOccurences.keys()]); - // If there's no outliers, use inliers as the unique keys - if (uniqueKeys.size === 0) { - uniqueKeys = new Set([...inlierValueOccurences.keys()]); + // Compute column metadata, property statistics, sorted/visible/hidden property lists. + // columnMeta is merged here (instead of a separate useMemo) so the denylist and + // cardinality checks can reference it during the same memoization pass. + const { + outlierValueOccurences, + inlierValueOccurences, + columnMeta, + visibleProperties, + hiddenProperties, + highlightIndex, + sampleRowCount, + } = useMemo(() => { + const columnMeta = (outlierData?.meta ?? + inlierData?.meta ?? + allSpansData?.meta ?? + []) as { + name: string; + type: string; + }[]; + + // When no selection: use allSpans as "outlier" data and empty for inliers. + // The sort will rank by frequency (delta = count - 0 = count). + const actualOutlierData = hasSelection + ? (outlierData?.data ?? []) + : (allSpansData?.data ?? []); + const actualInlierData = hasSelection ? (inlierData?.data ?? []) : []; + + const { + percentageOccurences: outlierValueOccurences, + propertyOccurences: outlierPropertyOccurences, + } = getPropertyStatistics(actualOutlierData); + + const { + percentageOccurences: inlierValueOccurences, + propertyOccurences: inlierPropertyOccurences, + } = getPropertyStatistics(actualInlierData); + + // Get all the unique keys from the outliers + let uniqueKeys = new Set([...outlierValueOccurences.keys()]); + // If there's no outliers, use inliers as the unique keys + if (uniqueKeys.size === 0) { + uniqueKeys = new Set([...inlierValueOccurences.keys()]); + } + // Sort properties by how useful they are for filtering/analysis. + // Comparison mode: sort by max difference between selection and background. + // Distribution mode (no selection): sort by deviation from uniform distribution — + // score = max(pct) - mean(pcts) + // This scores 0 for single-value fields (all-same) and for perfectly uniform + // multi-value fields, and scores high for skewed distributions. + const sortedProperties = Array.from(uniqueKeys) + .map(key => { + const inlierCount = + inlierValueOccurences.get(key) ?? new Map(); + const outlierCount = + outlierValueOccurences.get(key) ?? new Map(); + + let sortScore: number; + if (hasSelection) { + // Comparison mode: sort by proportional distribution difference. + // Normalizes each group's percentages before comparing, so fields + // with identical proportions (e.g., 100% "message" in both) score 0 + // regardless of coverage rate differences between groups. + sortScore = computeComparisonScore(outlierCount, inlierCount); + } else { + // Distribution mode: sort by how useful the field is for filtering. + // Fields with actual variance (multiple values, unequal distribution) + // always rank above single-value or perfectly uniform fields. + const baseScore = + DISTRIBUTION_SCORING === 'entropy' + ? computeEntropyScore(outlierCount) + : computeDistributionScore(outlierCount); + // Semantic boost only applies when the field has actual variance + // (baseScore > 0). Scaled to 0.1 so it acts as a tiebreaker — + // never overrides a genuinely more interesting distribution. + const boost = baseScore > 0 ? semanticBoost(key) * 0.1 : 0; + sortScore = baseScore + boost; + } + + return [key, sortScore] as const; + }) + .sort((a, b) => b[1] - a[1]) + .map(a => a[0]); + + // Split properties into visible (shown in charts) and hidden (denylist or high cardinality) + const visibleProperties: string[] = []; + const hiddenProperties: string[] = []; + + sortedProperties.forEach(key => { + if (isDenylisted(key, columnMeta)) { + hiddenProperties.push(key); + } else if ( + isHighCardinality( + key, + outlierValueOccurences, + inlierValueOccurences, + outlierPropertyOccurences, + inlierPropertyOccurences, + ) + ) { + hiddenProperties.push(key); + } else { + visibleProperties.push(key); } - // Now process the keys to find the ones with the highest delta between outlier and inlier percentages - const sortedProperties = Array.from(uniqueKeys) - .map(key => { - const inlierCount = - inlierValueOccurences.get(key) ?? new Map(); - const outlierCount = - outlierValueOccurences.get(key) ?? new Map(); - - const mergedArray = mergeValueStatisticsMaps( - outlierCount, - inlierCount, - ); - let maxValueDelta = 0; - mergedArray.forEach(item => { - const delta = Math.abs(item.outlierCount - item.inlierCount); - if (delta > maxValueDelta) { - maxValueDelta = delta; - } - }); - - return [key, maxValueDelta] as const; - }) - .sort((a, b) => b[1] - a[1]) - .map(a => a[0]); - - return { - sortedProperties, - outlierValueOccurences, - inlierValueOccurences, - }; - }, [outlierData?.data, inlierData?.data]); + }); + + // Build a pre-indexed lookup for hover-based timestamp highlighting. + // Structure: property → value → HighlightPoint[] + // This replaces the O(n) scan per hover with an O(1) Map lookup. + const flattenedRawData = [ + ...actualOutlierData.map(flattenData), + ...actualInlierData.map(flattenData), + ]; + + // Find the first non-array DateTime64 column (typically 'Timestamp') + const tsColName = columnMeta.find( + c => + (stripTypeWrappers(c.type).startsWith('DateTime64(') || + c.type === 'DateTime64') && + !stripTypeWrappers(c.type).startsWith('Array('), + )?.name; + + const highlightIndex = new Map>(); + if (tsColName) { + for (const flat of flattenedRawData) { + const ts = flat[tsColName]; + if (ts == null) continue; + const tsMs = new Date(ts as string).getTime(); + if (isNaN(tsMs)) continue; + const yValue = computeYValue(valueExpr, flat); + const point: HighlightPoint = { tsMs, yValue }; + + for (const [key, val] of Object.entries(flat)) { + if (key === tsColName) continue; + const strVal = String(val); + let valueMap = highlightIndex.get(key); + if (!valueMap) { + valueMap = new Map(); + highlightIndex.set(key, valueMap); + } + let points = valueMap.get(strVal); + if (!points) { + points = []; + valueMap.set(strVal, points); + } + points.push(point); + } + } + } + + // Row counts for the sample-size annotation in the legend + const sampleRowCount = actualOutlierData.length + actualInlierData.length; + + return { + outlierValueOccurences, + inlierValueOccurences, + columnMeta, + visibleProperties, + hiddenProperties, + highlightIndex, + sampleRowCount, + }; + }, [outlierData, inlierData, allSpansData, hasSelection, valueExpr]); + + // Wrap onAddFilter to convert flattened dot-notation keys (from flattenData) + // into ClickHouse bracket notation for Map columns: ColName['map.key']. + // This matches the search bar format and produces valid ClickHouse SQL. + const handleAddFilter = useCallback>( + (property, value, action) => { + if (!onAddFilter) return; + onAddFilter(flattenedKeyToFilterKey(property, columnMeta), value, action); + }, + [onAddFilter, columnMeta], + ); + + // Track hovered attribute value for correlation highlighting on the heatmap + const [hoveredAttributeValue, setHoveredAttributeValue] = useState<{ + property: string; + value: string; + } | null>(null); + + const handleHoverValue = useCallback( + (property: string, value: string | null) => { + setHoveredAttributeValue(value != null ? { property, value } : null); + }, + [], + ); + + // O(1) lookup: retrieve pre-indexed {tsMs, yValue} pairs for the hovered attribute. + const highlightPoints = useMemo((): HighlightPoint[] | null => { + if (!hoveredAttributeValue) return null; + const { property, value } = hoveredAttributeValue; + const points = highlightIndex.get(property)?.get(value); + return points?.length ? points : null; + }, [hoveredAttributeValue, highlightIndex]); + + // Propagate highlight points to parent (e.g., DBSearchHeatmapChart) + useEffect(() => { + onHighlightPoints?.(highlightPoints); + }, [highlightPoints, onHighlightPoints]); const [activePage, setPage] = useState(1); @@ -513,7 +498,7 @@ export default function DBDeltaChart({ if (error) { return ( - + Error Message: @@ -542,33 +527,145 @@ export default function DBDeltaChart({ )} - + ); } - const totalPages = Math.ceil(sortedProperties.length / PAGE_SIZE); + // Paginate by ROWS, not by item count, because visible and hidden fields are + // rendered in separate CSS grids. An incomplete last row of the visible + // section must not be "filled" with hidden items — each section always starts + // from column 1. + const visibleRows = Math.ceil(visibleProperties.length / columns); + const hiddenRows = Math.ceil(hiddenProperties.length / columns); + const totalRows = visibleRows + hiddenRows; + const totalPages = Math.ceil(totalRows / rows); + + const pageRowStart = (activePage - 1) * rows; + const pageRowEnd = activePage * rows; + + // Rows occupied by the visible section on this page + const visRowStart = Math.min(pageRowStart, visibleRows); + const visRowEnd = Math.min(pageRowEnd, visibleRows); + const visibleOnPage = visibleProperties.slice( + visRowStart * columns, + Math.min(visRowEnd * columns, visibleProperties.length), + ); + + // Rows occupied by the hidden section on this page + const hidRowStart = Math.max(0, pageRowStart - visibleRows); + const hidRowEnd = Math.min(hiddenRows, Math.max(0, pageRowEnd - visibleRows)); + const hiddenOnPage = hiddenProperties.slice( + hidRowStart * columns, + Math.min(hidRowEnd * columns, hiddenProperties.length), + ); + + // Show a divider when both sections appear on the same page + const showDivider = visibleOnPage.length > 0 && hiddenOnPage.length > 0; + // Show a header when ONLY hidden fields appear on this page (no divider above) + const showHiddenHeader = + hiddenOnPage.length > 0 && visibleOnPage.length === 0; return ( -
- {sortedProperties - .slice((activePage - 1) * PAGE_SIZE, activePage * PAGE_SIZE) - .map(property => ( + {/* Legend */} + + {hasSelection ? ( + <> + + + + Selection + + + + + + Background + + + {!isLoading && sampleRowCount > 0 && ( + + (n={sampleRowCount.toLocaleString()} + {totalCount > 0 + ? ` of ${totalCount.toLocaleString()}` + : ''}{' '} + sampled) + + )} + + ) : ( + <> + + + + All spans + + {!isLoading && sampleRowCount > 0 && ( + + (n={sampleRowCount.toLocaleString()} sampled) + + )} + + + {isLoading + ? 'Loading…' + : 'Select an area on the chart above to enable comparisons'} + + + )} + + {/* Loading state */} + {isLoading && visibleOnPage.length === 0 && hiddenOnPage.length === 0 && ( + + + Loading attribute distributions… + + + )} + {/* Primary fields — own grid so empty trailing cells don't interact with divider */} + {visibleOnPage.length > 0 && ( +
+ {visibleOnPage.map(property => ( ))} -
+
+ )} + {/* Divider between primary and lower-priority fields */} + {(showDivider || showHiddenHeader) && ( + + Lower-priority fields ({hiddenProperties.length}) + + } + labelPosition="left" + /> + )} + {/* Lower-priority fields — separate grid so rows align independently */} + {hiddenOnPage.length > 0 && ( +
+ {hiddenOnPage.map(key => ( + + ))} +
+ )} void; + onClearSelection?: () => void; title?: React.ReactNode; toolbarPrefix?: React.ReactNode[]; toolbarSuffix?: React.ReactNode[]; + highlightPoints?: { tsMs: number; yValue: number | null }[] | null; }) { const dateRange = config.dateRange; const granularity = convertDateRangeToGranularityString(dateRange, 245); @@ -555,6 +559,8 @@ function HeatmapContainer({ data={[time, bucket, count]} numberFormat={config.numberFormat} onFilter={onFilter} + onClearSelection={onClearSelection} + highlightPoints={highlightPoints} /> )} @@ -666,10 +672,14 @@ function Heatmap({ data, numberFormat, onFilter, + onClearSelection, + highlightPoints, }: { data: Mode2DataArray; numberFormat?: NumberFormat; onFilter?: (xMin: number, xMax: number, yMin: number, yMax: number) => void; + onClearSelection?: () => void; + highlightPoints?: { tsMs: number; yValue: number | null }[] | null; }) { const [selectingInfo, setSelectingInfo] = useState< | { @@ -687,6 +697,60 @@ function Heatmap({ | undefined >(undefined); + // After the user clicks "Filter by Selection", hide that button so only "X" remains. + // Resets to false when the selection is cleared. + const [hasFiltered, setHasFiltered] = useState(false); + + // Refs for correlation highlight overlay: uPlot instance + latest highlight timestamps + const uplotRef = useRef(null); + + // Clears the React selection state AND the uPlot selection rectangle + const clearSelectionAndRect = useCallback(() => { + setSelectingInfo(undefined); + setHasFiltered(false); + if (uplotRef.current) { + try { + uplotRef.current.setSelect( + { left: 0, top: 0, width: 0, height: 0 }, + false, + ); + } catch (err) { + console.warn( + 'clearSelectionAndRect: failed to reset uPlot selection:', + err, + ); + } + } + onClearSelection?.(); + }, [onClearSelection]); + const highlightPointsRef = useRef< + { tsMs: number; yValue: number | null }[] | null + >(null); + // Keep ref in sync with latest prop value on every render + highlightPointsRef.current = highlightPoints ?? null; + + // Trigger a uPlot redraw when highlight points change so the draw hook re-runs. + // Wrapped in requestAnimationFrame to coalesce rapid hover events (e.g., mouse + // moving across multiple bars in quick succession) into a single frame repaint. + const rafIdRef = useRef(null); + useEffect(() => { + if (uplotRef.current) { + if (rafIdRef.current != null) { + cancelAnimationFrame(rafIdRef.current); + } + rafIdRef.current = requestAnimationFrame(() => { + rafIdRef.current = null; + uplotRef.current?.redraw(false); + }); + } + return () => { + if (rafIdRef.current != null) { + cancelAnimationFrame(rafIdRef.current); + rafIdRef.current = null; + } + }; + }, [highlightPoints]); + const [highlightedPoint, setHighlightedPoint] = useState< | { xVal: number; @@ -702,6 +766,10 @@ function Heatmap({ | undefined >(undefined); + // Gate tooltip display on actual mouse interaction. uPlot fires setCursor + // on init (before user hovers), which would show the tooltip on page load. + const mouseInsideRef = useRef(false); + const { ref, width, height } = useElementSize(); const tickFormatter = useCallback( @@ -769,6 +837,9 @@ function Heatmap({ xSize, ySize, }) => { + // Only show tooltip after the user has actually hovered the chart. + // uPlot fires setCursor on init which would trigger this on page load. + if (!mouseInsideRef.current) return; setHighlightedPoint({ xVal, yVal, @@ -785,6 +856,14 @@ function Heatmap({ { hooks: { setSelect: u => { + // Ignore zero-size selections (single-click, or when clearSelectionAndRect() + // calls u.setSelect({width:0,height:0}) to erase the visual rectangle). + // clearSelectionAndRect() calls setSelectingInfo(undefined) directly before + // calling u.setSelect, so we don't need to do anything here. + if (u.select.width <= 0 || u.select.height <= 0) { + return; + } + // Calculate offset from parent so we can render tooltip // relative to the parent pixels const { offsetLeft, offsetTop } = u.over; @@ -794,8 +873,8 @@ function Heatmap({ const yMax = u.posToVal(u.select.top, 'y'); const yMin = u.posToVal(u.select.top + u.select.height, 'y'); - // This ensures we set the timeout after all click handlers - // to prevent our state from being wiped by onclick handler + // Small timeout to ensure this fires after uPlot completes its + // synchronous drag-end processing. setTimeout(() => { setSelectingInfo({ top: u.select.top + offsetTop, @@ -811,20 +890,119 @@ function Heatmap({ }, }, }, + { + // Correlation highlight: draws filled rectangles on the heatmap canvas + // at cells corresponding to spans matching the hovered attribute value. + // Mirrors heatmapPaths coordinate calculation exactly so highlights align + // with the heatmap cells they represent. + hooks: { + init: (u: uPlot) => { + // eslint-disable-next-line react-hooks/exhaustive-deps + uplotRef.current = u; + }, + draw: (u: uPlot) => { + const pts = highlightPointsRef.current; + if (!pts?.length) return; + + // Derive bin geometry from the mode-2 heatmap data + const [xs, ys] = u.data[1] as unknown as Mode2DataArray; + if (!xs?.length || !ys?.length || xs.length < 2 || ys.length < 2) + return; + + const dlen = xs.length; + const yBinQty = dlen - ys.lastIndexOf(ys[0]); + if (yBinQty < 2) return; + const xBinQty = Math.floor(dlen / yBinQty); + const yBinIncr = ys[1] - ys[0]; // positive value increment per bin + const xBinIncr = xs[yBinQty] - xs[0]; // ms increment per time bucket + if (yBinIncr === 0 || xBinIncr === 0) return; + + // Compute pixel cell size using actual data points (same as heatmapPaths). + // Using xs[0]/ys[0] as reference avoids issues with values far outside + // the visible scale range (e.g. valToPos(0) on a ms-epoch time axis). + const xSizePx = Math.abs( + u.valToPos(xs[0] + xBinIncr, 'x', true) - + u.valToPos(xs[0], 'x', true), + ); + const ySizePx = Math.abs( + u.valToPos(ys[0] + yBinIncr, 'y', true) - + u.valToPos(ys[0], 'y', true), + ); + + u.ctx.save(); + u.ctx.rect(u.bbox.left, u.bbox.top, u.bbox.width, u.bbox.height); + u.ctx.clip(); + u.ctx.fillStyle = 'rgba(255, 220, 50, 0.6)'; + + // Draw one cell per unique (xi, yi) bucket position occupied by a + // matching span. This avoids the previous min-to-max range approach + // which always extended to yi=0 (the bottom of the chart) because any + // matching span with a near-zero duration pulled yiMin to 0 — even for + // attributes that only appear in slow spans. + // + // With per-cell drawing: if error=true only appears in 500ms+ spans, + // highlighted cells cluster near the top. If service=A appears in 100% + // of spans (all durations), cells are distributed across the full Y + // range, accurately reflecting the attribute's actual distribution. + const cellSet = new Set(); // encoded as xi * yBinQty + yi + + for (const { tsMs, yValue } of pts) { + const xi = Math.max( + 0, + Math.min(xBinQty - 1, Math.round((tsMs - xs[0]) / xBinIncr)), + ); + + if (yValue == null) { + // Can't determine Y position (e.g. complex expression): draw all + // Y cells in this X column as fallback so there's visual feedback. + for (let yi = 0; yi < yBinQty; yi++) { + cellSet.add(xi * yBinQty + yi); + } + continue; + } + + const yi = Math.max( + 0, + Math.min( + yBinQty - 1, + Math.round((yValue - ys[0]) / yBinIncr), + ), + ); + cellSet.add(xi * yBinQty + yi); + } + + for (const cell of cellSet) { + const xi = Math.floor(cell / yBinQty); + const yi = cell % yBinQty; + const xPx = u.valToPos(xs[xi * yBinQty], 'x', true); + const cx = Math.round(xPx - xSizePx / 2); + // Y axis is inverted: higher value = smaller pixel position (top). + const cy = Math.round( + u.valToPos(ys[yi], 'y', true) - ySizePx / 2, + ); + u.ctx.fillRect(cx, cy, xSizePx, ySizePx); + } + + u.ctx.restore(); + }, + }, + }, ], }; + // uplotRef and highlightPointsRef are stable refs — not included in deps + // eslint-disable-next-line react-hooks/exhaustive-deps }, [width, height, tickFormatter]); return (
{ - if (selectingInfo != null) { - setSelectingInfo(undefined); - } + onMouseEnter={() => { + mouseInsideRef.current = true; }} onMouseLeave={() => { + mouseInsideRef.current = false; setHighlightedPoint(undefined); }} > @@ -853,18 +1031,20 @@ function Heatmap({ style={{ position: 'absolute', top: highlightedPoint.yCoord + 5, - ...(highlightedPoint.xCoord > (width * 2) / 3 + ...(highlightedPoint.xCoord > width / 2 ? { right: width - highlightedPoint.xCoord + 10, } : { left: highlightedPoint.xCoord + 10, }), + maxWidth: '50%', backdropFilter: 'blur(8px)', - backgroundColor: 'rgba(#1A1D23 0.75)', + backgroundColor: 'rgba(26, 29, 35, 0.75)', border: '1px solid #5F6776', borderRadius: 2, pointerEvents: 'none', + whiteSpace: 'nowrap', }} > @@ -887,30 +1067,79 @@ function Heatmap({
)} - {selectingInfo != null && onFilter != null && ( + {selectingInfo != null && (
{ - e.stopPropagation(); - onFilter?.( - selectingInfo.xMin / 1000, - selectingInfo.xMax / 1000, - selectingInfo.yMin, - selectingInfo.yMax, - ); + bottom: height - selectingInfo.top + 4, + left: selectingInfo.left, }} - role="button" > - Filter by Selection + {onFilter != null && !hasFiltered && ( +
{ + e.stopPropagation(); + onFilter( + selectingInfo.xMin / 1000, + selectingInfo.xMax / 1000, + selectingInfo.yMin, + selectingInfo.yMax, + ); + // Hide the Filter button — only the X (clear) button remains. + setHasFiltered(true); + }} + onKeyDown={e => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + onFilter( + selectingInfo.xMin / 1000, + selectingInfo.xMax / 1000, + selectingInfo.yMin, + selectingInfo.yMax, + ); + setHasFiltered(true); + } + }} + > + Filter by Selection +
+ )} +
{ + e.stopPropagation(); + clearSelectionAndRect(); + }} + onKeyDown={e => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + clearSelectionAndRect(); + } + }} + > + ✕ +
)} diff --git a/packages/app/src/components/PropertyComparisonChart.tsx b/packages/app/src/components/PropertyComparisonChart.tsx new file mode 100644 index 0000000000..af6d9862dd --- /dev/null +++ b/packages/app/src/components/PropertyComparisonChart.tsx @@ -0,0 +1,407 @@ +import { memo, useEffect, useRef, useState } from 'react'; +import { createPortal } from 'react-dom'; +import { withErrorBoundary } from 'react-error-boundary'; +import type { TooltipProps } from 'recharts'; +import { + Bar, + BarChart, + Cell, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; +import { Flex, Text } from '@mantine/core'; +import { IconCopy, IconFilter, IconFilterX } from '@tabler/icons-react'; + +import { + getChartColorError, + getChartColorSuccess, + truncateMiddle, +} from '@/utils'; + +import { DBRowTableIconButton } from './DBTable/DBRowTableIconButton'; +import { + AddFilterFn, + ALL_SPANS_COLOR, + applyTopNAggregation, + mergeValueStatisticsMaps, + OTHER_BUCKET_COLOR, +} from './deltaChartUtils'; + +import styles from '../../styles/HDXLineChart.module.scss'; + +// Layout constants for dynamic grid calculation. +// CHART_WIDTH is the minimum chart width used to determine how many columns fit; actual rendered +// width expands to fill the container (charts use width: '100%' inside a CSS grid). +// CHART_HEIGHT must match PropertyComparisonChart's outer div height. +// CHART_GAP is used both in the column/row formula and as the CSS grid gap. +export const CHART_WIDTH = 340; // minimum column width threshold (px) +export const CHART_HEIGHT = 120; // must match PropertyComparisonChart outer div height (px) +export const CHART_GAP = 16; // px; used in grid gap and layout math +// Space reserved for the pagination row: Pagination control (~32px) + top padding (16px). +// Always reserved (even when pagination is hidden via visibility:hidden) so rows count is stable. +export const PAGINATION_HEIGHT = 48; + +type TooltipPayloadItem = { + dataKey: string; + name: string; + value: number; + payload?: Record; +}; + +type TooltipContentProps = TooltipProps & { + title?: string; +}; + +// Hover-only tooltip: shows value name and percentages, no action buttons. +// Actions are handled by the click popover in PropertyComparisonChart. +const HDXBarChartTooltip = withErrorBoundary( + memo(({ active, payload, label, title }: TooltipContentProps) => { + if (active && payload && payload.length) { + return ( +
+
+ {title && ( + + {title} + + )} + + {String(label).length === 0 ? Empty String : String(label)} + + {(payload as TooltipPayloadItem[]) + .sort((a, b) => b.value - a.value) + .map(p => ( +
+ {p.name}: {p.value.toFixed(2)}% +
+ ))} +
+
+ ); + } + return null; + }), + { + onError: console.error, + fallback: ( +
+ An error occurred while rendering the tooltip. +
+ ), + }, +); + +type TickProps = { + x?: number; + y?: number; + payload?: { value: string | number }; +}; + +// Custom XAxis tick that truncates long labels and adds a native SVG tooltip. +function TruncatedTick({ x = 0, y = 0, payload }: TickProps) { + const value = String(payload?.value ?? ''); + const MAX_CHARS = 12; + const displayValue = + value.length > MAX_CHARS ? value.slice(0, MAX_CHARS) + '…' : value; + return ( + + {value} + + {displayValue} + + + ); +} + +export function PropertyComparisonChart({ + name, + outlierValueOccurences, + inlierValueOccurences, + onAddFilter, + hasSelection, + onHoverValue, +}: { + name: string; + outlierValueOccurences: Map; + inlierValueOccurences: Map; + onAddFilter?: AddFilterFn; + hasSelection: boolean; + onHoverValue?: (property: string, value: string | null) => void; +}) { + const mergedValueStatistics = mergeValueStatisticsMaps( + outlierValueOccurences, + inlierValueOccurences, + ); + const chartData = applyTopNAggregation(mergedValueStatistics); + + const [clickedBar, setClickedBar] = useState<{ + value: string; + clientX: number; + clientY: number; + } | null>(null); + const [copiedValue, setCopiedValue] = useState(false); + // Local hover state for bar dimming — dims non-hovered bars for prominence + const [hoveredBar, setHoveredBar] = useState(null); + const popoverRef = useRef(null); + const chartWrapperRef = useRef(null); + // Track last hovered bar value to avoid firing onHoverValue on every pixel move + const lastHoveredBarRef = useRef(null); + + // Dismiss popover on click outside, scroll, or Escape key + useEffect(() => { + if (!clickedBar) return; + const handleClickOutside = (e: MouseEvent) => { + if ( + popoverRef.current && + !popoverRef.current.contains(e.target as Node) + ) { + setClickedBar(null); + } + }; + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape') setClickedBar(null); + }; + const handleScroll = () => setClickedBar(null); + document.addEventListener('mousedown', handleClickOutside); + document.addEventListener('keydown', handleKeyDown); + window.addEventListener('scroll', handleScroll, true); + return () => { + document.removeEventListener('mousedown', handleClickOutside); + document.removeEventListener('keydown', handleKeyDown); + window.removeEventListener('scroll', handleScroll, true); + }; + }, [clickedBar]); + + const handleChartClick = (data: any, event: any) => { + if (!data?.activePayload?.length) { + setClickedBar(null); + return; + } + if (data.activePayload[0]?.payload?.isOther) { + setClickedBar(null); + return; + } + // Reset copy confirmation so it doesn't carry over to the new bar's popover + setCopiedValue(false); + setClickedBar({ + value: String(data.activeLabel ?? ''), + clientX: event.clientX, + clientY: event.clientY, + }); + }; + + return ( +
+ + {name} + + + { + const label = data?.activeLabel; + const isOther = data?.activePayload?.[0]?.payload?.isOther; + const newVal = label != null && !isOther ? String(label) : null; + if (newVal !== lastHoveredBarRef.current) { + lastHoveredBarRef.current = newVal; + onHoverValue?.(name, newVal); + setHoveredBar(newVal); + } + }} + onMouseLeave={() => { + if (lastHoveredBarRef.current !== null) { + lastHoveredBarRef.current = null; + onHoverValue?.(name, null); + } + setHoveredBar(null); + }} + > + } /> + + } + allowEscapeViewBox={{ x: true, y: true }} + wrapperStyle={{ zIndex: 9998 }} + /> + + {chartData.map((entry, index) => ( + + ))} + + {hasSelection && ( + + {chartData.map((entry, index) => ( + + ))} + + )} + + + {clickedBar && + createPortal( +
+ + {truncateMiddle(name, 40)} + + + {clickedBar.value.length === 0 ? ( + Empty String + ) : ( + clickedBar.value + )} + + {hasSelection ? ( + + + Selection:{' '} + {(outlierValueOccurences.get(clickedBar.value) ?? 0).toFixed( + 1, + )} + % + + + Background:{' '} + {(inlierValueOccurences.get(clickedBar.value) ?? 0).toFixed( + 1, + )} + % + + + ) : ( + + Distribution:{' '} + {(outlierValueOccurences.get(clickedBar.value) ?? 0).toFixed(1)} + % + + )} + + {onAddFilter && ( + <> + { + onAddFilter(name, clickedBar.value, 'include'); + setClickedBar(null); + }} + > + + + { + onAddFilter(name, clickedBar.value, 'exclude'); + setClickedBar(null); + }} + > + + + + )} + { + try { + await navigator.clipboard.writeText(clickedBar.value); + setCopiedValue(true); + setTimeout(() => setCopiedValue(false), 2000); + } catch (err) { + console.error('Failed to copy:', err); + } + }} + > + + + +
, + document.body, + )} +
+ ); +} diff --git a/packages/app/src/components/Search/DBSearchHeatmapChart.tsx b/packages/app/src/components/Search/DBSearchHeatmapChart.tsx index 8ffa81ed33..1cff1f1b20 100644 --- a/packages/app/src/components/Search/DBSearchHeatmapChart.tsx +++ b/packages/app/src/components/Search/DBSearchHeatmapChart.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { parseAsFloat, parseAsString, useQueryStates } from 'nuqs'; import { useForm } from 'react-hook-form'; import { z } from 'zod'; @@ -12,15 +12,12 @@ import { DisplayType, TSource, } from '@hyperdx/common-utils/dist/types'; -import { Box, Flex } from '@mantine/core'; -import { Button } from '@mantine/core'; -import { Center } from '@mantine/core'; -import { Text } from '@mantine/core'; +import { Box, Button, Flex } from '@mantine/core'; import { IconPlayerPlay } from '@tabler/icons-react'; import { getDurationMsExpression } from '@/source'; -import DBDeltaChart from '../DBDeltaChart'; +import DBDeltaChart, { AddFilterFn, HighlightPoint } from '../DBDeltaChart'; import DBHeatmapChart from '../DBHeatmapChart'; import { SQLInlineEditorControlled } from '../SQLInlineEditor'; @@ -33,10 +30,12 @@ export function DBSearchHeatmapChart({ chartConfig, source, isReady, + onAddFilter, }: { chartConfig: ChartConfigWithDateRange; source: TSource; isReady: boolean; + onAddFilter?: AddFilterFn; }) { const [fields, setFields] = useQueryStates({ value: parseAsString.withDefault(getDurationMsExpression(source)), @@ -49,6 +48,44 @@ export function DBSearchHeatmapChart({ }); const [container, setContainer] = useState(null); + // Highlight points from hovering an attribute value in the delta charts. + // Passed to the heatmap to draw filled cell overlays at the correct X+Y position. + const [highlightPoints, setHighlightPoints] = useState< + HighlightPoint[] | null + >(null); + + const handleClearSelection = useCallback(() => { + setFields({ xMin: null, xMax: null, yMin: null, yMax: null }); + }, [setFields]); + + // After applying a filter, clear the heatmap selection so the delta chart + // resets to "all spans" distribution mode instead of staying in comparison mode. + const handleAddFilterAndClearSelection = useCallback< + NonNullable + >( + (property, value, action) => { + setFields({ xMin: null, xMax: null, yMin: null, yMax: null }); + onAddFilter?.(property, value, action); + }, + [onAddFilter, setFields], + ); + + // When the user changes the timeframe, reset any existing selection so the + // delta chart returns to distribution-only mode (no comparison). + // Use primitive timestamp values as deps (not the Date array reference) so + // the effect only runs when the actual date values change, not on every render. + const dateRangeStart = chartConfig.dateRange[0].getTime(); + const dateRangeEnd = chartConfig.dateRange[1].getTime(); + const isFirstDateRangeRender = useRef(true); + useEffect(() => { + if (isFirstDateRangeRender.current) { + isFirstDateRangeRender.current = false; + return; + } + setFields({ xMin: null, xMax: null, yMin: null, yMax: null }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [dateRangeStart, dateRangeEnd]); + return ( { - // Simply store the coordinates - DBDeltaChart will handle the logic setFields({ xMin, xMax, @@ -103,31 +140,22 @@ export function DBSearchHeatmapChart({ yMax, }); }} + onClearSelection={handleClearSelection} /> - {fields.xMin != null && - fields.xMax != null && - fields.yMin != null && - fields.yMax != null ? ( - - ) : ( -
- - Please highlight an outlier range in the heatmap to view the delta - chart. - -
- )} + ); } diff --git a/packages/app/src/components/__tests__/DBDeltaChart.test.ts b/packages/app/src/components/__tests__/DBDeltaChart.test.ts new file mode 100644 index 0000000000..49e8490cd3 --- /dev/null +++ b/packages/app/src/components/__tests__/DBDeltaChart.test.ts @@ -0,0 +1,1090 @@ +import { + applyTopNAggregation, + computeComparisonScore, + computeDistributionScore, + computeEffectiveSampleSize, + computeEntropyScore, + computeYValue, + flattenedKeyToFilterKey, + flattenedKeyToSqlExpression, + isDenylisted, + isHighCardinality, + isIdField, + isTimestampArrayField, + MAX_CHART_VALUES, + MAX_CHART_VALUES_UPPER, + MAX_SAMPLE_SIZE, + MIN_SAMPLE_SIZE, + SAMPLE_RATIO, + SAMPLE_SIZE, + semanticBoost, +} from '../deltaChartUtils'; + +const traceColumnMeta = [ + { name: 'Timestamp', type: 'DateTime64(9)' }, + { name: 'TraceId', type: 'String' }, + { name: 'SpanId', type: 'String' }, + { name: 'ParentSpanId', type: 'String' }, + { name: 'ResourceAttributes', type: 'Map(String, String)' }, + { name: 'SpanAttributes', type: 'Map(String, String)' }, + { name: 'Events.Timestamp', type: 'Array(DateTime64(9))' }, + { name: 'Events.Name', type: 'Array(String)' }, + { name: 'Events.Attributes', type: 'Array(Map(String, String))' }, + { name: 'Links.TraceId', type: 'Array(String)' }, + { name: 'Links.SpanId', type: 'Array(String)' }, + { name: 'Links.Timestamp', type: 'Array(DateTime64(9))' }, + { name: 'Links.Attributes', type: 'Array(Map(String, String))' }, +]; + +describe('flattenedKeyToSqlExpression', () => { + it('converts Map column dot-notation to bracket notation', () => { + expect( + flattenedKeyToSqlExpression( + 'ResourceAttributes.service.name', + traceColumnMeta, + ), + ).toBe("ResourceAttributes['service.name']"); + }); + + it('converts SpanAttributes dot-notation to bracket notation', () => { + expect( + flattenedKeyToSqlExpression( + 'SpanAttributes.http.method', + traceColumnMeta, + ), + ).toBe("SpanAttributes['http.method']"); + }); + + it('converts Array(Map) dot-notation with 0-based index to 1-based bracket notation', () => { + expect( + flattenedKeyToSqlExpression( + 'Events.Attributes[0].message.type', + traceColumnMeta, + ), + ).toBe("Events.Attributes[1]['message.type']"); + }); + + it('increments the array index from 0-based JS to 1-based ClickHouse', () => { + expect( + flattenedKeyToSqlExpression('Events.Attributes[4].key', traceColumnMeta), + ).toBe("Events.Attributes[5]['key']"); + }); + + it('handles Links.Attributes Array(Map) correctly', () => { + expect( + flattenedKeyToSqlExpression( + 'Links.Attributes[0].some.key', + traceColumnMeta, + ), + ).toBe("Links.Attributes[1]['some.key']"); + }); + + it('returns simple columns unchanged', () => { + expect(flattenedKeyToSqlExpression('TraceId', traceColumnMeta)).toBe( + 'TraceId', + ); + }); + + it('returns non-map nested columns unchanged (e.g., Arrays of primitives)', () => { + expect(flattenedKeyToSqlExpression('Events.Name[0]', traceColumnMeta)).toBe( + 'Events.Name[0]', + ); + }); + + it('returns key unchanged when no matching column found', () => { + expect( + flattenedKeyToSqlExpression('SomeUnknownColumn.key', traceColumnMeta), + ).toBe('SomeUnknownColumn.key'); + }); + + it('handles LowCardinality(Map) wrapped types', () => { + const meta = [ + { name: 'LogAttributes', type: 'LowCardinality(Map(String, String))' }, + ]; + expect(flattenedKeyToSqlExpression('LogAttributes.level', meta)).toBe( + "LogAttributes['level']", + ); + }); + + it('handles Nullable(Map) wrapped types', () => { + const meta = [{ name: 'Attrs', type: 'Nullable(Map(String, String))' }]; + expect(flattenedKeyToSqlExpression('Attrs.some.key', meta)).toBe( + "Attrs['some.key']", + ); + }); + + it('returns key unchanged for empty columnMeta', () => { + expect( + flattenedKeyToSqlExpression('ResourceAttributes.service.name', []), + ).toBe('ResourceAttributes.service.name'); + }); + + it('escapes single quotes in Map column keys to prevent SQL injection', () => { + expect( + flattenedKeyToSqlExpression( + "ResourceAttributes.it's.key", + traceColumnMeta, + ), + ).toBe("ResourceAttributes['it''s.key']"); + }); + + it('escapes single quotes in Array(Map) column keys', () => { + expect( + flattenedKeyToSqlExpression( + "Events.Attributes[0].it's.key", + traceColumnMeta, + ), + ).toBe("Events.Attributes[1]['it''s.key']"); + }); +}); + +describe('flattenedKeyToFilterKey', () => { + it('converts Map column keys to bracket notation (matching search bar format)', () => { + expect( + flattenedKeyToFilterKey( + 'ResourceAttributes.service.name', + traceColumnMeta, + ), + ).toBe("ResourceAttributes['service.name']"); + }); + + it('converts SpanAttributes Map keys to bracket notation', () => { + expect( + flattenedKeyToFilterKey('SpanAttributes.http.method', traceColumnMeta), + ).toBe("SpanAttributes['http.method']"); + }); + + it('handles multi-segment dotted Map keys as single bracket key', () => { + expect( + flattenedKeyToFilterKey( + 'ResourceAttributes.service.instance.id', + traceColumnMeta, + ), + ).toBe("ResourceAttributes['service.instance.id']"); + }); + + it('escapes single quotes in Map keys', () => { + expect( + flattenedKeyToFilterKey("ResourceAttributes.it's.key", traceColumnMeta), + ).toBe("ResourceAttributes['it''s.key']"); + }); + + it('returns simple columns unchanged', () => { + expect(flattenedKeyToFilterKey('TraceId', traceColumnMeta)).toBe('TraceId'); + }); + + it('returns simple columns unchanged for non-Map types', () => { + expect(flattenedKeyToFilterKey('Timestamp', traceColumnMeta)).toBe( + 'Timestamp', + ); + }); + + it('falls back to SQL expression for Array(Map) columns', () => { + // Array(Map) sub-keys don't have sidebar facets, so use SQL expression + expect( + flattenedKeyToFilterKey( + 'Events.Attributes[0].message.type', + traceColumnMeta, + ), + ).toBe("Events.Attributes[1]['message.type']"); + }); + + it('returns unknown column keys unchanged', () => { + expect( + flattenedKeyToFilterKey('SomeUnknownColumn.key', traceColumnMeta), + ).toBe('SomeUnknownColumn.key'); + }); +}); + +describe('isIdField', () => { + it('identifies top-level String columns ending in Id', () => { + expect(isIdField('TraceId', traceColumnMeta)).toBe(true); + expect(isIdField('SpanId', traceColumnMeta)).toBe(true); + expect(isIdField('ParentSpanId', traceColumnMeta)).toBe(true); + }); + + it('identifies Array(String) column elements whose name ends in Id', () => { + expect(isIdField('Links.TraceId[0]', traceColumnMeta)).toBe(true); + expect(isIdField('Links.SpanId[0]', traceColumnMeta)).toBe(true); + expect(isIdField('Links.TraceId[5]', traceColumnMeta)).toBe(true); + }); + + it('identifies plain Array(String) column reference ending in Id', () => { + expect(isIdField('Links.TraceId', traceColumnMeta)).toBe(true); + expect(isIdField('Links.SpanId', traceColumnMeta)).toBe(true); + }); + + it('does not match non-ID String columns', () => { + expect(isIdField('Timestamp', traceColumnMeta)).toBe(false); + expect(isIdField('Events.Name[0]', traceColumnMeta)).toBe(false); + }); + + it('does not match Map or Array(Map) columns even if name ends in Id', () => { + const meta = [{ name: 'MyMapId', type: 'Map(String, String)' }]; + expect(isIdField('MyMapId', meta)).toBe(false); + }); + + it('does not match keys with sub-keys after array index (Array(Map) paths)', () => { + expect(isIdField('Events.Attributes[0].spanId', traceColumnMeta)).toBe( + false, + ); + }); + + it('returns false for unknown columns', () => { + expect(isIdField('UnknownId', traceColumnMeta)).toBe(false); + }); + + it('returns false for empty columnMeta', () => { + expect(isIdField('TraceId', [])).toBe(false); + }); +}); + +describe('isTimestampArrayField', () => { + it('identifies Array(DateTime64) column elements by index', () => { + expect(isTimestampArrayField('Events.Timestamp[0]', traceColumnMeta)).toBe( + true, + ); + expect(isTimestampArrayField('Events.Timestamp[23]', traceColumnMeta)).toBe( + true, + ); + expect(isTimestampArrayField('Links.Timestamp[0]', traceColumnMeta)).toBe( + true, + ); + }); + + it('identifies plain Array(DateTime64) column reference', () => { + expect(isTimestampArrayField('Events.Timestamp', traceColumnMeta)).toBe( + true, + ); + expect(isTimestampArrayField('Links.Timestamp', traceColumnMeta)).toBe( + true, + ); + }); + + it('does not match non-DateTime64 array columns', () => { + expect(isTimestampArrayField('Events.Name[0]', traceColumnMeta)).toBe( + false, + ); + expect(isTimestampArrayField('Links.TraceId[0]', traceColumnMeta)).toBe( + false, + ); + expect(isTimestampArrayField('Events.Attributes[0]', traceColumnMeta)).toBe( + false, + ); + }); + + it('does not match non-array DateTime64 columns', () => { + expect(isTimestampArrayField('Timestamp', traceColumnMeta)).toBe(false); + }); + + it('does not match Array(Map) sub-key paths', () => { + expect( + isTimestampArrayField('Events.Attributes[0].timestamp', traceColumnMeta), + ).toBe(false); + }); + + it('returns false for unknown columns', () => { + expect(isTimestampArrayField('Unknown.Timestamp[0]', traceColumnMeta)).toBe( + false, + ); + }); + + it('handles Array(DateTime64) with timezone parameter', () => { + const meta = [ + { name: 'MyTimestamps', type: "Array(DateTime64(9, 'UTC'))" }, + ]; + expect(isTimestampArrayField('MyTimestamps[0]', meta)).toBe(true); + }); +}); + +describe('isDenylisted', () => { + it('denylists ID fields', () => { + expect(isDenylisted('TraceId', traceColumnMeta)).toBe(true); + expect(isDenylisted('SpanId', traceColumnMeta)).toBe(true); + expect(isDenylisted('ParentSpanId', traceColumnMeta)).toBe(true); + expect(isDenylisted('Links.TraceId[0]', traceColumnMeta)).toBe(true); + }); + + it('denylists timestamp array fields', () => { + expect(isDenylisted('Events.Timestamp[0]', traceColumnMeta)).toBe(true); + expect(isDenylisted('Links.Timestamp[3]', traceColumnMeta)).toBe(true); + }); + + it('does not denylist useful fields', () => { + expect( + isDenylisted('ResourceAttributes.service.name', traceColumnMeta), + ).toBe(false); + expect(isDenylisted('SpanAttributes.http.method', traceColumnMeta)).toBe( + false, + ); + expect(isDenylisted('Events.Name[0]', traceColumnMeta)).toBe(false); + }); +}); + +describe('isHighCardinality', () => { + it('identifies high cardinality fields (all unique values)', () => { + // 1000 unique values out of 1000 total occurrences + const outlierValues = new Map(); + for (let i = 0; i < 1000; i++) { + outlierValues.set(`value-${i}`, 0.1); + } + const outlierValueOccurences = new Map([['TraceId', outlierValues]]); + const outlierPropertyOccurences = new Map([['TraceId', 1000]]); + + expect( + isHighCardinality( + 'TraceId', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + new Map(), + ), + ).toBe(true); + }); + + it('keeps low cardinality fields visible (few distinct values)', () => { + const outlierValues = new Map([ + ['GET', 80], + ['POST', 20], + ]); + const outlierValueOccurences = new Map([['http.method', outlierValues]]); + const outlierPropertyOccurences = new Map([['http.method', 1000]]); + + expect( + isHighCardinality( + 'http.method', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + new Map(), + ), + ).toBe(false); + }); + + it('uses min of both groups — keeps visible if either group has low cardinality', () => { + // Outliers: 2 unique values (low cardinality) + const outlierValues = new Map([ + ['GET', 80], + ['POST', 20], + ]); + const outlierValueOccurences = new Map([['method', outlierValues]]); + const outlierPropertyOccurences = new Map([['method', 1000]]); + + // Inliers: 500 unique values (high cardinality) + const inlierValues = new Map(); + for (let i = 0; i < 500; i++) inlierValues.set(`v${i}`, 0.2); + const inlierValueOccurences = new Map([['method', inlierValues]]); + const inlierPropertyOccurences = new Map([['method', 500]]); + + // outlierUniqueness = 2/1000 = 0.002, inlierUniqueness = 500/500 = 1.0 + // min = 0.002 < 0.9 → field is visible + expect( + isHighCardinality( + 'method', + outlierValueOccurences, + inlierValueOccurences, + outlierPropertyOccurences, + inlierPropertyOccurences, + ), + ).toBe(false); + }); + + it('hides field when BOTH groups have high cardinality', () => { + const makeHighCardinalityMap = (n: number) => { + const m = new Map(); + for (let i = 0; i < n; i++) m.set(`v${i}`, 100 / n); + return m; + }; + + const outlierValues = makeHighCardinalityMap(500); + const inlierValues = makeHighCardinalityMap(400); + const outlierValueOccurences = new Map([['url', outlierValues]]); + const inlierValueOccurences = new Map([['url', inlierValues]]); + const outlierPropertyOccurences = new Map([['url', 500]]); + const inlierPropertyOccurences = new Map([['url', 400]]); + + // min(500/500, 400/400) = 1.0 > 0.9 → hidden + expect( + isHighCardinality( + 'url', + outlierValueOccurences, + inlierValueOccurences, + outlierPropertyOccurences, + inlierPropertyOccurences, + ), + ).toBe(true); + }); + + it('keeps visible when combined sample size is <= 20', () => { + const outlierValues = new Map(); + for (let i = 0; i < 10; i++) outlierValues.set(`v${i}`, 10); + const outlierValueOccurences = new Map([['field', outlierValues]]); + const outlierPropertyOccurences = new Map([['field', 10]]); + const inlierPropertyOccurences = new Map([['field', 10]]); + + // combined = 10 + 10 = 20, threshold is > 20 + expect( + isHighCardinality( + 'field', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + inlierPropertyOccurences, + ), + ).toBe(false); + }); + + it('uses single group uniqueness when other group has no data', () => { + // Only outlier data, high cardinality + const outlierValues = new Map(); + for (let i = 0; i < 100; i++) outlierValues.set(`v${i}`, 1); + const outlierValueOccurences = new Map([['id', outlierValues]]); + const outlierPropertyOccurences = new Map([['id', 100]]); + + expect( + isHighCardinality( + 'id', + outlierValueOccurences, + new Map(), + outlierPropertyOccurences, + new Map(), + ), + ).toBe(true); + }); + + it('returns false for field not present in either group', () => { + expect( + isHighCardinality( + 'unknownField', + new Map(), + new Map(), + new Map(), + new Map(), + ), + ).toBe(false); + }); +}); + +describe('applyTopNAggregation', () => { + const makeData = (names: string[]) => + names.map((name, i) => ({ + name, + outlierCount: 100 - i, // descending counts so order is deterministic + inlierCount: 5, + })); + + it('returns data unchanged when at or below MAX_CHART_VALUES', () => { + const data = makeData(['a', 'b', 'c']); + expect(applyTopNAggregation(data)).toEqual(data); + }); + + it('returns data unchanged when exactly MAX_CHART_VALUES entries', () => { + const data = makeData(['a', 'b', 'c', 'd', 'e', 'f']); + expect(data.length).toBe(MAX_CHART_VALUES); + const result = applyTopNAggregation(data); + expect(result.length).toBe(MAX_CHART_VALUES); + expect(result.some(r => r.isOther)).toBe(false); + }); + + it('returns data unchanged when between MAX_CHART_VALUES and MAX_CHART_VALUES_UPPER (adaptive)', () => { + // 7 and 8 values should be shown in full — no "Other" bucket + const data7 = makeData(['a', 'b', 'c', 'd', 'e', 'f', 'g']); + expect(data7.length).toBeGreaterThan(MAX_CHART_VALUES); + expect(data7.length).toBeLessThanOrEqual(MAX_CHART_VALUES_UPPER); + expect(applyTopNAggregation(data7)).toEqual(data7); + + const data8 = makeData(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']); + expect(data8.length).toBe(MAX_CHART_VALUES_UPPER); + const result8 = applyTopNAggregation(data8); + expect(result8.some(r => r.isOther)).toBe(false); + }); + + it('collapses values beyond MAX_CHART_VALUES_UPPER into an Other bucket', () => { + const data = makeData(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']); + expect(data.length).toBeGreaterThan(MAX_CHART_VALUES_UPPER); + const result = applyTopNAggregation(data); + expect(result.length).toBe(MAX_CHART_VALUES + 1); + const other = result[result.length - 1]; + expect(other.isOther).toBe(true); + expect(other.name).toBe('Other (3)'); + }); + + it('keeps top N entries by combined count', () => { + const data = makeData(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']); + const result = applyTopNAggregation(data); + const topNames = result.slice(0, MAX_CHART_VALUES).map(r => r.name); + expect(topNames).toContain('a'); + expect(topNames).toContain('b'); + expect(topNames).not.toContain('i'); + expect(topNames).not.toContain('j'); + }); + + it('Other bucket accumulates outlierCount and inlierCount from dropped values', () => { + // 12 items > MAX_CHART_VALUES_UPPER(8) → top 6 kept, bottom 6 collapsed + const data = [ + { name: 'a', outlierCount: 50, inlierCount: 50 }, // kept (100) + { name: 'b', outlierCount: 40, inlierCount: 40 }, // kept (80) + { name: 'c', outlierCount: 30, inlierCount: 30 }, // kept (60) + { name: 'd', outlierCount: 20, inlierCount: 20 }, // kept (40) + { name: 'e', outlierCount: 10, inlierCount: 10 }, // kept (20) + { name: 'f', outlierCount: 5, inlierCount: 5 }, // kept (10) + { name: 'g', outlierCount: 4, inlierCount: 4 }, // dropped + { name: 'h', outlierCount: 3, inlierCount: 3 }, // dropped + { name: 'i', outlierCount: 3, inlierCount: 2 }, // dropped + { name: 'j', outlierCount: 1, inlierCount: 1 }, // dropped + { name: 'k', outlierCount: 0, inlierCount: 1 }, // dropped + { name: 'l', outlierCount: 0, inlierCount: 0 }, // dropped + ]; + const result = applyTopNAggregation(data); + const other = result.find(r => r.isOther); + expect(other).toBeDefined(); + expect(other!.outlierCount).toBe(11); // 4 + 3 + 3 + 1 + 0 + 0 + expect(other!.inlierCount).toBe(11); // 4 + 3 + 2 + 1 + 1 + 0 + }); + + it('Other bucket name shows the count of aggregated values', () => { + const data = makeData([ + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + ]); + const result = applyTopNAggregation(data); + const other = result.find(r => r.isOther); + expect(other?.name).toBe('Other (5)'); + }); + + it('non-Other entries do not have isOther set', () => { + const data = makeData(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']); + const result = applyTopNAggregation(data); + result.slice(0, MAX_CHART_VALUES).forEach(entry => { + expect(entry.isOther).toBeFalsy(); + }); + }); +}); + +describe('computeComparisonScore', () => { + it('returns 0 when both groups have identical single-value proportions', () => { + // Events.Name[3] = "message" at different coverage rates but same proportion + // Outlier: 80% coverage, Inlier: 27% coverage — but both are 100% "message" + const outlier = new Map([['message', 80]]); + const inlier = new Map([['message', 27]]); + expect(computeComparisonScore(outlier, inlier)).toBeCloseTo(0); + }); + + it('returns 0 when both groups have identical multi-value proportions', () => { + // Same 60/40 split in both groups, different total coverage + const outlier = new Map([ + ['GET', 48], + ['POST', 32], + ]); + const inlier = new Map([ + ['GET', 18], + ['POST', 12], + ]); + expect(computeComparisonScore(outlier, inlier)).toBeCloseTo(0); + }); + + it('returns high score for genuinely different distributions', () => { + // Selection has mostly errors, background has mostly successes + const outlier = new Map([ + ['error', 70], + ['success', 10], + ]); + const inlier = new Map([ + ['error', 5], + ['success', 80], + ]); + expect(computeComparisonScore(outlier, inlier)).toBeGreaterThan(50); + }); + + it('returns high score when a value exists in one group but not the other', () => { + const outlier = new Map([['error', 50]]); + const inlier = new Map([['success', 80]]); + expect(computeComparisonScore(outlier, inlier)).toBeGreaterThan(50); + }); + + it('returns 0 when both groups are empty', () => { + expect(computeComparisonScore(new Map(), new Map())).toBe(0); + }); + + it('uses raw delta as fallback when one group has no data', () => { + const outlier = new Map([['error', 50]]); + expect(computeComparisonScore(outlier, new Map())).toBe(50); + }); + + it('ranks genuinely different fields above same-proportion fields', () => { + // Same proportion (100% "message" at different rates) + const sameScore = computeComparisonScore( + new Map([['message', 80]]), + new Map([['message', 27]]), + ); + // Different distribution (90% error in selection vs 10% in background) + const diffScore = computeComparisonScore( + new Map([ + ['error', 90], + ['ok', 10], + ]), + new Map([ + ['error', 10], + ['ok', 90], + ]), + ); + expect(diffScore).toBeGreaterThan(sameScore); + }); +}); + +describe('computeDistributionScore', () => { + it('returns 0 for empty map', () => { + expect(computeDistributionScore(new Map())).toBe(0); + }); + + it('returns 0 for a single-value field (all spans have same value)', () => { + // 1 value at 100% — boring, not useful for filtering + expect(computeDistributionScore(new Map([['production', 100]]))).toBe(0); + }); + + it('returns 0 for a perfectly uniform 2-value field', () => { + // 50% / 50% — uniform, nothing stands out + expect( + computeDistributionScore( + new Map([ + ['GET', 50], + ['POST', 50], + ]), + ), + ).toBe(0); + }); + + it('returns high score for a skewed 2-value field', () => { + // 90% / 10% → score = 90 - 50 = 40 + const score = computeDistributionScore( + new Map([ + ['fraud-detection', 90], + ['auth', 10], + ]), + ); + expect(score).toBeCloseTo(40); + }); + + it('ranks more-skewed fields higher than less-skewed fields', () => { + // 80%/20% → score = 80 - 50 = 30 + const scoreA = computeDistributionScore( + new Map([ + ['a', 80], + ['b', 20], + ]), + ); + // 60%/40% → score = 60 - 50 = 10 + const scoreB = computeDistributionScore( + new Map([ + ['a', 60], + ['b', 40], + ]), + ); + expect(scoreA).toBeGreaterThan(scoreB); + }); + + it('returns 0 for a perfectly uniform 3-value field', () => { + // 33%/33%/33% → score = 33 - 33 = 0 + const score = computeDistributionScore( + new Map([ + ['a', 33.33], + ['b', 33.33], + ['c', 33.33], + ]), + ); + expect(score).toBeCloseTo(0, 1); + }); + + it('returns positive score for a skewed 3-value field', () => { + // 70%/20%/10% → score = 70 - 33.3 ≈ 36.7 + const score = computeDistributionScore( + new Map([ + ['a', 70], + ['b', 20], + ['c', 10], + ]), + ); + expect(score).toBeCloseTo(36.67, 1); + }); + + it('single-value field scores lower than two-value skewed field', () => { + const singleValue = computeDistributionScore(new Map([['only', 100]])); + const skewed = computeDistributionScore( + new Map([ + ['dominant', 95], + ['other', 5], + ]), + ); + expect(skewed).toBeGreaterThan(singleValue); + }); + + it('works correctly when percentages do not sum to 100 (all-spans mode)', () => { + // After the percentage fix, values represent % of ALL spans. + // Property "env" appears in 60% of spans: prod=40%, staging=20%. + // Score should be positive (skewed), not zero. + const score = computeDistributionScore( + new Map([ + ['prod', 40], + ['staging', 20], + ]), + ); + // mean = 30, max = 40, score = 10 + expect(score).toBeCloseTo(10); + }); + + it('returns 0 for a single-valued property that appears in only 30% of spans', () => { + // 1 value at 30% (of all spans) — nValues=1, returns 0 regardless + expect(computeDistributionScore(new Map([['production', 30]]))).toBe(0); + }); +}); + +describe('computeEntropyScore', () => { + it('returns 0 for empty map', () => { + expect(computeEntropyScore(new Map())).toBe(0); + }); + + it('returns 0 for a single-value field', () => { + expect(computeEntropyScore(new Map([['production', 100]]))).toBe(0); + }); + + it('returns 0 for a perfectly uniform 2-value field', () => { + // H = log2(2) = 1, maxH = log2(2) = 1 → 1 - 1/1 = 0 + expect( + computeEntropyScore( + new Map([ + ['GET', 50], + ['POST', 50], + ]), + ), + ).toBeCloseTo(0); + }); + + it('returns high score for a skewed 2-value field', () => { + const score = computeEntropyScore( + new Map([ + ['ok', 99], + ['error', 1], + ]), + ); + // Very skewed → low entropy → high score (close to 1) + expect(score).toBeGreaterThan(0.5); + }); + + it('ranks more-skewed fields higher than less-skewed fields', () => { + const scoreA = computeEntropyScore( + new Map([ + ['a', 95], + ['b', 5], + ]), + ); + const scoreB = computeEntropyScore( + new Map([ + ['a', 60], + ['b', 40], + ]), + ); + expect(scoreA).toBeGreaterThan(scoreB); + }); + + it('returns 0 for a perfectly uniform 3-value field', () => { + const score = computeEntropyScore( + new Map([ + ['a', 33.33], + ['b', 33.33], + ['c', 33.34], + ]), + ); + expect(score).toBeCloseTo(0, 2); + }); + + it('returns positive score for a skewed 3-value field', () => { + const score = computeEntropyScore( + new Map([ + ['a', 90], + ['b', 5], + ['c', 5], + ]), + ); + expect(score).toBeGreaterThan(0.3); + }); + + it('handles power-law distributions better than skewness scorer', () => { + // Power-law: 50, 25, 12, 6, 4, 2, 1 — entropy captures this spread + const powerLaw = new Map([ + ['v1', 50], + ['v2', 25], + ['v3', 12], + ['v4', 6], + ['v5', 4], + ['v6', 2], + ['v7', 1], + ]); + const score = computeEntropyScore(powerLaw); + expect(score).toBeGreaterThan(0.1); + expect(score).toBeLessThan(1); + }); + + it('works when percentages do not sum to 100', () => { + // Property appears in 60% of spans: prod=40%, staging=20% + const score = computeEntropyScore( + new Map([ + ['prod', 40], + ['staging', 20], + ]), + ); + // Normalized by sum, so this is really 67%/33% — moderate skew + expect(score).toBeGreaterThan(0); + expect(score).toBeLessThan(0.5); + }); +}); + +describe('semanticBoost', () => { + it('boosts well-known OTel attributes', () => { + expect(semanticBoost('ResourceAttributes.service.name')).toBe(1); + expect(semanticBoost('SpanAttributes.http.method')).toBe(1); + expect(semanticBoost('SpanAttributes.http.status_code')).toBe(1); + expect(semanticBoost('SpanAttributes.error')).toBe(1); + expect(semanticBoost('ResourceAttributes.deployment.environment')).toBe(1); + }); + + it('boosts new OTel semconv attribute names', () => { + expect(semanticBoost('SpanAttributes.http.request.method')).toBe(1); + expect(semanticBoost('SpanAttributes.http.response.status_code')).toBe(1); + }); + + it('returns 0 for non-boosted attributes', () => { + expect(semanticBoost('SpanAttributes.custom.field')).toBe(0); + expect(semanticBoost('ResourceAttributes.host.name')).toBe(0); + expect(semanticBoost('TraceId')).toBe(0); + }); + + it('is case-insensitive', () => { + expect(semanticBoost('ResourceAttributes.Service.Name')).toBe(1); + expect(semanticBoost('SpanAttributes.HTTP.METHOD')).toBe(1); + }); +}); + +describe('scoring integration: entropy + semantic boost as tiebreaker', () => { + // These tests verify the intended scoring behavior in DBDeltaChart: + // sortScore = baseScore > 0 ? baseScore + semanticBoost(key) * 0.1 : 0 + const computeSortScore = ( + key: string, + valuePercentages: Map, + ) => { + const baseScore = computeEntropyScore(valuePercentages); + const boost = baseScore > 0 ? semanticBoost(key) * 0.1 : 0; + return baseScore + boost; + }; + + it('single-value boosted attribute scores 0 (not boosted above multi-value fields)', () => { + // service.name with 100% "payment" — completely useless for outlier detection + const score = computeSortScore( + 'ResourceAttributes.service.name', + new Map([['payment', 100]]), + ); + expect(score).toBe(0); + }); + + it('multi-value non-boosted field with unequal distribution beats single-value boosted field', () => { + const boostedSingleValue = computeSortScore( + 'ResourceAttributes.service.name', + new Map([['payment', 100]]), + ); + const nonBoostedSkewed = computeSortScore( + 'SpanAttributes.loyalty.level', + new Map([ + ['bronze', 60], + ['silver', 20], + ['gold', 15], + ['platinum', 5], + ]), + ); + expect(nonBoostedSkewed).toBeGreaterThan(boostedSingleValue); + }); + + it('among fields with similar variance, boosted attribute ranks higher', () => { + const skewedValues = new Map([ + ['Error', 80], + ['Unset', 20], + ]); + const boostedScore = computeSortScore('SpanAttributes.error', skewedValues); + const nonBoostedScore = computeSortScore( + 'SpanAttributes.custom.flag', + skewedValues, + ); + expect(boostedScore).toBeGreaterThan(nonBoostedScore); + // But the difference is small (tiebreaker only) + expect(boostedScore - nonBoostedScore).toBeCloseTo(0.1, 1); + }); + + it('boost never overrides a genuinely more interesting distribution', () => { + // Boosted but mildly skewed + const boostedMild = computeSortScore( + 'SpanAttributes.http.method', + new Map([ + ['GET', 55], + ['POST', 45], + ]), + ); + // Non-boosted but highly skewed + const nonBoostedStrong = computeSortScore( + 'SpanAttributes.custom.field', + new Map([ + ['dominant', 95], + ['rare', 5], + ]), + ); + expect(nonBoostedStrong).toBeGreaterThan(boostedMild); + }); +}); + +describe('computeEffectiveSampleSize', () => { + it('returns SAMPLE_SIZE when totalCount is 0 (fallback)', () => { + expect(computeEffectiveSampleSize(0)).toBe(SAMPLE_SIZE); + }); + + it('returns SAMPLE_SIZE when totalCount is negative', () => { + expect(computeEffectiveSampleSize(-1)).toBe(SAMPLE_SIZE); + }); + + it('returns MIN_SAMPLE_SIZE for small datasets', () => { + // 100 rows * 0.01 = 1 → clamped to MIN_SAMPLE_SIZE + expect(computeEffectiveSampleSize(100)).toBe(MIN_SAMPLE_SIZE); + }); + + it('returns SAMPLE_RATIO * totalCount for mid-size datasets', () => { + // 200,000 * 0.01 = 2000 → between MIN and MAX + const result = computeEffectiveSampleSize(200_000); + expect(result).toBe(Math.ceil(200_000 * SAMPLE_RATIO)); + expect(result).toBeGreaterThan(MIN_SAMPLE_SIZE); + expect(result).toBeLessThan(MAX_SAMPLE_SIZE); + }); + + it('caps at MAX_SAMPLE_SIZE for very large datasets', () => { + // 10,000,000 * 0.01 = 100,000 → capped to MAX_SAMPLE_SIZE + expect(computeEffectiveSampleSize(10_000_000)).toBe(MAX_SAMPLE_SIZE); + }); + + it('returns exact 1% for datasets where 1% falls in the valid range', () => { + // 100,000 * 0.01 = 1000 → between MIN(500) and MAX(5000) + expect(computeEffectiveSampleSize(100_000)).toBe(1000); + }); +}); + +describe('computeYValue', () => { + it('returns value for simple column reference', () => { + expect(computeYValue('Duration', { Duration: 5000000 })).toBe(5000000); + }); + + it('handles string-encoded numbers (ClickHouse UInt64)', () => { + expect(computeYValue('Duration', { Duration: '5000000' })).toBe(5000000); + }); + + it('handles division expression "Col / N"', () => { + expect( + computeYValue('Duration / 1000000', { Duration: 5000000 }), + ).toBeCloseTo(5); + }); + + it('handles parenthesized column in division "(Col)/N" (getDurationMsExpression format)', () => { + expect(computeYValue('(Duration)/1e6', { Duration: 5000000 })).toBeCloseTo( + 5, + ); + expect( + computeYValue('(Duration) / 1e6', { Duration: 5000000 }), + ).toBeCloseTo(5); + expect( + computeYValue('(Duration)/1000000', { Duration: 5000000 }), + ).toBeCloseTo(5); + }); + + it('handles simple parenthesized column reference "(Col)"', () => { + expect(computeYValue('(Duration)', { Duration: 5000000 })).toBe(5000000); + }); + + it('handles multiplication expression "Col * N"', () => { + expect( + computeYValue('Duration * 0.001', { Duration: 5000000 }), + ).toBeCloseTo(5000); + }); + + it('handles parenthesized column in multiplication "(Col) * N"', () => { + expect( + computeYValue('(Duration) * 0.001', { Duration: 5000000 }), + ).toBeCloseTo(5000); + expect( + computeYValue('(Duration)*0.001', { Duration: 5000000 }), + ).toBeCloseTo(5000); + }); + + it('handles scientific notation divisor', () => { + expect(computeYValue('Duration / 1e6', { Duration: 5000000 })).toBeCloseTo( + 5, + ); + }); + + it('returns null when column is missing', () => { + expect(computeYValue('Duration / 1000000', { OtherCol: 42 })).toBeNull(); + }); + + it('returns null for complex expressions it cannot parse', () => { + expect(computeYValue('count()', {})).toBeNull(); + expect(computeYValue('toUnixTimestamp(Timestamp)', {})).toBeNull(); + }); + + it('returns null when divisor is zero', () => { + expect(computeYValue('Duration / 0', { Duration: 5000 })).toBeNull(); + }); +}); + +describe('field split logic (visible vs hidden)', () => { + it('correctly classifies a mix of ID, timestamp, cardinality, and useful fields', () => { + // TraceId → denylist (ID field, String) + expect(isDenylisted('TraceId', traceColumnMeta)).toBe(true); + + // Events.Timestamp[0] → denylist (timestamp array) + expect(isDenylisted('Events.Timestamp[0]', traceColumnMeta)).toBe(true); + + // ResourceAttributes.service.name → not denylisted + expect( + isDenylisted('ResourceAttributes.service.name', traceColumnMeta), + ).toBe(false); + + // High cardinality field with 1000 unique values in 1000 rows → hidden + const hcValues = new Map(); + for (let i = 0; i < 1000; i++) hcValues.set(`trace-${i}`, 0.1); + expect( + isHighCardinality( + 'trace.id', + new Map([['trace.id', hcValues]]), + new Map(), + new Map([['trace.id', 1000]]), + new Map(), + ), + ).toBe(true); + + // Low cardinality field → visible + const lcValues = new Map([ + ['production', 70], + ['staging', 30], + ]); + expect( + isHighCardinality( + 'deployment.env', + new Map([['deployment.env', lcValues]]), + new Map(), + new Map([['deployment.env', 1000]]), + new Map(), + ), + ).toBe(false); + }); +}); diff --git a/packages/app/src/components/deltaChartUtils.ts b/packages/app/src/components/deltaChartUtils.ts new file mode 100644 index 0000000000..3242154863 --- /dev/null +++ b/packages/app/src/components/deltaChartUtils.ts @@ -0,0 +1,636 @@ +/** + * Utility functions and types for DBDeltaChart. + * Pure helpers with no React dependencies — safe to import from tests. + */ + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +export function stripTypeWrappers(type: string): string { + let t = type.trim(); + let changed = true; + while (changed) { + changed = false; + if (t.startsWith('LowCardinality(') && t.endsWith(')')) { + t = t.slice('LowCardinality('.length, -1).trim(); + changed = true; + } else if (t.startsWith('Nullable(') && t.endsWith(')')) { + t = t.slice('Nullable('.length, -1).trim(); + changed = true; + } + } + return t; +} + +/** + * Converts a flattened dot-notation property key (produced by flattenData()) + * into a valid ClickHouse SQL expression for use in filter conditions. + * + * flattenData() uses JavaScript's object/array iteration, producing keys like: + * "ResourceAttributes.service.name" for Map(String, String) columns + * "Events.Attributes[0].message.type" for Array(Map(String, String)) columns + * + * These must be converted to bracket notation for ClickHouse Map access: + * "ResourceAttributes['service.name']" + * "Events.Attributes[1]['message.type']" (note: 0-based JS → 1-based CH index) + */ +export function flattenedKeyToSqlExpression( + key: string, + columnMeta: { name: string; type: string }[], +): string { + for (const col of columnMeta) { + const baseType = stripTypeWrappers(col.type); + + if (baseType.startsWith('Map(')) { + // Simple Map column: "MapCol.some.key" → "MapCol['some.key']" + if (key.startsWith(col.name + '.')) { + const mapKey = key.slice(col.name.length + 1).replace(/'/g, "''"); + return `${col.name}['${mapKey}']`; + } + } else if (baseType.startsWith('Array(')) { + const innerType = stripTypeWrappers(baseType.slice('Array('.length, -1)); + if (innerType.startsWith('Map(')) { + // Array(Map) column: "ColName[N].key" → "ColName[N+1]['key']" + // flattenData() uses 0-based JS indexing; ClickHouse SQL uses 1-based. + const pattern = new RegExp( + `^${escapeRegExp(col.name)}\\[(\\d+)\\]\\.(.+)$`, + ); + const match = key.match(pattern); + if (match) { + const chIndex = parseInt(match[1]) + 1; + const mapKey = match[2].replace(/'/g, "''"); + return `${col.name}[${chIndex}]['${mapKey}']`; + } + } + } + } + return key; +} + +/** + * Returns true if the field is a structural ID field that should always be hidden. + * + * Matches: + * - Top-level String columns whose name ends in "Id" or "ID" (e.g., TraceId, SpanId) + * - Array(String) column elements or plain column references whose name ends in + * "Id" or "ID" (e.g., Links.TraceId[0] from a Links.TraceId Array(String) column) + */ +export function isIdField( + key: string, + columnMeta: { name: string; type: string }[], +): boolean { + // Extract base column name: + // "ColName[N]" → colName is "ColName" + // "ColName" (no brackets) → colName is the key itself + // "ColName[N].subkey" → has brackets but doesn't end with ], skip + const arrMatch = key.match(/^([^[]+)\[(\d+)\]$/); + const colName = arrMatch ? arrMatch[1] : key.includes('[') ? null : key; + if (!colName) return false; + if (!/(Id|ID)$/.test(colName)) return false; + + const col = columnMeta.find(c => c.name === colName); + if (!col) return false; + const baseType = stripTypeWrappers(col.type); + if (baseType === 'String') return true; + if (baseType.startsWith('Array(')) { + const innerType = stripTypeWrappers(baseType.slice('Array('.length, -1)); + return innerType === 'String'; + } + return false; +} + +/** + * Returns true if the field is a per-index timestamp array element (e.g., + * Events.Timestamp[0]) from a column of type Array(DateTime64(...)), or the + * plain column reference itself (e.g., Events.Timestamp). + */ +export function isTimestampArrayField( + key: string, + columnMeta: { name: string; type: string }[], +): boolean { + const arrMatch = key.match(/^([^[]+)\[(\d+)\]$/); + const colName = arrMatch ? arrMatch[1] : key.includes('[') ? null : key; + if (!colName) return false; + + const col = columnMeta.find(c => c.name === colName); + if (!col) return false; + const baseType = stripTypeWrappers(col.type); + if (!baseType.startsWith('Array(')) return false; + const innerType = stripTypeWrappers(baseType.slice('Array('.length, -1)); + return innerType.startsWith('DateTime64('); +} + +/** + * Returns true if the field should always be hidden per the structural denylist: + * - ID fields (TraceId, SpanId, ParentSpanId, Links.TraceId[N], Links.SpanId[N], etc.) + * - Per-index timestamp array elements (Events.Timestamp[N], Links.Timestamp[N], etc.) + */ +export function isDenylisted( + key: string, + columnMeta: { name: string; type: string }[], +): boolean { + return isIdField(key, columnMeta) || isTimestampArrayField(key, columnMeta); +} + +/** + * Returns true if the field should be hidden due to high cardinality (most values are + * unique, meaning it provides little analytical value in the comparison view). + * + * Takes the percentage occurrence maps (value → percentage 0–100) produced by + * getPropertyStatistics, and the raw property occurrence counts. Unique value count is + * derived from the map's size. + * + * A field is considered high cardinality when: + * min(outlierUniqueness, inlierUniqueness) > 0.9 AND combined sample size > 20 + * + * "min" ensures that if either group clusters (low cardinality), the field is kept visible. + * If only one group has data, that group's uniqueness alone is used. + */ +export function isHighCardinality( + key: string, + outlierValueOccurences: Map>, + inlierValueOccurences: Map>, + outlierPropertyOccurences: Map, + inlierPropertyOccurences: Map, +): boolean { + const outlierTotal = outlierPropertyOccurences.get(key) ?? 0; + const inlierTotal = inlierPropertyOccurences.get(key) ?? 0; + const combinedSampleSize = outlierTotal + inlierTotal; + if (combinedSampleSize <= 20) return false; + + const outlierUniqueValues = outlierValueOccurences.get(key)?.size ?? 0; + const inlierUniqueValues = inlierValueOccurences.get(key)?.size ?? 0; + + const outlierUniqueness = + outlierTotal > 0 ? outlierUniqueValues / outlierTotal : null; + const inlierUniqueness = + inlierTotal > 0 ? inlierUniqueValues / inlierTotal : null; + + let effectiveUniqueness: number; + if (outlierUniqueness !== null && inlierUniqueness !== null) { + effectiveUniqueness = Math.min(outlierUniqueness, inlierUniqueness); + } else if (outlierUniqueness !== null) { + effectiveUniqueness = outlierUniqueness; + } else if (inlierUniqueness !== null) { + effectiveUniqueness = inlierUniqueness; + } else { + return false; + } + + return effectiveUniqueness > 0.9; +} + +// Recursively flattens nested objects/arrays into dot-notation keys. +// Empty objects produce an empty {} entry; empty arrays produce an empty [] entry. +// Based on https://stackoverflow.com/a/19101235 +export function flattenData(data: Record) { + const result: Record = {}; + function recurse(cur: Record, prop: string) { + if (Object(cur) !== cur) { + result[prop] = cur; + } else if (Array.isArray(cur)) { + let l; + for (let i = 0, l = cur.length; i < l; i++) + recurse(cur[i], prop + '[' + i + ']'); + if (l == 0) result[prop] = []; + } else { + let isEmpty = true; + for (const p in cur) { + isEmpty = false; + recurse(cur[p], prop ? prop + '.' + p : p); + } + if (isEmpty && prop) result[prop] = {}; + } + } + recurse(data, ''); + return result; +} + +export function getPropertyStatistics(data: Record[]) { + const flattened = data.map(flattenData); + const propertyOccurences = new Map(); + + const MIN_PROPERTY_OCCURENCES = 5; + const commonProperties = new Set(); + + flattened.forEach(item => { + Object.entries(item).forEach(([key, value]) => { + const count = propertyOccurences.get(key) || 0; + propertyOccurences.set(key, count + 1); + + if (count + 1 >= MIN_PROPERTY_OCCURENCES) { + commonProperties.add(key); + } + }); + }); + + // property -> (value -> count) + const valueOccurences = new Map>(); + flattened.forEach(item => { + Object.entries(item).forEach(([key, value]) => { + if (commonProperties.has(key)) { + let valuesMap = valueOccurences.get(key); + if (!valuesMap) { + valuesMap = new Map(); + valueOccurences.set(key, valuesMap); + } + + const valueCount = valuesMap.get(value) || 0; + valuesMap.set(value, valueCount + 1); + } + }); + }); + + // Divide by total rows so percentages represent "fraction of ALL spans", + // not "fraction of spans that have this property". This ensures a single-valued + // field that only appears in 30% of spans shows 30%, not 100%. + const totalRows = data.length || 1; + const percentageOccurences = new Map>(); + valueOccurences.forEach((valuesMap, property) => { + const percentageMap = new Map(); + valuesMap.forEach((valueCount, value) => { + percentageMap.set(value, (valueCount / totalRows) * 100); + }); + percentageOccurences.set(property, percentageMap); + }); + + return { + percentageOccurences, + propertyOccurences, + }; +} + +export function mergeValueStatisticsMaps( + outlierValues: Map, // value -> count + inlierValues: Map, +) { + const mergedArray: { + name: string; + outlierCount: number; + inlierCount: number; + }[] = []; + // Collect all value names for this property + // we sort them so timestamps are ordered + const allValues = Array.from( + new Set([...outlierValues.keys(), ...inlierValues.keys()]), + ).sort(); + + allValues.forEach(value => { + const count1 = outlierValues.get(value) || 0; + const count2 = inlierValues.get(value) || 0; + mergedArray.push({ + name: value, + outlierCount: count1, + inlierCount: count2, + }); + }); + + return mergedArray; +} + +/** + * Computes a comparison score for sorting properties in selection mode. + * + * Normalizes each group's percentages to sum to 100% before computing delta, + * so the score measures *proportional distribution difference*, not coverage + * rate difference. This prevents fields like Events.Name[N] (same "message" + * value in both groups but different coverage rates due to sample composition) + * from sorting above fields with genuinely different value distributions. + * + * Returns max(|normalizedOutlierPct - normalizedInlierPct|) across all values. + * Score is 0 when both groups have identical proportional distributions, + * regardless of how many spans in each group have this property. + */ +export function computeComparisonScore( + outlierValues: Map, + inlierValues: Map, +): number { + const allValues = new Set([...outlierValues.keys(), ...inlierValues.keys()]); + if (allValues.size === 0) return 0; + + // Sum each group's raw percentages (may not be 100% since denominator is totalRows) + let outlierSum = 0; + let inlierSum = 0; + outlierValues.forEach(v => (outlierSum += v)); + inlierValues.forEach(v => (inlierSum += v)); + + // If either group has no data for this property, use raw delta as fallback + if (outlierSum === 0 && inlierSum === 0) return 0; + if (outlierSum === 0 || inlierSum === 0) { + // Property exists in one group but not the other — genuinely different + let maxDelta = 0; + allValues.forEach(value => { + const delta = Math.abs( + (outlierValues.get(value) ?? 0) - (inlierValues.get(value) ?? 0), + ); + if (delta > maxDelta) maxDelta = delta; + }); + return maxDelta; + } + + // Normalize to proportions (0-100) and compute max delta + let maxDelta = 0; + allValues.forEach(value => { + const outlierNorm = ((outlierValues.get(value) ?? 0) / outlierSum) * 100; + const inlierNorm = ((inlierValues.get(value) ?? 0) / inlierSum) * 100; + const delta = Math.abs(outlierNorm - inlierNorm); + if (delta > maxDelta) maxDelta = delta; + }); + return maxDelta; +} + +/** + * Computes a distribution skewness score for sorting properties in "all spans" mode. + * + * Score = max(pct) - mean(pcts): + * - 0 for single-value fields (all spans share the same value → not useful for filtering) + * - 0 for perfectly uniform multi-value fields + * - High for skewed distributions where one value dominates others + * + * Uses mean(pcts) as the uniform baseline so the score works correctly even when + * percentages don't sum to 100 (e.g., when each value's % is computed relative to + * ALL spans, not just spans with this property). + * + * @param valuePercentages - Map from value string to its percentage (0–100) of occurrences + */ +export function computeDistributionScore( + valuePercentages: Map, +): number { + const nValues = valuePercentages.size; + if (nValues <= 1) return 0; + let totalPct = 0; + let maxPct = 0; + valuePercentages.forEach(pct => { + totalPct += pct; + if (pct > maxPct) maxPct = pct; + }); + if (totalPct === 0) return 0; + const uniformExpected = totalPct / nValues; + return maxPct - uniformExpected; +} + +/** + * Shannon entropy-based distribution score for sorting properties. + * + * Returns a score in [0, 1] where: + * - 1 = maximally useful for filtering (low entropy, one dominant value among several) + * - 0 = not useful (single value, empty, or perfectly uniform) + * + * Formula: 1 - H(p) / log2(N), where H(p) = -Σ(p_i * log2(p_i)) and N = number of values. + * This is the normalized redundancy (1 - normalized entropy). + * + * Unlike computeDistributionScore, entropy naturally handles multi-modal distributions + * and power-law distributions without bias toward the single largest value. + */ +export function computeEntropyScore( + valuePercentages: Map, +): number { + const nValues = valuePercentages.size; + if (nValues <= 1) return 0; + + let totalPct = 0; + valuePercentages.forEach(pct => { + totalPct += pct; + }); + if (totalPct === 0) return 0; + + let entropy = 0; + valuePercentages.forEach(pct => { + const p = pct / totalPct; + if (p > 0) { + entropy -= p * Math.log2(p); + } + }); + + const maxEntropy = Math.log2(nValues); + if (maxEntropy === 0) return 0; + + // Normalized redundancy: 1 when fully skewed, 0 when perfectly uniform + return 1 - entropy / maxEntropy; +} + +// Well-known OpenTelemetry attribute suffixes that are almost always useful +// for debugging and filtering. These get a score boost to appear first. +const BOOSTED_ATTRIBUTE_SUFFIXES = [ + 'service.name', + 'http.method', + 'http.request.method', + 'http.status_code', + 'http.response.status_code', + 'error', + 'error.type', + 'deployment.environment', + 'deployment.environment.name', + 'rpc.method', + 'rpc.service', + 'db.system', + 'db.operation', + 'messaging.system', + 'messaging.operation', +]; + +/** + * Returns a semantic priority boost for well-known OTel attributes. + * Returns 1 for known attributes, 0 otherwise. + * + * IMPORTANT: This is a raw signal — callers must scale it down to a tiebreaker + * and only apply it when the field has actual variance (baseScore > 0). + * A single-value service.name (entropy=0) must NOT be boosted above a + * multi-value field with genuine distribution differences. + */ +export function semanticBoost(key: string): number { + const lowerKey = key.toLowerCase(); + for (const suffix of BOOSTED_ATTRIBUTE_SUFFIXES) { + if (lowerKey.endsWith(suffix)) return 1; + } + return 0; +} + +// Scoring strategy for "all spans" distribution mode. +// 'entropy' uses Shannon entropy + semantic boosting (better for multi-modal distributions). +// 'skewness' uses the original max(pct) - mean(pcts) formula. +export type ScoringStrategy = 'entropy' | 'skewness'; +export const DISTRIBUTION_SCORING: ScoringStrategy = 'entropy'; + +/** + * Converts a flattened dot-notation property key into a filter key using + * ClickHouse bracket notation for Map columns: ColName['map.key']. + * This matches the search bar format (WHERE ResourceAttributes['k8s.pod.name'] = ...). + * For simple (non-Map) columns, returns the key unchanged. + */ +export function flattenedKeyToFilterKey( + key: string, + columnMeta: { name: string; type: string }[], +): string { + for (const col of columnMeta) { + const baseType = stripTypeWrappers(col.type); + + if (baseType.startsWith('Map(')) { + if (key.startsWith(col.name + '.')) { + const mapKey = key.slice(col.name.length + 1).replace(/'/g, "''"); + // Use bracket notation — the same format used by the search bar + // when filtering from trace detail: ResourceAttributes['k8s.pod.name'] + // This is the only format that reliably works in ClickHouse SQL for + // Map columns with dotted keys. toString() and backtick approaches + // fail because ClickHouse interprets dots as nested tuple access. + return `${col.name}['${mapKey}']`; + } + } else if (baseType.startsWith('Array(')) { + const innerType = stripTypeWrappers(baseType.slice('Array('.length, -1)); + if (innerType.startsWith('Map(')) { + // Array(Map) columns: keep the SQL expression format for these + // since the sidebar doesn't have facets for per-index array Map sub-keys + return flattenedKeyToSqlExpression(key, columnMeta); + } + } + } + return key; +} + +export type AddFilterFn = ( + property: string, + value: string, + action?: 'only' | 'exclude' | 'include', +) => void; + +export type HighlightPoint = { tsMs: number; yValue: number | null }; + +/** + * Tries to compute the heatmap Y-axis value for a span from a raw flattened row. + * Handles simple SQL expressions: "ColName", "ColName / N", "ColName * N". + * Returns null if the expression is too complex or the column is missing. + */ +export function computeYValue( + valueExpr: string, + flatRow: Record, +): number | null { + const trimmed = valueExpr.trim(); + + // Identifier pattern (with optional surrounding parentheses): + // Matches "ColName" and "(ColName)" + const identPat = '\\(?([A-Za-z_][A-Za-z0-9_]*)\\)?'; + + // Simple column reference: "Duration" or "(Duration)" + const simpleMatch = trimmed.match(new RegExp(`^${identPat}$`)); + if (simpleMatch) { + const v = flatRow[simpleMatch[1]]; + if (v == null) return null; + const n = Number(v); + return isNaN(n) ? null : n; + } + + // Division: "Duration / 1000000", "(Duration)/1e6", "(Duration) / 1e6" + const numPat = '([0-9]+(?:\\.[0-9]+)?(?:e[+-]?[0-9]+)?)'; + const divMatch = trimmed.match( + new RegExp(`^${identPat}\\s*\\/\\s*${numPat}$`, 'i'), + ); + if (divMatch) { + const v = flatRow[divMatch[1]]; + if (v == null) return null; + const n = Number(v); + const d = parseFloat(divMatch[2]); + return isNaN(n) || isNaN(d) || d === 0 ? null : n / d; + } + + // Multiplication: "Duration * 0.001", "(Duration) * 0.001" + const mulMatch = trimmed.match( + new RegExp(`^${identPat}\\s*\\*\\s*${numPat}$`, 'i'), + ); + if (mulMatch) { + const v = flatRow[mulMatch[1]]; + if (v == null) return null; + const n = Number(v); + const m = parseFloat(mulMatch[2]); + return isNaN(n) || isNaN(m) ? null : n * m; + } + + return null; +} + +// Sampling configuration for the delta chart queries. +// +// SAMPLE_SIZE: default number of rows sampled when the total count is unknown. +// MIN_SAMPLE_SIZE / MAX_SAMPLE_SIZE: bounds for adaptive sampling. +// SAMPLE_RATIO: fraction of total rows to sample (e.g., 0.01 = 1%). +// +// Adaptive formula: clamp(MIN, ceil(totalCount * SAMPLE_RATIO), MAX) +// Falls back to SAMPLE_SIZE when total count is not yet available. +// +// STABLE_SAMPLE_EXPR: ClickHouse expression used for ORDER BY in sample queries. +// 'cityHash64(SpanId)' gives deterministic ordering — same data always produces +// the same sample, so hover highlights are stable across re-renders. +// Set to 'rand()' to restore non-deterministic sampling. +export const SAMPLE_SIZE = 1000; +export const MIN_SAMPLE_SIZE = 500; +export const MAX_SAMPLE_SIZE = 5000; +export const SAMPLE_RATIO = 0.01; +export const STABLE_SAMPLE_EXPR = 'cityHash64(SpanId)'; + +/** + * Computes the effective sample size based on total row count. + * Returns SAMPLE_SIZE as fallback when totalCount is 0 or unavailable. + */ +export function computeEffectiveSampleSize(totalCount: number): number { + if (totalCount <= 0) return SAMPLE_SIZE; + return Math.min( + MAX_SAMPLE_SIZE, + Math.max(MIN_SAMPLE_SIZE, Math.ceil(totalCount * SAMPLE_RATIO)), + ); +} + +// Maximum number of distinct values to show in a chart before collapsing +// the rest into an "Other (N)" bucket. The effective limit adapts: when the +// actual unique count is at most MAX_CHART_VALUES_UPPER, all values are shown +// without aggregation. This avoids the awkward "Other (1)" or "Other (2)" cases +// for attributes like http.status_code that naturally have 4-8 values. +export const MAX_CHART_VALUES = 6; +export const MAX_CHART_VALUES_UPPER = 8; + +// Color for the "All spans" distribution bar (no selection / comparison mode off). +// Uses Mantine's blue-6 CSS variable so it adapts to light/dark themes. +export const ALL_SPANS_COLOR = 'var(--mantine-color-blue-6)'; + +// Color for the "Other (N)" aggregated bucket — neutral gray. +export const OTHER_BUCKET_COLOR = 'var(--mantine-color-gray-5)'; + +// Aggregates chart data beyond the effective limit into a single "Other (N)" entry. +// Sorts by combined count (outlier + inlier) descending so the most frequent +// values are kept. The effective limit adapts: if the total unique count is at +// most MAX_CHART_VALUES_UPPER, all values are shown without aggregation. +export function applyTopNAggregation( + data: { name: string; outlierCount: number; inlierCount: number }[], +): { + name: string; + outlierCount: number; + inlierCount: number; + isOther?: boolean; +}[] { + // Adaptive: show all values when they fit within the upper bound + if (data.length <= MAX_CHART_VALUES_UPPER) return data; + + const sorted = [...data].sort( + (a, b) => b.outlierCount + b.inlierCount - (a.outlierCount + a.inlierCount), + ); + const top = sorted.slice(0, MAX_CHART_VALUES); + const rest = sorted.slice(MAX_CHART_VALUES); + + const otherOutlierCount = rest.reduce( + (sum, item) => sum + item.outlierCount, + 0, + ); + const otherInlierCount = rest.reduce( + (sum, item) => sum + item.inlierCount, + 0, + ); + + return [ + ...top, + { + name: `Other (${rest.length})`, + outlierCount: otherOutlierCount, + inlierCount: otherInlierCount, + isOther: true, + }, + ]; +} diff --git a/packages/app/styles/app.scss b/packages/app/styles/app.scss index 5083629608..f17394bb84 100644 --- a/packages/app/styles/app.scss +++ b/packages/app/styles/app.scss @@ -293,6 +293,13 @@ a, color: #2e224788; } +// Heatmap selection uses the "error" color (red) to match the "Selection" +// legend and bar color in the delta comparison charts below. +.heatmap-selection-container .u-select { + border: 1px solid var(--color-chart-error, #ff6b6b); + background: color-mix(in srgb, var(--color-chart-error, #ff6b6b) 10%, transparent) !important; +} + .hdx-markdown { h1, h2,