feat(app): exemplar overlay for metric and PromQL time charts

.changeset/exemplar-overlay.md

@@ -0,0 +1,27 @@
+---
+'@hyperdx/common-utils': minor
+'@hyperdx/app': minor
+'@hyperdx/api': patch
+---
+
+feat: exemplar overlay for metric and PromQL time charts
+
+Time charts on metric and PromQL sources can overlay exemplars — individual
+trace-linked data points — via the "Show exemplars" toggle in the chart
+editor's display settings. Hovering a marker shows the exemplar's own value and time plus trace metadata
+from a configurable trace source, with a button to open the trace.
+
+Off by default for the whole deployment behind `NEXT_PUBLIC_ENABLE_EXEMPLARS`,
+and per-chart behind the toggle. How many markers a chart draws is a team
+setting, Max Exemplars per Chart.
+
+Markers are sampled the way Grafana samples them: bucketed at the chart's
+granularity, keeping the slowest trace in each bucket plus any further trace more
+than 2σ below it. The overlay shows the shape of the latency distribution rather
+than tracing the top of the chart.
+
+A marker sits at the trace's own measurement, so it is only shown where that is
+honest: a single non-ratio histogram series with no group by, aggregated in a way
+that leaves the axis on the observation scale, and for PromQL an expression that
+plots a duration. Markers outside the rendered window, or below a fitted y-axis
+floor, are dropped rather than moved — and the count is surfaced on the chart.

packages/api/src/models/team.ts

@@ -46,6 +46,7 @@ export default mongoose.model<ITeam>(
       fieldMetadataDisabled: Boolean,
       parallelizeWhenPossible: Boolean,
       filterKeysFetchLimit: Number,
+      maxExemplars: Number,
     },
     {
       timestamps: true,

packages/api/src/routers/external-api/v2/utils/__tests__/alertChartConfig.test.ts

@@ -346,6 +346,33 @@ describe('convertAlertChartConfigToExternal', () => {
     }
   });
 
+  it('still emits a number config carrying exemplar settings', () => {
+    // The chart explorer can author an inline alert off a chart with the
+    // exemplar overlay switched on. Markers are drawn client-side and the
+    // alert task never reads either field, so dropping them beats blanking
+    // chartConfig. Asserted on the number variant because its schema has no
+    // spelling for either field however the line/bar dialect evolves.
+    const external = convertAlertChartConfigToExternal(
+      baseInternal({
+        displayType: DisplayType.Number,
+        enableExemplars: true,
+        exemplarTraceSourceId: '65f5e4a3b9e77c001a222222',
+        select: [
+          {
+            aggFn: 'count',
+            aggCondition: '',
+            aggConditionLanguage: 'lucene',
+            valueExpression: '',
+          },
+        ],
+      }),
+    );
+
+    expect(external).toMatchObject({ displayType: DisplayType.Number });
+    expect(external).not.toHaveProperty('enableExemplars');
+    expect(external).not.toHaveProperty('exemplarTraceSourceId');
+  });
+
   it.each([
     [
       'filters',

packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts

@@ -115,6 +115,12 @@ export const EVALUATION_INERT_CONFIG_KEYS: ReadonlySet<string> = new Set([
   'heatmap',
   // Client-side null-bucket rendering; the alert task ignores it.
   'fillNulls',
+  // Exemplar markers are a chart-rendering overlay; the alert task never reads
+  // them. Needed even where the tile dialect carries them: neither the number
+  // schema nor the raw SQL schemas an alert can use derive from
+  // externalDashboardTimeChartConfigSchema.
+  'enableExemplars',
+  'exemplarTraceSourceId',
   // Evaluation-relevant only with a groupBy, which (on variants whose shape
   // lacks seriesLimit, i.e. number) is itself refused when present.
   'seriesLimit',

packages/app/package.json

@@ -14,7 +14,7 @@
     "build:clickhouse": "NEXT_PUBLIC_THEME=clickstack NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_PUBLIC_CLICKHOUSE_BUILD=true next build --webpack && node scripts/prepare-clickhouse-build-export.js",
     "run:clickhouse": "test -d out && npx rimraf tmp && mkdir tmp && cp -r out tmp/clickstack && echo 'visit http://localhost:3000/clickstack to start' && npx serve tmp -l 3000 || echo 'run build:clickhouse first'",
     "start": "next start",
-    "lint": "npx eslint . --ext .ts,.tsx --max-warnings 568",
+    "lint": "npx eslint . --ext .ts,.tsx --max-warnings 573",
     "lint:fix": "npx eslint . --ext .ts,.tsx --fix",
     "lint:styles": "stylelint **/*/*.{css,scss}",
     "ci:lint": "yarn lint && yarn tsc --noEmit && yarn lint:styles --quiet",

packages/app/src/HDXMultiSeriesTimeChart/MemoChart.tsx

@@ -17,6 +17,7 @@ import {
   Customized,
   Legend,
   ReferenceArea,
+  ReferenceDot,
   ReferenceLine,
   ResponsiveContainer,
   Tooltip,
@@ -25,6 +26,7 @@ import {
 } from 'recharts';
 import {
   DisplayType,
+  Exemplar,
   LineInterpolation,
 } from '@hyperdx/common-utils/dist/types';
 
@@ -38,6 +40,7 @@ import { AnnotationTooltip } from '@/components/charts/AnnotationTooltip';
 import { ChartAnnotation } from '@/components/charts/chartAnnotations';
 import { ChartOverlayControls } from '@/components/charts/ChartOverlayControls';
 import { toViewportPoint } from '@/components/charts/ChartTooltip';
+import { ExemplarDot } from '@/components/Exemplars';
 import type { NumberFormat } from '@/types';
 import { useFormatTime } from '@/useFormatTime';
 import { useContentFontSize } from '@/useUserPreferences';
@@ -67,6 +70,7 @@ import {
   Y_AXIS_WIDTH,
 } from './constants';
 import { getLineCurveType, useChartScales } from './useChartScales';
+import { useExemplarMarkers } from './useExemplarMarkers';
 
 import styles from '@styles/HDXLineChart.module.scss';
 
@@ -132,6 +136,15 @@ export const MemoChart = memo(function MemoChart({
   granularity,
   dateRangeEndInclusive = true,
   fitYAxisToData = false,
+  exemplars,
+  maxExemplars = 12,
+  onExemplarHover,
+  onExemplarHoverEnd,
+  onExemplarSelect,
+  pinnedExemplarKey = null,
+  onExemplarPinEnd,
+  onActiveExemplarMoved,
+  onExemplarsDropped,
   lineInterpolation,
 }: {
   // Matches what useChartScales narrows to, so the hook's stricter type is
@@ -179,11 +192,33 @@ export const MemoChart = memo(function MemoChart({
    * (with padding) instead of zero.
    **/
   fitYAxisToData?: boolean;
+  /** Exemplar markers to overlay on the chart (linked to traces). */
+  exemplars?: Exemplar[];
+  /** Target number of exemplar markers to show (0 = unlimited). */
+  maxExemplars?: number;
+  /** Invoked when the cursor enters an exemplar marker, with its pixel coords. */
+  onExemplarHover?: (exemplar: Exemplar, cx: number, cy: number) => void;
+  /** Invoked when the cursor leaves an exemplar marker. */
+  onExemplarHoverEnd?: () => void;
+  /** Invoked when an exemplar marker is clicked, with its pixel coords. */
+  onExemplarSelect?: (exemplar: Exemplar, cx: number, cy: number) => void;
+  /**
+   * Key of the exemplar whose card is pinned open, or null. A key rather than a
+   * boolean so the chart can tell when that marker stops being rendered — see
+   * the reset effect below. A pin also suppresses the series tooltip.
+   */
+  pinnedExemplarKey?: string | null;
+  /** Invoked when the pinned marker is no longer in the rendered set. */
+  onExemplarPinEnd?: () => void;
+  /** See useExemplarMarkers: re-anchors the open card when its marker moves. */
+  onActiveExemplarMoved?: (cx: number, cy: number) => void;
+  /** How many markers the render-layer clamps dropped; see useExemplarMarkers. */
+  onExemplarsDropped?: (count: number) => void;
   /** Unset follows the display type's default (see getLineCurveType). */
   lineInterpolation?: LineInterpolation;
 }) {
-  const _id = useId();
-  const id = _id.replace(/:/g, '');
+  const rawId = useId();
+  const id = rawId.replace(/:/g, '');
 
   // The y-axis width budget is expressed in characters (see MAX_AXIS_MANTISSA),
   // so it has to grow with the tick font or the widest labels clip.
@@ -316,24 +351,31 @@ export const MemoChart = memo(function MemoChart({
 
   const [containerWidth, setContainerWidth] = useState(0);
 
-  // Axis domains and annotation elements — see useChartScales.
-  const { yAxisBounds, xAxisDomain, annotationElements, laidOutAnnotations } =
-    useChartScales({
-      annotations,
-      containerWidth,
-      dateRange,
-      granularity,
-      dateRangeEndInclusive,
-      displayType,
-      fitYAxisToData,
-      graphResults,
-      lineData,
-      visibleLineData,
-      selectedSeriesNames,
-      referenceLineValues,
-      axisNumberFormat,
-      yAxisWidth,
-    });
+  // Axis domains, the exemplar clamp range, and annotation elements — see
+  // useChartScales.
+  const {
+    yAxisBounds,
+    exemplarYBounds,
+    xAxisDomain,
+    annotationElements,
+    laidOutAnnotations,
+  } = useChartScales({
+    annotations,
+    containerWidth,
+    dateRange,
+    granularity,
+    dateRangeEndInclusive,
+    displayType,
+    fitYAxisToData,
+    graphResults,
+    lineData,
+    visibleLineData,
+    selectedSeriesNames,
+    referenceLineValues,
+    axisNumberFormat,
+    yAxisWidth,
+    hasExemplars: !!exemplars?.length,
+  });
   const yAxisDomain = yAxisBounds.domain;
 
   // The chart's outer positioned container. Used to convert a pointer's
@@ -730,6 +772,31 @@ export const MemoChart = memo(function MemoChart({
   const [hoveredAnnotation, setHoveredAnnotation] =
     useState<HoveredAnnotation | null>(null);
 
+  // Exemplar marker layer — see useExemplarMarkers.
+  const {
+    activeExemplarKey,
+    exemplarPoints,
+    isExemplarHovered,
+    handleExemplarHoverStart,
+    handleExemplarHoverEnd,
+    handleExemplarSelect,
+  } = useExemplarMarkers({
+    exemplars,
+    maxExemplars,
+    granularity,
+    pinnedExemplarKey,
+    xAxisDomain,
+    exemplarYBounds,
+    onExemplarHover,
+    onExemplarHoverEnd,
+    onExemplarSelect,
+    onExemplarPinEnd,
+    onActiveExemplarMoved,
+    onExemplarsDropped,
+    suppressNextClickRef,
+    brushOriginRef: mouseDownPosRef,
+  });
+
   return (
     <div
       ref={containerRef}
@@ -862,12 +929,14 @@ export const MemoChart = memo(function MemoChart({
               Hidden once a point is clicked, where the pinned tooltip takes over.
               Portaled to body so HDXLineChartTooltip can self-position (see its
               docblock) and escape the chart's bounds near an edge. */}
-          {isClickActive == null && (
-            <Tooltip
-              content={hoverTooltipContent}
-              portal={typeof document !== 'undefined' ? document.body : null}
-            />
-          )}
+          {isClickActive == null &&
+            !isExemplarHovered &&
+            pinnedExemplarKey == null && (
+              <Tooltip
+                content={hoverTooltipContent}
+                portal={typeof document !== 'undefined' ? document.body : null}
+              />
+            )}
           {referenceLines}
           {annotationElements}
           {laidOutAnnotations != null && (
@@ -880,6 +949,29 @@ export const MemoChart = memo(function MemoChart({
               }
             />
           )}
+          {exemplarPoints.map(p => (
+            <ReferenceDot
+              key={p.key}
+              // Already placed inside the x-domain by useExemplarMarkers, which
+              // also drops markers that belong to a different window.
+              x={p.x}
+              // Already placed inside the y-range by useExemplarMarkers.
+              y={p.y}
+              // Stated rather than assumed: both clamps exist because the default
+              // is "discard", and recharts 3 landed here recently.
+              ifOverflow="discard"
+              shape={
+                <ExemplarDot
+                  exemplar={p.exemplar}
+                  onHoverStart={handleExemplarHoverStart}
+                  onHoverEnd={handleExemplarHoverEnd}
+                  onSelect={handleExemplarSelect}
+                  isActive={p.key === activeExemplarKey}
+                  onPositionChange={onActiveExemplarMoved}
+                />
+              }
+            />
+          ))}
           {highlightStart && highlightEnd ? (
             <ReferenceArea
               // yAxisId="1"

packages/app/src/HDXMultiSeriesTimeChart/__tests__/useChartScales.test.ts

@@ -43,6 +43,7 @@ const baseArgs = {
   lineData,
   selectedSeriesNames: undefined as Set<string> | undefined,
   referenceLineValues: [] as number[],
+  hasExemplars: false,
   yAxisWidth: 40,
 };
 
@@ -135,6 +136,52 @@ describe('useChartScales y-domain', () => {
   });
 });
 
+describe('useChartScales exemplar clamp', () => {
+  it('bounds markers by the top of the drawn axis', () => {
+    expect(scales({ hasExemplars: true }).exemplarYBounds).toEqual({
+      min: 0,
+      max: 42,
+    });
+  });
+
+  // A reference line leaves the upper bound 'auto', so the clamp falls back to
+  // the visible series max: an outlier marker pins to the top of the series
+  // range instead of stretching the axis and flattening every line.
+  it('falls back to the visible series max when the axis top is auto', () => {
+    expect(
+      scales({ hasExemplars: true, referenceLineValues: [100] })
+        .exemplarYBounds,
+    ).toEqual({ min: 0, max: 40 });
+  });
+
+  it('follows the legend selection rather than every series', () => {
+    // A selection gives the axis numeric bounds, so the clamp takes those
+    // directly ([9, 31]) instead of falling back to the series max. Either way
+    // it tracks what is on screen — with B shown too it would be [8.5, 41.5].
+    expect(
+      scales({ hasExemplars: true, selectedSeriesNames: new Set(['A']) })
+        .exemplarYBounds,
+    ).toEqual({ min: 9, max: 31 });
+  });
+
+  it('skips the O(rows x series) scan when no marker can draw', () => {
+    // Every time chart in the app pays for this pass otherwise, including
+    // deployments running with the overlay switched off entirely.
+    expect(
+      scales({ hasExemplars: false, referenceLineValues: [100] })
+        .exemplarYBounds.max,
+    ).toBe(0);
+  });
+
+  it('takes a numeric upper bound from the axis when there is one', () => {
+    // Fitting to data gives a real number, and the marker should respect the
+    // axis the chart actually drew rather than the raw series max.
+    expect(
+      scales({ hasExemplars: true, fitYAxisToData: true }).exemplarYBounds,
+    ).toEqual({ min: 8.5, max: 41.5 });
+  });
+});
+
 describe('useChartScales x-domain', () => {
   it('spans the requested range in seconds', () => {
     const [start, end] = scales().xAxisDomain;