diff --git a/.changeset/exemplar-mode-metrics.md b/.changeset/exemplar-mode-metrics.md new file mode 100644 index 0000000000..51cb5cae48 --- /dev/null +++ b/.changeset/exemplar-mode-metrics.md @@ -0,0 +1,33 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +feat: add exemplar overlay for metric and PromQL charts + +Time charts on metric and PromQL sources can now overlay exemplars — +individual data points linked to a trace — via the "Exemplars" toggle in the +chart editor (next to "As Ratio" for metric charts, in the PromQL editor for +PromQL charts). Each marker sits at the trace's own measurement rather than on +the series line, so its height matches what the linked trace reports; hovering a +marker shows trace metadata (service, span, duration, status) from a +configurable exemplar trace source, with a button to open the trace directly. + +Markers are sampled the way Grafana samples exemplars: bucketed at the chart's +granularity, keeping the slowest trace in each bucket plus any further trace +more than 2σ below it, with the marker budget spread evenly across the time +range. The overlay therefore shows the shape of the latency distribution +instead of tracing the top of the chart. + +For structured metric sources, exemplars are read directly from the OTel metric +tables' `Exemplars.*` columns (`renderMetricExemplarsChartConfig`), honoring the +chart's time range, metric name, and filters. For PromQL sources backed by a +real Prometheus endpoint, the new `/v1/prometheus/query_exemplars` route proxies +to Prometheus's native `/api/v1/query_exemplars`. The overlay is opt-in and runs +its query in parallel only when enabled, so charts that don't use it are +unaffected. Trace-source exemplar generation lands in a follow-up. + +The overlay is off by default for the whole deployment and is enabled with +`NEXT_PUBLIC_ENABLE_EXEMPLARS=true`; with it unset, the chart-editor toggle is +hidden and no exemplar query runs even for charts that have the flag saved. diff --git a/.changeset/span-metrics-connector.md b/.changeset/span-metrics-connector.md new file mode 100644 index 0000000000..6e6725de76 --- /dev/null +++ b/.changeset/span-metrics-connector.md @@ -0,0 +1,23 @@ +--- +"@hyperdx/api": minor +"@hyperdx/otel-collector": minor +--- + +feat: optional spanmetrics connector for metric exemplars + +Adds the `spanmetricsconnector` to the collector build and wires it into the +OpAMP-generated collector config, gated on the `ENABLE_SPAN_METRICS` env flag +(off by default). When enabled, the collector derives `traces.span.metrics.*` +(calls + duration histogram) from spans with **exemplars enabled**, so the +duration histogram lands in ClickHouse with `Exemplars.*` pointing back at the +spans they were measured from — giving coherent, fully-OTLP metric exemplars +without any direct ClickHouse writes. Enabled in local dev to back the new +`telemetry-generator` service. + +The duration histogram uses exponential (OTLP) buckets rather than a fixed +ladder, so it lands in `otel_metrics_exponential_histogram` and remote-writes +to Prometheus as a native histogram. A fixed ladder's wide top bucket makes +high quantiles interpolate well past the slowest request that actually +happened — a p99 reading 10s off a 5s–10s bucket when nothing took longer than +6s — which no exemplar can ever sit on. Exemplar overlays now accept +exponential histograms alongside explicit-bucket ones. diff --git a/.gitignore b/.gitignore index 31c6b70198..1f3ae85a33 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,7 @@ docker-compose.prod.yml # webstorm .idea + +# Stryker mutation-testing output +packages/*/reports/ +packages/*/.stryker-tmp/ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 95e882eca0..c364199dfd 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -41,6 +41,9 @@ services: OTEL_SUPERVISOR_LOGS: 'true' HYPERDX_OTEL_EXPORTER_TABLES_TTL: '24h' ENABLE_PROMQL: 'true' + # Note: SPAN_METRICS_PROM_RW_ENDPOINT belongs to the API, not here — the + # API resolves it and inlines it into the generated collector config. It + # lives in packages/api/.env.development. volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl @@ -157,9 +160,17 @@ services: hdx.dev.service: prometheus hdx.dev.port: '${HDX_DEV_PROMETHEUS_PORT:-9090}' hdx.dev.url: 'http://localhost:${HDX_DEV_PROMETHEUS_PORT:-9090}' - profiles: - - prometheus image: prom/prometheus:latest + # exemplar-storage: query_exemplars data; remote-write receiver: ingest the + # collector's span-derived metrics (with exemplars) from real generated data. + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + # native-histograms: the collector remote-writes the span-derived metrics + # as native histograms; without the flag Prometheus drops those samples. + - '--enable-feature=exemplar-storage,native-histograms' + - '--web.enable-remote-write-receiver' + - '--web.enable-lifecycle' ports: - '${HDX_DEV_PROMETHEUS_PORT:-9090}:9090' volumes: @@ -169,5 +180,25 @@ services: - internal restart: on-failure + # Synthetic traces (via OTLP) + coherent metric exemplars (to ClickHouse) for + # local dev + e2e. See telemetry-generator/README.md. + telemetry-generator: + labels: + <<: *hdx-labels + hdx.dev.service: telemetry-generator + build: + context: ./telemetry-generator + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + GEN_OTLP_API_KEY: '${INGESTION_API_KEY:-super-secure-ingestion-api-key}' + GEN_BACKFILL_MINUTES: '30' + GEN_RATE_PER_SEC: '20' + networks: + - internal + restart: on-failure + depends_on: + otel-collector: + condition: service_started + networks: internal: diff --git a/docker-compose.yml b/docker-compose.yml index bdd7339e03..7a2ae855c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,8 @@ services: # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics) # ENABLE_PROMQL: 'true' # NEXT_PUBLIC_ENABLE_PROMQL: 'true' + # Uncomment to enable exemplar overlays (trace markers on time charts) + # NEXT_PUBLIC_ENABLE_EXEMPLARS: 'true' DEFAULT_CONNECTIONS: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 94877ac65d..ff6643b21f 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -18,3 +18,7 @@ scrape_configs: static_configs: - targets: ['ch-server:9363'] metrics_path: '/metrics' + + # Note: the span-derived request metrics (with exemplars) arrive via + # remote-write from the OTel collector, not a scrape — see the collector's + # SPAN_METRICS_PROM_RW_ENDPOINT and the spanmetrics connector. diff --git a/knip.json b/knip.json index 8e9cf032c8..420285cfc7 100644 --- a/knip.json +++ b/knip.json @@ -39,6 +39,7 @@ ".github/scripts/**", ".github/actions/**", "docker/hyperdx/**", + "telemetry-generator/**", ".claude/**" ], "ignoreBinaries": ["make", "migrate"], diff --git a/packages/api/.env.development b/packages/api/.env.development index 8ba0a6c0c8..edf825773d 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -20,9 +20,14 @@ REDIS_URL=redis://localhost:6379 USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true -DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] +DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""},{"name":"Local Prometheus","host":"http://localhost:${HDX_DEV_PROMETHEUS_PORT}","username":"","password":"","isPrometheusEndpoint":true}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","exponential histogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"},{"from":{"databaseName":"prometheus","tableName":"prometheus"},"kind":"promql","timestampValueExpression":"timestamp","name":"Prometheus","connection":"Local Prometheus","traceSourceId":"Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" ENABLE_PROMQL=true +ENABLE_SPAN_METRICS=true +ENABLE_SPAN_METRICS_PROM_RW=true +# Resolved API-side and inlined into the generated collector config, so this is +# the collector's view of the network: container DNS, not a host port. +SPAN_METRICS_PROM_RW_ENDPOINT=http://prometheus:9090/api/v1/write diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 4c076f92fa..9875f4b8d9 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -1642,6 +1642,16 @@ "type": "boolean", "description": "Overlay the equivalent previous time period for comparison.", "default": false + }, + "enableExemplars": { + "type": "boolean", + "description": "Overlay exemplars: markers for individual trace-linked data points. Only renders when the tile is exemplar-eligible — a single non-ratio histogram metric series with no groupBy — and is accepted but inert otherwise.\n", + "default": false + }, + "exemplarTraceSourceId": { + "type": "string", + "description": "ID of the Trace source an exemplar marker links to. Defaults to the chart source's linked trace source when omitted.\n", + "example": "65f5e4a3b9e77c001a222222" } } }, @@ -1700,6 +1710,16 @@ "numberFormat": { "$ref": "#/components/schemas/NumberFormat", "description": "Number formatting options for displayed values." + }, + "enableExemplars": { + "type": "boolean", + "description": "Overlay exemplars: markers for individual trace-linked data points. Only renders when the tile is exemplar-eligible — a single non-ratio histogram metric series with no groupBy — and is accepted but inert otherwise.\n", + "default": false + }, + "exemplarTraceSourceId": { + "type": "string", + "description": "ID of the Trace source an exemplar marker links to. Defaults to the chart source's linked trace source when omitted.\n", + "example": "65f5e4a3b9e77c001a222222" } } }, diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 7c6a242969..28da4e564b 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -57,6 +57,21 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; +// Opt-in: have the collector derive request metrics (with trace exemplars) from +// spans via the spanmetrics connector. Off by default; enabled in dev so the +// telemetry-generator's traces produce coherent metric exemplars end-to-end. +export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; + +// Opt-in: also remote-write the span-derived metrics (with exemplars) to a +// Prometheus endpoint so the native Prometheus query_exemplars path can be +// tested against real data. The endpoint is resolved here (API side) and +// inlined into the generated collector config, so the collector container does +// not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the +// endpoint to be set; without it the feature stays disabled. +export const SPAN_METRICS_PROM_RW_ENDPOINT = env.SPAN_METRICS_PROM_RW_ENDPOINT; +export const IS_SPAN_METRICS_PROM_RW_ENABLED = + env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT; + // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index e0e92e830f..561136756d 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -534,6 +534,22 @@ const mcpLineTileSchema = mcpTileLayoutSchema.extend({ .describe( 'Scale the y-axis to the data range instead of starting at zero.', ), + enableExemplars: z + .boolean() + .optional() + .describe( + 'Overlay exemplars: markers for individual trace-linked data points. ' + + 'Only renders on an exemplar-eligible tile — a single non-ratio ' + + 'histogram metric series with no groupBy — and is inert otherwise.', + ), + exemplarTraceSourceId: z + .string() + .optional() + .describe( + 'Trace source an exemplar marker links to. Must be a Trace source: use ' + + 'clickstack_list_sources and pick one whose kind is "trace". Defaults ' + + "to the chart source's linked trace source when omitted.", + ), }), }); @@ -551,6 +567,22 @@ const mcpBarTileSchema = mcpTileLayoutSchema.extend({ numberFormat: mcpNumberFormatSchema .optional() .describe(tileLevelNumberFormatDescription), + enableExemplars: z + .boolean() + .optional() + .describe( + 'Overlay exemplars: markers for individual trace-linked data points. ' + + 'Only renders on an exemplar-eligible tile — a single non-ratio ' + + 'histogram metric series with no groupBy — and is inert otherwise.', + ), + exemplarTraceSourceId: z + .string() + .optional() + .describe( + 'Trace source an exemplar marker links to. Must be a Trace source: use ' + + 'clickstack_list_sources and pick one whose kind is "trace". Defaults ' + + "to the chart source's linked trace source when omitted.", + ), }), }); diff --git a/packages/api/src/models/team.ts b/packages/api/src/models/team.ts index 4d6b620162..259c695f2c 100644 --- a/packages/api/src/models/team.ts +++ b/packages/api/src/models/team.ts @@ -42,6 +42,7 @@ export default mongoose.model( fieldMetadataDisabled: Boolean, parallelizeWhenPossible: Boolean, filterKeysFetchLimit: Number, + maxExemplars: Number, }, { timestamps: true, diff --git a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts index f86baa8039..2a2f17833d 100644 --- a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts +++ b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts @@ -7,6 +7,9 @@ const configState = { INGESTION_API_KEY: '' as string, IS_PROMQL_ENABLED: false, ENABLE_DATADOG_RECEIVER: false, + IS_SPAN_METRICS_ENABLED: false, + IS_SPAN_METRICS_PROM_RW_ENABLED: false, + SPAN_METRICS_PROM_RW_ENDPOINT: undefined as string | undefined, }; jest.mock('@/config', () => ({ @@ -28,8 +31,20 @@ jest.mock('@/config', () => ({ get ENABLE_DATADOG_RECEIVER() { return configState.ENABLE_DATADOG_RECEIVER; }, + get IS_SPAN_METRICS_ENABLED() { + return configState.IS_SPAN_METRICS_ENABLED; + }, + get IS_SPAN_METRICS_PROM_RW_ENABLED() { + return configState.IS_SPAN_METRICS_PROM_RW_ENABLED; + }, + get SPAN_METRICS_PROM_RW_ENDPOINT() { + return configState.SPAN_METRICS_PROM_RW_ENDPOINT; + }, })); +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + import { buildOtelCollectorConfig } from '@/opamp/controllers/opampController'; const resetConfig = () => { @@ -39,6 +54,9 @@ const resetConfig = () => { configState.INGESTION_API_KEY = ''; configState.IS_PROMQL_ENABLED = false; configState.ENABLE_DATADOG_RECEIVER = false; + configState.IS_SPAN_METRICS_ENABLED = false; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = false; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = undefined; }; describe('opampController', () => { @@ -123,4 +141,136 @@ describe('opampController', () => { expect(cfg.service.extensions).toContain('bearertokenauth/hyperdx'); }); }); + + describe('buildOtelCollectorConfig span metrics', () => { + it('omits the span metrics connector when the flag is off (default)', () => { + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + expect(cfg.connectors?.spanmetrics).toBeUndefined(); + expect(cfg.service.pipelines['metrics/spanmetrics']).toBeUndefined(); + expect(cfg.service.pipelines.traces.exporters).not.toContain( + 'spanmetrics', + ); + }); + + it('derives span metrics as exponential histograms with exemplars', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + // Exponential rather than explicit buckets: a fixed ladder's wide top + // bucket makes high quantiles interpolate well past the slowest real + // request, so no exemplar can ever sit on the plotted line. + expect(cfg.connectors?.spanmetrics?.histogram).toEqual({ + unit: 'ms', + exponential: { max_size: 160 }, + }); + expect(cfg.connectors?.spanmetrics?.exemplars).toEqual({ + enabled: true, + }); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + ]); + // The connector has to be wired at both ends or it silently does nothing. + expect(cfg.service.pipelines.traces.exporters).toContain('spanmetrics'); + expect(cfg.service.pipelines['metrics/spanmetrics']?.receivers).toEqual([ + 'spanmetrics', + ]); + }); + + it('adds a remote-write exporter for the derived metrics when enabled', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = + 'http://prometheus:9090/api/v1/write'; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + expect( + cfg.exporters?.['prometheusremotewrite/spanmetrics'], + ).toMatchObject({ endpoint: 'http://prometheus:9090/api/v1/write' }); + expect(cfg.service.pipelines['metrics/spanmetrics']?.exporters).toEqual([ + 'clickhouse', + 'prometheusremotewrite/spanmetrics', + ]); + }); + + // A config naming a component type the binary doesn't register fails to + // decode as a whole, and docker/otel-collector/config.yaml defines no + // pipelines of its own — so a bad type name here leaves the collector with + // no usable config at all rather than just disabling one feature. Pin every + // generated component id against what builder-config.yaml actually builds. + it('only references component types the collector build registers', () => { + configState.IS_SPAN_METRICS_ENABLED = true; + configState.IS_SPAN_METRICS_PROM_RW_ENABLED = true; + configState.SPAN_METRICS_PROM_RW_ENDPOINT = + 'http://prometheus:9090/write'; + configState.IS_PROMQL_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + const builderConfig = readFileSync( + resolve(__dirname, '../../../../../otel-collector/builder-config.yaml'), + 'utf8', + ); + // `/` ids share the base type, which is what gets registered. + const baseType = (id: string) => id.split('/')[0]; + // gomod lines end in `` — e.g. `.../connector/spanmetricsconnector` + // for type `spanmetrics`. Matching the suffixed forms is what makes + // `span_metrics` (the bug this test exists for) fail to resolve. + const registers = (id: string) => + ['connector', 'exporter', 'receiver', 'processor'].some(kind => + builderConfig.includes(`/${baseType(id)}${kind}`), + ); + + // Core components shipped in the collector binary rather than declared as + // contrib gomods in builder-config.yaml. + const CORE_COMPONENTS = ['nop', 'debug', 'memory_limiter', 'batch']; + + const declaredIds = [ + ...Object.keys(cfg.connectors ?? {}), + ...Object.keys(cfg.exporters ?? {}), + ...Object.keys(cfg.receivers ?? {}), + ]; + // Assert we actually looked at something — an empty set would make every + // loop below vacuously green, which is how a bad type name shipped. + expect(declaredIds.length).toBeGreaterThan(0); + + for (const id of declaredIds) { + if (CORE_COMPONENTS.includes(baseType(id))) continue; + expect(registers(id)).toBe(true); + } + + // Every pipeline reference must resolve to a declared component. Processors + // are included: they're named here but declared in + // docker/otel-collector/config.yaml, so they can't be checked against the + // declared set — but they still have to be types the build registers, since + // an unregistered processor fails the whole config to decode just like an + // unregistered connector. + const declared = new Set(declaredIds); + let pipelineRefs = 0; + for (const pipeline of Object.values(cfg.service.pipelines)) { + for (const id of [ + ...(pipeline?.receivers ?? []), + ...(pipeline?.exporters ?? []), + ]) { + expect(declared).toContain(id); + pipelineRefs++; + } + for (const id of pipeline?.processors ?? []) { + if (CORE_COMPONENTS.includes(baseType(id))) continue; + expect(registers(id)).toBe(true); + pipelineRefs++; + } + } + expect(pipelineRefs).toBeGreaterThan(0); + }); + }); }); diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 6dfb5f989c..67a9b77c5f 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -105,6 +105,13 @@ type CollectorConfig = { pipelines: string[]; }>; }; + spanmetrics?: { + histogram: { unit: string; exponential: { max_size: number } }; + dimensions: Array<{ name: string }>; + exemplars: { enabled: boolean }; + metrics_flush_interval: string; + namespace?: string; + }; }; exporters?: { nop?: null; @@ -155,6 +162,15 @@ type CollectorConfig = { enabled: boolean; }; }; + 'prometheusremotewrite/spanmetrics'?: { + endpoint: string; + tls: { + insecure: boolean; + }; + resource_to_telemetry_conversion: { + enabled: boolean; + }; + }; }; service: { extensions: string[]; @@ -345,6 +361,72 @@ export const buildOtelCollectorConfig = ( }; } + if ( + config.IS_SPAN_METRICS_ENABLED && + otelCollectorConfig.connectors && + otelCollectorConfig.exporters + ) { + // Derive request metrics (with trace exemplars) from spans. The connector + // consumes the traces pipeline and feeds a dedicated metrics pipeline, so + // the resulting `traces.span.metrics.*` land in ClickHouse with + // `Exemplars.*` pointing back at the spans they were measured from. + // + // The key MUST be `spanmetrics` — that is the component type + // spanmetricsconnector registers, and a config naming an unregistered type + // fails to decode wholesale. Because docker/otel-collector/config.yaml + // supplies no pipelines of its own, a rejected remote config leaves the + // collector with nothing to run, so a typo here takes ingestion down + // rather than just disabling this feature. + // + // Requires a collector built from the current builder-config.yaml (which + // added spanmetricsconnector). Enabling this flag against collectors built + // before that will have them reject the config for the same reason, so + // roll the collector image out first. + otelCollectorConfig.connectors.spanmetrics = { + histogram: { + unit: 'ms', + // Exponential (OTLP) / native (Prometheus) buckets rather than a fixed + // ladder. Explicit bounds put everything slow into one wide top bucket + // — with a 5s–10s bucket, a p99 interpolates towards 10s while the + // slowest real request was half that, and no exemplar can ever sit on + // the line. max_size is the connector's default; scale adapts to the + // observed range, giving sub-percent quantile error at any latency. + exponential: { max_size: 160 }, + }, + dimensions: [ + { name: 'http.route' }, + { name: 'http.method' }, + { name: 'host.region' }, + { name: 'app.tenant_id' }, + { name: 'http.status_code' }, + ], + exemplars: { enabled: true }, + metrics_flush_interval: '15s', + }; + otelCollectorConfig.service.pipelines.traces.exporters.push( + 'spanmetrics', + ); + + const spanMetricsExporters = ['clickhouse']; + // Optionally also remote-write the derived metrics (with exemplars) to a + // Prometheus endpoint, so the native Prometheus `query_exemplars` path can + // be exercised against the same real, generated data. + if (config.IS_SPAN_METRICS_PROM_RW_ENABLED) { + otelCollectorConfig.exporters['prometheusremotewrite/spanmetrics'] = { + // Guaranteed set by IS_SPAN_METRICS_PROM_RW_ENABLED above. + endpoint: config.SPAN_METRICS_PROM_RW_ENDPOINT!, + tls: { insecure: true }, + resource_to_telemetry_conversion: { enabled: true }, + }; + spanMetricsExporters.push('prometheusremotewrite/spanmetrics'); + } + otelCollectorConfig.service.pipelines['metrics/spanmetrics'] = { + receivers: ['spanmetrics'], + processors: ['memory_limiter', 'batch'], + exporters: spanMetricsExporters, + }; + } + if (collectorAuthenticationEnforced) { if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) { // should never happen diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts index dc48dd8499..b7c0b4bdd4 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts @@ -9,7 +9,13 @@ const mockFetch = global.fetch as jest.Mock; // The proxy now streams the upstream response straight through (no // `await resp.json()`), so test mocks must expose the fields the pipeline // actually reads: `status`, `headers.get()`, and a web `ReadableStream` body. -function fakeUpstreamResponse(payload: unknown, status = 200) { +// Returned as `Response` so callers can hand it straight to `mockResolvedValue` +// without an `as any` at every site — the proxy only touches the fields below. +function fakeUpstreamResponse( + payload: unknown, + status = 200, + contentType = 'application/json', +): Response { const body = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(JSON.stringify(payload))); @@ -19,11 +25,11 @@ function fakeUpstreamResponse(payload: unknown, status = 200) { return { ok: status >= 200 && status < 300, status, - headers: new Headers({ 'content-type': 'application/json' }), + headers: new Headers({ 'content-type': contentType }), body, text: jest.fn().mockResolvedValue(JSON.stringify(payload)), json: jest.fn().mockResolvedValue(payload), - }; + } as unknown as Response; } describe('prometheus router', () => { @@ -36,7 +42,7 @@ describe('prometheus router', () => { afterEach(async () => { await server.clearDBs(); mockFetch.mockReset(); - mockFetch.mockResolvedValue(fakeUpstreamResponse({}) as any); + mockFetch.mockResolvedValue(fakeUpstreamResponse({})); }); afterAll(async () => { @@ -111,6 +117,59 @@ describe('prometheus router', () => { }); }); + // The connection host is member-configured, so its response body is untrusted + // output on our own origin: /api/* is same-origin-proxied by the app and the + // session cookie is sameSite lax. A text/html body forwarded verbatim would + // render as script here. + it('never forwards a non-JSON upstream content-type, and always sends nosniff', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse( + '', + 200, + 'text/html', + ), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.headers['content-type']).toContain('application/json'); + expect(res.headers['content-type']).not.toContain('text/html'); + expect(res.headers['x-content-type-options']).toBe('nosniff'); + }); + + it('passes a JSON upstream content-type through unchanged', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse({ status: 'success', data: [] }, 200), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.headers['content-type']).toContain('application/json'); + expect(res.headers['x-content-type-options']).toBe('nosniff'); + }); + it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { const { agent, team } = await getLoggedInAgent(server); const conn = await seedPrometheusConnection(team._id); @@ -119,9 +178,7 @@ describe('prometheus router', () => { status: 'success', data: { resultType: 'matrix', result: [] }, }; - mockFetch.mockResolvedValueOnce( - fakeUpstreamResponse(promResponse) as any, - ); + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); const res = await agent .get('/v1/prometheus/query_range') @@ -220,9 +277,7 @@ describe('prometheus router', () => { status: 'success', data: { resultType: 'vector', result: [] }, }; - mockFetch.mockResolvedValueOnce( - fakeUpstreamResponse(promResponse) as any, - ); + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); const res = await agent .get('/v1/prometheus/query') @@ -256,9 +311,7 @@ describe('prometheus router', () => { const conn = await seedPrometheusConnection(team._id); const promResponse = { status: 'success', data: ['up', 'requests'] }; - mockFetch.mockResolvedValueOnce( - fakeUpstreamResponse(promResponse) as any, - ); + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); const res = await agent .get('/v1/prometheus/label/__name__/values') @@ -270,4 +323,93 @@ describe('prometheus router', () => { expect(calledUrl).toContain('/api/v1/label/__name__/values'); }); }); + + describe('GET /v1/prometheus/query_exemplars', () => { + it('returns 400 when query parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ connectionId: new Types.ObjectId().toString() }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('query'), + }); + }); + + it('returns 400 when connectionId is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ query: 'up' }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('connectionId'), + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('returns 404 for a connection owned by another team', async () => { + const { agent } = await getLoggedInAgent(server); + // A real, resolvable connection — just not this team's. The 404 must come + // from the team scoping, not from the id simply not existing. + const otherTeamConn = await seedPrometheusConnection( + new Types.ObjectId(), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ query: 'up', connectionId: otherTeamConn._id.toString() }) + .expect(404); + expect(res.body).toMatchObject({ + status: 'error', + error: 'Connection not found', + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + const promResponse = { status: 'success', data: [] }; + mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual(promResponse); + const calledUrl = mockFetch.mock.calls[0][0] as string; + expect(calledUrl).toContain('/api/v1/query_exemplars'); + expect(calledUrl).toContain('query=up'); + }); + + it('returns an empty result for ClickHouse-backed connections (no native exemplar table function)', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedClickHouseConnection(team._id); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual({ status: 'success', data: [] }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/api/src/routers/api/__tests__/prometheus.test.ts b/packages/api/src/routers/api/__tests__/prometheus.test.ts index 6f5b0b7063..5c7df04a0b 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.test.ts @@ -3,6 +3,7 @@ import { formatVectorResponse, parseDuration, parseTimestamp, + resolveExemplarWindow, } from '@/routers/api/prometheus'; describe('parseTimestamp', () => { @@ -133,3 +134,63 @@ describe('formatVectorResponse', () => { ]); }); }); + +describe('resolveExemplarWindow', () => { + const DAY = 24 * 60 * 60; + const end = 1_700_000_000; + + it('passes a window inside the cap through untouched', () => { + expect(resolveExemplarWindow(String(end - DAY), String(end))).toEqual({ + start: end - DAY, + end, + }); + }); + + it('narrows an over-wide window instead of rejecting it', () => { + // A 30-day dashboard range is an ordinary request, and Prometheus keeps + // exemplars in a small recent buffer, so the older part has nothing to + // return. Rejecting would surface as the chart's exemplar error indicator on + // a perfectly healthy chart. + const result = resolveExemplarWindow(String(end - 30 * DAY), String(end)); + expect(result).toEqual({ start: end - 7 * DAY, end }); + }); + + it('keeps the requested end when narrowing', () => { + const result = resolveExemplarWindow(String(end - 30 * DAY), String(end)); + expect('end' in result && result.end).toBe(end); + }); + + it('rejects an inverted range', () => { + expect(resolveExemplarWindow(String(end), String(end - DAY))).toEqual({ + error: 'invalid or missing start/end parameters', + }); + }); + + it('rejects a missing or unparseable bound', () => { + for (const [start, endArg] of [ + [undefined, String(end)], + [String(end - DAY), undefined], + ['', String(end)], + ['not-a-time', String(end)], + ] as [string | undefined, string | undefined][]) { + expect(resolveExemplarWindow(start, endArg)).toEqual({ + error: 'invalid or missing start/end parameters', + }); + } + }); + + it('accepts an ISO timestamp, matching parseTimestamp', () => { + const result = resolveExemplarWindow( + '2023-11-14T22:13:20Z', + '2023-11-14T22:14:20Z', + ); + expect('start' in result).toBe(true); + }); + + it('honours an explicit cap', () => { + expect(resolveExemplarWindow(String(end - 100), String(end), 10)).toEqual({ + start: end - 10, + end, + }); + }); +}); diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 07f446816f..214750c878 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -147,9 +147,16 @@ const PROMETHEUS_CH_TIMEOUT_MS = 30_000; const PROMETHEUS_MAX_EXECUTION_SEC = 30; const PROMETHEUS_MAX_RESULT_ROWS = '100000'; const PROMETHEUS_MAX_RESOLUTION = 11_000; +// Widest window /query_exemplars will proxy. Prometheus's exemplar store is a +// small circular buffer, so a wider range mostly costs a bigger streamed body +// for no extra markers — the chart's own thinning caps what's rendered anyway. +const PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC = 7 * 24 * 60 * 60; // Forwards the response straight from the upstream Prometheus to the -// HyperDX client. The response can be multi-megabyte (e.g. `/label/__name__/ +// HyperDX client. Returns the HTTP status it wrote, so callers can record an +// error metric: this helper handles its own failures by writing 400/502/504 and +// returning normally, so a caller's `catch` never sees an upstream outage and +// would otherwise report zero errors while still recording duration. The response can be multi-megabyte (e.g. `/label/__name__/ // values` on a large Prometheus), so we avoid `await resp.json()` + // `res.json(...)` which would parse + re-serialize the whole body in memory. // Prometheus's native response shape (`{status, data}` / `{status, errorType, @@ -160,7 +167,7 @@ async function proxyToPrometheus( path: string, params: Record, res: express.Response, -): Promise { +): Promise { let url: URL; try { url = new URL(path, upstreamHost); @@ -170,7 +177,7 @@ async function proxyToPrometheus( errorType: 'bad_data', error: `Connection host is not a valid URL: ${JSON.stringify(upstreamHost)}`, }); - return; + return 400; } for (const [k, v] of Object.entries(params)) { if (['connectionId', 'database', 'table'].includes(k)) continue; @@ -190,7 +197,7 @@ async function proxyToPrometheus( errorType: 'timeout', error: `Prometheus request to ${target} timed out after ${PROMETHEUS_PROXY_TIMEOUT_MS}ms`, }); - return; + return 504; } // Node's fetch wraps the real network error (ECONNREFUSED, ENOTFOUND, TLS, // …) in `.cause`. Without unwrapping it the client only sees a useless @@ -210,16 +217,30 @@ async function proxyToPrometheus( errorType: 'unavailable', error: `Failed to reach Prometheus at ${target} (${detail})`, }); - return; + return 502; } res.status(upstreamResp.status); + + // The connection host is member-configured, so its response is untrusted output + // on our own origin: the app same-origin-proxies /api/*, and the session cookie + // is sameSite lax. Forwarding the upstream content-type verbatim would let a + // text/html body render as script here. Send nosniff unconditionally and pass + // the content-type through only when it is JSON; anything else is relabelled, + // which keeps a genuine Prometheus error body readable while making a hostile + // one inert. + res.setHeader('x-content-type-options', 'nosniff'); const contentType = upstreamResp.headers.get('content-type'); - if (contentType) res.setHeader('content-type', contentType); + res.setHeader( + 'content-type', + contentType && /^application\/(json|[\w.+-]+\+json)\b/i.test(contentType) + ? contentType + : 'application/json', + ); if (!upstreamResp.body) { res.end(); - return; + return upstreamResp.status; } try { @@ -231,6 +252,28 @@ async function proxyToPrometheus( if (!res.writableEnded) { res.destroy(err instanceof Error ? err : new Error(String(err))); } + return 502; + } + return upstreamResp.status; +} + +/** + * Records an error for a proxied response that failed server-side. + * proxyToPrometheus never throws, so this is the only place the error counter + * gets incremented on the proxy path. + * + * 5xx only. An upstream 4xx is almost always a malformed PromQL expression the + * user typed, and counting those would make this counter track user typos rather + * than backend health — which is what alerts and SLOs read it for. The helper's + * own failure statuses (502 unreachable, 504 timeout) are 5xx and so are counted. + */ +function recordProxyOutcome( + status: number, + endpoint: string, + backend: PrometheusBackend, +) { + if (status >= 500) { + prometheusQueryErrors.add(1, { endpoint, backend }); } } @@ -281,12 +324,13 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { // directly to connection.host instead of running a ClickHouse query. if (connection.isPrometheusEndpoint) { backend = 'prometheus'; - await proxyToPrometheus( + const status = await proxyToPrometheus( connection.host, '/api/v1/query_range', params, res, ); + recordProxyOutcome(status, 'query_range', backend); return; } @@ -414,7 +458,13 @@ const queryHandler: express.RequestHandler = async (req, res) => { if (connection.isPrometheusEndpoint) { backend = 'prometheus'; - await proxyToPrometheus(connection.host, '/api/v1/query', params, res); + const status = await proxyToPrometheus( + connection.host, + '/api/v1/query', + params, + res, + ); + recordProxyOutcome(status, 'query', backend); return; } @@ -478,6 +528,140 @@ const queryHandler: express.RequestHandler = async (req, res) => { router.get('/query', queryHandler); router.post('/query', queryHandler); +/** + * Resolve the exemplar window a /query_exemplars request should be proxied with. + * + * Returns either the bounded window or the reason it is unusable, so the handler + * stays a thin translation to HTTP and this logic can be tested without a route. + * Over-wide windows are narrowed rather than rejected: a 30d dashboard range is an + * ordinary request, and Prometheus keeps exemplars in a small recent buffer, so + * the older part has nothing to return anyway. + */ +export function resolveExemplarWindow( + rawStart: string | undefined, + rawEnd: string | undefined, + maxWindowSec = PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC, +): { start: number; end: number } | { error: string } { + const parse = (v: string | undefined) => { + if (v == null || v === '') return NaN; + try { + return parseTimestamp(v); + } catch { + return NaN; + } + }; + const start = parse(rawStart); + const end = parse(rawEnd); + if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) { + return { error: 'invalid or missing start/end parameters' }; + } + return { start: Math.max(start, end - maxWindowSec), end }; +} + +// -------------------------- +// GET|POST /query_exemplars +// -------------------------- + +// Native Prometheus exposes exemplars via /api/v1/query_exemplars. We proxy +// straight through for Prometheus-backed connections. ClickHouse-backed metric +// exemplars are read directly from the OTel metric tables' `Exemplars.*` +// columns in the app (via renderMetricExemplarsChartConfig), so there is no +// ClickHouse table function to call here — return an empty result instead. +const queryExemplarsHandler: express.RequestHandler = async (req, res) => { + const startedAt = performance.now(); + let backend: PrometheusBackend = 'unknown'; + try { + const { teamId } = getNonNullUserWithTeam(req); + const params = getParams(req); + + const query = params.query; + if (!query) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: query', + }); + } + + const connectionId = params.connectionId; + if (!connectionId) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: connectionId', + }); + } + + const connection = await getConnectionById( + teamId.toString(), + connectionId, + true, + ); + if (!connection) { + return res.status(404).json({ + status: 'error', + errorType: 'bad_data', + error: 'Connection not found', + }); + } + + if (connection.isPrometheusEndpoint) { + backend = 'prometheus'; + + // Bound the requested window before proxying. Unlike /query_range there is + // no `step` to cap the result size with, and Prometheus's /query_exemplars + // takes no limit parameter — the window is the only lever, and the response + // streams through this process. Parsed (not just forwarded) so a malformed + // timestamp fails here rather than upstream. + // + // Deliberately inside this branch: the ClickHouse-backed branch below never + // reaches Prometheus and answers with an empty success, so bounding it there + // would turn a healthy wide-range chart into a 400 (and light up the chart's + // exemplar error indicator) for a request that does no upstream work. + // + // parseTimestamp throws on a missing/unparseable value; a bad client param + // is a 400, not an upstream error, so it is resolved here, not in the catch. + const window = resolveExemplarWindow(params.start, params.end); + if ('error' in window) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: window.error, + }); + } + + const status = await proxyToPrometheus( + connection.host, + '/api/v1/query_exemplars', + { ...params, start: String(window.start) }, + res, + ); + recordProxyOutcome(status, 'query_exemplars', backend); + return; + } + + // ClickHouse-backed PromQL: no native exemplar table function. Exemplars + // for structured metric charts are fetched app-side from the metric table. + backend = 'clickhouse'; + return res.json({ status: 'success', data: [] }); + } catch (e) { + prometheusQueryErrors.add(1, { endpoint: 'query_exemplars', backend }); + logger.error(e, 'Prometheus query_exemplars error'); + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: e instanceof Error ? e.message : String(e), + }); + } finally { + prometheusQueryDuration.record(performance.now() - startedAt, { + endpoint: 'query_exemplars', + backend, + }); + } +}; +router.get('/query_exemplars', queryExemplarsHandler); +router.post('/query_exemplars', queryExemplarsHandler); + // -------------------------- // GET /label/:name/values // -------------------------- @@ -528,12 +712,13 @@ router.get('/label/:name/values', async (req, res) => { // Proxy to Prometheus if endpoint is set if (connection.isPrometheusEndpoint) { backend = 'prometheus'; - await proxyToPrometheus( + const status = await proxyToPrometheus( connection.host, `/api/v1/label/${labelName}/values`, params, res, ); + recordProxyOutcome(status, 'label_values', backend); return; } diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 3bb6749daf..aac3d7a18a 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -629,6 +629,20 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: boolean * description: Overlay the equivalent previous time period for comparison. * default: false + * enableExemplars: + * type: boolean + * description: > + * Overlay exemplars: markers for individual trace-linked data points. + * Only renders when the tile is exemplar-eligible — a single + * non-ratio histogram metric series with no groupBy — and is + * accepted but inert otherwise. + * default: false + * exemplarTraceSourceId: + * type: string + * description: > + * ID of the Trace source an exemplar marker links to. Defaults to + * the chart source's linked trace source when omitted. + * example: "65f5e4a3b9e77c001a222222" * * BarBuilderChartConfig: * type: object @@ -676,6 +690,20 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * numberFormat: * $ref: '#/components/schemas/NumberFormat' * description: Number formatting options for displayed values. + * enableExemplars: + * type: boolean + * description: > + * Overlay exemplars: markers for individual trace-linked data points. + * Only renders when the tile is exemplar-eligible — a single + * non-ratio histogram metric series with no groupBy — and is + * accepted but inert otherwise. + * default: false + * exemplarTraceSourceId: + * type: string + * description: > + * ID of the Trace source an exemplar marker links to. Defaults to + * the chart source's linked trace source when omitted. + * example: "65f5e4a3b9e77c001a222222" * * TableBuilderChartConfig: * type: object diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 638efbcb1b..b608f7633f 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -300,6 +300,8 @@ const convertToExternalTileChartConfig = ( : [DEFAULT_SELECT_ITEM], compareToPreviousPeriod: config.compareToPreviousPeriod, numberFormat: config.numberFormat, + enableExemplars: config.enableExemplars, + exemplarTraceSourceId: config.exemplarTraceSourceId, }; case DisplayType.StackedBar: return { @@ -316,6 +318,8 @@ const convertToExternalTileChartConfig = ( ? config.select.map(convertToExternalSelectItem) : [DEFAULT_SELECT_ITEM], numberFormat: config.numberFormat, + enableExemplars: config.enableExemplars, + exemplarTraceSourceId: config.exemplarTraceSourceId, }; case DisplayType.Number: return { @@ -693,6 +697,8 @@ export function convertToInternalTileConfig( 'alignDateRangeToGranularity', 'compareToPreviousPeriod', 'fitYAxisToData', + 'enableExemplars', + 'exemplarTraceSourceId', ]), displayType: externalConfig.displayType === 'stacked_bar' diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 42f11afeab..2ffbd57e59 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -261,6 +261,11 @@ const externalDashboardTimeChartConfigSchema = z.object({ alignDateRangeToGranularity: z.boolean().optional(), fillNulls: z.boolean().optional(), numberFormat: NumberFormatSchema.optional(), + // Exemplar overlay (trace-linked markers). Rendering additionally requires an + // exemplar-eligible shape — single non-ratio histogram series, no group by — + // so setting this on an ineligible tile is inert rather than an error. + enableExemplars: z.boolean().optional(), + exemplarTraceSourceId: objectIdSchema.optional(), }); const externalDashboardLineChartConfigSchema = diff --git a/packages/app/.env.development b/packages/app/.env.development index 7fd59cdfde..6c2de5a184 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -13,3 +13,4 @@ NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PO # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT= NEXT_PUBLIC_ENABLE_PROMQL=true +NEXT_PUBLIC_ENABLE_EXEMPLARS=true diff --git a/packages/app/public/whats-new.json b/packages/app/public/whats-new.json new file mode 100644 index 0000000000..05ad0d0f11 --- /dev/null +++ b/packages/app/public/whats-new.json @@ -0,0 +1,189 @@ +{ + "releases": [ + { + "version": "2.31.0", + "features": [ + { + "scope": "general", + "text": "Convert current builder config to SQL during editor switch" + }, + { + "scope": "dashboards", + "text": "Add background area sparklines to the Browser RUM dashboard number tiles" + }, + { + "scope": "dashboards", + "text": "Revamp the out-of-the-box Browser RUM dashboard" + }, + { + "scope": "charts", + "text": "Add per-series actions to the chart drill-down menu" + }, + { + "scope": "charts", + "text": "Clicking a time-chart point now locks the tooltip in place instead of opening a separate drill-down menu" + }, + { + "scope": "app", + "text": "Make the selected source clearer in the source picker" + } + ], + "highlight": { + "title": "Interactive charts & a revamped Browser RUM dashboard", + "blurb": "Click a time-chart point to **pin its tooltip** and drill in per series, and explore the out-of-the-box **Browser RUM dashboard** — now reorganised into Core Web Vitals, Load Time, Traffic, and Errors with colour-coded tiles." + } + }, + { + "version": "2.30.1", + "features": [ + { + "scope": "general", + "text": "Allow freeform text in VirtualMultiSelect" + } + ] + }, + { + "version": "2.30.0", + "features": [ + { + "scope": "general", + "text": "Add categorical bar chart display type" + }, + { + "scope": "general", + "text": "Add event patterns as a first-class dashboard tile type" + }, + { + "scope": "dashboards", + "text": "Overlay alert firing/recovery markers on tile charts" + }, + { + "scope": "general", + "text": "Add a custom ORDER BY input for Bar and Pie charts" + }, + { + "scope": "general", + "text": "Allow specifying a limit on pie and bar chart series" + }, + { "scope": "trace", "text": "Add a trace minimap above the waterfall" }, + { + "scope": "general", + "text": "Redesign the trace waterfall — per-service span colors, vertical service color bar, child counts, duration outside the bar with the span body on hover, expand/collapse depth controls" + } + ] + }, + { + "version": "2.29.0", + "features": [ + { "scope": "general", "text": "Add Browser RUM dashboard template" }, + { + "scope": "general", + "text": "Lazy-load dashboard tiles based on viewport visibility" + }, + { + "scope": "dashboards", + "text": "Add a background trend sparkline to number tiles" + }, + { + "scope": "general", + "text": "Add an \"external link\" row-click action for dashboard table tiles" + }, + { "scope": "general", "text": "Trace panel inline split detail" }, + { + "scope": "general", + "text": "Add option to fit time chart y-axis lower bound" + }, + { + "scope": "general", + "text": "Add a \"Connect your AI assistant\" section to Team Settings" + }, + { + "scope": "dashboard", + "text": "Auto-resize font in number tiles to fit container" + }, + { + "scope": "chart-explorer", + "text": "Duplicate a series in the chart builder" + }, + { + "scope": "search", + "text": "Make active filter pills editable in place" + }, + { "scope": "general", "text": "Use optimistic updates for favorites" }, + { + "scope": "general", + "text": "Allow selecting the column or SQL expression used for event pattern grouping (with shareable URL state)" + }, + { + "scope": "charts", + "text": "The time-chart series limit is now configured per chart in the Display Settings drawer instead of as a workspace-wide team setting (the team \"Time Chart Series Limit\" setting is removed)" + }, + { + "scope": "mcp", + "text": "Add denoise option to clickstack_search tool" + }, + { + "scope": "general", + "text": "Auto-fill metric table dropdowns when creating a Metrics source" + }, + { "scope": "general", "text": "Add source field suggestions" }, + { + "scope": "service-map", + "text": "Server-side filtering, latency percentiles, throughput & focus" + }, + { "scope": "source-picker", "text": "Chip + kebab menu UX" }, + { + "scope": "general", + "text": "Add an optional Section field to data sources" + }, + { + "scope": "general", + "text": "Add Known Columns List setting for distributed tables" + }, + { + "scope": "charts", + "text": "Cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys" + }, + { + "scope": "general", + "text": "Emphasize the series nearest the cursor in multi-series time charts" + } + ] + }, + { + "version": "2.28.0", + "features": [ + { "scope": "general", "text": "Experimental promql support" }, + { + "scope": "general", + "text": "Emit Lucene conditions from sidebar/dashboard filters to enable KV items direct_read optimization on Map columns" + }, + { "scope": "general", "text": "Upgrade to clickhouse-server 26.5" }, + { + "scope": "dashboard", + "text": "Add Table of Contents right rail with bulk collapse/expand" + }, + { + "scope": "general", + "text": "Default the direct_read map column optimization on supported ClickHouse versions" + }, + { + "scope": "general", + "text": "Preserve compatible filters when switching sources" + }, + { + "scope": "general", + "text": "Add source scoping to dashboard filters" + }, + { + "scope": "general", + "text": "Add option for force-enabling/disabling text index support" + }, + { + "scope": "general", + "text": "Filters reflect all values, not search aware; filters use metadata MVs if available" + } + ] + } + ] +} diff --git a/packages/app/src/HDXMultiSeriesTimeChart/ChartLegend.tsx b/packages/app/src/HDXMultiSeriesTimeChart/ChartLegend.tsx new file mode 100644 index 0000000000..683e8e420c --- /dev/null +++ b/packages/app/src/HDXMultiSeriesTimeChart/ChartLegend.tsx @@ -0,0 +1,173 @@ +import { memo, useMemo, useState } from 'react'; +import cx from 'classnames'; +import { Popover } from '@mantine/core'; + +import { type LineData } from '@/ChartUtils'; +import { truncateMiddle } from '@/utils'; + +import { MAX_LEGEND_ITEMS } from './constants'; + +import styles from '@styles/HDXLineChart.module.scss'; + +function ExpandableLegendItem({ + entry, + expanded, + isSelected, + isDisabled, + onToggle, +}: { + entry: any; + expanded?: boolean; + isSelected?: boolean; + isDisabled?: boolean; + onToggle?: (isShiftKey: boolean) => void; +}) { + const [_expanded, setExpanded] = useState(false); + const isExpanded = _expanded || expanded; + + return ( + { + if (onToggle) { + onToggle(e.shiftKey); + } else { + setExpanded(v => !v); + } + }} + title={ + isSelected + ? 'Click to show all (Shift+click to deselect)' + : 'Click to show only this (Shift+click for multi-select)' + } + > +
+ + + +
+ {isExpanded || isSelected + ? entry.value + : truncateMiddle(`${entry.value}`, 35)} + + ); +} + +export const LegendRenderer = memo<{ + payload?: { + dataKey: string; + value: string; + color: string; + }[]; + lineDataMap: { [key: string]: LineData }; + allLineData?: LineData[]; + selectedSeries?: Set; + onToggleSeries?: (seriesName: string, isShiftKey?: boolean) => void; +}>(props => { + const { payload, lineDataMap, allLineData, selectedSeries, onToggleSeries } = + props; + + const hasSelection = !!selectedSeries && selectedSeries.size > 0; + + // Use allLineData to ensure all series are always shown in legend + const allSeriesPayload = useMemo(() => { + if (allLineData?.length) { + return allLineData.map(ld => ({ + dataKey: ld.dataKey, + value: ld.displayName || ld.dataKey, + color: ld.color, + payload: { strokeDasharray: ld.isDashed ? '4 3' : '0' }, + })); + } + return payload ?? []; + }, [allLineData, payload]); + + const sortedLegendItems = useMemo(() => { + // Order items such that current and previous period lines are consecutive + const currentPeriodKeyIndex = new Map(); + allSeriesPayload.forEach((line, index) => { + const currentPeriodKey = + lineDataMap[line.dataKey]?.currentPeriodKey || ''; + if (!currentPeriodKeyIndex.has(currentPeriodKey)) { + currentPeriodKeyIndex.set(currentPeriodKey, index); + } + }); + + // Copy before sorting: when this comes from Recharts' legend payload it is + // kept in the Immer-backed store and frozen, so an in-place sort throws. + return [...allSeriesPayload].sort((a, b) => { + const keyA = lineDataMap[a.dataKey]?.currentPeriodKey ?? ''; + const keyB = lineDataMap[b.dataKey]?.currentPeriodKey ?? ''; + + const indexA = currentPeriodKeyIndex.get(keyA) ?? 0; + const indexB = currentPeriodKeyIndex.get(keyB) ?? 0; + + return indexB - indexA || a.dataKey.localeCompare(b.dataKey); + }); + }, [allSeriesPayload, lineDataMap]); + + const shownItems = sortedLegendItems.slice(0, MAX_LEGEND_ITEMS); + const restItems = sortedLegendItems.slice(MAX_LEGEND_ITEMS); + + return ( +
+ {shownItems.map((entry, index) => { + const isSelected = !!selectedSeries?.has(entry.value); + const isDisabled = hasSelection && !isSelected; + return ( + onToggleSeries?.(entry.value, isShiftKey)} + /> + ); + })} + {restItems.length ? ( + + +
+ +{restItems.length} more +
+ + +
+ {restItems.map((entry, index) => { + const isSelected = !!selectedSeries?.has(entry.value); + const isDisabled = hasSelection && !isSelected; + return ( + + onToggleSeries?.(entry.value, isShiftKey) + } + /> + ); + })} +
+ + + ) : null} +
+ ); +}); diff --git a/packages/app/src/HDXMultiSeriesTimeChart/ChartTooltipContent.tsx b/packages/app/src/HDXMultiSeriesTimeChart/ChartTooltipContent.tsx new file mode 100644 index 0000000000..58497eea43 --- /dev/null +++ b/packages/app/src/HDXMultiSeriesTimeChart/ChartTooltipContent.tsx @@ -0,0 +1,162 @@ +import { memo, useMemo } from 'react'; +import { withErrorBoundary } from 'react-error-boundary'; + +import { findNearestSeriesKey, type LineData } from '@/ChartUtils'; +import { + ChartTooltipContainer, + ChartTooltipHeader, + toViewportPoint, + useChartTooltipZIndex, +} from '@/components/charts/ChartTooltip'; +import type { NumberFormat } from '@/types'; + +import { + NEAREST_SERIES_MAX_DISTANCE_PX, + TOOLTIP_POINT_OFFSET_PX, +} from './constants'; +import { TooltipItem, type TooltipPayload } from './TooltipItem'; + +import styles from '@styles/HDXLineChart.module.scss'; + +type HDXLineChartTooltipProps = { + lineDataMap: { [keyName: string]: LineData }; + previousPeriodOffsetSeconds?: number; + numberFormat?: NumberFormat; + numberFormatByKey: Map; + /** Per-series active-point pixel Y, captured by the Area active dots. */ + activePointYByKeyRef: React.MutableRefObject>; + /** The chart's outer container; its viewport rect anchors this tooltip. */ + containerRef: React.MutableRefObject; +} & Record; + +/** + * The recharts `` content used for the HOVER tooltip (on the hovered + * chart and its synced followers). Clicking pins ChartSeriesTooltip instead. + * + * Because it's given `portal={document.body}`, recharts skips its own transform + * positioning, so this content self-anchors at the active point with + * `position: fixed` (container rect + `coordinate`) — matching the pinned + * tooltip's anchor, and escaping the chart's bounds so edges aren't clipped. + */ +export const HDXLineChartTooltip = withErrorBoundary( + memo((props: HDXLineChartTooltipProps) => { + const { + active, + payload, + label, + numberFormat, + numberFormatByKey, + lineDataMap, + previousPeriodOffsetSeconds, + activePointYByKeyRef, + containerRef, + } = props; + const typedPayload = payload as TooltipPayload[]; + + const tooltipZIndex = useChartTooltipZIndex(); + + const payloadByKey = useMemo( + () => new Map(typedPayload.map(p => [p.dataKey, p])), + [typedPayload], + ); + + if (active && payload && payload.length) { + // No onClose: hover renders the X hidden (kept for layout parity). + const header = ( + + ); + + // Bold the line nearest the cursor by comparing pointer Y to each series' + // active-dot Y. The dots write their positions earlier in this same render + // (Recharts draws graphical items before the tooltip), so it's current. + const pointerY: number | undefined = props.coordinate?.y; + // eslint-disable-next-line react-hooks/refs + const activePointYByKey = activePointYByKeyRef?.current ?? undefined; + const nearestSeriesKey = + typedPayload.length > 1 + ? findNearestSeriesKey( + activePointYByKey, + typedPayload.map(p => p.dataKey), + pointerY, + NEAREST_SERIES_MAX_DISTANCE_PX, + ) + : undefined; + + // Anchor at the active point (see the component docblock for why fixed). + const pointX = props.coordinate?.x; + const pointY = props.coordinate?.y; + // eslint-disable-next-line react-hooks/refs + const containerRect = containerRef?.current?.getBoundingClientRect(); + const anchor = + typeof pointX === 'number' && + typeof pointY === 'number' && + containerRect != null + ? toViewportPoint(containerRect, { x: pointX, y: pointY }) + : undefined; + const anchorStyle: React.CSSProperties = + anchor != null + ? { + position: 'fixed', + left: anchor.x, + top: anchor.y + TOOLTIP_POINT_OFFSET_PX, + transform: 'translateX(-50%)', + pointerEvents: 'none', + // z-index must live here: recharts leaves the portaled wrapper + // `position: static`, where z-index has no effect. + zIndex: tooltipZIndex, + } + : {}; + + return ( +
+ + {/* Copy before sorting: Recharts 3 freezes the payload, so an + in-place sort throws "this object has been frozen". */} + {[...payload] + .sort((a: TooltipPayload, b: TooltipPayload) => b.value - a.value) + .map((p: TooltipPayload) => { + const previousKey = lineDataMap[p.dataKey]?.previousPeriodKey; + const isPreviousPeriod = previousKey === p.dataKey; + const previousPayload = + !isPreviousPeriod && previousKey + ? payloadByKey.get(previousKey) + : undefined; + const valueColumnName = + lineDataMap[p.dataKey]?.valueColumnName ?? p.dataKey; + const numberFormatForKey = + numberFormatByKey.get(valueColumnName) ?? numberFormat; + + return ( + + ); + })} + +
+ ); + } + return null; + }), + { + onError: console.error, + fallback: ( +
+ An error occurred while rendering the tooltip. +
+ ), + }, +); diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart/MemoChart.tsx similarity index 52% rename from packages/app/src/HDXMultiSeriesTimeChart.tsx rename to packages/app/src/HDXMultiSeriesTimeChart/MemoChart.tsx index b334956ecb..6c526f87b4 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart/MemoChart.tsx @@ -7,600 +7,59 @@ import { useRef, useState, } from 'react'; -import cx from 'classnames'; -import { add, isSameSecond, sub } from 'date-fns'; -import { withErrorBoundary } from 'react-error-boundary'; import { Area, AreaChart, Bar, BarChart, - BarProps, CartesianGrid, Legend, ReferenceArea, + ReferenceDot, ReferenceLine, ResponsiveContainer, Tooltip, XAxis, YAxis, } from 'recharts'; -import { AxisDomain } from 'recharts/types/util/types'; -import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils'; -import { DisplayType } from '@hyperdx/common-utils/dist/types'; -import { Button, Popover, Tooltip as MantineTooltip } from '@mantine/core'; +import { DisplayType, Exemplar } from '@hyperdx/common-utils/dist/types'; +import { Button, Tooltip as MantineTooltip } from '@mantine/core'; import { IconZoomReset } from '@tabler/icons-react'; +import { useChartSyncId } from '@/chartSync'; +import { findNearestSeriesKey, LineData } from '@/ChartUtils'; +import { ChartAnnotation } from '@/components/charts/chartAnnotations'; +import { toViewportPoint } from '@/components/charts/ChartTooltip'; +import { ExemplarDot } from '@/components/Exemplars'; import type { NumberFormat } from '@/types'; -import { COLORS, formatNumber, truncateMiddle } from '@/utils'; +import { useFormatTime } from '@/useFormatTime'; +import { COLORS, formatNumber } from '@/utils'; import { - ChartAnnotation, - getAnnotationElements, -} from './components/charts/chartAnnotations'; + type ActiveClickPayload, + buildActiveClickSeries, + getActiveLabel, + getSeriesDisplayName, + getVisibleLineData, +} from './chartData'; +import { LegendRenderer } from './ChartLegend'; +import { CaptureActiveDot, StackedBarWithOverlap } from './chartPrimitives'; +import { HDXLineChartTooltip } from './ChartTooltipContent'; import { - ChartTooltipContainer, - ChartTooltipHeader, - ChartTooltipItem, - toViewportPoint, - useChartTooltipZIndex, -} from './components/charts/ChartTooltip'; -import { useChartSyncId } from './chartSync'; -import { - findNearestSeriesKey, - LineData, - MAX_TIME_CHART_SERIES, - toStartOfInterval, -} from './ChartUtils'; -import { useFormatTime } from './useFormatTime'; - -import styles from '@styles/HDXLineChart.module.scss'; - -const MAX_LEGEND_ITEMS = 4; - -// Vertical pixel distance within which a series' line counts as "near" the -// cursor for tooltip highlighting. Beyond this, no row is emphasized so the -// tooltip is not misleading when the pointer is in empty space. -const NEAREST_SERIES_MAX_DISTANCE_PX = 30; - -// Gap below the data point for the hover tooltip. Kept equal to the pinned -// tooltip's Popover `offset` so both land in the same spot. -const TOOLTIP_POINT_OFFSET_PX = 12; - -const Y_AXIS_WIDTH = 40; -const SINGLE_POINT_BAR_RIGHT_PADDING = 10; -const SINGLE_POINT_BAR_WIDTH_RATIO = 0.8; -// Top margin (px) reserved above the plot for annotation labels ("Alert"/"OK"), -// added only when a chart is showing annotations so other charts keep their -// tighter default headroom. -const ANNOTATION_LABEL_HEADROOM = 18; - -type TooltipPayload = { - dataKey: string; - name: string; - value: number; - color?: string; - stroke?: string; - strokeWidth?: number; - strokeDasharray?: string; - opacity?: number; -}; - -export const TooltipItem = memo( - ({ - p, - previous, - numberFormat, - highlighted, - dimmed, - }: { - p: TooltipPayload; - previous?: TooltipPayload; - numberFormat?: NumberFormat; - highlighted?: boolean; - dimmed?: boolean; - }) => { - return ( - - ); - }, -); - -type HDXLineChartTooltipProps = { - lineDataMap: { [keyName: string]: LineData }; - previousPeriodOffsetSeconds?: number; - numberFormat?: NumberFormat; - numberFormatByKey: Map; - /** Per-series active-point pixel Y, captured by the Area active dots. */ - activePointYByKeyRef: React.MutableRefObject>; - /** The chart's outer container; its viewport rect anchors this tooltip. */ - containerRef: React.MutableRefObject; -} & Record; - -/** - * The recharts `` content used for the HOVER tooltip (on the hovered - * chart and its synced followers). Clicking pins ChartSeriesTooltip instead. - * - * Because it's given `portal={document.body}`, recharts skips its own transform - * positioning, so this content self-anchors at the active point with - * `position: fixed` (container rect + `coordinate`) — matching the pinned - * tooltip's anchor, and escaping the chart's bounds so edges aren't clipped. - */ -const HDXLineChartTooltip = withErrorBoundary( - memo((props: HDXLineChartTooltipProps) => { - const { - active, - payload, - label, - numberFormat, - numberFormatByKey, - lineDataMap, - previousPeriodOffsetSeconds, - activePointYByKeyRef, - containerRef, - } = props; - const typedPayload = payload as TooltipPayload[]; - - const tooltipZIndex = useChartTooltipZIndex(); - - const payloadByKey = useMemo( - () => new Map(typedPayload.map(p => [p.dataKey, p])), - [typedPayload], - ); - - if (active && payload && payload.length) { - // No onClose: hover renders the X hidden (kept for layout parity). - const header = ( - - ); - - // Bold the line nearest the cursor by comparing pointer Y to each series' - // active-dot Y. The dots write their positions earlier in this same render - // (Recharts draws graphical items before the tooltip), so it's current. - const pointerY: number | undefined = props.coordinate?.y; - // eslint-disable-next-line react-hooks/refs - const activePointYByKey = activePointYByKeyRef?.current ?? undefined; - const nearestSeriesKey = - typedPayload.length > 1 - ? findNearestSeriesKey( - activePointYByKey, - typedPayload.map(p => p.dataKey), - pointerY, - NEAREST_SERIES_MAX_DISTANCE_PX, - ) - : undefined; - - // Anchor at the active point (see the component docblock for why fixed). - const pointX = props.coordinate?.x; - const pointY = props.coordinate?.y; - // eslint-disable-next-line react-hooks/refs - const containerRect = containerRef?.current?.getBoundingClientRect(); - const anchor = - typeof pointX === 'number' && - typeof pointY === 'number' && - containerRect != null - ? toViewportPoint(containerRect, { x: pointX, y: pointY }) - : undefined; - const anchorStyle: React.CSSProperties = - anchor != null - ? { - position: 'fixed', - left: anchor.x, - top: anchor.y + TOOLTIP_POINT_OFFSET_PX, - transform: 'translateX(-50%)', - pointerEvents: 'none', - // z-index must live here: recharts leaves the portaled wrapper - // `position: static`, where z-index has no effect. - zIndex: tooltipZIndex, - } - : {}; - - return ( -
- - {/* Copy before sorting: Recharts 3 freezes the payload, so an - in-place sort throws "this object has been frozen". */} - {[...payload] - .sort((a: TooltipPayload, b: TooltipPayload) => b.value - a.value) - .map((p: TooltipPayload) => { - const previousKey = lineDataMap[p.dataKey]?.previousPeriodKey; - const isPreviousPeriod = previousKey === p.dataKey; - const previousPayload = - !isPreviousPeriod && previousKey - ? payloadByKey.get(previousKey) - : undefined; - const valueColumnName = - lineDataMap[p.dataKey]?.valueColumnName ?? p.dataKey; - const numberFormatForKey = - numberFormatByKey.get(valueColumnName) ?? numberFormat; - - return ( - - ); - })} - -
- ); - } - return null; - }), - { - onError: console.error, - fallback: ( -
- An error occurred while rendering the tooltip. -
- ), - }, -); - -function ExpandableLegendItem({ - entry, - expanded, - isSelected, - isDisabled, - onToggle, -}: { - entry: any; - expanded?: boolean; - isSelected?: boolean; - isDisabled?: boolean; - onToggle?: (isShiftKey: boolean) => void; -}) { - const [_expanded, setExpanded] = useState(false); - const isExpanded = _expanded || expanded; - - return ( - { - if (onToggle) { - onToggle(e.shiftKey); - } else { - setExpanded(v => !v); - } - }} - title={ - isSelected - ? 'Click to show all (Shift+click to deselect)' - : 'Click to show only this (Shift+click for multi-select)' - } - > -
- - - -
- {isExpanded || isSelected - ? entry.value - : truncateMiddle(`${entry.value}`, 35)} - - ); -} - -const LegendRenderer = memo<{ - payload?: { - dataKey: string; - value: string; - color: string; - }[]; - lineDataMap: { [key: string]: LineData }; - allLineData?: LineData[]; - selectedSeries?: Set; - onToggleSeries?: (seriesName: string, isShiftKey?: boolean) => void; -}>(props => { - const { payload, lineDataMap, allLineData, selectedSeries, onToggleSeries } = - props; - - const hasSelection = !!selectedSeries && selectedSeries.size > 0; - - // Use allLineData to ensure all series are always shown in legend - const allSeriesPayload = useMemo(() => { - if (allLineData?.length) { - return allLineData.map(ld => ({ - dataKey: ld.dataKey, - value: ld.displayName || ld.dataKey, - color: ld.color, - payload: { strokeDasharray: ld.isDashed ? '4 3' : '0' }, - })); - } - return payload ?? []; - }, [allLineData, payload]); - - const sortedLegendItems = useMemo(() => { - // Order items such that current and previous period lines are consecutive - const currentPeriodKeyIndex = new Map(); - allSeriesPayload.forEach((line, index) => { - const currentPeriodKey = - lineDataMap[line.dataKey]?.currentPeriodKey || ''; - if (!currentPeriodKeyIndex.has(currentPeriodKey)) { - currentPeriodKeyIndex.set(currentPeriodKey, index); - } - }); - - // Copy before sorting: when this comes from Recharts' legend payload it is - // kept in the Immer-backed store and frozen, so an in-place sort throws. - return [...allSeriesPayload].sort((a, b) => { - const keyA = lineDataMap[a.dataKey]?.currentPeriodKey ?? ''; - const keyB = lineDataMap[b.dataKey]?.currentPeriodKey ?? ''; - - const indexA = currentPeriodKeyIndex.get(keyA) ?? 0; - const indexB = currentPeriodKeyIndex.get(keyB) ?? 0; - - return indexB - indexA || a.dataKey.localeCompare(b.dataKey); - }); - }, [allSeriesPayload, lineDataMap]); - - const shownItems = sortedLegendItems.slice(0, MAX_LEGEND_ITEMS); - const restItems = sortedLegendItems.slice(MAX_LEGEND_ITEMS); - - return ( -
- {shownItems.map((entry, index) => { - const isSelected = !!selectedSeries?.has(entry.value); - const isDisabled = hasSelection && !isSelected; - return ( - onToggleSeries?.(entry.value, isShiftKey)} - /> - ); - })} - {restItems.length ? ( - - -
- +{restItems.length} more -
- - -
- {restItems.map((entry, index) => { - const isSelected = !!selectedSeries?.has(entry.value); - const isDisabled = hasSelection && !isSelected; - return ( - - onToggleSeries?.(entry.value, isShiftKey) - } - /> - ); - })} -
- - - ) : null} -
- ); -}); + ANNOTATION_LABEL_HEADROOM, + NEAREST_SERIES_MAX_DISTANCE_PX, + SINGLE_POINT_BAR_RIGHT_PADDING, + SINGLE_POINT_BAR_WIDTH_RATIO, + Y_AXIS_WIDTH, +} from './constants'; +import { useChartScales } from './useChartScales'; +import { useExemplarMarkers } from './useExemplarMarkers'; -export const HARD_LINES_LIMIT = MAX_TIME_CHART_SERIES; - -// Debounce (ms) for the chart's ResponsiveContainer resize observer. Without // it the observer fires on every frame, and a resize → re-render → resize // cycle can keep the chart (and the form controls around it in the tile // editor) from ever settling. const RESPONSIVE_CONTAINER_DEBOUNCE_MS = 50; -/** One series entry in a tooltip's per-bucket payload (hover or click-frozen). */ -export type ActiveClickSeries = { - value?: number; - dataKey?: string; - name?: string; - /** Series color, matching the legend swatch. */ - color?: string; - /** Previous-period value at the same bucket, for the percent-change chip. */ - previousValue?: number; - /** Whether this series is a dashed previous-period line. */ - isPreviousPeriod?: boolean; - /** Result column the values came from, for per-column number formatting. */ - valueColumnName?: string; -}; - -/** - * State for the pinned (click-locked) tooltip. Produced by MemoChart's onClick - * and rendered by DBTimeChart via ChartSeriesTooltip. (Hover uses recharts' own - * ; recharts' is also kept for its synced cursor.) - */ -export type ActiveClickPayload = { - /** Active point in viewport coords; the Popover anchor. */ - viewportX: number; - viewportY: number; - activeLabel: string; - activePayload?: ActiveClickSeries[]; -}; - -/** Series label shown in the legend, tooltip, and line `name`. */ -const getSeriesDisplayName = (ld: LineData) => ld.displayName || ld.dataKey; - -/** Normalize a chart event's active label (number | string) to a string. */ -const getActiveLabel = (state?: { - activeLabel?: string | number; -}): string | undefined => - state?.activeLabel != null ? String(state.activeLabel) : undefined; - -/** - * Build the per-series payload for a click-frozen tooltip from the data row at - * the clicked bucket. Only the visible series (legend selection + - * HARD_LINES_LIMIT) with a numeric value at that bucket are included, so the - * drill-down popover mirrors exactly what is drawn. Exported for unit testing. - */ -export function buildActiveClickSeries( - visibleLineData: LineData[], - activeRow: Record | undefined, -): ActiveClickSeries[] { - if (activeRow == null) return []; - return visibleLineData.flatMap(ld => { - const value = activeRow[ld.dataKey]; - if (typeof value !== 'number') return []; - const isPreviousPeriod = ld.previousPeriodKey === ld.dataKey; - // Pair each current-period series with its previous-period value for the - // percent-change chip. Only current-period rows carry a comparison. - const previousRaw = - !isPreviousPeriod && ld.previousPeriodKey - ? activeRow[ld.previousPeriodKey] - : undefined; - return [ - { - dataKey: ld.dataKey, - name: getSeriesDisplayName(ld), - value, - color: ld.color, - isPreviousPeriod, - valueColumnName: ld.valueColumnName, - previousValue: - typeof previousRaw === 'number' ? previousRaw : undefined, - }, - ]; - }); -} - -/** - * The series actually drawn on the chart. Without a selection, the first - * HARD_LINES_LIMIT of lineData. With a selection (legend isolate, checkbox - * filter, or table search), the selection is applied FIRST and then capped, so - * an explicitly chosen series always draws even if it ranks beyond the limit. - * Applying the cap first would slice out a chosen low-ranked series, leaving an - * empty chart while its stats still show in the legend table. The rendered - * lines and the drill-down click payload both derive from this same set so they - * never diverge. Exported for unit testing. - */ -export function getVisibleLineData( - lineData: LineData[], - selectedSeriesNames: Set | undefined, -): LineData[] { - const hasSelection = !!selectedSeriesNames && selectedSeriesNames.size > 0; - if (hasSelection) { - return lineData - .filter(ld => selectedSeriesNames.has(getSeriesDisplayName(ld))) - .slice(0, HARD_LINES_LIMIT); - } - return lineData.slice(0, HARD_LINES_LIMIT); -} - -const StackedBarWithOverlap = (props: BarProps) => { - const { x, y, width, fill } = props; - // `height` may arrive as a string, so coerce it to a number before the - // arithmetic below. - const height = - typeof props.height === 'number' ? props.height : Number(props.height ?? 0); - // Add a tiny bit to the height to create overlap. Otherwise there's a gap - return ( - 0 ? height + 0.5 : 0} - fill={fill} - /> - ); -}; - -type CaptureActiveDotProps = { - /** - * Called with each series' active-point pixel Y. This is a stable callback - * (not the ref itself) so Recharts, which stores this element's props in its - * Immer-backed store and freezes them, never freezes the underlying Map — - * the write happens on the ref captured in the callback's closure instead. - */ - onCapture: (dataKey: string, cy: number) => void; - cx?: number; - cy?: number; - dataKey?: string | number; - r?: number; - fill?: string; - stroke?: string; - strokeWidth?: number; -}; - -/** - * Active dot for an Area series. Records the active point's pixel Y (`cy`) - * via `onCapture`, keyed by dataKey, then draws the same dot Recharts - * renders by default. Recharts clones this element with the active-point - * props (cx, cy, dataKey, r, fill, stroke, strokeWidth) during the render - * that precedes the tooltip, so the capture is current when the tooltip reads - * it to find the series nearest the cursor. - */ -function CaptureActiveDot({ - onCapture, - cx, - cy, - dataKey, - r, - fill, - stroke, - strokeWidth, -}: CaptureActiveDotProps) { - if (dataKey != null && typeof cy === 'number' && Number.isFinite(cy)) { - // Written synchronously during render so the tooltip, which Recharts - // renders after the graphical items in the same commit, reads the - // current frame's positions rather than the previous frame's. - onCapture(String(dataKey), cy); - } - if (typeof cx !== 'number' || typeof cy !== 'number') { - return null; - } - return ( - - ); -} - /** * Compute the unique set of hexes referenced by `` defs * inside MemoChart. Exported so a unit test can pin the dedup-and-union @@ -650,6 +109,14 @@ export const MemoChart = memo(function MemoChart({ granularity, dateRangeEndInclusive = true, fitYAxisToData = false, + exemplars, + maxExemplars = 12, + onExemplarHover, + onExemplarHoverEnd, + onExemplarSelect, + pinnedExemplarKey = null, + onExemplarPinEnd, + onExemplarsDropped, }: { graphResults: any[]; setIsClickActive: (v: ActiveClickPayload | undefined) => void; @@ -683,9 +150,29 @@ export const MemoChart = memo(function MemoChart({ * (with padding) instead of zero. **/ fitYAxisToData?: boolean; + /** Exemplar markers to overlay on the chart (linked to traces). */ + exemplars?: Exemplar[]; + /** Target number of exemplar markers to show (0 = unlimited). */ + maxExemplars?: number; + /** Invoked when the cursor enters an exemplar marker, with its pixel coords. */ + onExemplarHover?: (exemplar: Exemplar, cx: number, cy: number) => void; + /** Invoked when the cursor leaves an exemplar marker. */ + onExemplarHoverEnd?: () => void; + /** Invoked when an exemplar marker is clicked, with its pixel coords. */ + onExemplarSelect?: (exemplar: Exemplar, cx: number, cy: number) => void; + /** + * Key of the exemplar whose card is pinned open, or null. A key rather than a + * boolean so the chart can tell when that marker stops being rendered — see + * the reset effect below. A pin also suppresses the series tooltip. + */ + pinnedExemplarKey?: string | null; + /** Invoked when the pinned marker is no longer in the rendered set. */ + onExemplarPinEnd?: () => void; + /** How many markers the render-layer clamps dropped; see useExemplarMarkers. */ + onExemplarsDropped?: (count: number) => void; }) { - const _id = useId(); - const id = _id.replace(/:/g, ''); + const rawId = useId(); + const id = rawId.replace(/:/g, ''); // recharts sync group, scoped via context (see chartSync). const syncId = useChartSyncId(); @@ -786,65 +273,22 @@ export const MemoChart = memo(function MemoChart({ captureActivePointY, ]); - const yAxisDomain: AxisDomain = useMemo(() => { - const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0; - - // Fitting the y-axis lower bound to the data only applies to line charts. - // Bar charts are always anchored at zero so the bar lengths stay - // proportional to their values. - const shouldFitYAxis = - fitYAxisToData && displayType !== DisplayType.StackedBar; - - // The data min/max is only needed to either zoom into a selection or to - // fit the lower bound to the data. When neither applies, let Recharts - // auto-calculate the upper bound while pinning the lower bound to zero. - if (!hasSelection && !shouldFitYAxis) { - return [0, 'auto']; - } - - // Calculate domain based on visible series (all series when there's no - // explicit selection). - let minValue = Infinity; - let maxValue = -Infinity; - - graphResults.forEach(dataPoint => { - lineData.forEach(ld => { - const seriesName = ld.displayName || ld.dataKey; - // Only consider visible series - if (!hasSelection || selectedSeriesNames.has(seriesName)) { - const value = dataPoint[ld.dataKey]; - if (typeof value === 'number' && !isNaN(value)) { - minValue = Math.min(minValue, value); - maxValue = Math.max(maxValue, value); - } - } - }); + // Axis domains, the exemplar clamp range, and annotation elements — see + // useChartScales. + const { yAxisDomain, exemplarYBounds, xAxisDomain, annotationElements } = + useChartScales({ + annotations, + dateRange, + granularity, + dateRangeEndInclusive, + displayType, + fitYAxisToData, + graphResults, + lineData, + selectedSeriesNames, + hasExemplars: !!exemplars?.length, }); - // If we found valid values, return them with some padding - if (minValue !== Infinity && maxValue !== -Infinity) { - const padding = (maxValue - minValue) * 0.05; // 5% padding - // When fitting to data, allow the lower bound to follow the data - // minimum; otherwise keep it pinned at zero. The 5% padding must not - // drag the axis below zero unless the data itself is negative, so - // clamp at zero whenever the minimum is non-negative. - const lowerBound = - shouldFitYAxis && minValue < 0 - ? minValue - padding - : Math.max(0, minValue - padding); - const upperBound = maxValue + padding; - return [lowerBound, upperBound]; - } - - return ['auto', 'auto']; - }, [ - graphResults, - lineData, - selectedSeriesNames, - fitYAxisToData, - displayType, - ]); - const [containerWidth, setContainerWidth] = useState(0); // The chart's outer positioned container. Used to convert a pointer's @@ -999,40 +443,28 @@ export const MemoChart = memo(function MemoChart({ return map; }, [lineData]); - const xAxisDomain: AxisDomain = useMemo(() => { - let startTime = toStartOfInterval(dateRange[0], granularity); - let endTime = toStartOfInterval(dateRange[1], granularity); - const endTimeIsBoundaryAligned = isSameSecond(dateRange[1], endTime); - if (endTimeIsBoundaryAligned && !dateRangeEndInclusive) { - endTime = sub(endTime, { - seconds: convertGranularityToSeconds(granularity), - }); - } - - // For bar charts, extend the domain in both directions by half a granularity unit - // so that the full bar width is within the bounds of the chart - if (displayType === DisplayType.StackedBar) { - const halfGranularitySeconds = - convertGranularityToSeconds(granularity) / 2; - startTime = sub(startTime, { seconds: halfGranularitySeconds }); - endTime = add(endTime, { seconds: halfGranularitySeconds }); - } - - return [startTime.getTime() / 1000, endTime.getTime() / 1000]; - }, [dateRange, granularity, dateRangeEndInclusive, displayType]); - - // Alert/event markers as dashed lines, clamped to the chart's x-axis domain so - // an edge marker (e.g. an alert already firing at window open) stays visible - // instead of being dropped. Labels float in the reserved top headroom. - const annotationElements = useMemo(() => { - if (!annotations?.length) { - return null; - } - // xAxisDomain is a [min, max] tuple at runtime (declared as AxisDomain). - return getAnnotationElements(annotations, { - domain: xAxisDomain as [number, number], - }); - }, [annotations, xAxisDomain]); + // Exemplar marker layer — see useExemplarMarkers. + const { + exemplarPoints, + isExemplarHovered, + handleExemplarHoverStart, + handleExemplarHoverEnd, + handleExemplarSelect, + } = useExemplarMarkers({ + exemplars, + maxExemplars, + granularity, + pinnedExemplarKey, + xAxisDomain, + exemplarYBounds, + onExemplarHover, + onExemplarHoverEnd, + onExemplarSelect, + onExemplarPinEnd, + onExemplarsDropped, + suppressNextClickRef, + brushOriginRef: mouseDownPosRef, + }); return ( + ); +} diff --git a/packages/app/src/components/Exemplars/__tests__/ExemplarDot.test.tsx b/packages/app/src/components/Exemplars/__tests__/ExemplarDot.test.tsx new file mode 100644 index 0000000000..bcb328a4dc --- /dev/null +++ b/packages/app/src/components/Exemplars/__tests__/ExemplarDot.test.tsx @@ -0,0 +1,66 @@ +import React from 'react'; +import { Exemplar } from '@hyperdx/common-utils/dist/types'; +import { fireEvent, render } from '@testing-library/react'; + +import { ExemplarDot } from '@/components/Exemplars/ExemplarDot'; + +const exemplar: Exemplar = { timestamp: 1000, value: 42, traceId: 'abc' }; + +/** + * The dot lives inside the chart's SVG, and the chart puts an onClick on its + * wrapper to pin a drill-down tooltip. This stands in for that wrapper. + */ +function renderDot(props: Partial>) { + const onParentClick = jest.fn(); + const { container } = render( + + + , + ); + // The transparent hit circle is the marker's click target. + const hitTarget = container.querySelector('circle')!; + return { onParentClick, hitTarget }; +} + +describe('ExemplarDot', () => { + it('reports a click and keeps it from reaching the chart', () => { + const onSelect = jest.fn(); + const { onParentClick, hitTarget } = renderDot({ onSelect }); + + fireEvent.click(hitTarget); + + // Without the stopPropagation this fix adds, the chart's own onClick also + // fires and opens the drill-down tooltip over the exemplar's menu. + expect(onSelect).toHaveBeenCalledWith(exemplar, 10, 20); + expect(onParentClick).not.toHaveBeenCalled(); + }); + + it('swallows the click even with no handler attached', () => { + const { onParentClick, hitTarget } = renderDot({}); + + fireEvent.click(hitTarget); + + expect(onParentClick).not.toHaveBeenCalled(); + }); + + it('reports hover enter/leave with the marker position', () => { + const onHoverStart = jest.fn(); + const onHoverEnd = jest.fn(); + const { hitTarget } = renderDot({ onHoverStart, onHoverEnd }); + + fireEvent.mouseEnter(hitTarget.parentElement!); + fireEvent.mouseLeave(hitTarget.parentElement!); + + expect(onHoverStart).toHaveBeenCalledWith(exemplar, 10, 20); + expect(onHoverEnd).toHaveBeenCalled(); + }); + + it('renders nothing until recharts supplies coordinates', () => { + const { container } = render( + + + , + ); + expect(container.querySelector('path')).toBeNull(); + }); +}); diff --git a/packages/app/src/components/Exemplars/__tests__/ExemplarHoverCard.test.tsx b/packages/app/src/components/Exemplars/__tests__/ExemplarHoverCard.test.tsx new file mode 100644 index 0000000000..2e81e22fb1 --- /dev/null +++ b/packages/app/src/components/Exemplars/__tests__/ExemplarHoverCard.test.tsx @@ -0,0 +1,89 @@ +import { Exemplar } from '@hyperdx/common-utils/dist/types'; + +import { ExemplarHoverCard } from '@/components/Exemplars/ExemplarHoverCard'; + +const exemplar: Exemplar = { + timestamp: 1704067200000, // 2024-01-01T00:00:00Z + value: 1234.5, + traceId: 'abc123def456789012345678', +}; + +const props = { + hovered: { exemplar, x: 10, y: 20 }, + isLoading: false, + traceSourceConfigured: false, + onInspect: jest.fn(), + onMouseEnter: jest.fn(), + onMouseLeave: jest.fn(), +}; + +/** + * The marker's drawn position is deliberately not the truth: clampExemplarY pins + * it into the y-domain and clampExemplarX into the x-domain. Both clamps are + * justified in code by the card reporting the real value and time, so these are + * the assertions that keep that justification honest. + */ +describe('ExemplarHoverCard', () => { + it('always shows the exemplar value and time', () => { + const { getByText } = renderWithMantine(); + expect(getByText(/Value:/).textContent).toContain('1234.5'); + // Rendered through the user's time preference (local vs UTC, 12h vs 24h), so + // assert a clock is present rather than pinning a timezone-dependent string. + expect(getByText(/Time:/).textContent).toMatch(/\d{1,2}:\d{2}/); + }); + + it('shows them even with no trace source configured', () => { + // The value/time rows must not live inside the trace-source branch — a chart + // with no exemplar trace source still needs to explain its markers. + const { getByText } = renderWithMantine( + , + ); + expect(getByText(/Set an exemplar trace source/)).toBeInTheDocument(); + expect(getByText(/Value:/).textContent).toContain('1234.5'); + }); + + it('shows them while trace metadata is still loading', () => { + const { getByText } = renderWithMantine( + , + ); + expect(getByText(/Loading trace/)).toBeInTheDocument(); + expect(getByText(/Value:/).textContent).toContain('1234.5'); + }); + + it('shows them when the trace is not found in the source', () => { + const { getByText } = renderWithMantine( + , + ); + expect(getByText(/Trace not found/)).toBeInTheDocument(); + expect(getByText(/Value:/).textContent).toContain('1234.5'); + }); + + it('formats the value with the chart number format', () => { + // Otherwise a milliseconds axis and a "1234.5" card disagree about units. + const { getByText } = renderWithMantine( + , + ); + expect(getByText(/Value:/).textContent).toContain('ms'); + }); + + it('distinguishes a failed trace lookup from a missing trace', () => { + // Both used to read "Trace not found in source", blaming the data for a + // misconfigured source kind or a query error. + const { getByText, queryByText } = renderWithMantine( + , + ); + expect(getByText(/could not be loaded/)).toBeInTheDocument(); + expect(queryByText(/Trace not found/)).toBeNull(); + expect(getByText(/Value:/).textContent).toContain('1234.5'); + }); + + it('renders nothing when no marker is hovered', () => { + const { queryByText } = renderWithMantine( + , + ); + expect(queryByText(/Value:/)).toBeNull(); + }); +}); diff --git a/packages/app/src/components/Exemplars/__tests__/exemplarPoints.test.ts b/packages/app/src/components/Exemplars/__tests__/exemplarPoints.test.ts new file mode 100644 index 0000000000..50f9468a6b --- /dev/null +++ b/packages/app/src/components/Exemplars/__tests__/exemplarPoints.test.ts @@ -0,0 +1,385 @@ +import { Exemplar } from '@hyperdx/common-utils/dist/types'; + +import { + clampExemplarX, + clampExemplarY, + computeExemplarPoints, + computeExemplarYBounds, +} from '@/components/Exemplars/exemplarPoints'; + +const ex = ( + over: Partial & { timestamp: number; value: number }, +): Exemplar => ({ + traceId: `t-${over.timestamp}-${over.value}`, + ...over, +}); + +const MINUTE = 60_000; + +describe('computeExemplarPoints', () => { + const opts = { maxExemplars: 12, granularity: '1 minute' }; + + it('returns [] for empty/undefined', () => { + expect(computeExemplarPoints(undefined, opts)).toEqual([]); + expect(computeExemplarPoints([], opts)).toEqual([]); + }); + + it('maps timestamp (ms) to seconds on the x-axis and value to y', () => { + const [p] = computeExemplarPoints( + [ex({ timestamp: 1_700_000_000_000, value: 42 })], + opts, + ); + expect(p.x).toBe(1_700_000_000); // ms -> s + expect(p.y).toBe(42); + }); + + it('unlimited (maxExemplars <= 0): keeps all, deduped by trace id + timestamp', () => { + const points = computeExemplarPoints( + [ + ex({ traceId: 'a', timestamp: 1000, value: 1 }), + ex({ traceId: 'a', timestamp: 1000, value: 1 }), // dup + ex({ traceId: 'b', timestamp: 2000, value: 2 }), + ], + { ...opts, maxExemplars: 0 }, + ); + expect(points).toHaveLength(2); + }); + + it('skips exemplars with a non-finite value', () => { + const points = computeExemplarPoints( + [ex({ timestamp: 1000, value: NaN }), ex({ timestamp: 2000, value: 3 })], + { ...opts, maxExemplars: 0 }, + ); + expect(points).toHaveLength(1); + expect(points[0].y).toBe(3); + }); + + it('skips exemplars with a non-finite timestamp', () => { + // timestamp feeds both the bucket key and the x coordinate, so a non-finite + // one collapses every affected exemplar into a single "@NaN" bucket and + // reaches recharts as a NaN x, rendering an invalid SVG path. + const points = computeExemplarPoints( + [ + ex({ timestamp: NaN, value: 5 }), + ex({ timestamp: Infinity, value: 6 }), + ex({ timestamp: 2000, value: 3 }), + ], + { ...opts, maxExemplars: 0 }, + ); + expect(points).toHaveLength(1); + expect(points[0].x).toBe(2); + expect(Number.isFinite(points[0].x)).toBe(true); + }); + + it('keeps the highest-value exemplar of a bucket', () => { + // Two exemplars in the same bucket, values too close to be separate + // samples -> only the max is plotted. + const points = computeExemplarPoints( + [ + ex({ traceId: 'low', timestamp: 1000, value: 8 }), + ex({ traceId: 'high', timestamp: 1000, value: 9 }), + ], + opts, + ); + expect(points).toHaveLength(1); + expect(points[0].y).toBe(9); + }); + + it('also keeps a value more than 2σ below the bucket max', () => { + // A bucket holding a cluster of typical traces plus one big outlier should + // surface both ends, so the overlay shows the spread and not just a max + // envelope. Values within the cluster stay collapsed to one marker. + const typical = Array.from({ length: 10 }, (_, i) => + ex({ traceId: `typical-${i}`, timestamp: 1000, value: 10 + i }), + ); + const points = computeExemplarPoints( + [ex({ traceId: 'slow', timestamp: 1000, value: 1000 }), ...typical], + opts, + ); + expect(points.map(p => p.y)).toEqual([1000, 19]); + }); + + it('separates buckets by series (groupKey) so distinct series both survive', () => { + // Same time bucket, different groupKey -> both kept (one per series). + const points = computeExemplarPoints( + [ + ex({ traceId: 'a', timestamp: 1000, value: 5, groupKey: 'svc=a' }), + ex({ traceId: 'b', timestamp: 1000, value: 5, groupKey: 'svc=b' }), + ], + opts, + ); + expect(points).toHaveLength(2); + }); + + it('buckets at the chart granularity, not at a budget-derived width', () => { + // Adjacent minutes are distinct buckets even though the budget is far from + // exhausted, so a marker always lands in the bucket it explains. + const points = computeExemplarPoints( + [ + ex({ traceId: 'a', timestamp: 0, value: 1 }), + ex({ traceId: 'b', timestamp: MINUTE, value: 2 }), + ex({ traceId: 'c', timestamp: 2 * MINUTE, value: 3 }), + ], + opts, + ); + expect(points).toHaveLength(3); + }); + + it('spreads the marker budget across the range instead of over the spikes', () => { + // 20 buckets, budget of 4. The slowest traces are all bunched at the start; + // a value-ranked cut would return only those and leave the rest empty. + const exemplars = Array.from({ length: 20 }, (_, i) => + ex({ + traceId: `t${i}`, + timestamp: i * MINUTE, + value: i < 5 ? 1000 + i : 1, + }), + ); + const points = computeExemplarPoints(exemplars, { + ...opts, + maxExemplars: 4, + }); + expect(points.length).toBeLessThanOrEqual(4); + const times = points.map(p => p.x).sort((a, b) => a - b); + // Markers reach the far end of the range, not just the leading spike. + expect(times[times.length - 1]).toBeGreaterThanOrEqual( + (15 * MINUTE) / 1000, + ); + }); + + it('renders 2σ companions instead of spending the whole budget on rank 0', () => { + // The regression this guards: taking one window per budget slot fills the + // budget on every bucket's maximum, so the 2σ sampling never places a + // second marker and the overlay is the max envelope it exists to replace. + // 30 buckets, budget 12 — the ordinary case (more buckets than budget). + // A realistic latency shape: mostly fast requests with one slow outlier per + // bucket. (σ is measured across the whole set, so a 50/50 bimodal split + // would push 2σ above the in-bucket gap and legitimately keep one marker.) + const exemplars = Array.from({ length: 30 }, (_, i) => [ + ex({ traceId: `slow-${i}`, timestamp: i * MINUTE, value: 5000 }), + ...Array.from({ length: 10 }, (_, j) => + ex({ + traceId: `typical-${i}-${j}`, + timestamp: i * MINUTE + j + 1, + value: 10 + j, + }), + ), + ]).flat(); + + const points = computeExemplarPoints(exemplars, opts); + + // At least one bucket contributed both its peak and its typical trace. + const byBucket = new Map(); + for (const p of points) { + const bucket = Math.floor(p.exemplar.timestamp / MINUTE); + byBucket.set(bucket, (byBucket.get(bucket) ?? 0) + 1); + } + expect(Math.max(...byBucket.values())).toBeGreaterThan(1); + // A typical trace made it onto the chart, not just the bucket maxima. + expect(points.some(p => p.y < 100)).toBe(true); + expect(points.length).toBeLessThanOrEqual(12); + }); + + it('picks the peak of each window, not a fixed stride', () => { + // Spikes sit at minutes 3 and 8. A stride of 5 would sample minutes 0 and 5 + // and miss both — on a latency chart those are the only markers worth + // having. + const exemplars = Array.from({ length: 10 }, (_, i) => + ex({ + traceId: `t${i}`, + timestamp: i * MINUTE, + value: i === 3 || i === 8 ? 5000 : 10, + }), + ); + const points = computeExemplarPoints(exemplars, { + ...opts, + maxExemplars: 2, + }); + expect(points.map(p => p.x)).toEqual([ + (3 * MINUTE) / 1000, + (8 * MINUTE) / 1000, + ]); + }); + + it('never returns more than the marker budget', () => { + const exemplars = Array.from({ length: 50 }, (_, i) => + ex({ traceId: `t${i}`, timestamp: i * MINUTE, value: i * 100 }), + ); + expect( + computeExemplarPoints(exemplars, { ...opts, maxExemplars: 7 }).length, + ).toBeLessThanOrEqual(7); + }); + + it('treats a fractional or non-finite budget as a usable one', () => { + // maxExemplars comes from a team setting; a value below 1 used to produce + // zero windows and silently empty the overlay. + const exemplars = Array.from({ length: 5 }, (_, i) => + ex({ traceId: `t${i}`, timestamp: i * MINUTE, value: i + 1 }), + ); + expect( + computeExemplarPoints(exemplars, { ...opts, maxExemplars: 0.5 }).length, + ).toBeGreaterThan(0); + expect( + computeExemplarPoints(exemplars, { ...opts, maxExemplars: NaN }).length, + ).toBeGreaterThan(0); + }); + + it('drops non-finite values so one Infinity cannot disable 2σ sampling', () => { + // Infinity used to pass the filter, make the standard deviation NaN, and + // switch the spread rule off for the entire chart. + const points = computeExemplarPoints( + [ + ex({ traceId: 'inf', timestamp: 1000, value: Infinity }), + ex({ traceId: 'ok', timestamp: 1000, value: 5 }), + ], + opts, + ); + expect(points.map(p => p.y)).toEqual([5]); + }); +}); + +describe('computeExemplarPoints window split threshold', () => { + // The split reserved a quarter of the budget for 2-sigma companions, but it also + // GATED on that reduced number — so 12 populated buckets at the default budget of + // 12 rendered 9 markers and left three buckets bare. + it('emits one marker per bucket when the buckets fit the budget', () => { + const exemplars: Exemplar[] = Array.from({ length: 12 }, (_, i) => ({ + timestamp: i * 60_000, + value: 100 + i, + traceId: `t${i}`, + })); + const points = computeExemplarPoints(exemplars, { + maxExemplars: 12, + granularity: '1 minute', + }); + expect(points).toHaveLength(12); + }); + + it('still splits when the buckets outnumber the budget', () => { + const exemplars: Exemplar[] = Array.from({ length: 40 }, (_, i) => ({ + timestamp: i * 60_000, + value: 100 + i, + traceId: `t${i}`, + })); + const points = computeExemplarPoints(exemplars, { + maxExemplars: 12, + granularity: '1 minute', + }); + expect(points.length).toBeLessThanOrEqual(12); + expect(points.length).toBeGreaterThan(1); + }); +}); + +describe('computeExemplarYBounds', () => { + it('uses both numeric domain bounds when the axis is fitted to data', () => { + expect(computeExemplarYBounds([120, 480], 400)).toEqual({ + min: 120, + max: 480, + }); + }); + + it("falls back to the visible series max for an 'auto' upper bound", () => { + expect(computeExemplarYBounds([0, 'auto'], 400)).toEqual({ + min: 0, + max: 400, + }); + }); + + it("floors at 0 for an 'auto' lower bound", () => { + expect(computeExemplarYBounds(['auto', 'auto'], 400)).toEqual({ + min: 0, + max: 400, + }); + }); + + it('tolerates a non-array domain (recharts allows a function)', () => { + expect(computeExemplarYBounds(() => [0, 1], 400)).toEqual({ + min: 0, + max: 400, + }); + }); +}); + +describe('clampExemplarY', () => { + it('pins an outlier to the top of the domain instead of overflowing it', () => { + // Without this, recharts' default ifOverflow="discard" drops the marker and + // the overlay silently loses the slowest trace. + expect(clampExemplarY(9000, { min: 0, max: 400 })).toBe(400); + }); + + it('drops a marker below a fitted axis floor rather than raising it', () => { + // fitYAxisToData puts the floor at the data's own minimum, which can sit well + // above a fast request. Raising that marker to the floor would draw it level + // with the slowest points — the opposite of the truth — so it is not drawn. + expect(clampExemplarY(12, { min: 120, max: 480 })).toBeNull(); + }); + + it('keeps a marker exactly on the floor', () => { + expect(clampExemplarY(120, { min: 120, max: 480 })).toBe(120); + }); + + it('leaves an in-domain value untouched', () => { + expect(clampExemplarY(200, { min: 120, max: 480 })).toBe(200); + }); + + it('leaves the value alone when there is no numeric series data yet', () => { + // visibleSeriesMax is -Infinity before any data arrives. + expect(clampExemplarY(200, { min: 0, max: -Infinity })).toBe(200); + }); + + it('leaves the value alone for inverted bounds', () => { + expect(clampExemplarY(200, { min: 480, max: 120 })).toBe(200); + }); +}); + +describe('clampExemplarX', () => { + const domain: [number, number] = [1_700_000_000, 1_700_000_060]; + const bucket = 60; // one granularity, in chart time units (seconds) + + // The domain's upper bound is the last *bucket start* when + // dateRangeEndInclusive is false, so an exemplar inside that final bucket sits + // past it and recharts' ifOverflow="discard" drops it — losing the newest + // window, which is the one a live investigation is watching. + it('nudges a marker inside the final partial bucket onto the last bucket', () => { + expect(clampExemplarX(1_700_000_090, domain, bucket)).toBe(1_700_000_060); + }); + + it('nudges a marker just before the domain start onto the first bucket', () => { + expect(clampExemplarX(1_699_999_970, domain, bucket)).toBe(1_700_000_000); + }); + + it('leaves an in-domain marker exactly where it is', () => { + expect(clampExemplarX(1_700_000_030, domain, bucket)).toBe(1_700_000_030); + }); + + // The regression this guards: placeholderData deliberately keeps the previous + // range's exemplars across a range change, so a zoom hands this markers from a + // wider window. Dragging those to the axis edge would draw real, clickable + // traces on buckets they never occurred in. + it('drops a marker more than one bucket past the domain end', () => { + expect(clampExemplarX(1_700_000_600, domain, bucket)).toBeNull(); + }); + + it('drops a marker more than one bucket before the domain start', () => { + expect(clampExemplarX(1_699_999_000, domain, bucket)).toBeNull(); + }); + + it('keeps the boundary case exactly one bucket out', () => { + expect(clampExemplarX(1_700_000_120, domain, bucket)).toBe(1_700_000_060); + expect(clampExemplarX(1_699_999_940, domain, bucket)).toBe(1_700_000_000); + }); + + it('treats a non-finite bucket width as zero tolerance', () => { + // An unparseable granularity must not widen the window silently. + expect(clampExemplarX(1_700_000_061, domain, NaN)).toBeNull(); + expect(clampExemplarX(1_700_000_030, domain, NaN)).toBe(1_700_000_030); + }); + + it('leaves x untouched for a degenerate or inverted domain', () => { + // Better an occasionally-discarded marker than one pinned to a meaningless + // position — same rule as clampExemplarY. + expect(clampExemplarX(42, [NaN, 100], bucket)).toBe(42); + expect(clampExemplarX(42, [100, NaN], bucket)).toBe(42); + expect(clampExemplarX(42, [100, 0], bucket)).toBe(42); + }); +}); diff --git a/packages/app/src/components/Exemplars/__tests__/promqlSeriesLabels.test.ts b/packages/app/src/components/Exemplars/__tests__/promqlSeriesLabels.test.ts new file mode 100644 index 0000000000..1e27709e41 --- /dev/null +++ b/packages/app/src/components/Exemplars/__tests__/promqlSeriesLabels.test.ts @@ -0,0 +1,167 @@ +import { + labelDistinguishesSeries, + promqlSeriesLabelRule, +} from '@/components/Exemplars/promqlSeriesLabels'; + +/** + * The rule decides whether two raw Prometheus series are the same *plotted* line. + * Both failure directions are silent, so both are pinned here: + * + * - too strict (`all` when the query really did aggregate) empties the overlay; + * - too loose (dropping a label that really does distinguish lines) merges + * distinct series and renders markers against a line they don't belong to. + * + * The second is worse, so every ambiguous shape must resolve to `all`. + */ +describe('promqlSeriesLabelRule', () => { + const distinguishing = (expression: string, labels: string[]) => { + const rule = promqlSeriesLabelRule(expression); + return labels.filter(l => labelDistinguishesSeries(rule, l)); + }; + + it('keeps only the `by` labels', () => { + // The canonical latency query: one plotted line, so `instance` must stop + // counting or the multi-target fan-out empties the overlay. + expect( + distinguishing( + 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le))', + ['le', 'instance', 'pod'], + ), + ).toEqual(['le']); + }); + + it('reads the prefix form of `by` as well as the suffix form', () => { + expect( + distinguishing( + 'histogram_quantile(0.95, sum by (le) (rate(http_latency_bucket[5m])))', + ['le', 'instance'], + ), + ).toEqual(['le']); + }); + + it('excludes only the `without` labels when there is no `by`', () => { + // The regression this covers: a `without`-only expression previously fell + // through to "every label counts", so the without-spelling of the query above + // still dropped the whole overlay. + expect( + distinguishing( + 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) without (instance))', + ['le', 'instance', 'pod'], + ), + ).toEqual(['le', 'pod']); + }); + + it('treats a bare selector as fully distinguishing', () => { + // No aggregation: this really does draw one line per instance. + expect( + distinguishing( + 'histogram_quantile(0.95, rate(http_latency_bucket[5m]))', + ['le', 'instance'], + ), + ).toEqual(['le', 'instance']); + }); + + describe('falls back to fully-distinguishing on anything ambiguous', () => { + // Each of these could otherwise shrink the identity and merge real series. + it.each([ + [ + 'both by and without present', + 'sum(rate(a[5m])) by (le) + sum(rate(b[5m])) without (instance)', + ], + [ + 'differing by sets (nested aggregation)', + 'max by (service) (histogram_quantile(0.95, sum by (le, service, pod) (rate(x[5m]))))', + ], + [ + 'top-level arithmetic', + 'histogram_quantile(0.95, sum(rate(a[5m])) by (le)) * 1000', + ], + [ + 'top-level comparison', + 'histogram_quantile(0.95, sum(rate(a[5m])) by (le)) > 0.5', + ], + ['two without clauses', 'sum(sum(rate(a[5m])) without (x)) without (y)'], + ])('%s', (_name, expression) => { + expect(promqlSeriesLabelRule(expression)).toEqual({ mode: 'all' }); + }); + }); + + it('ignores clause-like text inside string literals', () => { + // A label *value* containing `by (` must not be read as syntax. + expect(promqlSeriesLabelRule('rate(x{job="sum by (le)"}[5m])')).toEqual({ + mode: 'all', + }); + expect( + distinguishing('sum(rate(x{path="/a+b"}[5m])) by (le)', [ + 'le', + 'instance', + ]), + ).toEqual(['le']); + }); + + it('does not read a rate interval or label list as division', () => { + // `[5m]` and the `/` in a path matcher previously risked tripping the + // binary-operator bail-out, which would silently disable the whole fix. + expect( + distinguishing('sum(rate(http_latency_bucket[5m])) by (le)', [ + 'le', + 'instance', + ]), + ).toEqual(['le']); + }); + + // A label matcher is not a comparison. Leaving `!=` in the operator test made a + // routine matcher suppress the overlay on a query that does aggregate to one + // line — reported as a P2 fail-closed. + it('does not read a label matcher as a top-level operator', () => { + expect( + distinguishing( + 'histogram_quantile(0.95, sum(rate(http_latency_bucket{code!="200"}) by (le)))', + ['le', 'instance'], + ), + ).toEqual(['le']); + expect( + distinguishing('sum(rate(x{path=~"/api/.*",code!="200"}[5m])) by (le)', [ + 'le', + 'instance', + ]), + ).toEqual(['le']); + }); + + // The opposite direction: a clause entry that is not a bare label name means we + // misread the expression, and building a key from it would match no real label + // and collapse every series into one group — the fail-open the module forbids. + it('bails out when a clause entry is not a bare label name', () => { + expect( + promqlSeriesLabelRule( + 'histogram_quantile(0.95, sum(rate(x_bucket[5m])) by ("le"))', + ), + ).toEqual({ mode: 'all' }); + expect(promqlSeriesLabelRule('sum(rate(x[5m])) by (`le`)')).toEqual({ + mode: 'all', + }); + }); + + it('does not treat a `#` inside a label value as a comment', () => { + // Regression: stripping comments before strings truncated the expression at a + // URL fragment and ate the `by (le)` clause, suppressing the overlay. + expect( + distinguishing( + 'histogram_quantile(0.95, sum(rate(x_bucket{path="/v1#frag"}[5m])) by (le))', + ['le', 'instance'], + ), + ).toEqual(['le']); + }); + + it('ignores a comment that would otherwise hide a clause', () => { + // `#` starts a PromQL comment; a `by (...)` inside one is not real syntax. + expect(promqlSeriesLabelRule('sum(rate(x[5m])) # by (le)')).toEqual({ + mode: 'all', + }); + }); + + it('is fully distinguishing for an empty or missing expression', () => { + expect(promqlSeriesLabelRule(undefined)).toEqual({ mode: 'all' }); + expect(promqlSeriesLabelRule('')).toEqual({ mode: 'all' }); + }); +}); diff --git a/packages/app/src/components/Exemplars/exemplarPoints.ts b/packages/app/src/components/Exemplars/exemplarPoints.ts new file mode 100644 index 0000000000..2797fc6a3f --- /dev/null +++ b/packages/app/src/components/Exemplars/exemplarPoints.ts @@ -0,0 +1,272 @@ +import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils'; +import { Exemplar } from '@hyperdx/common-utils/dist/types'; + +/** An exemplar plus the on-screen position of its marker, for the hover card. */ +export type PositionedExemplar = { exemplar: Exemplar; x: number; y: number }; + +/** A single exemplar plotted on the chart: x in chart time units, y = value. */ +type ExemplarPoint = { + x: number; + y: number; + exemplar: Exemplar; + key: string; +}; + +function finiteOrNull(v: unknown): number | null { + // Finite, not merely non-NaN: a single Infinity would otherwise reach + // standardDeviation, make the spread NaN, and silently switch the 2σ rule off + // for the whole chart (`spread > 0` is false for NaN). + return typeof v === 'number' && Number.isFinite(v) ? v : null; +} + +/** Sample standard deviation, 0 when there isn't enough data to have one. */ +function standardDeviation(values: number[]): number { + if (values.length < 2) return 0; + const mean = values.reduce((a, b) => a + b, 0) / values.length; + const variance = + values.reduce((acc, v) => acc + (v - mean) ** 2, 0) / (values.length - 1); + return Math.sqrt(variance); +} + +/** + * Turn raw exemplars into plotted points, thinned to keep the chart legible. + * + * - `maxExemplars <= 0`: no thinning — every exemplar is a point (deduped by + * trace id + timestamp). + * - `maxExemplars > 0`: bucket at the chart granularity, per series + * (`groupKey`), then within each bucket keep the highest value plus any + * further value that sits more than 2σ below the last one kept (σ measured + * across the whole set). A busy bucket therefore contributes both a typical + * trace and its outlier, while a quiet one contributes a single marker — + * the overlay reads as the latency distribution rather than as a max + * envelope. If that leaves more buckets than the marker budget, the range is + * split into evenly spaced windows and the highest bucket in each survives — + * markers stay spread across the range *and* land on its peaks, where a + * globally value-ranked cut would bunch them all on one spike. + * + * Bucketing at the chart's own granularity (rather than a width derived from + * the marker budget) keeps a marker inside the bucket of the series point it + * explains. Same approach as Grafana's StandardDeviationSampler. + * + * Pure and side-effect free so the thinning behaviour can be unit-tested without + * a recharts render. + */ +export function computeExemplarPoints( + exemplars: Exemplar[] | undefined, + opts: { + maxExemplars: number; + granularity: string; + }, +): ExemplarPoint[] { + if (!exemplars?.length) return []; + const { granularity } = opts; + // The budget arrives from a team setting. Floor it to a whole number of + // markers, and treat a non-finite value as unlimited rather than as zero: a + // fractional value below 1 (or NaN) would otherwise make the window split + // produce no windows and empty the overlay even though exemplars exist. + const maxExemplars = !Number.isFinite(opts.maxExemplars) + ? 0 + : opts.maxExemplars <= 0 + ? 0 + : Math.max(1, Math.floor(opts.maxExemplars)); + + const toPoint = (exemplar: Exemplar, value: number): ExemplarPoint => ({ + x: exemplar.timestamp / 1000, // ms -> seconds (chart x unit) + y: value, + exemplar, + key: `exemplar-${exemplar.traceId}-${exemplar.timestamp}`, + }); + + const points: ExemplarPoint[] = []; + for (const exemplar of exemplars) { + const value = finiteOrNull(exemplar.value); + // `timestamp` needs the same guard as `value`: it feeds both the bucket key + // and the x coordinate, so a non-finite one collapses every affected exemplar + // into a single `@NaN` bucket and reaches recharts as a NaN x. The normalizers + // parse through ExemplarSchema (`.finite()`) so this shouldn't fire in the + // app, but this function is the pure, independently-tested boundary. + if (value != null && finiteOrNull(exemplar.timestamp) != null) { + points.push(toPoint(exemplar, value)); + } + } + if (!points.length) return []; + + if (maxExemplars <= 0) { + const all = new Map(); + for (const p of points) all.set(p.key, p); // dedupe identical trace+time + return Array.from(all.values()); + } + + const bucketMs = convertGranularityToSeconds(granularity) * 1000 || 1; + const buckets = new Map< + string, + { bucket: number; max: number; points: ExemplarPoint[] } + >(); + for (const p of points) { + const bucket = Math.floor(p.exemplar.timestamp / bucketMs); + const key = `${p.exemplar.groupKey ?? ''}@${bucket}`; + const existing = buckets.get(key); + if (existing) { + existing.points.push(p); + existing.max = Math.max(existing.max, p.y); + } else { + buckets.set(key, { bucket, max: p.y, points: [p] }); + } + } + + // More buckets than the budget allows: split them into evenly spaced windows + // and keep the most notable bucket in each. Coverage still spans the range, + // but the surviving markers land on the peaks — an even stride is blind to + // where the spikes are and skips straight past the marker you wanted. + // + // Take fewer windows than the budget so the leftover slots can hold the 2σ + // companions below. Filling one window per slot would exhaust the budget on + // rank 0 and the spread sampling would never render a second marker — the max + // envelope this function exists to avoid. + // + // Two different thresholds, which is the point. The split only happens when the + // buckets genuinely outnumber the budget; gating it on windowCount instead meant + // 12 populated buckets at the default budget of 12 rendered 9 markers and left + // three buckets bare, which is not "more buckets than the marker budget". + const windowCount = Math.max(1, Math.ceil(maxExemplars * 0.75)); + const ordered = Array.from(buckets.values()).sort( + (a, b) => a.bucket - b.bucket, + ); + const chosen = + ordered.length <= maxExemplars + ? ordered + : Array.from({ length: windowCount }, (_, i) => + ordered + .slice( + Math.floor((i * ordered.length) / windowCount), + Math.floor(((i + 1) * ordered.length) / windowCount), + ) + // >= so ties resolve to the *later* bucket: on a flat series every + // bucket max is equal, and preferring the earlier one would leave + // the right-hand edge of the chart bare. + .reduce((best, b) => (b.max >= best.max ? b : best)), + ); + + const spread = standardDeviation(points.map(p => p.y)) * 2; + const sampled = chosen.map(({ points: inBucket }) => { + const byValue = [...inBucket].sort((a, b) => b.y - a.y); + const kept = [byValue[0]]; + for (const p of byValue.slice(1)) { + if (spread > 0 && kept[kept.length - 1].y - p.y > spread) kept.push(p); + } + return kept; + }); + + // Round-robin by rank so every surviving bucket gets its most notable marker + // before any bucket gets a second one. + const out: ExemplarPoint[] = []; + for (let rank = 0; out.length < maxExemplars; rank++) { + let placed = false; + for (const kept of sampled) { + if (rank >= kept.length) continue; + out.push(kept[rank]); + placed = true; + if (out.length >= maxExemplars) break; + } + if (!placed) break; + } + return out; +} + +/** The y range an exemplar marker may be drawn in. */ +export type ExemplarYBounds = { min: number; max: number }; + +/** + * Derive the y range a marker may be drawn in, from the domain the chart actually + * renders. A recharts `ReferenceDot` defaults to `ifOverflow="discard"`, so a + * marker outside the domain vanishes with no explanation. + * + * The two bounds are used differently by clampExemplarY, which is where the + * reasoning lives: `max` is a ceiling markers are pinned to, `min` is a threshold + * below which they are dropped. `min` is NOT a lift target — raising a fast + * request to a fitted floor would draw it level with the slowest ones. + * + * Note `min` goes numeric on more than "Fit Y Axis to Data": useChartScales also + * fits the floor whenever a legend selection is active, so isolating one series + * can start dropping below-floor markers too. + * + * `'auto'` bounds are resolved by recharts against the series data, so the + * series max is in-domain by construction and 0 is a safe floor for the + * non-negative durations exemplars are scoped to today. + */ +export function computeExemplarYBounds( + yAxisDomain: unknown, + visibleSeriesMax: number, +): ExemplarYBounds { + const [lower, upper] = Array.isArray(yAxisDomain) + ? yAxisDomain + : [undefined, undefined]; + return { + min: typeof lower === 'number' ? lower : 0, + max: typeof upper === 'number' ? upper : visibleSeriesMax, + }; +} + +/** + * Place an exemplar's value inside `bounds` so recharts keeps drawing it, or + * return null to drop it. + * + * The two directions are not equivalent, so they are handled differently. + * + * Above `max`: pinned to the top. An outlier can be many times the plotted + * quantile, and letting it set the axis would flatten the series into a line at + * the bottom. A marker at the ceiling reads as "at least this high", and the + * hover card carries the real number. + * + * Below `min`: dropped. With `fitYAxisToData` the floor is the data's own + * minimum, so it can sit well above a fast request — and raising that marker to + * the floor would draw it level with the slowest points, saying the opposite of + * the truth. There is no honest position for it on this axis, so it is not drawn. + * + * Degenerate bounds (no numeric data yet, or inverted) leave the value untouched. + */ +export function clampExemplarY( + y: number, + bounds: ExemplarYBounds, +): number | null { + const { min, max } = bounds; + if (!Number.isFinite(max) || !Number.isFinite(min) || min > max) return y; + if (y < min) return null; + return Math.min(y, max); +} + +/** + * Place an exemplar's x (chart time units) inside the rendered x-domain, or + * return null to drop it. + * + * `ReferenceDot` defaults to `ifOverflow="discard"`, so a marker outside the + * domain silently vanishes. That loses the newest window: when + * `dateRangeEndInclusive` is false the domain's upper bound is the *last bucket + * start*, so an exemplar occurring inside that final bucket sits past the bound + * and disappears — the window a live investigation is watching. Nudging it onto + * the boundary of the bucket whose series point it explains fixes that. + * + * But only within one bucket. Anything further out is not an edge case, it is a + * marker that belongs to a different window: `placeholderData` deliberately keeps + * the previous range's exemplars across a range change, so a zoom hands this + * function markers from the old, wider window. Pulling those to the axis edge + * would draw real, clickable traces on buckets they did not occur in, with + * nothing on screen saying so. Dropping them is the honest answer — the overlay + * refills when the new range's query lands. + * + * `bucketSeconds` is the tolerance, in the chart's own time units. Degenerate or + * inverted domains leave x untouched. + */ +export function clampExemplarX( + x: number, + domain: [number, number], + bucketSeconds: number, +): number | null { + const [min, max] = domain; + if (!Number.isFinite(min) || !Number.isFinite(max) || min > max) return x; + const tolerance = Number.isFinite(bucketSeconds) + ? Math.max(0, bucketSeconds) + : 0; + if (x < min - tolerance || x > max + tolerance) return null; + return Math.min(Math.max(x, min), max); +} diff --git a/packages/app/src/components/Exemplars/index.ts b/packages/app/src/components/Exemplars/index.ts new file mode 100644 index 0000000000..a59f4c38f4 --- /dev/null +++ b/packages/app/src/components/Exemplars/index.ts @@ -0,0 +1,10 @@ +export { ExemplarDot } from './ExemplarDot'; +export { ExemplarHoverCard } from './ExemplarHoverCard'; +export { + clampExemplarX, + clampExemplarY, + computeExemplarPoints, + computeExemplarYBounds, + type ExemplarYBounds, + type PositionedExemplar, +} from './exemplarPoints'; diff --git a/packages/app/src/components/Exemplars/promqlSeriesLabels.ts b/packages/app/src/components/Exemplars/promqlSeriesLabels.ts new file mode 100644 index 0000000000..837a261fac --- /dev/null +++ b/packages/app/src/components/Exemplars/promqlSeriesLabels.ts @@ -0,0 +1,163 @@ +/** + * Working out which labels distinguish the *plotted* lines of a PromQL query. + * + * Prometheus resolves `/query_exemplars` against the raw selector, so it returns + * one entry per underlying series while the chart draws the aggregated result. + * The canonical latency query + * `histogram_quantile(0.95, sum(rate(x_bucket[5m])) by (le))` draws a single line + * but comes back split across every scrape target *and* every `le` bucket. An + * exemplar overlay that decides "is this one series?" from the raw label + * cardinality therefore sees N series and drops itself on any metric scraped from + * more than one pod — i.e. on essentially every real deployment. + * + * Extracted from useExemplars so this rule has its own tests: the failure it + * guards against is silent (an empty overlay, or worse, markers attributed to the + * wrong line), so its edge cases need to be pinned rather than eyeballed. + * + * This is a regex approximation, not a PromQL parse. That is a deliberate + * trade-off, and the direction of the approximation is what matters: when the + * expression is anything this cannot read confidently, it reports `all` — every + * label distinguishes a line — which OVER-counts series and so drops the overlay. + * Dropping a legitimate overlay is a visible absence; rendering markers against + * the wrong line is a silent lie. Always fail towards `all`. + */ + +/** + * How to decide whether two raw series belong to the same plotted line. + * + * - `all` — no usable aggregation info; every label counts (conservative). + * - `keep` — only `labels` count (from a `by (...)` clause). + * - `drop` — every label except `labels` counts (from a `without (...)` clause). + */ +export type SeriesLabelRule = + | { mode: 'all' } + | { mode: 'keep'; labels: Set } + | { mode: 'drop'; labels: Set }; + +const ALL: SeriesLabelRule = { mode: 'all' }; + +// `by (a, b)` / `by(a,b)` and the `without` equivalent. PromQL allows the clause +// either before or after the argument list (`sum by (le) (x)` and +// `sum(x) by (le)`), and both spellings match here since we only need the label +// list, not the position. +const BY_CLAUSE = /\bby\s*\(([^)]*)\)/g; +const WITHOUT_CLAUSE = /\bwithout\s*\(([^)]*)\)/g; + +// A top-level arithmetic or comparison operator means the plotted value is a +// derived quantity, and the aggregation clauses we can see may belong to either +// operand. Bail to `all` rather than guess which side governs the result. +const BINARY_OPERATOR = /[-+*/%]|==|!=|>=|<=|>|<|\b(and|or|unless)\b/; + +const parseLabelList = (clause: string) => + clause + .split(',') + .map(s => s.trim()) + .filter(Boolean); + +const sameLabels = (a: Set, b: Set) => + a.size === b.size && [...a].every(l => b.has(l)); + +/** A bare PromQL label name. Anything else in a `by`/`without` list is not one. */ +const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +/** + * Strip string literals and comments so their contents cannot be read as syntax — + * e.g. `{job="sum by (le)"}` or `{path="/a+b"}`. Replaced with `""` rather than + * removed so adjacent tokens do not run together. + * + * Backticks are PromQL raw strings and `#` starts a comment; both were missed + * before, which let either hide a clause or an operator from the checks below. + */ +function stripStringLiterals(expression: string): string { + // Strings first, comments second. A `#` is only a comment OUTSIDE a string, so + // stripping comments first truncates at a `#` inside a label value (a URL + // fragment, say) and eats the rest of the expression — including any `by (...)` + // clause, which silently suppresses the overlay. + return expression + .replace(/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`(?:[^`\\]|\\.)*`/g, '""') + .replace(/#[^\n]*/g, ''); +} + +/** + * Derive the series-identity rule for `expression`. + * + * Handles the two shapes that actually occur on latency charts: + * - one or more `by (...)` clauses -> `keep` their intersection (a label must + * survive every aggregation to still distinguish a line). + * - a `without (...)` clause and no `by` -> `drop` those labels. Without this, + * `sum without (instance) (...)` — the same query the `by (le)` form expresses + * — would fall through to `all` and empty the overlay, which is exactly the bug + * this module exists to fix. + * + * Anything ambiguous reports `all`: + * - both `by` and `without` present (they may govern different aggregations, and + * subtracting one from the other can shrink the identity to nothing and merge + * genuinely distinct lines — a fail-open we refuse). + * - several `by` clauses with differing label sets, or several `without` clauses. + * - a top-level binary operator. + */ +export function promqlSeriesLabelRule( + expression: string | undefined, +): SeriesLabelRule { + if (!expression) return ALL; + const src = stripStringLiterals(expression); + + const byClauses = [...src.matchAll(BY_CLAUSE)].map(m => parseLabelList(m[1])); + const withoutClauses = [...src.matchAll(WITHOUT_CLAUSE)].map(m => + parseLabelList(m[1]), + ); + + // Mixed forms are not reconcilable without knowing which aggregation each + // clause belongs to. + if (byClauses.length > 0 && withoutClauses.length > 0) return ALL; + + // Remove everything that can legitimately contain an operator character before + // testing for a top-level operator. Label matchers matter most: `!=` and `=~` + // are ordinary matcher syntax, so leaving `{code!="200"}` in would read as a + // comparison and suppress the overlay on a query that does aggregate to one + // line. + const operatorCandidate = src + .replace(BY_CLAUSE, '') + .replace(WITHOUT_CLAUSE, '') + .replace(/\{[^}]*\}/g, '') // label matchers: {code!="200"} + .replace(/\[[^\]]*\]/g, ''); // range selectors: [5m], [1h:5m] + if (BINARY_OPERATOR.test(operatorCandidate)) return ALL; + + // A clause entry that is not a bare label name means we misread the expression + // (a quoted name, say, which stripStringLiterals has already rewritten to `""`). + // Keeping it would build a key that matches no real label, collapsing every raw + // series into one group and slipping past the multiple-series guard — the exact + // fail-open this module promises not to have. + const allBare = [...byClauses, ...withoutClauses] + .flat() + .every(l => LABEL_NAME.test(l)); + if (!allBare) return ALL; + + if (byClauses.length > 0) { + const sets = byClauses.map(l => new Set(l)); + // Differing `by` sets mean nested aggregations we are not confident reading. + if (!sets.every(s => sameLabels(s, sets[0]))) return ALL; + return { mode: 'keep', labels: sets[0] }; + } + + if (withoutClauses.length === 1) { + return { mode: 'drop', labels: new Set(withoutClauses[0]) }; + } + + return ALL; +} + +/** Whether `label` distinguishes one plotted line from another under `rule`. */ +export function labelDistinguishesSeries( + rule: SeriesLabelRule, + label: string, +): boolean { + switch (rule.mode) { + case 'keep': + return rule.labels.has(label); + case 'drop': + return !rule.labels.has(label); + case 'all': + return true; + } +} diff --git a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx index e685112003..cfba1f5564 100644 --- a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx +++ b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx @@ -19,8 +19,10 @@ import { IconHelpCircle, IconPencil } from '@tabler/icons-react'; import api from '@/api'; import SelectControlled from '@/components/SelectControlled'; +import { IS_EXEMPLARS_ENABLED } from '@/config'; import { DEFAULT_FILTER_KEYS_FETCH_LIMIT, + DEFAULT_MAX_EXEMPLARS, DEFAULT_QUERY_TIMEOUT, DEFAULT_SEARCH_ROW_LIMIT, } from '@/defaults'; @@ -344,6 +346,30 @@ export default function TeamQueryConfigSection() { /> + + {IS_EXEMPLARS_ENABLED && ( + <> + + Chart Settings + + + + + + + + + )} ); } diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx index 6174961a7f..53d7ae4793 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx @@ -26,7 +26,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx index 28e8031937..346e29b1e9 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx @@ -24,7 +24,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, @@ -56,7 +56,7 @@ const LINK = { const mockUseRowData = jest.fn(); jest.mock('../DBRowDataPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useRowData: (args: unknown) => mockUseRowData(args), ROW_DATA_ALIASES: { DURATION_MS: '__hdx_duration', @@ -82,14 +82,14 @@ const TRACE_SOURCE = { jest.mock('@/source', () => ({ __esModule: true, getEventBody: () => undefined, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSource: ({ id }: { id: string | null }) => id === 'trace-src' ? { data: TRACE_SOURCE } : { data: undefined }, })); jest.mock('../DBSessionPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }), DBSessionPanel: () => null, })); diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx index 2b7a988e3c..c6f2a8b421 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx @@ -38,7 +38,7 @@ const LINKED_SPAN_NAME = 'consume order.created'; const mockUseRowData = jest.fn(); jest.mock('../DBRowDataPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useRowData: (args: unknown) => mockUseRowData(args), ROW_DATA_ALIASES: { DURATION_MS: '__hdx_duration', @@ -62,14 +62,14 @@ const TRACE_SOURCE = { jest.mock('@/source', () => ({ __esModule: true, getEventBody: () => undefined, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSource: ({ id }: { id: string | null }) => id === 'trace-src' ? { data: TRACE_SOURCE } : { data: undefined }, })); jest.mock('../DBSessionPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }), DBSessionPanel: () => null, })); diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx index 471f270e14..e6f3314222 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx @@ -26,7 +26,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index be36368c43..92a92a501d 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -66,3 +66,7 @@ export const IS_METRICS_ENABLED = true; export const IS_MTVIEWS_ENABLED = false; export const IS_SESSIONS_ENABLED = true; export const IS_PROMQL_ENABLED = env('NEXT_PUBLIC_ENABLE_PROMQL') === 'true'; +// Exemplar overlay (trace markers on time charts). Off by default while the +// feature is being tested; set NEXT_PUBLIC_ENABLE_EXEMPLARS=true to enable. +export const IS_EXEMPLARS_ENABLED = + env('NEXT_PUBLIC_ENABLE_EXEMPLARS') === 'true'; diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 95d151307f..d38ed01118 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -5,6 +5,8 @@ export const DEFAULT_SEARCH_ROW_LIMIT = 200; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; export const DEFAULT_SERIES_LIMIT = 100; +// Target number of exemplar markers shown per chart (0 = unlimited). +export const DEFAULT_MAX_EXEMPLARS = 12; export function searchChartConfigDefaults( team: any | undefined | null, diff --git a/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx b/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx index 530c859712..69c7dbfbcf 100644 --- a/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx @@ -6,7 +6,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: () => [false, mockSetKioskMode], }; }); diff --git a/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx b/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx index 24fcd69cf1..8314a8a111 100644 --- a/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx +++ b/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx @@ -24,7 +24,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, diff --git a/packages/app/src/hooks/useExemplars/__tests__/useExemplars.test.tsx b/packages/app/src/hooks/useExemplars/__tests__/useExemplars.test.tsx new file mode 100644 index 0000000000..fd3d6003d7 --- /dev/null +++ b/packages/app/src/hooks/useExemplars/__tests__/useExemplars.test.tsx @@ -0,0 +1,786 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import React from 'react'; +import { EXEMPLAR_QUERY_LIMIT } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { + ChartConfigWithOptDateRange, + DisplayType, + Exemplar, + MetricsDataType, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { act, renderHook, waitFor } from '@testing-library/react'; + +import { prometheusApi } from '@/api'; +import { useClickhouseClient } from '@/clickhouse'; +import { + capExemplarsPerBucket, + normalizePrometheusExemplars, + useExemplars, +} from '@/hooks/useExemplars'; + +// Flipped per-test to exercise the deployment feature gate. A getter (rather +// than a literal) so the hook reads the current value on each render. +let isExemplarsEnabled = true; +jest.mock('@/config', () => ({ + get IS_EXEMPLARS_ENABLED() { + return isExemplarsEnabled; + }, +})); + +jest.mock('@/api', () => ({ + __esModule: true, + prometheusApi: { queryExemplars: jest.fn() }, +})); + +jest.mock('@/clickhouse', () => ({ + __esModule: true, + useClickhouseClient: jest.fn(), +})); + +jest.mock('@/hooks/useMetadata', () => ({ + __esModule: true, + useMetadataWithSettings: jest.fn().mockReturnValue({ + getColumns: jest.fn().mockResolvedValue([]), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null), + getColumn: jest.fn().mockResolvedValue(undefined), + getTableMetadata: jest.fn().mockResolvedValue({ primary_key: 'TimeUnix' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + }), +})); + +jest.mock('@/source', () => ({ + __esModule: true, + getDurationMsExpression: jest.fn().mockReturnValue('Duration / 1e6'), +})); + +describe('normalizePrometheusExemplars', () => { + it('returns [] for undefined/empty input', () => { + expect(normalizePrometheusExemplars(undefined).exemplars).toEqual([]); + expect(normalizePrometheusExemplars([]).exemplars).toEqual([]); + }); + + it('maps trace/span ids, value, and seconds→ms timestamp', () => { + const { exemplars: result } = normalizePrometheusExemplars([ + { + seriesLabels: { __name__: 'http_latency', service: 'api' }, + exemplars: [ + { + labels: { trace_id: 'abc', span_id: 'def' }, + value: '1.5', + timestamp: 1700000000, + }, + ], + }, + ]); + expect(result).toEqual([ + { + timestamp: 1700000000 * 1000, + value: 1.5, + traceId: 'abc', + spanId: 'def', + groupKey: 'service="api"', + }, + ]); + }); + + it('accepts alternate label spellings (traceID/spanID)', () => { + const { + exemplars: [ex], + } = normalizePrometheusExemplars([ + { + seriesLabels: {}, + exemplars: [ + { + labels: { traceID: 'xyz', spanID: 's1' }, + value: '2', + timestamp: 1, + }, + ], + }, + ]); + expect(ex.traceId).toBe('xyz'); + expect(ex.spanId).toBe('s1'); + expect(ex.groupKey).toBeUndefined(); + }); + + it('merges the per-`le` entries a histogram query returns', () => { + // /query_exemplars reports one entry per underlying series, so a + // histogram_quantile chart — a single plotted line — comes back split + // across its bucket series. Those are the same series to the overlay. + const histogramBuckets = [ + { + seriesLabels: { __name__: 'http_latency_bucket', le: '0.1' }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { __name__: 'http_latency_bucket', le: '0.5' }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 }, + ], + }, + ]; + const { exemplars: result } = normalizePrometheusExemplars( + histogramBuckets, + 'histogram_quantile(0.99, rate(http_latency_bucket[5m]))', + ); + expect(result.map(e => e.traceId)).toEqual(['a', 'b']); + expect(result.every(e => e.groupKey === undefined)).toBe(true); + }); + + it('keeps `le` significant when the expression does not collapse buckets', () => { + // `rate(x_bucket[5m])` genuinely draws one line per bucket, so merging the + // entries would plot markers that belong to no drawn line. + const perBucketLines = [ + { + seriesLabels: { __name__: 'http_latency_bucket', le: '0.1' }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { __name__: 'http_latency_bucket', le: '0.5' }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 }, + ], + }, + ]; + expect( + normalizePrometheusExemplars( + perBucketLines, + 'rate(http_latency_bucket[5m])', + ), + ).toEqual({ exemplars: [], dropped: 'multiple-series' }); + }); + + it('drops the overlay when entries span different metrics', () => { + // Both entries carry no labels beyond __name__, so a label-only identity + // would collapse them to one series and plot markers from an unrelated + // metric against the drawn line. + const twoMetrics = [ + { + seriesLabels: { __name__: 'foo' }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '1', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { __name__: 'bar' }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '2', timestamp: 1700000001 }, + ], + }, + ]; + expect(normalizePrometheusExemplars(twoMetrics)).toEqual({ + exemplars: [], + dropped: 'multiple-series', + }); + }); + + it('drops the overlay entirely when the query returns multiple series', () => { + // Exemplars are single-series only; multi-series markers can't be attributed + // or scaled meaningfully, so the whole set is dropped rather than rendered. + const multiSeries = [ + { + seriesLabels: { service: 'api' }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '1', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { service: 'web' }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '2', timestamp: 1700000000 }, + ], + }, + ]; + expect(normalizePrometheusExemplars(multiSeries)).toEqual({ + exemplars: [], + dropped: 'multiple-series', + }); + }); + + it('skips exemplars without a trace id', () => { + expect( + normalizePrometheusExemplars([ + { + seriesLabels: {}, + exemplars: [{ labels: { foo: 'bar' }, value: '1', timestamp: 1 }], + }, + ]).exemplars, + ).toEqual([]); + }); + + // The canonical latency query. /query_exemplars resolves the raw selector, so + // it returns one entry per scrape target *and* per `le` bucket, while the chart + // draws a single aggregated line. Keying the single-series guard on the full + // selector label set therefore emptied the overlay on any metric scraped from + // more than one instance — i.e. essentially always. + it('merges entries that differ only by labels the aggregation drops', () => { + const acrossInstances = [ + { + seriesLabels: { + __name__: 'http_latency_bucket', + le: '0.1', + instance: 'pod-a', + }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { + __name__: 'http_latency_bucket', + le: '0.5', + instance: 'pod-b', + }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 }, + ], + }, + ]; + const { exemplars, dropped } = normalizePrometheusExemplars( + acrossInstances, + 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le))', + ); + expect(dropped).toBeUndefined(); + expect(exemplars.map(e => e.traceId)).toEqual(['a', 'b']); + // `le` is collapsed by histogram_quantile and `instance` by the `by (le)`, + // so nothing distinguishes the plotted line. + expect(exemplars.every(e => e.groupKey === undefined)).toBe(true); + }); + + it('still drops the overlay for labels the aggregation keeps', () => { + // `by (le, service)` draws one line per service, so these are genuinely two + // plotted series and their markers can't be attributed. + const twoServices = [ + { + seriesLabels: { + __name__: 'http_latency_bucket', + le: '0.1', + service: 'api', + instance: 'pod-a', + }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { + __name__: 'http_latency_bucket', + le: '0.1', + service: 'web', + instance: 'pod-b', + }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 }, + ], + }, + ]; + expect( + normalizePrometheusExemplars( + twoServices, + 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le, service))', + ), + ).toEqual({ exemplars: [], dropped: 'multiple-series' }); + }); + + it('merges entries under a `without` aggregation too', () => { + // The `without` spelling of the same canonical query. This previously fell + // through to "every label distinguishes a line" and dropped the whole overlay + // — the exact bug the `by (...)` handling was written to fix. + const acrossInstances = [ + { + seriesLabels: { + __name__: 'http_latency_bucket', + le: '0.1', + instance: 'pod-a', + }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { + __name__: 'http_latency_bucket', + le: '0.5', + instance: 'pod-b', + }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 }, + ], + }, + ]; + const { exemplars, dropped } = normalizePrometheusExemplars( + acrossInstances, + 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) without (instance))', + ); + expect(dropped).toBeUndefined(); + expect(exemplars.map(e => e.traceId)).toEqual(['a', 'b']); + }); + + it('treats `histogram_quantile (` with a space as collapsing buckets', () => { + // isPromqlExemplarEligible allows whitespace before the paren, so a literal + // substring test here disagreed with it about one expression: the toggle + // allowed it, `le` stayed in the group key, and the overlay came back + // suppressed telling the user to aggregate to a single line they already had. + const { exemplars, dropped } = normalizePrometheusExemplars( + [ + { + seriesLabels: { __name__: 'http_latency_bucket', le: '0.1' }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { __name__: 'http_latency_bucket', le: '0.5' }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 }, + ], + }, + ], + 'histogram_quantile (0.95, sum(rate(http_latency_bucket[5m])) by (le))', + ); + expect(dropped).toBeUndefined(); + expect(exemplars.map(e => e.traceId)).toEqual(['a', 'b']); + }); + + it('rejects exemplars whose value or timestamp is not finite', () => { + // ExemplarSchema is `.finite()` on both: a NaN timestamp collapses every + // affected exemplar into one bucket and emits a NaN SVG coordinate. + const { exemplars } = normalizePrometheusExemplars([ + { + seriesLabels: { __name__: 'http_latency' }, + exemplars: [ + { labels: { trace_id: 'ok' }, value: '1', timestamp: 1700000000 }, + { + labels: { trace_id: 'bad-value' }, + value: 'not-a-number', + timestamp: 1700000000, + }, + { + labels: { trace_id: 'bad-ts' }, + value: '1', + timestamp: Number.POSITIVE_INFINITY, + }, + ], + }, + ]); + expect(exemplars.map(e => e.traceId)).toEqual(['ok']); + }); +}); + +describe('capExemplarsPerBucket', () => { + const start = new Date('2026-01-01T00:00:00Z'); + const end = new Date('2026-01-01T01:00:00Z'); + const at = (offsetMs: number, value: number): Exemplar => ({ + timestamp: start.getTime() + offsetMs, + value, + traceId: `t-${offsetMs}-${value}`, + }); + + // The regression this guards: an inclusive range put a timestamp exactly on + // `end` one bucket past the last, giving 201 buckets. perBucket floored to 1, + // the total exceeded the budget, and the trailing slice trimmed a time-sorted + // list — so the exemplar it dropped was the newest, at the chart's right edge. + it('keeps the newest exemplar when the range is inclusive', () => { + const rangeMs = end.getTime() - start.getTime(); + const many: Exemplar[] = []; + for (let i = 0; i <= EXEMPLAR_QUERY_LIMIT; i++) { + many.push(at(Math.round((i * rangeMs) / EXEMPLAR_QUERY_LIMIT), i)); + } + const newest = many[many.length - 1]; + const capped = capExemplarsPerBucket(many, start, end); + + expect(capped.length).toBeLessThanOrEqual(EXEMPLAR_QUERY_LIMIT); + expect(capped.map(e => e.traceId)).toContain(newest.traceId); + // And the result is still chronological for the render layer. + expect(capped.map(e => e.timestamp)).toEqual( + [...capped.map(e => e.timestamp)].sort((a, b) => a - b), + ); + }); + + it('returns the set untouched when it is already within the limit', () => { + const few = [at(0, 1), at(1000, 2)]; + expect(capExemplarsPerBucket(few, start, end)).toBe(few); + }); + + it('keeps the peak of each bucket rather than a value-blind stride', () => { + // One slow trace early on, buried among many fast ones. A uniform index + // stride would drop it with ~98% probability; it is the whole reason the + // overlay exists. + const spanMs = end.getTime() - start.getTime(); + const many = Array.from({ length: EXEMPLAR_QUERY_LIMIT * 5 }, (_, i) => + at((i * spanMs) / (EXEMPLAR_QUERY_LIMIT * 5), i === 7 ? 9999 : 1), + ); + const capped = capExemplarsPerBucket(many, start, end); + expect(capped.length).toBeLessThanOrEqual(EXEMPLAR_QUERY_LIMIT); + expect(capped.some(ex => ex.value === 9999)).toBe(true); + // Still chronological and still spanning the range. + expect(capped.map(ex => ex.timestamp)).toEqual( + [...capped.map(ex => ex.timestamp)].sort((a, b) => a - b), + ); + expect(capped[capped.length - 1].timestamp).toBeGreaterThan( + start.getTime() + spanMs * 0.75, + ); + }); + + it('falls back to the highest values when the range is degenerate', () => { + const many = Array.from({ length: EXEMPLAR_QUERY_LIMIT + 10 }, (_, i) => + at(0, i), + ); + const capped = capExemplarsPerBucket(many, start, start); + expect(capped).toHaveLength(EXEMPLAR_QUERY_LIMIT); + expect(Math.min(...capped.map(ex => ex.value))).toBe(10); + }); +}); + +describe('useExemplars', () => { + const mockQuery = jest.fn(); + const mockQueryExemplars = prometheusApi.queryExemplars as jest.Mock; + + const wrapper = ({ children }: { children: React.ReactNode }) => ( + + {children} + + ); + + const metricSource = { kind: SourceKind.Metric } as TSource; + const promqlSource = { kind: SourceKind.Promql } as TSource; + + const histogramConfig = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { databaseName: 'default', tableName: '' }, + select: [ + { + aggFn: 'quantile', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + level: 0.95, + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + enableExemplars: true, + } as ChartConfigWithOptDateRange; + + const promqlConfig = { + configType: 'promql', + displayType: DisplayType.Line, + connection: 'test-connection', + promqlExpression: 'histogram_quantile(0.95, http_latency)', + from: { databaseName: 'default', tableName: 'metrics' }, + select: '', + where: '', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + enableExemplars: true, + } as unknown as ChartConfigWithOptDateRange; + + beforeAll(() => { + // The stubbed metadata can't resolve column types, which the SQL renderer + // warns about — expected here, and noisy. + jest.spyOn(console, 'warn').mockImplementation(jest.fn()); + jest.spyOn(console, 'error').mockImplementation(jest.fn()); + }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + isExemplarsEnabled = true; + jest.clearAllMocks(); + (useClickhouseClient as jest.Mock).mockReturnValue({ query: mockQuery }); + }); + + describe('gating', () => { + // These assert that NO query is issued. A synchronous assertion on the first + // render proves nothing: `exemplars` is [] because data is undefined, and the + // fetch sits behind an await either way — the whole block passed with the + // `enabled` gate hardcoded to true. So every case now flushes the microtask + // queue first, and a control case proves the harness does fetch when it should. + const flush = () => act(async () => {}); + + const renderGated = async ( + config: Parameters[0], + source: Parameters[1], + ) => { + const rendered = renderHook(() => useExemplars(config, source), { + wrapper, + }); + await flush(); + return rendered; + }; + + it('fetches when nothing gates it (control for the cases below)', async () => { + const { result } = await renderGated(histogramConfig, metricSource); + expect(mockQuery).toHaveBeenCalled(); + expect(result.current.exemplars).toEqual([]); + }); + + it('does not fetch when the chart has not opted in', async () => { + await renderGated( + { ...histogramConfig, enableExemplars: undefined }, + metricSource, + ); + expect(mockQuery).not.toHaveBeenCalled(); + expect(mockQueryExemplars).not.toHaveBeenCalled(); + }); + + it('does not fetch while the deployment feature flag is off', async () => { + isExemplarsEnabled = false; + await renderGated(histogramConfig, metricSource); + expect(mockQuery).not.toHaveBeenCalled(); + expect(mockQueryExemplars).not.toHaveBeenCalled(); + }); + + it('does not fetch for source kinds that cannot produce exemplars', async () => { + await renderGated(histogramConfig, { + kind: SourceKind.Log, + } as TSource); + expect(mockQuery).not.toHaveBeenCalled(); + expect(mockQueryExemplars).not.toHaveBeenCalled(); + }); + + it('does not fetch without a source', async () => { + await renderGated(histogramConfig, undefined); + expect(mockQuery).not.toHaveBeenCalled(); + expect(mockQueryExemplars).not.toHaveBeenCalled(); + }); + + it('does not fetch for a PromQL expression that plots no duration', async () => { + // promqlEligible was only ever fed an eligible expression, so the guard that + // stops duration markers landing on a requests/sec axis went unexercised. + await renderGated( + { + ...promqlConfig, + promqlExpression: 'rate(http_requests_total[5m])', + } as typeof promqlConfig, + promqlSource, + ); + expect(mockQueryExemplars).not.toHaveBeenCalled(); + }); + + it('drops the overlay when the chart draws more than one series', async () => { + // The rendered series count comes from the main query, not the exemplar + // payload — a multi-line chart must not get markers of unknown provenance. + // Must wait for real data: asserting [] before the query settles is true + // whatever the count is, which is how the first version of this test + // certified nothing. + mockQuery.mockResolvedValue({ + json: async () => ({ + data: [{ timestamp: '1700000000000', value: '1', traceId: 'a' }], + }), + }); + + const single = renderHook( + () => useExemplars(histogramConfig, metricSource, 1), + { wrapper }, + ); + await waitFor(() => + expect(single.result.current.exemplars).toHaveLength(1), + ); + + const many = renderHook( + () => useExemplars(histogramConfig, metricSource, 3), + { wrapper }, + ); + await waitFor(() => expect(many.result.current.isLoading).toBe(false)); + expect(many.result.current.exemplars).toEqual([]); + expect(many.result.current.dropped).toBe('multiple-series'); + }); + }); + + // placeholderData is scoped to the same chart on purpose: TanStack keeps its + // last-defined data on the observer, which outlives a key change, so an + // unscoped `prev => prev` hands the PREVIOUS metric's exemplars — real, + // clickable trace ids — to the new chart while isLoading and isError both say + // settled. Nothing exercised that comparison, so its removal was silent. + describe('placeholder scoping across key changes', () => { + const rows = (traceId: string) => ({ + json: async () => ({ + data: [{ timestamp: '1700000000000', value: '1', traceId }], + }), + }); + + it('keeps the overlay across a range-only change', async () => { + mockQuery.mockResolvedValue(rows('from-first-range')); + const { result, rerender } = renderHook( + ({ config }) => useExemplars(config, metricSource), + { wrapper, initialProps: { config: histogramConfig } }, + ); + await waitFor(() => expect(result.current.exemplars).toHaveLength(1)); + + // Same chart, later window: the markers should survive the refetch. + mockQuery.mockImplementation(() => new Promise(() => {})); + rerender({ + config: { + ...histogramConfig, + dateRange: [ + new Date('2025-02-12T01:00:00Z'), + new Date('2025-02-12T02:00:00Z'), + ], + } as typeof histogramConfig, + }); + expect(result.current.exemplars).toHaveLength(1); + }); + + it('blanks the overlay when the chart itself changes', async () => { + mockQuery.mockResolvedValue(rows('from-first-metric')); + const { result, rerender } = renderHook( + ({ config }) => useExemplars(config, metricSource), + { wrapper, initialProps: { config: histogramConfig } }, + ); + await waitFor(() => expect(result.current.exemplars).toHaveLength(1)); + + // A different chart (its filter changed, so it plots different data): the + // previous chart's traces must not be shown here. + mockQuery.mockImplementation(() => new Promise(() => {})); + rerender({ + config: { + ...histogramConfig, + where: "ServiceName = 'a-different-service'", + }, + }); + expect(result.current.exemplars).toEqual([]); + }); + }); + + describe('metric source (ClickHouse)', () => { + it('maps exemplar rows and drops rows without a trace id', async () => { + mockQuery.mockResolvedValue({ + json: async () => ({ + data: [ + { + timestamp: '1700000000000', + value: '1.5', + traceId: 'a', + spanId: 's1', + }, + // No trace id → nothing to link to, must be dropped. + { timestamp: '1700000001000', value: '2.5', traceId: '' }, + { timestamp: '1700000002000', value: '3.5', traceId: 'b' }, + ], + }), + }); + + const { result } = renderHook( + () => useExemplars(histogramConfig, metricSource), + { wrapper }, + ); + + await waitFor(() => expect(result.current.exemplars).toHaveLength(2)); + expect(result.current.exemplars).toEqual([ + { timestamp: 1700000000000, value: 1.5, traceId: 'a', spanId: 's1' }, + { + timestamp: 1700000002000, + value: 3.5, + traceId: 'b', + spanId: undefined, + }, + ]); + }); + + it('returns [] without querying when the config is not exemplar-eligible', async () => { + // A Group By makes markers unattributable, so the renderer returns no SQL. + const { result } = renderHook( + () => + useExemplars( + { ...histogramConfig, groupBy: 'ServiceName' }, + metricSource, + ), + { wrapper }, + ); + await waitFor(() => expect(result.current.isLoading).toBe(false)); + expect(result.current.exemplars).toEqual([]); + expect(mockQuery).not.toHaveBeenCalled(); + }); + }); + + describe('promql source', () => { + it('caps the result at EXEMPLAR_QUERY_LIMIT, thinning evenly across time', async () => { + const exemplars = Array.from( + { length: EXEMPLAR_QUERY_LIMIT + 50 }, + (_, i) => ({ + labels: { trace_id: `t${i}` }, + value: String(i), + timestamp: 1700000000 + i, + }), + ); + mockQueryExemplars.mockResolvedValue({ + status: 'success', + data: [{ seriesLabels: {}, exemplars }], + }); + + const { result } = renderHook( + () => useExemplars(promqlConfig, promqlSource), + { wrapper }, + ); + + await waitFor(() => + expect(result.current.exemplars).toHaveLength(EXEMPLAR_QUERY_LIMIT), + ); + // Chronological and spanning the range — a value-ranked cap over the + // whole set would keep only the slowest traces and leave most of the + // range bare — while still keeping the peak within each bucket, which a + // uniform index stride would discard. + const timestamps = result.current.exemplars.map(ex => ex.timestamp); + expect(timestamps).toEqual([...timestamps].sort((a, b) => a - b)); + // Markers survive at both ends of the response, so neither the start nor + // the tail of the range is left bare. + const start = 1700000000 * 1000; + const inputSpan = (EXEMPLAR_QUERY_LIMIT + 49) * 1000; + expect(timestamps[0]).toBeLessThan(start + inputSpan * 0.25); + expect(timestamps[timestamps.length - 1]).toBeGreaterThan( + start + inputSpan * 0.75, + ); + // The single slowest exemplar survives the cap. + const slowest = EXEMPLAR_QUERY_LIMIT + 49; + expect(result.current.exemplars.some(ex => ex.value === slowest)).toBe( + true, + ); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('surfaces an error when the proxy reports a non-success status', async () => { + mockQueryExemplars.mockResolvedValue({ + status: 'error', + error: 'upstream exploded', + }); + + const { result } = renderHook( + () => useExemplars(promqlConfig, promqlSource), + { wrapper }, + ); + + // The hook sets retry: 1, so allow for the retry's backoff delay. + await waitFor(() => expect(result.current.isError).toBe(true), { + timeout: 5000, + }); + expect(result.current.exemplars).toEqual([]); + }); + }); +}); diff --git a/packages/app/src/hooks/useExemplars/exemplarNormalize.ts b/packages/app/src/hooks/useExemplars/exemplarNormalize.ts new file mode 100644 index 0000000000..54b8d9cf47 --- /dev/null +++ b/packages/app/src/hooks/useExemplars/exemplarNormalize.ts @@ -0,0 +1,224 @@ +import { EXEMPLAR_QUERY_LIMIT } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { Exemplar, ExemplarSchema } from '@hyperdx/common-utils/dist/types'; + +import { type PrometheusExemplarsResult } from '@/api'; +import { + labelDistinguishesSeries, + promqlSeriesLabelRule, + type SeriesLabelRule, +} from '@/components/Exemplars/promqlSeriesLabels'; + +// Native Prometheus exporters disagree on the trace/span id label name; accept +// the common spellings. +const TRACE_ID_LABELS = ['trace_id', 'traceID', 'traceId', 'trace.id']; +const SPAN_ID_LABELS = ['span_id', 'spanID', 'spanId', 'span.id']; + +/** + * Whether the expression collapses a histogram's `le` buckets into one line. + * Only then is `le` a non-series label: `rate(x_bucket[5m])` genuinely plots one + * line per bucket, and treating those as one series would render markers that + * belong to no drawn line. + */ +// Whitespace-tolerant, matching isPromqlExemplarEligible. A literal substring +// test disagreed with the eligibility gate about `histogram_quantile (0.95, ...)`: +// the toggle allowed it, this kept `le` in the group key, and the overlay came +// back suppressed with a notice telling the user to aggregate to a single line +// they already had. +const HISTOGRAM_QUANTILE_CALL = /\bhistogram_quantile\s*\(/; + +function collapsesHistogramBuckets(expression: string | undefined): boolean { + return !!expression && HISTOGRAM_QUANTILE_CALL.test(expression); +} + +function pick(labels: Record, keys: string[]) { + for (const k of keys) { + if (labels[k]) return labels[k]; + } + return undefined; +} + +/** + * Stable identity for the plotted series an exemplar belongs to. `__name__` is + * deliberately excluded from the *label* key and counted separately by the + * caller — two different metrics with no other labels would otherwise both + * produce an empty key and merge into one overlay. + * + * `rule` restricts the key to the labels the query's aggregation actually keeps + * — see promqlSeriesLabelRule. + */ +function seriesGroupKey( + labels: Record, + ignoreLe: boolean, + rule: SeriesLabelRule, +): string | undefined { + return ( + Object.entries(labels) + .filter( + ([k]) => + k !== '__name__' && + !(ignoreLe && k === 'le') && + labelDistinguishesSeries(rule, k), + ) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${k}="${v}"`) + .join(', ') || undefined + ); +} + +/** + * Why an otherwise-populated exemplar overlay was suppressed, for the UI. + * Deliberately not exported: consumers reach it via NormalizedExemplars.dropped + * and compare against the literal, so exporting the alias only adds dead surface. + */ +type ExemplarDropReason = 'multiple-series'; + +export type NormalizedExemplars = { + exemplars: Exemplar[]; + /** Set when exemplars existed but the overlay was deliberately suppressed. */ + dropped?: ExemplarDropReason; +}; + +/** + * Normalize a native Prometheus /query_exemplars response into the shared + * Exemplar shape. Exported for testing — label naming varies by exporter. + * + * Prometheus returns one entry per *underlying* series, so a + * `histogram_quantile(...)` query — a single plotted line — comes back split + * across its `le` buckets and across every scrape target. Entries that the + * query's aggregation collapses into one line are merged; a genuine fan-out + * across *plotted* series, or across different metrics, is dropped rather than + * rendered as unattributable markers. + * + * Every candidate is parsed through ExemplarSchema rather than coerced: the body + * is an untrusted upstream response that `prometheusFetch` only type-asserts, and + * a `Number()` of a malformed value yields NaN coordinates downstream. + */ +export function normalizePrometheusExemplars( + data: PrometheusExemplarsResult[] | undefined, + expression?: string, +): NormalizedExemplars { + if (!data) return { exemplars: [] }; + const ignoreLe = collapsesHistogramBuckets(expression); + const rule = promqlSeriesLabelRule(expression); + const out: Exemplar[] = []; + const seenSeries = new Set(); + const seenMetrics = new Set(); + for (const series of data) { + const labels = series.seriesLabels ?? {}; + const groupKey = seriesGroupKey(labels, ignoreLe, rule); + for (const ex of series.exemplars ?? []) { + const traceId = pick(ex.labels ?? {}, TRACE_ID_LABELS); + if (!traceId) continue; + const parsed = ExemplarSchema.safeParse({ + timestamp: ex.timestamp * 1000, // prometheus exemplar ts is unix seconds + value: Number(ex.value), + traceId, + spanId: pick(ex.labels ?? {}, SPAN_ID_LABELS), + groupKey, + }); + if (!parsed.success) continue; + seenSeries.add(groupKey ?? ''); + seenMetrics.add(labels.__name__ ?? ''); + out.push(parsed.data); + } + } + // Exemplars are a single-series feature today: their y-position is the trace's + // own value on the chart's shared axis, so markers from multiple series can't + // be attributed or coloured yet. + // + // This payload check catches an exemplar response that spans several series or + // metrics. It is NOT sufficient on its own: Prometheus only returns series that + // carry a sampled exemplar, so it answers "how many series had exemplars", not + // "how many lines does the chart draw". The caller supplies the rendered series + // count for that — see useExemplars. Metric name is counted separately from the + // label key because it is excluded from that key, so two metrics with no other + // labels would both produce an empty key and merge. + if (seenSeries.size > 1 || seenMetrics.size > 1) { + return { exemplars: [], dropped: 'multiple-series' }; + } + return { exemplars: out }; +} + +/** + * Bound a Prometheus exemplar set to EXEMPLAR_QUERY_LIMIT, mirroring what the + * ClickHouse scan's `LIMIT n BY bucket` does server-side: bucket by time, then + * keep the highest-value exemplars in each bucket. + * + * Exported for testing. A uniform index stride would be simpler but throws away + * the slowest traces — on a 10k-exemplar response the p99 trace the overlay + * exists to surface has a ~2% chance of surviving. + */ +export function capExemplarsPerBucket( + sorted: Exemplar[], + start: Date, + end: Date, +): Exemplar[] { + if (sorted.length <= EXEMPLAR_QUERY_LIMIT) return sorted; + const rangeMs = end.getTime() - start.getTime(); + // Degenerate range: no meaningful buckets to spread across, so fall back to + // the highest-value exemplars overall. + if (!(rangeMs > 0)) { + return [...sorted] + .sort((a, b) => b.value - a.value) + .slice(0, EXEMPLAR_QUERY_LIMIT) + .sort((a, b) => a.timestamp - b.timestamp); + } + const bucketMs = rangeMs / EXEMPLAR_QUERY_LIMIT; + const byBucket = new Map(); + for (const ex of sorted) { + // Clamped to the last bucket. An inclusive range puts a timestamp exactly on + // `end` at index EXEMPLAR_QUERY_LIMIT — one past the last bucket — giving 201 + // buckets, which drove `perBucket` to 1 and pushed the total over the budget. + // The trailing slice then trimmed a *time-sorted* list, so the exemplar it + // dropped was the newest one, at the chart's right-hand edge. + const bucket = Math.min( + EXEMPLAR_QUERY_LIMIT - 1, + Math.max(0, Math.floor((ex.timestamp - start.getTime()) / bucketMs)), + ); + const inBucket = byBucket.get(bucket); + if (inBucket) inBucket.push(ex); + else byBucket.set(bucket, [ex]); + } + const perBucket = Math.max( + 1, + Math.floor(EXEMPLAR_QUERY_LIMIT / byBucket.size), + ); + return ( + Array.from(byBucket.keys()) + .sort((a, b) => a - b) + .flatMap(k => + [...byBucket.get(k)!] + .sort((a, b) => b.value - a.value) + .slice(0, perBucket), + ) + // buckets <= LIMIT after the clamp above, and perBucket = floor(LIMIT/buckets), + // so buckets * perBucket <= LIMIT: the slice is a backstop that no longer has + // anything to drop. Chronological order is restored last, after the budget is + // already satisfied, so it can never decide *which* exemplars survive. + .slice(0, EXEMPLAR_QUERY_LIMIT) + .sort((a, b) => a.timestamp - b.timestamp) + ); +} + +/** + * Map raw ClickHouse exemplar rows (renderMetricExemplarsChartConfig) → + * Exemplar[]. Parsed rather than coerced for the same reason as the Prometheus + * normalizer: a row that can't produce a finite timestamp/value is dropped + * instead of reaching recharts as a NaN coordinate. + */ +export function mapClickhouseExemplars( + rows: Record[], +): Exemplar[] { + const out: Exemplar[] = []; + for (const r of rows) { + if (!r.traceId) continue; + const parsed = ExemplarSchema.safeParse({ + timestamp: Number(r.timestamp), + value: Number(r.value), + traceId: String(r.traceId), + spanId: r.spanId ? String(r.spanId) : undefined, + }); + if (parsed.success) out.push(parsed.data); + } + return out; +} diff --git a/packages/app/src/hooks/useExemplars/index.ts b/packages/app/src/hooks/useExemplars/index.ts new file mode 100644 index 0000000000..4c22c45da4 --- /dev/null +++ b/packages/app/src/hooks/useExemplars/index.ts @@ -0,0 +1,14 @@ +/** + * Public surface of the exemplars hooks. Split out of a single 430-line module; + * consumers (and the several test files that `jest.mock('@/hooks/useExemplars')`) + * import from here, so the internal file layout stays free to change. + */ +export { + capExemplarsPerBucket, + normalizePrometheusExemplars, +} from './exemplarNormalize'; +export { useExemplars } from './useExemplars'; +export { + type ExemplarTraceMeta, + useExemplarTraceMeta, +} from './useExemplarTraceMeta'; diff --git a/packages/app/src/hooks/useExemplars/quantize.ts b/packages/app/src/hooks/useExemplars/quantize.ts new file mode 100644 index 0000000000..eaf4c216f2 --- /dev/null +++ b/packages/app/src/hooks/useExemplars/quantize.ts @@ -0,0 +1,28 @@ +/** + * Quantisation of an exemplar query's time window. + * + * Its own module because the exemplar hooks are mocked wholesale in several test + * files (`jest.mock('@/hooks/useExemplars')`), and these are pure helpers that + * callers still need for real. + */ + +// Live-tail charts advance `dateRange` continuously. Rounding the range to this +// bucket keeps sub-minute ticks on one cache entry — without it every tick mints a +// new key, empties the overlay, and force-closes the hover card the user is +// reaching for. +const EXEMPLAR_KEY_QUANTUM_MS = 30_000; + +// Floor the start and ceil the end, so the quantised window always *contains* the +// rendered one. Rounding both ends could produce a zero-width window and served +// markers offset from the chart's. +// +// Load-bearing detail: the quantised window is what gets FETCHED, not just what +// gets keyed. Keying on it while fetching the raw range would make the cache entry +// hold whichever raw window arrived first, so two windows sharing a key would show +// each other's markers. Fetching the quantised window makes the entry a genuine +// superset of every raw window that maps to it, and the render layer then trims to +// the drawn x-domain (see clampExemplarX). +export const quantizeStart = (d: Date) => + Math.floor(d.getTime() / EXEMPLAR_KEY_QUANTUM_MS) * EXEMPLAR_KEY_QUANTUM_MS; +export const quantizeEnd = (d: Date) => + Math.ceil(d.getTime() / EXEMPLAR_KEY_QUANTUM_MS) * EXEMPLAR_KEY_QUANTUM_MS; diff --git a/packages/app/src/hooks/useExemplars/useExemplarTraceMeta.ts b/packages/app/src/hooks/useExemplars/useExemplarTraceMeta.ts new file mode 100644 index 0000000000..fa7d2d6adb --- /dev/null +++ b/packages/app/src/hooks/useExemplars/useExemplarTraceMeta.ts @@ -0,0 +1,69 @@ +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { useQuery } from '@tanstack/react-query'; + +import { useClickhouseClient } from '@/clickhouse'; +import { getDurationMsExpression } from '@/source'; + +export type ExemplarTraceMeta = { + service?: string; + spanName?: string; + statusCode?: string; + durationMs?: number; + timestamp?: string; +}; + +/** + * Fetches a one-row summary of a trace (root/first span) from the given trace + * source, for the exemplar hover card. Enabled only while a trace id is hovered + * and a trace source is configured. + */ +export function useExemplarTraceMeta( + traceId: string | undefined, + traceSource: TSource | undefined, +) { + const clickhouseClient = useClickhouseClient(); + const isTrace = !!traceSource && traceSource.kind === SourceKind.Trace; + + return useQuery({ + queryKey: ['exemplarTraceMeta', traceId, traceSource?.id], + enabled: !!traceId && isTrace, + staleTime: 5 * 60 * 1000, + queryFn: async context => { + if (!traceId || !traceSource || traceSource.kind !== SourceKind.Trace) { + return null; + } + const s = traceSource; + const from = s.from.databaseName + ? `\`${s.from.databaseName}\`.\`${s.from.tableName}\`` + : `\`${s.from.tableName}\``; + const traceIdExpr = s.traceIdExpression || 'TraceId'; + const parentExpr = s.parentSpanIdExpression || 'ParentSpanId'; + const tsExpr = s.timestampValueExpression || 'Timestamp'; + const sql = ` + SELECT + ${s.serviceNameExpression || 'ServiceName'} AS service, + ${s.spanNameExpression || 'SpanName'} AS spanName, + ${s.statusCodeExpression || 'StatusCode'} AS statusCode, + ${getDurationMsExpression(s)} AS durationMs, + ${tsExpr} AS timestamp + FROM ${from} + WHERE ${traceIdExpr} = {traceId:String} + ORDER BY (${parentExpr} = '') DESC, ${tsExpr} ASC + LIMIT 1`; + const resp = await clickhouseClient.query({ + query: sql, + query_params: { traceId }, + format: 'JSON', + abort_signal: context.signal, + connectionId: s.connection, + }); + const json = await resp.json(); + const row = json.data?.[0]; + if (!row) return null; + return { + ...row, + durationMs: row.durationMs != null ? Number(row.durationMs) : undefined, + }; + }, + }); +} diff --git a/packages/app/src/hooks/useExemplars/useExemplars.tsx b/packages/app/src/hooks/useExemplars/useExemplars.tsx new file mode 100644 index 0000000000..f89ea4d2d4 --- /dev/null +++ b/packages/app/src/hooks/useExemplars/useExemplars.tsx @@ -0,0 +1,219 @@ +import { useMemo } from 'react'; +import { isEqual } from 'lodash'; +import { + isPromqlExemplarEligible, + renderMetricExemplarsChartConfig, +} from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + ChartConfigWithOptDateRange, + Exemplar, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { useQuery } from '@tanstack/react-query'; + +import { prometheusApi } from '@/api'; +import { useClickhouseClient } from '@/clickhouse'; +import { IS_EXEMPLARS_ENABLED } from '@/config'; +import { + capExemplarsPerBucket, + mapClickhouseExemplars, + type NormalizedExemplars, + normalizePrometheusExemplars, +} from '@/hooks/useExemplars/exemplarNormalize'; +import { quantizeEnd, quantizeStart } from '@/hooks/useExemplars/quantize'; +import { useMetadataWithSettings } from '@/hooks/useMetadata'; + +// Source kinds that can produce exemplars today: native metric/promql sources. +// Trace-generated exemplars are added in a follow-up. +const EXEMPLAR_SUPPORTED_KINDS: SourceKind[] = [ + SourceKind.Metric, + SourceKind.Promql, +]; + +// Stable identity for "no exemplars". DBTimeChart forwards this straight into +// memo(MemoChart)'s props, so a fresh [] on every render would fail the shallow +// compare and re-render recharts for every time chart in the app on any parent +// state change — including with the feature flag off. +const NO_EXEMPLARS: Exemplar[] = []; + +// The exemplar overlay is a coarse annotation layer, not the series itself, so it +// tolerates being a little stale in exchange for not refiring on every mount. +const EXEMPLAR_STALE_TIME_MS = 60_000; + +/** + * Fetches exemplars for a chart in parallel with the main series query. A no-op + * (disabled query) unless `config.enableExemplars` is set and the source kind + * supports exemplars, so it adds zero cost to charts that don't use the overlay. + */ +export function useExemplars( + config: ChartConfigWithOptDateRange, + source: TSource | undefined, + /** + * How many series the chart actually draws, from the main query's result. + * + * The exemplar response cannot answer this: Prometheus only returns series that + * carry a sampled exemplar, so a genuinely multi-line chart whose buffer + * happened to hold one series' worth would look single-series and get markers + * attributed to a line they may not belong to. + * + * An absent or zero count permits the overlay. That is safe rather than + * deliberate: DBTimeChart does not mount the chart until it has data, so no + * markers reach the screen while the count is still unknown. + */ + plottedSeriesCount?: number, +) { + const clickhouseClient = useClickhouseClient(); + const metadata = useMetadataWithSettings(); + + const isPromql = isPromqlChartConfig(config); + const supported = !!source && EXEMPLAR_SUPPORTED_KINDS.includes(source.kind); + // A PromQL expression only carries exemplars when it plots a duration — see + // isPromqlExemplarEligible. Same rule the PromQL editor gates its toggle on, so + // a config saved before the rule tightened (or written via the API) doesn't + // plot duration markers on a requests/sec axis. + const promqlEligible = + !isPromql || isPromqlExemplarEligible(config.promqlExpression); + // Global feature gate: even a config with enableExemplars set fetches nothing + // while the feature is disabled for the deployment. + // A marker's y is the trace's own value on the chart's shared axis, so with + // more than one line drawn there is no way to say which line a marker belongs + // to. Decided from the rendered series count rather than the exemplar payload — + // see the parameter's note. + const tooManySeries = plottedSeriesCount != null && plottedSeriesCount > 1; + + // Everything except the series count: "the user asked for exemplars and this + // chart could carry them". Kept separate so the multi-series notice can fire + // without a fetch. + const wantsExemplars = + IS_EXEMPLARS_ENABLED && + config.enableExemplars === true && + supported && + promqlEligible; + + // No point paying for a proxy round-trip per window on a chart that can never + // show the result. + const enabled = wantsExemplars && !tooManySeries; + + // `config` minus the raw dateRange. This identifies the *chart* — the metric or + // PromQL expression, filters, connection — independently of the window being + // viewed, which is what makes it safe to carry a placeholder across a range + // change but not across a chart change (see placeholderData below). + const keyConfig = useMemo( + () => ({ ...config, dateRange: undefined }), + [config], + ); + + // The window actually fetched, and the window keyed. Must be the same value — + // see the note on quantizeStart above. + const fetchRange = useMemo( + () => + config.dateRange + ? ([ + new Date(quantizeStart(config.dateRange[0])), + new Date(quantizeEnd(config.dateRange[1])), + ] as [Date, Date]) + : undefined, + [config.dateRange], + ); + + const query = useQuery({ + // The raw dateRange is replaced by its quantized form so a live-tail tick + // doesn't invalidate the overlay every second. + queryKey: ['exemplars', keyConfig, fetchRange?.map(d => d.getTime())], + queryFn: async context => { + // PromQL → native Prometheus exemplars via the API proxy. + if (isPromqlChartConfig(config) && fetchRange) { + const [startDate, endDate] = fetchRange; + const resp = await prometheusApi.queryExemplars( + { + query: config.promqlExpression, + start: startDate.getTime() / 1000, + end: endDate.getTime() / 1000, + connectionId: config.connection, + database: config.from?.databaseName, + table: config.from?.tableName, + }, + context.signal, + ); + if (resp.status !== 'success') { + throw new Error(resp.error ?? 'query_exemplars failed'); + } + const { exemplars, dropped } = normalizePrometheusExemplars( + resp.data, + config.promqlExpression, + ); + // Native Prometheus /query_exemplars has no result-limit parameter, so + // bound the set client-side to keep an unbounded upstream response from + // ballooning downstream thinning/render work. + const all = [...exemplars].sort((a, b) => a.timestamp - b.timestamp); + return { + exemplars: capExemplarsPerBucket(all, startDate, endDate), + dropped, + }; + } + + // Structured metric source → exemplars stored on the OTel metric table. + const exemplarSql = await renderMetricExemplarsChartConfig( + // Same quantized window as the key and the PromQL branch, so one cache + // entry means one fetched window on both backends. + fetchRange ? { ...config, dateRange: fetchRange } : config, + metadata, + ); + if (!exemplarSql) return { exemplars: [] }; + + const resp = await clickhouseClient.query({ + query: exemplarSql.sql, + query_params: exemplarSql.params, + format: 'JSON', + abort_signal: context.signal, + connectionId: config.connection, + }); + const json = await resp.json>(); + return { exemplars: mapClickhouseExemplars(json.data ?? []) }; + }, + enabled, + retry: 1, + refetchOnWindowFocus: false, + staleTime: EXEMPLAR_STALE_TIME_MS, + // Keep the previous overlay visible across a *time-range* key change (a + // live-tail tick or a range nudge) instead of blanking it — the main series + // query does the same, and blanking here would also force-close an open hover + // card. + // + // Scoped to the same chart on purpose. TanStack keeps its last-defined-data on + // the observer instance, which outlives a key change, so an unscoped + // `prev => prev` hands the PREVIOUS metric's exemplars to the new chart: real, + // clickable trace ids, clamped onto the new axes, while isLoading/isError both + // report settled. The user clicks a marker and lands on an unrelated trace. + // Comparing the chart identity means a genuine chart switch blanks the overlay + // (correct) while a range tick keeps it (the point of the placeholder). + placeholderData: (prev, prevQuery) => + isEqual(prevQuery?.queryKey?.[1], keyConfig) ? prev : undefined, + }); + + return { + exemplars: + enabled && !tooManySeries + ? (query.data?.exemplars ?? NO_EXEMPLARS) + : NO_EXEMPLARS, + /** The overlay was asked for but suppressed — see ExemplarDropReason. */ + dropped: tooManySeries + ? wantsExemplars + ? ('multiple-series' as const) + : undefined + : enabled + ? query.data?.dropped + : undefined, + isLoading: query.isLoading, + isError: query.isError, + /** + * The upstream failure message, when there is one. Surfaced verbatim because + * the API phrases these actionably (e.g. the /query_exemplars window bound + * tells the user to narrow the chart's range), and a generic "could not load" + * would throw that guidance away. + */ + error: query.error instanceof Error ? query.error.message : undefined, + }; +} diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index c3592f0b78..8f9b7fa045 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -2,7 +2,12 @@ import { chSql, ColumnMeta, parameterizedQueryToSql } from '@/clickhouse'; import { Metadata } from '@/core/metadata'; import { ChartConfigWithOptDateRangeEx, + EXEMPLAR_QUERY_LIMIT, + exemplarScanBucketing, + isExemplarEligible, + isPromqlExemplarEligible, renderChartConfig, + renderMetricExemplarsChartConfig, timeFilterExpr, } from '@/core/renderChartConfig'; import { @@ -3508,3 +3513,521 @@ describe('renderChartConfig', () => { // gauge / sum / histogram snapshots earlier in this file plus the // cross-scope integration test in packages/api/src/clickhouse/__tests__. }); + +describe('isExemplarEligible', () => { + // The single rule the chart-editor toggle and the SQL renderer both delegate + // to; each clause below is a case where a marker can't be attributed to a + // point on the chart's y-axis. + const eligible = { + seriesCount: 1, + seriesReturnType: 'column' as const, + metricType: MetricsDataType.Histogram, + aggFn: 'quantile', + hasGroupBy: false, + }; + + it('accepts a single, non-ratio, non-grouped histogram series', () => { + expect(isExemplarEligible(eligible)).toBe(true); + }); + + it('accepts an unset seriesReturnType (defaults to a plain column series)', () => { + expect( + isExemplarEligible({ ...eligible, seriesReturnType: undefined }), + ).toBe(true); + }); + + it('rejects multi-series and zero-series charts', () => { + expect(isExemplarEligible({ ...eligible, seriesCount: 2 })).toBe(false); + expect(isExemplarEligible({ ...eligible, seriesCount: 0 })).toBe(false); + }); + + it('rejects a ratio chart', () => { + expect(isExemplarEligible({ ...eligible, seriesReturnType: 'ratio' })).toBe( + false, + ); + }); + + it('rejects a grouped chart', () => { + expect(isExemplarEligible({ ...eligible, hasGroupBy: true })).toBe(false); + }); + + it('rejects non-histogram and missing metric types', () => { + expect( + isExemplarEligible({ ...eligible, metricType: MetricsDataType.Gauge }), + ).toBe(false); + expect(isExemplarEligible({ ...eligible, metricType: undefined })).toBe( + false, + ); + }); + + it('rejects aggregations that leave the axis in another unit', () => { + // A count of observations or their sum is not on the same scale as one + // observation, so a duration marker clamped into that domain reads as a real + // point on it — the failure the PromQL path already guards against. + expect(isExemplarEligible({ ...eligible, aggFn: 'count' })).toBe(false); + expect(isExemplarEligible({ ...eligible, aggFn: 'sum' })).toBe(false); + expect(isExemplarEligible({ ...eligible, aggFn: 'count_distinct' })).toBe( + false, + ); + expect(isExemplarEligible({ ...eligible, aggFn: 'increase' })).toBe(false); + }); + + it('accepts aggregations that stay on the observation scale', () => { + for (const aggFn of ['avg', 'max', 'min', 'quantile', 'none']) { + expect(isExemplarEligible({ ...eligible, aggFn })).toBe(true); + } + }); + + it('rejects a missing aggregation', () => { + // A caller that cannot say what it plots cannot promise a duration axis. + expect(isExemplarEligible({ ...eligible, aggFn: undefined })).toBe(false); + }); + + it('rejects a differently-cased metric type', () => { + // The comparison is by value, so a caller passing the app package's legacy + // MetricsDataType ('Histogram') must not silently read as eligible. + expect(isExemplarEligible({ ...eligible, metricType: 'Histogram' })).toBe( + false, + ); + }); +}); + +describe('isPromqlExemplarEligible', () => { + // The PromQL counterpart: an exemplar's value is a duration, so it may only be + // plotted on an axis that is also a duration. + it('accepts expressions that plot a duration', () => { + expect( + isPromqlExemplarEligible( + 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le))', + ), + ).toBe(true); + expect(isPromqlExemplarEligible('histogram_avg(rate(latency[5m]))')).toBe( + true, + ); + }); + + it('rejects a count-valued expression', () => { + // The regression this guards: markers valued in milliseconds clamped into a + // requests/sec axis, reading as real points on that scale. + expect(isPromqlExemplarEligible('rate(http_requests_total[5m])')).toBe( + false, + ); + // A raw `_bucket` series is observation *counts*, not durations. + expect(isPromqlExemplarEligible('rate(http_latency_bucket[5m])')).toBe( + false, + ); + expect(isPromqlExemplarEligible('sum(up)')).toBe(false); + }); + + it('rejects an empty or missing expression', () => { + expect(isPromqlExemplarEligible(undefined)).toBe(false); + expect(isPromqlExemplarEligible('')).toBe(false); + }); + + it('rejects a duration expression wrapped in arithmetic', () => { + // The axis is then milliseconds while the exemplars are still seconds, so the + // markers would plot 1000x low and clamp to the axis floor as if real. + expect( + isPromqlExemplarEligible( + 'histogram_quantile(0.95, sum(rate(a[5m])) by (le)) * 1000', + ), + ).toBe(false); + expect( + isPromqlExemplarEligible( + '1000 * histogram_quantile(0.95, sum(rate(a[5m])) by (le))', + ), + ).toBe(false); + }); + + it('rejects the function name appearing inside a label matcher', () => { + expect( + isPromqlExemplarEligible('rate(x{note="histogram_quantile("}[5m])'), + ).toBe(false); + }); + + it('accepts leading whitespace before the outermost call', () => { + expect( + isPromqlExemplarEligible(' histogram_quantile(0.95, rate(a[5m]))'), + ).toBe(true); + }); +}); + +describe('exemplarScanBucketing', () => { + const hours = (n: number): [Date, Date] => [ + new Date('2026-01-01T00:00:00Z'), + new Date(new Date('2026-01-01T00:00:00Z').getTime() + n * 3600_000), + ]; + + it('keeps the chart granularity when it fits the row budget', () => { + // 1 hour at 1 minute = 60 buckets, comfortably inside the 200-row budget. + expect(exemplarScanBucketing('1 minute', hours(1))).toEqual({ + interval: '1 minute', + perBucket: 3, + }); + }); + + it('widens the bucket when the granularity would outrun the budget', () => { + const { interval, perBucket } = exemplarScanBucketing( + '1 minute', + hours(48), + ); + expect(interval).toBe('869 second'); + expect(perBucket).toBe(1); + }); + + it('leaves room for the extra epoch-aligned bucket', () => { + // toStartOfInterval aligns to the epoch, not the range start, so an + // unaligned range spans one more bucket than ceil(range / bucket). Without + // headroom the trailing LIMIT clips the newest bucket. + const [start, end] = hours(48); + const { interval, perBucket } = exemplarScanBucketing('1 minute', [ + start, + end, + ]); + const bucketSeconds = Number(interval.split(' ')[0]); + const rangeSeconds = (end.getTime() - start.getTime()) / 1000; + const worstCaseBuckets = Math.ceil(rangeSeconds / bucketSeconds) + 1; + expect(worstCaseBuckets * perBucket).toBeLessThanOrEqual( + EXEMPLAR_QUERY_LIMIT, + ); + }); + + it('falls back safely for degenerate inputs', () => { + expect(exemplarScanBucketing('1 minute', undefined)).toEqual({ + interval: '1 minute', + perBucket: 1, + }); + const [start] = hours(1); + expect(exemplarScanBucketing('1 minute', [start, start])).toEqual({ + interval: '1 minute', + perBucket: 1, + }); + expect(exemplarScanBucketing('1 minute', [start, new Date(NaN)])).toEqual({ + interval: '1 minute', + perBucket: 1, + }); + }); + + it('rejects a granularity that is not a valid SQL interval', () => { + // The interval is spliced into the query as raw SQL, and granularity + // reaches this function from a URL param. + const { interval } = exemplarScanBucketing( + '1 minute) UNION ALL SELECT' as never, + hours(1), + ); + expect(interval).toBe('1 minute'); + }); +}); + +describe('renderMetricExemplarsChartConfig', () => { + let mockMetadata: jest.Mocked; + + beforeAll(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); + }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + mockMetadata = { + getColumns: jest.fn().mockResolvedValue([]), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null), + getColumn: jest.fn().mockResolvedValue(undefined), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'TimeUnix' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + } as unknown as jest.Mocked; + }); + + const histogramConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { databaseName: 'default', tableName: '' }, + select: [ + { + aggFn: 'quantile', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + level: 0.95, + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + }; + + it('builds an ARRAY JOIN exemplar query against the metric-type table', async () => { + const generated = await renderMetricExemplarsChartConfig( + histogramConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + + // Surfaces the exemplar columns + expect(sql).toContain('ARRAY JOIN'); + expect(sql).toContain('`Exemplars.TraceId` AS ex_TraceId'); + expect(sql).toContain('toUnixTimestamp64Milli(ex_TimeUnix)'); + // Points at the histogram table and filters by metric name + time + expect(sql).toContain('otel_metrics_histogram'); + expect(sql).toContain("MetricName = 'http.server.duration'"); + expect(sql).toContain('TimeUnix'); + // Drops empty trace ids and caps the result set + expect(sql).toContain('notEmpty(ex_TraceId)'); + expect(sql).toContain(`LIMIT ${EXEMPLAR_QUERY_LIMIT}`); + }); + + // renderWhere bounds the row's TimeUnix, but the projected and bucketed value is + // the ARRAY JOINed ex_TimeUnix, and an exemplar's own timestamp falls in the + // collection interval before its data point's. Without an explicit ex_TimeUnix + // bound the scan returned pre-window exemplars and missed in-window ones whose + // data point landed just after the range end. + it('bounds the ARRAY JOINed exemplar time, not just the row time', async () => { + const generated = await renderMetricExemplarsChartConfig( + histogramConfig, + mockMetadata, + ); + const sql = parameterizedQueryToSql(generated!); + + const start = new Date('2025-02-12').getTime(); + const end = new Date('2025-02-14').getTime(); + expect(sql).toContain(`ex_TimeUnix >= fromUnixTimestamp64Milli(${start})`); + expect(sql).toContain(`ex_TimeUnix <= fromUnixTimestamp64Milli(${end})`); + }); + + it('widens the row-level bound past the range end by one interval', async () => { + // So an exemplar inside the window whose data point lands just after it is + // still reachable; the exact ex_TimeUnix bound above then trims the overshoot. + const generated = await renderMetricExemplarsChartConfig( + histogramConfig, + mockMetadata, + ); + const sql = parameterizedQueryToSql(generated!); + // 1-minute granularity, so the row bound ends 60s after the range end. + const widened = new Date('2025-02-14').getTime() + 60_000; + expect(sql).toContain(`TimeUnix <= fromUnixTimestamp64Milli(${widened})`); + }); + + it('floors the row-bound widening at one minute on a fine granularity', async () => { + // A short window resolves to a 15s or 30s granularity, finer than a typical + // scrape — widening by that much would leave the newest exemplars out of reach + // again, which is the whole point of the widening. + const generated = await renderMetricExemplarsChartConfig( + { ...histogramConfig, granularity: '15 second' }, + mockMetadata, + ); + const sql = parameterizedQueryToSql(generated!); + const widened = new Date('2025-02-14').getTime() + 60_000; + expect(sql).toContain(`TimeUnix <= fromUnixTimestamp64Milli(${widened})`); + }); + + it('returns null without a date range', async () => { + // renderWhere emits no time predicate then, leaving a whole-table ARRAY JOIN. + const { dateRange: _dropped, ...noRange } = histogramConfig; + const generated = await renderMetricExemplarsChartConfig( + noRange as typeof histogramConfig, + mockMetadata, + ); + expect(generated).toBeNull(); + }); + + it('spends the row budget per time bucket, not on the slowest traces overall', async () => { + const generated = await renderMetricExemplarsChartConfig( + // 1 hour at 1-minute granularity: 60 buckets, so the budget stretches to + // several rows each. + { + ...histogramConfig, + dateRange: [ + new Date('2025-02-12T00:00:00Z'), + new Date('2025-02-12T01:00:00Z'), + ], + }, + mockMetadata, + ); + const sql = parameterizedQueryToSql(generated!); + expect(sql).toContain( + 'toStartOfInterval(toDateTime(ex_TimeUnix), INTERVAL 1 minute)', + ); + // Ordering by bucket first is what makes the trailing LIMIT a backstop + // rather than a value-ranked cut of the whole range. + expect(sql).toContain('ORDER BY bucket ASC, value DESC'); + expect(sql).toContain('LIMIT 3 BY bucket'); + }); + + it('widens the scan bucket when the granularity would outrun the row budget', async () => { + const generated = await renderMetricExemplarsChartConfig( + // 2 days at 1-minute granularity is 2880 buckets — far more than the + // budget covers. Widening keeps markers spread over the whole range + // instead of running out partway along the x-axis. + histogramConfig, + mockMetadata, + ); + const sql = parameterizedQueryToSql(generated!); + expect(sql).toContain( + 'toStartOfInterval(toDateTime(ex_TimeUnix), INTERVAL 869 second)', + ); + expect(sql).toContain('LIMIT 1 BY bucket'); + }); + + it("scopes exemplars to the series' aggCondition so markers match the plotted line", async () => { + const filteredConfig = { + ...histogramConfig, + select: [ + { + aggFn: 'quantile', + level: 0.95, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + aggCondition: "ServiceName = 'api'", + aggConditionLanguage: 'sql', + }, + ], + } as ChartConfigWithOptDateRange; + const generated = await renderMetricExemplarsChartConfig( + filteredConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + expect(sql).toContain("ServiceName = 'api'"); + }); + + it('ANDs the metric-name predicate even when the chart uses OR filters', async () => { + const orConfig = { + ...histogramConfig, + filtersLogicalOperator: 'OR', + filters: [ + { type: 'sql', condition: "ServiceName = 'api'" }, + { type: 'sql', condition: "ServiceName = 'web'" }, + ], + } as ChartConfigWithOptDateRange; + const generated = await renderMetricExemplarsChartConfig( + orConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + // The required metric-name check must be ANDed as its own group, never + // folded into the user OR group (which would let the scan match other + // metrics whenever a user filter matches). + expect(sql).toContain("AND (MetricName = 'http.server.duration')"); + // The user filters are still present and OR'd within their own group. + expect(sql).toContain("ServiceName = 'api'"); + expect(sql).toContain("ServiceName = 'web'"); + expect(sql).not.toMatch(/OR\s*\(?\s*MetricName/); + }); + + it('returns null for a ratio config (exemplars are meaningless on a ratio axis)', async () => { + const ratioConfig = { + ...histogramConfig, + seriesReturnType: 'ratio', + select: [histogramConfig.select[0], histogramConfig.select[0]], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(ratioConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null for a multi-series config', async () => { + const multiConfig = { + ...histogramConfig, + select: [histogramConfig.select[0], histogramConfig.select[0]], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(multiConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null when the chart has a Group By', async () => { + // A grouped chart draws one line per group, but the exemplar scan is not + // group-aware: it would pool exemplars from every group into one + // unattributable set. Drop the overlay instead. + const groupedConfig = { + ...histogramConfig, + groupBy: 'ServiceName', + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(groupedConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null for a non-histogram metric (value is on a different scale)', async () => { + const gaugeConfig = { + ...histogramConfig, + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'system.memory.usage', + metricType: MetricsDataType.Gauge, + }, + ], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(gaugeConfig, mockMetadata), + ).toBeNull(); + }); + + it('scans the exponential histogram table for a native-histogram metric', async () => { + // Exponential (OTLP) / native (Prometheus) histograms carry exemplars in + // the same `Exemplars.*` columns, on their own metric-type table. + const exponentialConfig = { + ...histogramConfig, + select: [ + { + aggFn: 'quantile', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + level: 0.95, + metricName: 'traces.span.metrics.duration', + metricType: MetricsDataType.ExponentialHistogram, + }, + ], + } as ChartConfigWithOptDateRange; + const generated = await renderMetricExemplarsChartConfig( + exponentialConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + expect(sql).toContain('otel_metrics_exponential_histogram'); + expect(sql).toContain('`Exemplars.TraceId` AS ex_TraceId'); + }); + + it('returns null for a non-metric config', async () => { + const logConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + }; + expect( + await renderMetricExemplarsChartConfig(logConfig, mockMetadata), + ).toBeNull(); + }); +}); diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 2f22cbabcd..033f35f0b7 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -55,6 +55,7 @@ import { SortSpecificationList, SqlAstFilter, SQLInterval, + SQLIntervalSchema, } from '@/types'; /** @@ -2283,6 +2284,356 @@ export async function renderChartConfig( ]); } +/** Overall cap on exemplar markers returned for a single chart, so a wide + * time range can't flood the chart overlay with thousands of points. */ +export const EXEMPLAR_QUERY_LIMIT = 200; + +/** Ceiling on rows kept per time bucket, so a single busy bucket can't eat the + * whole budget when the range is short. */ +const EXEMPLAR_MAX_PER_BUCKET = 4; + +/** + * Pick the bucket width and per-bucket row count for the exemplar scan so its + * {@link EXEMPLAR_QUERY_LIMIT} budget is spent evenly across the range, instead + * of being emptied into whichever buckets happen to hold the slowest traces. + * + * The scan bucket is the chart granularity, widened only if that granularity + * would produce more buckets than the budget can cover — one marker per plotted + * point is the ideal, but running out of budget a third of the way along the + * x-axis is worse than a coarser sample that spans it. Widening here costs no + * accuracy: exemplars keep their raw timestamps, and the chart re-buckets at the + * true granularity when it decides what to draw. + */ +export function exemplarScanBucketing( + granularity: SQLInterval, + dateRange: [Date, Date] | undefined, +): { interval: SQLInterval; perBucket: number } { + // Re-validate rather than trust the type: granularity reaches here from a URL + // param, and `interval` is spliced into the query as raw SQL by timeBucketExpr. + const safeGranularity: SQLInterval = SQLIntervalSchema.safeParse(granularity) + .success + ? granularity + : '1 minute'; + const granularitySeconds = convertGranularityToSeconds(safeGranularity); + const rangeSeconds = dateRange + ? (dateRange[1].getTime() - dateRange[0].getTime()) / 1000 + : 0; + if ( + granularitySeconds <= 0 || + !Number.isFinite(rangeSeconds) || + rangeSeconds <= 0 + ) { + return { interval: safeGranularity, perBucket: 1 }; + } + + // Divide by LIMIT - 1, not LIMIT: toStartOfInterval aligns to epoch multiples + // of the interval rather than to the range start, so a range that isn't a + // multiple of the widened bucket spills into one extra group. Leaving room + // for it keeps buckets * perBucket inside the trailing LIMIT — otherwise that + // LIMIT clips the newest bucket, the right-hand edge of the chart. + const bucketSeconds = Math.max( + granularitySeconds, + Math.ceil(rangeSeconds / (EXEMPLAR_QUERY_LIMIT - 1)), + ); + const buckets = Math.ceil(rangeSeconds / bucketSeconds) + 1; + return { + interval: + bucketSeconds === granularitySeconds + ? safeGranularity + : `${bucketSeconds} second`, + perBucket: Math.min( + EXEMPLAR_MAX_PER_BUCKET, + Math.max(1, Math.floor(EXEMPLAR_QUERY_LIMIT / buckets)), + ), + }; +} + +/** + * The exemplar-eligibility rule, in one place. An exemplar marker sits at a + * single trace's raw measurement on the chart's y-axis, so it is only + * attributable when the chart draws exactly one series of a compatible unit: + * + * - single series, no Group By — a marker can't be attributed (or scaled) across + * series, and a grouped scan pools exemplars from every group. + * - not a ratio — the exemplar value isn't on a ratio axis. + * - histogram, explicit-bucket or exponential — the exemplar value is a + * duration, which only shares the y-axis unit on a latency histogram + * (counts/gauges/rates are a different scale). Both kinds store exemplars in + * the same `Exemplars.*` columns, and the scan resolves its table from + * `metricTables[metricType]`, so neither the query nor the marker changes. + * + * The chart-editor toggle and the SQL renderer both delegate here so they can't + * drift apart. + */ +// Aggregations whose result is on the same scale as a single observation, and so +// share an axis with `Exemplars.Value`. A quantile, average, min or max of +// durations is itself a duration; a count is a number of observations and a sum is +// their total, both of which put the axis in a different unit — a duration marker +// clamped into a count domain reads as a real point on that scale. `none` is the +// raw value, so it qualifies. +const EXEMPLAR_COMPATIBLE_AGG_FNS = new Set([ + 'avg', + 'max', + 'min', + 'quantile', + 'last_value', + 'any', + 'none', +]); + +export function isExemplarEligible({ + seriesCount, + seriesReturnType, + metricType, + aggFn, + hasGroupBy, +}: { + seriesCount: number; + seriesReturnType?: 'ratio' | 'column'; + // Deliberately `string`, and compared by value below. tsup emits this + // declaration without importing MetricsDataType, so any annotation naming the + // enum resolves against the *consumer's* scope: in packages/app it binds to + // that package's unrelated legacy MetricsDataType, whose values are + // capitalised ('Histogram'), and the build fails on the mismatch. + // Callers must pass a common-utils MetricsDataType value (lowercase + // 'histogram'); a legacy-enum value would compile but never match. + metricType?: string; + /** + * The series' aggregation. `string` for the same tsup reason as `metricType`. + * Undefined is treated as ineligible: a caller that cannot say what it plots + * cannot promise the axis is a duration. + */ + aggFn?: string; + hasGroupBy: boolean; +}): boolean { + return ( + seriesCount === 1 && + seriesReturnType !== 'ratio' && + (metricType === MetricsDataType.Histogram || + metricType === MetricsDataType.ExponentialHistogram) && + aggFn != null && + EXEMPLAR_COMPATIBLE_AGG_FNS.has(aggFn) && + !hasGroupBy + ); +} + +/** + * PromQL functions whose result shares the y-axis unit with an exemplar's value + * (a duration). A marker's height is the linked trace's own measurement, so it + * is only honest on an axis measured in the same unit. + * + * Notably absent: `rate(..._bucket[5m])` and friends. A `_bucket` series holds + * observation *counts*, so plotting duration-valued markers on it puts, say, a + * 250ms trace at "250 requests/sec" — clampExemplarY would then pin it inside + * the axis and it would read as a real point on that scale. + */ +const PROMQL_DURATION_VALUED_CALL = + /^\s*(histogram_quantile|histogram_avg)\s*\(/; + +/** + * The PromQL counterpart to isExemplarEligible: whether the expression plots a + * duration, and so can carry exemplar markers on its axis. The PromQL editor's + * toggle and the exemplar fetch both delegate here so they can't drift apart. + * + * Deliberately narrow rather than a full parse, and deliberately erring towards + * `false`: a false negative hides the toggle on a chart that could have shown + * markers, while a false positive plots duration markers on an axis measured in + * something else and clamps them into it, which reads as real data. Other + * duration-valued shapes (`avg_over_time(latency_seconds[5m])`, a + * `_sum / _count` ratio) are not recognised yet — extend this pattern when one + * is wanted rather than loosening it to a substring test. + */ +export function isPromqlExemplarEligible( + expression: string | undefined, +): boolean { + if (!expression) return false; + const opening = PROMQL_DURATION_VALUED_CALL.exec(expression); + if (!opening) return false; + + // The call must span the WHOLE expression, not merely start it. Walking to the + // matching close paren is what distinguishes `histogram_quantile(...)` from + // `histogram_quantile(...) * 1000`: the latter's axis is milliseconds while its + // exemplars are still seconds, so the markers would plot 1000x low and clamp to + // the axis floor as though they were real values. Quotes are skipped so a paren + // inside a label matcher doesn't unbalance the count. + let depth = 0; + let quote: string | null = null; + for (let i = opening[0].length - 1; i < expression.length; i++) { + const c = expression.charAt(i); + if (quote) { + if (c === '\\') i++; + else if (c === quote) quote = null; + continue; + } + if (c === '"' || c === "'") quote = c; + else if (c === '(') depth++; + else if (c === ')') { + depth--; + // Closed the outermost call: eligible only if nothing but whitespace is + // left, i.e. its value is what the axis renders. + if (depth === 0) return expression.slice(i + 1).trim() === ''; + } + } + // Unbalanced parens — a half-typed expression. Not eligible. + return false; +} + +/** + * Builds a ClickHouse query that surfaces native exemplars stored on an OTel + * metric table (`Exemplars.TraceId/SpanId/Value/TimeUnix`). Returns null when + * the config is not a single-metric chart we can resolve a table for. + * + * Reuses `renderWhere` so the exemplar scan honors the exact same time range, + * metric-name, and user filters as the rendered series. Exemplars keep their own + * raw timestamp/value (the marker sits at the exemplar's own measurement); the + * bucket column exists only to spread the result set evenly across the range — + * `LIMIT n BY bucket` keeps the top `n` of every bucket rather than the top `n` + * of the whole range, which would return nothing but the spikes. + */ +export async function renderMetricExemplarsChartConfig( + chartConfig: ChartConfigWithOptDateRangeEx, + metadata: Metadata, +): Promise { + if ( + isRawSqlChartConfig(chartConfig) || + isPromqlChartConfig(chartConfig) || + !isMetricChartConfig(chartConfig) || + !Array.isArray(chartConfig.select) || + // Without a range, renderWhere emits no time predicate and this becomes a + // whole-table ARRAY JOIN plus sort, bounded only by the trailing LIMIT. The + // PromQL branch already refuses this case; match it. + chartConfig.dateRange == null + ) { + return null; + } + const { metricTables, select } = chartConfig; + const { metricType, metricName, metricNameSql, aggFn } = select[0] ?? {}; + // Shared eligibility rule (single, non-ratio, non-grouped histogram series) — + // see isExemplarEligible. A Group By in particular would pool exemplars from + // every group into one unattributable set. + if ( + !isExemplarEligible({ + seriesCount: select.length, + seriesReturnType: chartConfig.seriesReturnType, + metricType, + aggFn, + hasGroupBy: isUsingGroupBy(chartConfig), + }) + ) { + return null; + } + const table = + metricType && metricTables ? metricTables[metricType] : undefined; + if (!metricType || !metricName || !table) { + return null; + } + + // Build a config that points at the concrete metric-type table and carries + // the metric-name predicate alongside the user filters, then let renderWhere + // assemble the time filter + filters exactly as the main query does. The + // guards above narrow chartConfig to the metric builder config, so no cast. + // Bucket the scan so every part of the range gets a fair shot at a marker. + // 'auto' is resolved here rather than left to timeBucketExpr so the bucket + // column and the per-bucket limit agree on the same width. + const granularity: SQLInterval = + chartConfig.granularity && chartConfig.granularity !== 'auto' + ? chartConfig.granularity + : chartConfig.dateRange + ? convertDateRangeToGranularityString(chartConfig.dateRange) + : '1 minute'; + const { interval, perBucket } = exemplarScanBucketing( + granularity, + chartConfig.dateRange, + ); + // How far an exemplar's own timestamp can trail its data point is the metric's + // collection interval, which we don't know here. The chart's granularity is a + // proxy, not the same thing — and on a short window it resolves to 15s or 30s, + // finer than a typical scrape, which would leave the newest exemplars out of + // reach again. So take the larger of the granularity and a one-minute floor. + // The exact ex_TimeUnix bound below plus `LIMIT n BY bucket` trim the overshoot, + // so erring wide is close to free. The derived bucket width is deliberately not + // used: exemplarScanBucketing can stretch it to many minutes on a multi-day + // range, which over-fetches for no benefit. + const COLLECTION_INTERVAL_FLOOR_MS = 60_000; + const collectionIntervalMs = Math.max( + convertGranularityToSeconds(granularity) * 1000, + COLLECTION_INTERVAL_FLOOR_MS, + ); + + const [rangeStart, rangeEnd] = chartConfig.dateRange; + + // renderWhere bounds the *row's* TimeUnix, but the value we project and bucket + // is the ARRAY JOINed `ex_TimeUnix`. Those are not the same instant: an + // exemplar is attached to the data point that covers it, so its own timestamp + // falls in the collection interval *before* that row's. So the row bound is + // widened forward by one interval — otherwise an exemplar inside the window + // whose data point lands just after it is never returned — and the exact + // window is then applied to `ex_TimeUnix` below, which also drops the + // pre-window exemplars the widened row bound now lets through. + const rowBoundEnd = new Date(rangeEnd.getTime() + collectionIntervalMs); + + const whereConfig: BuilderChartConfigWithOptDateRangeEx = { + ...chartConfig, + dateRange: [rangeStart, rowBoundEnd], + from: { ...chartConfig.from, tableName: table }, + timestampValueExpression: + chartConfig.timestampValueExpression || DEFAULT_METRIC_TABLE_TIME_COLUMN, + // Keep the original select so renderWhere applies the series' aggCondition — + // otherwise the exemplar scan would surface traces from other series (e.g. + // other services/routes/tenants) that share the same metric name. + }; + + const where = await renderWhere(whereConfig, metadata); + const from = renderFrom({ from: whereConfig.from }); + + // The metric-name predicate is REQUIRED and must always be ANDed. Appending it + // to `chartConfig.filters` would subject it to the chart's + // `filtersLogicalOperator`, so an 'OR' filter group would produce + // `userFilterA OR userFilterB OR MetricName = ...` and let the exemplar scan + // surface traces from other metrics. AND it separately from the user filters. + const metricNameCondition = createMetricNameFilter(metricName, metricNameSql); + + const bucketExpr = timeBucketExpr({ + interval, + timestampValueExpression: 'ex_TimeUnix', + dateRange: chartConfig.dateRange, + alias: 'bucket', + }); + + return concatChSql(' ', [ + chSql`SELECT + toUnixTimestamp64Milli(ex_TimeUnix) AS timestamp, + ex_Value AS value, + ex_TraceId AS traceId, + ex_SpanId AS spanId, + ${bucketExpr}`, + chSql`FROM ${from}`, + chSql`ARRAY JOIN + \`Exemplars.TimeUnix\` AS ex_TimeUnix, + \`Exemplars.Value\` AS ex_Value, + \`Exemplars.TraceId\` AS ex_TraceId, + \`Exemplars.SpanId\` AS ex_SpanId`, + chSql`WHERE ${where.sql ? where : chSql`1 = 1`} AND (${metricNameCondition}) AND notEmpty(ex_TraceId) AND ex_TimeUnix >= fromUnixTimestamp64Milli(${{ Int64: rangeStart.getTime() }}) AND ex_TimeUnix <= fromUnixTimestamp64Milli(${{ Int64: rangeEnd.getTime() }})`, + // Within a bucket the slowest traces are the interesting ones; across + // buckets we want coverage, so the per-bucket limit — not the value order — + // is what bounds the result set. exemplarScanBucketing keeps + // buckets * perBucket under the trailing LIMIT, which is a backstop only. + // + // Consequence worth knowing: this hands the client the top `perBucket` + // rows per bucket, so the chart's 2σ spread sampler can only choose among a + // bucket's slowest few — on this path the overlay is a narrow max envelope, + // not the full within-bucket distribution the Prometheus path can show. + // Widening that needs within-bucket sampling in SQL (a window function), + // which is a bigger change than this one. + chSql`ORDER BY bucket ASC, value DESC`, + // Raw rather than a bound parameter: `LIMIT n BY` is parsed before + // parameter substitution. `perBucket` (not the sibling `interval`) is a + // clamped integer from exemplarScanBucketing, never user input. + chSql`LIMIT ${{ UNSAFE_RAW_SQL: String(perBucket) }} BY bucket`, + chSql`LIMIT ${{ Int32: EXEMPLAR_QUERY_LIMIT }}`, + ]); +} + // EditForm -> translateToQueriedChartConfig -> QueriedChartConfig // renderFn(QueriedChartConfig) -> sql // query(sql) -> data diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 9ef47c1da4..f387b6a717 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1229,6 +1229,15 @@ const SharedChartSettingsSchema = z.object({ // number tiles have no time dimension to bucket). Other display types // ignore the field. Kept at shared level mirroring `color` / `colorRules`. backgroundChart: BackgroundChartSchema.optional(), + // Opt-in: overlay exemplar markers (individual traces linked to the series) + // on time charts. Sourced natively for metric/PromQL sources and generated + // per time bucket for trace sources. The UI gates the toggle on supported + // source kinds; off by default so the extra exemplar query never runs. + enableExemplars: z.boolean().optional(), + // Trace source that an exemplar's trace id resolves against — used to fetch + // hover metadata and to deep-link straight to the trace. When unset, the + // chart source's linked `traceSourceId` is used as a fallback. + exemplarTraceSourceId: z.string().optional(), // Zebra striping for table tiles: when true, the renderer tints alternating // rows so wide tables are easier to scan across. Applies to any table tile // (builder or raw SQL); the striping is purely presentational and keys off @@ -1366,6 +1375,30 @@ export const ChartConfigSchema = z.union([ export type ChartConfig = z.infer; +/** + * A single exemplar: an individual data point overlaid on a time chart that + * links back to a trace. Shared shape for both backends — metric/PromQL + * sources surface native exemplars, trace sources generate them per bucket. + */ +// Both numbers are `.finite()` because both reach the renderer as SVG +// coordinates: a non-finite timestamp collapses every affected exemplar into one +// `@NaN` bucket and emits a NaN x-coordinate, and a single Infinity value makes +// the 2σ spread NaN, silently switching the thinning rule off chart-wide. The +// normalizers parse untrusted upstream bodies through this schema, so rejecting +// here keeps both out of the render path. +export const ExemplarSchema = z.object({ + timestamp: z.number().finite(), // epoch ms, x-position on the chart + value: z.number().finite(), // exemplar's own value (metric Value / trace Duration in ms) + traceId: z.string(), + spanId: z.string().optional(), + // Matches the series the exemplar belongs to (LineData.displayName) so the + // overlay can attach markers to the right line. Undefined => applies to all. + groupKey: z.string().optional(), + attributes: z.record(z.string()).optional(), +}); + +export type Exemplar = z.infer; + export type DateRange = { dateRange: [Date, Date]; dateRangeStartInclusive?: boolean; // default true @@ -1669,6 +1702,9 @@ export const TeamClickHouseSettingsSchema = z.object({ metadataMaxRowsToRead: z.number().optional(), parallelizeWhenPossible: z.boolean().optional(), filterKeysFetchLimit: z.number().optional(), + // Target number of exemplar markers shown per chart (0 = unlimited). Not a + // ClickHouse setting, but lives in the same team-config bag for reuse. + maxExemplars: z.number().optional(), }); /** Accepts null to unset (reset to default) a setting. */ @@ -1679,6 +1715,10 @@ export const TeamClickHouseSettingsUpdateSchema = z.object({ metadataMaxRowsToRead: z.number().nullish(), parallelizeWhenPossible: z.boolean().nullish(), filterKeysFetchLimit: z.number().nullish(), + // Bounded here, not just in the settings form: the form's min/max are + // client-side only, and a fractional or negative value reaches the chart's + // marker-budget arithmetic directly. + maxExemplars: z.number().int().min(0).max(1000).nullish(), }); export type TeamClickHouseSettingsUpdate = z.infer< typeof TeamClickHouseSettingsUpdateSchema diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index ecd794a456..e84cad931a 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -135,6 +135,9 @@ connectors: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector + v__OTEL_COLLECTOR_VERSION__ extensions: # Core diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh index 938aaeb4d7..135eca7ef5 100755 --- a/scripts/dev-env.sh +++ b/scripts/dev-env.sh @@ -55,6 +55,7 @@ HDX_DEV_OTEL_GRPC_PORT=$((30800 + HDX_DEV_SLOT)) HDX_DEV_OTEL_HTTP_PORT=$((30900 + HDX_DEV_SLOT)) HDX_DEV_OTEL_METRICS_PORT=$((31000 + HDX_DEV_SLOT)) HDX_DEV_OTEL_JSON_HTTP_PORT=$((31100 + HDX_DEV_SLOT)) +HDX_DEV_PROMETHEUS_PORT=$((31200 + HDX_DEV_SLOT)) # --- Docker Compose project name (unique per slot) --- HDX_DEV_PROJECT="hdx-dev-${HDX_DEV_SLOT}" @@ -80,6 +81,7 @@ export HDX_DEV_OTEL_GRPC_PORT export HDX_DEV_OTEL_HTTP_PORT export HDX_DEV_OTEL_METRICS_PORT export HDX_DEV_OTEL_JSON_HTTP_PORT +export HDX_DEV_PROMETHEUS_PORT export HDX_DEV_PROJECT export NX_CACHE_DIRECTORY @@ -111,6 +113,7 @@ cat > "${HDX_DEV_SLOTS_DIR}/${HDX_DEV_SLOT}.json" < ClickHouse). Metric + * exemplars are written to ClickHouse directly because OTel JS (1.30) does not + * yet emit metric exemplars and our collector has no spanmetrics connector. + * See README for the spanmetrics upgrade path. + */ +'use strict'; + +const { + trace, + ROOT_CONTEXT, + SpanStatusCode, + SpanKind, + diag, + DiagConsoleLogger, + DiagLogLevel, +} = require('@opentelemetry/api'); +// Surface OTLP export failures (auth, connection) instead of dropping silently. +diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR); +const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { + OTLPTraceExporter, +} = require('@opentelemetry/exporter-trace-otlp-proto'); +const { Resource } = require('@opentelemetry/resources'); +const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions'); +const net = require('net'); + +// ── Config ────────────────────────────────────────────────────────── +const cfg = { + // OTLP HTTP base (port 4318). Traces are POSTed to /v1/traces. + otlpEndpoint: + process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://otel-collector:4318', + backfillMinutes: Number(process.env.GEN_BACKFILL_MINUTES || 30), + ratePerSec: Number(process.env.GEN_RATE_PER_SEC || 20), + // HyperDX collectors enforce bearer-token auth on OTLP ingest. Default to the + // dev INGESTION_API_KEY (packages/api/.env.development); override per env. + otlpApiKey: + process.env.GEN_OTLP_API_KEY || 'super-secure-ingestion-api-key', +}; + +// ── Weighted / uniform random helpers ─────────────────────────────── +function pickWeighted(choices) { + const total = choices.reduce((s, c) => s + c[1], 0); + let r = Math.random() * total; + for (const [value, weight] of choices) { + r -= weight; + if (r <= 0) return value; + } + return choices[choices.length - 1][0]; +} +function pickUniform(arr) { + return arr[Math.floor(Math.random() * arr.length)]; +} +// Box-Muller gaussian, clamped to >= 1ms +function gaussianMs(mean, stddev) { + let u = 0; + let v = 0; + while (u === 0) u = Math.random(); + while (v === 0) v = Math.random(); + const z = Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); + return Math.max(1, mean + z * stddev); +} + +// ── Attribute pools (cardinality + diversity) ─────────────────────── +const routes = [ + ['/api/users', 25], + ['/api/orders', 15], + ['/cart/checkout', 10], + ['/api/search', 25], + ['/api/products', 15], + ['/api/auth', 10], +]; +const methods = [ + ['GET', 60], + ['POST', 25], + ['PUT', 10], + ['DELETE', 5], +]; +const regions = [ + ['us-east-1', 40], + ['us-west-2', 30], + ['eu-west-1', 20], + ['ap-southeast-1', 10], +]; +const buildIDs = [ + ['build-7a1', 35], + ['build-7a2', 35], + ['build-7a3', 30], +]; +const platforms = [ + ['web', 50], + ['ios', 30], + ['android', 20], +]; +const featureFlags = [ + ['new-checkout-flow', 15], + ['dark-launch-search', 10], + ['legacy', 75], +]; +const tenants = [ + 'tenant-acme', + 'tenant-globex', + 'tenant-initech', + 'tenant-umbrella', +]; +const pods = Array.from({ length: 8 }, (_, i) => `pod-abc-${i + 1}`); +const userId = () => `user-${String(Math.floor(Math.random() * 500) + 1).padStart(4, '0')}`; + +function routeToService(route) { + switch (route) { + case '/api/orders': + case '/cart/checkout': + return 'order-service'; + case '/api/users': + case '/api/auth': + return 'user-service'; + case '/api/search': + case '/api/products': + return 'search-service'; + default: + return 'unknown-service'; + } +} +function serviceToDb(svc) { + if (svc === 'order-service' || svc === 'user-service') return 'postgres'; + if (svc === 'search-service') return 'elasticsearch'; + return 'none'; +} + +// ── Per-service tracer providers (ServiceName = resource attr) ─────── +const SERVICE_NAMES = [ + 'api-gateway', + 'order-service', + 'user-service', + 'search-service', + 'payment-service', + 'notification-service', +]; +const exporter = new OTLPTraceExporter({ + url: `${cfg.otlpEndpoint.replace(/\/$/, '')}/v1/traces`, + // HyperDX's bearertokenauth uses scheme:'' — send the raw token, no prefix. + headers: cfg.otlpApiKey ? { authorization: cfg.otlpApiKey } : undefined, +}); +const providers = new Map(); +const tracers = new Map(); +for (const name of SERVICE_NAMES) { + const provider = new NodeTracerProvider({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: name, + 'service.version': '1.0.0', + }), + spanProcessors: [ + // Large queue so the backfill burst isn't dropped before export. + new BatchSpanProcessor(exporter, { + maxExportBatchSize: 512, + maxQueueSize: 32768, + scheduledDelayMillis: 500, + }), + ], + }); + providers.set(name, provider); + tracers.set(name, provider.getTracer('telemetry-generator')); +} +const tracer = name => tracers.get(name); + +const jitter = () => Math.random() * 2; // ms + +function commonAttrs(a) { + return { + 'http.method': a.method, + 'http.route': a.route, + 'user.id': a.uid, + 'app.tenant_id': a.tenant, + 'host.region': a.region, + 'app.build_id': a.buildID, + 'app.platform': a.platform, + 'app.feature_flag': a.featureFlag, + 'k8s.pod.name': a.pod, + }; +} +function markError(span, type, message) { + span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.addEvent('exception', { + 'exception.type': type, + 'exception.message': message, + }); +} + +// ── Trace emission ────────────────────────────────────────────────── +// Emits a trace; the collector's spanmetrics connector derives the +// request-duration metric (with exemplars) from these spans. +function emitTrace(tsMs) { + const a = { + route: pickWeighted(routes), + method: pickWeighted(methods), + region: pickWeighted(regions), + buildID: pickWeighted(buildIDs), + platform: pickWeighted(platforms), + featureFlag: pickWeighted(featureFlags), + tenant: pickUniform(tenants), + uid: userId(), + pod: pickUniform(pods), + }; + const svc = routeToService(a.route); + const sc = detectScenario(a); + const plan = SCENARIOS[sc](a, svc); + + const root = tracer('api-gateway').startSpan( + `${a.method} ${a.route}`, + { + kind: SpanKind.SERVER, + startTime: tsMs, + attributes: { ...commonAttrs(a), 'http.status_code': plan.statusCode }, + }, + ROOT_CONTEXT, + ); + if (plan.error) markError(root, plan.error.type, plan.error.message); + else root.setStatus({ code: SpanStatusCode.OK }); + + const rootCtx = trace.setSpan(ROOT_CONTEXT, root); + const svcStart = tsMs + jitter(); + const svcSpan = tracer(svc).startSpan( + `${svc}.handle`, + { kind: SpanKind.INTERNAL, startTime: svcStart, attributes: svcAttrs(svc, a) }, + rootCtx, + ); + if (plan.error) markError(svcSpan, plan.error.type, plan.error.message); + const svcCtx = trace.setSpan(rootCtx, svcSpan); + + // Leaf spans (db / cache / downstream) per the scenario plan. + let cursor = svcStart + jitter(); + for (const leaf of plan.leaves) { + const leafSvc = leaf.service || svc; + const ls = tracer(leafSvc).startSpan( + leaf.name, + { + kind: SpanKind.CLIENT, + startTime: cursor, + attributes: { + [ATTR_SERVICE_NAME]: leafSvc, + 'host.region': a.region, + ...(leaf.attrs || {}), + }, + }, + svcCtx, + ); + if (leaf.error) markError(ls, leaf.error.type, leaf.error.message); + ls.end(cursor + leaf.durMs); + cursor += leaf.durMs + 1; + } + + svcSpan.end(svcStart + plan.svcDurMs); + root.end(tsMs + plan.rootDurMs); +} + +function svcAttrs(svc, a) { + return { + [ATTR_SERVICE_NAME]: svc, + 'http.route': a.route, + 'host.region': a.region, + 'app.build_id': a.buildID, + 'app.feature_flag': a.featureFlag, + 'app.platform': a.platform, + 'user.id': a.uid, + 'app.tenant_id': a.tenant, + 'k8s.pod.name': a.pod, + }; +} + +// ── Scenarios ─────────────────────────────────────────────────────── +// Each returns { rootDurMs, svcDurMs, statusCode, error?, leaves:[{name,durMs,service?,attrs?,error?}] } +const PAYMENT_TIMEOUT_RATE = 0.05; +const AUTH_LEAK_ERROR_RATE = 0.1; + +function detectScenario(a) { + const svc = routeToService(a.route); + if ( + a.route === '/cart/checkout' && + a.region === 'us-west-2' && + Math.random() < PAYMENT_TIMEOUT_RATE + ) + return 'paymentTimeout'; + if ( + a.route === '/cart/checkout' && + a.featureFlag === 'new-checkout-flow' && + a.region === 'eu-west-1' + ) + return 'slowCheckout'; + if (a.route === '/api/orders' && a.platform === 'ios' && a.buildID === 'build-7a3') + return 'iosOrderErrors'; + if ( + a.tenant === 'tenant-initech' && + a.featureFlag === 'dark-launch-search' && + a.route === '/api/search' + ) + return 'initechSearch'; + if ( + a.route === '/api/auth' && + a.buildID === 'build-7a3' && + (a.pod === 'pod-abc-7' || a.pod === 'pod-abc-8') + ) + return 'authMemoryLeak'; + if (a.region === 'ap-southeast-1' && svc === 'user-service') + return 'redisTimeoutApac'; + if (a.tenant === 'tenant-umbrella' && a.region === 'eu-west-1') + return 'umbrellaCompliance'; + return 'normal'; +} + +function normalStatus() { + const r = Math.random(); + if (r < 0.95) return 200; + if (r < 0.98) return 201; + return 404; +} + +const SCENARIOS = { + normal: (a, svc) => { + const db = serviceToDb(svc); + const leaves = []; + if (db === 'postgres') + leaves.push({ + name: 'postgres.query', + durMs: gaussianMs(10, 5), + attrs: { 'db.system': 'postgres', 'db.statement': `SELECT * FROM ${a.route.slice(5)}` }, + }); + else if (db === 'elasticsearch') + leaves.push({ + name: 'elasticsearch.search', + durMs: gaussianMs(10, 5), + attrs: { 'db.system': 'elasticsearch' }, + }); + if (a.route === '/cart/checkout') + leaves.push({ + name: 'payment-service.charge', + service: 'payment-service', + durMs: gaussianMs(10, 4), + }); + return { rootDurMs: gaussianMs(40, 20), svcDurMs: gaussianMs(25, 12), statusCode: normalStatus(), leaves }; + }, + slowCheckout: a => { + const leaves = []; + const n = 3 + Math.floor(Math.random() * 3); + for (let i = 0; i < n; i++) + leaves.push({ + name: 'postgres.query', + durMs: gaussianMs(200, 40), + attrs: { 'db.system': 'postgres', 'db.statement': 'SELECT * FROM orders WHERE id = ?' }, + }); + leaves.push({ name: 'payment-service.charge', service: 'payment-service', durMs: gaussianMs(200, 50) }); + return { rootDurMs: gaussianMs(1500, 400), svcDurMs: gaussianMs(1200, 350), statusCode: 200, leaves }; + }, + iosOrderErrors: () => ({ + rootDurMs: gaussianMs(250, 60), + svcDurMs: gaussianMs(100, 30), + statusCode: 500, + error: { type: 'ValidationError', message: 'malformed request body' }, + leaves: [], + }), + redisTimeoutApac: a => ({ + rootDurMs: gaussianMs(650, 120), + svcDurMs: gaussianMs(580, 100), + statusCode: 200, + leaves: [ + { name: 'redis.get', durMs: gaussianMs(550, 100), attrs: { 'db.system': 'redis', 'db.statement': `GET user:session:${a.uid}` } }, + { name: 'postgres.query', durMs: gaussianMs(30, 10), attrs: { 'db.system': 'postgres' } }, + ], + }), + initechSearch: () => ({ + rootDurMs: gaussianMs(3000, 500), + svcDurMs: gaussianMs(2800, 450), + statusCode: 500, + error: { type: 'TimeoutError', message: 'elasticsearch timeout' }, + leaves: [ + { + name: 'elasticsearch.search', + durMs: gaussianMs(2500, 400), + attrs: { 'db.system': 'elasticsearch' }, + error: { type: 'IOException', message: 'read tcp: i/o timeout' }, + }, + ], + }), + authMemoryLeak: a => { + const err = Math.random() < AUTH_LEAK_ERROR_RATE; + return { + rootDurMs: gaussianMs(800, 200), + svcDurMs: gaussianMs(700, 180), + statusCode: err ? 503 : 200, + error: err ? { type: 'ServiceUnavailableError', message: 'GC overhead' } : undefined, + leaves: [{ name: 'redis.get', durMs: gaussianMs(600, 150), attrs: { 'db.system': 'redis', 'db.statement': `GET auth:token:${a.uid}` } }], + }; + }, + paymentTimeout: () => ({ + rootDurMs: gaussianMs(5000, 500), + svcDurMs: gaussianMs(4800, 450), + statusCode: 504, + error: { type: 'TimeoutError', message: 'gateway timeout' }, + leaves: [ + { name: 'postgres.query', durMs: gaussianMs(20, 8), attrs: { 'db.system': 'postgres' } }, + { + name: 'payment-service.charge', + service: 'payment-service', + durMs: gaussianMs(4500, 300), + error: { type: 'TimeoutError', message: 'context deadline exceeded' }, + }, + ], + }), + umbrellaCompliance: (a, svc) => { + const overhead = gaussianMs(150, 40); + const db = serviceToDb(svc); + const leaves = [{ name: 'compliance.data_residency_check', durMs: overhead, attrs: { 'app.tenant_id': a.tenant } }]; + if (db === 'postgres') leaves.push({ name: 'postgres.query', durMs: gaussianMs(10, 5), attrs: { 'db.system': 'postgres' } }); + else if (db === 'elasticsearch') leaves.push({ name: 'elasticsearch.search', durMs: gaussianMs(10, 5), attrs: { 'db.system': 'elasticsearch' } }); + return { rootDurMs: gaussianMs(40, 20) + overhead, svcDurMs: gaussianMs(40, 20) + overhead, statusCode: normalStatus(), leaves }; + }, +}; + +// The collector's OTLP receiver is configured dynamically via OpAMP, so the +// port isn't open the instant the container starts — it appears only after the +// collector fetches its remote config. Wait for it before backfilling, else the +// initial burst hits ECONNREFUSED and is lost. +function waitForCollector(timeoutMs = 120_000) { + const u = new URL(cfg.otlpEndpoint); + const port = Number(u.port) || (u.protocol === 'https:' ? 443 : 80); + const host = u.hostname; + const deadline = Date.now() + timeoutMs; + const tryOnce = () => + new Promise(resolve => { + const sock = net.connect({ host, port }, () => { + sock.destroy(); + resolve(true); + }); + sock.on('error', () => { + sock.destroy(); + resolve(false); + }); + sock.setTimeout(2000, () => { + sock.destroy(); + resolve(false); + }); + }); + return (async () => { + while (Date.now() < deadline) { + if (await tryOnce()) return true; + await new Promise(r => setTimeout(r, 2000)); + } + return false; + })(); +} + +// ── Main ──────────────────────────────────────────────────────────── +async function main() { + console.log( + `telemetry-generator starting: otlp=${cfg.otlpEndpoint} ` + + `backfill=${cfg.backfillMinutes}m rate=${cfg.ratePerSec}/s`, + ); + + console.log('waiting for collector OTLP endpoint...'); + const ready = await waitForCollector(); + console.log(ready ? 'collector reachable' : 'collector wait timed out, proceeding anyway'); + + // Backfill historical traces. The spanmetrics connector derives the + // request-duration metric (with exemplars) from these spans. + const now = Date.now(); + const backfillStart = now - cfg.backfillMinutes * 60_000; + const backfillCount = cfg.backfillMinutes * 60 * cfg.ratePerSec; + console.log(`backfilling ~${backfillCount} requests over ${cfg.backfillMinutes}m...`); + for (let i = 0; i < backfillCount; i++) { + const ts = backfillStart + Math.random() * (now - backfillStart); + emitTrace(ts); + // Periodically yield so the span exporter can drain its queue instead of + // overflowing (which would drop traces, and thus their exemplars). + if (i % 500 === 0) await new Promise(r => setTimeout(r, 25)); + } + // Let the final span batches export before we declare backfill done. + await new Promise(r => setTimeout(r, 3000)); + console.log('backfill complete, starting live emission...'); + + // Live emission. + const intervalMs = Math.max(1, Math.floor(1000 / cfg.ratePerSec)); + setInterval(() => emitTrace(Date.now()), intervalMs); + + const shutdown = async () => { + console.log('shutting down...'); + await Promise.all( + [...providers.values()].map(p => p.shutdown().catch(() => {})), + ); + process.exit(0); + }; + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); +} + +main().catch(e => { + console.error(e); + process.exit(1); +});