diff --git a/.changeset/event-source-formulas.md b/.changeset/event-source-formulas.md new file mode 100644 index 0000000000..e6d3a42023 --- /dev/null +++ b/.changeset/event-source-formulas.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +--- + +Formulas now work on log and trace sources, not just metrics. Time series, table and number charts on event sources can define derived series via letter-ref arithmetic expressions (e.g. `A / B * 100`), with the same editor controls (Add Formula, series letter badges, Show input series) previously offered only on metric sources. Event formulas compile inline into the chart's single-scan SELECT — no per-series query fan-out — with the same missing-data semantics as the existing events ratio toggle. diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index b02bcd3ad7..1a3ab8aafb 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -70,6 +70,16 @@ const traceSource: TSource = { durationPrecision: 9, }; +const sessionSource: TSource = { + id: 'source-session', + name: 'Session Source', + kind: SourceKind.Session, + connection: 'conn-1', + from: { databaseName: 'db', tableName: 'sessions' }, + timestampValueExpression: 'Timestamp', + traceSourceId: 'source-trace', +}; + const seriesItem = { aggFn: 'count' as const, valueExpression: '*', @@ -1746,7 +1756,7 @@ describe('metric formulas (HDX-5080)', () => { ]); }); - it('skips formula validation for non-metric sources (formulas are stripped on save)', () => { + it('validates formulas for log event sources', () => { const setError = jest.fn(); const errors = validateChartForm( makeMetricForm({ @@ -1757,6 +1767,36 @@ describe('metric formulas (HDX-5080)', () => { logSource, setError, ); + expect(errors).toEqual([ + expect.objectContaining({ path: 'formulas.0.expression' }), + ]); + }); + + it('accepts a valid formula on a trace source', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeMetricForm({ + source: 'source-trace', + series: [seriesItem, seriesItem], + formulas: [{ expression: 'A / B * 100' }], + }), + traceSource, + setError, + ); + expect(errors).toHaveLength(0); + }); + + it('skips formula validation for formula-incapable source kinds (formulas are stripped on save)', () => { + const setError = jest.fn(); + const errors = validateChartForm( + makeMetricForm({ + source: 'source-session', + series: [seriesItem], + formulas: [{ expression: 'A +' }], + }), + sessionSource, + setError, + ); expect(errors).toHaveLength(0); }); @@ -1897,7 +1937,7 @@ describe('metric formulas (HDX-5080)', () => { expect(saved.showOperandSeries).toBeUndefined(); }); - it('strips formulas for non-metric sources', () => { + it('retains formulas for a log event source', () => { const saved = savedBuilderConfig( makeMetricForm({ source: 'source-log', @@ -1907,6 +1947,20 @@ describe('metric formulas (HDX-5080)', () => { }), logSource, ); + expect(saved.formulas).toEqual([{ expression: 'A * 100' }]); + expect(saved.showOperandSeries).toBe(false); + }); + + it('strips formulas for formula-incapable source kinds', () => { + const saved = savedBuilderConfig( + makeMetricForm({ + source: 'source-session', + series: [seriesItem], + formulas: [{ expression: 'A * 100' }], + showOperandSeries: false, + }), + sessionSource, + ); expect(saved.formulas).toBeUndefined(); expect(saved.showOperandSeries).toBeUndefined(); }); diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 87a8499c6b..ae155c8a2e 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -83,13 +83,13 @@ function normalizeChartConfig< >, >(config: C, source: TSource): C { const isMetricSource = source.kind === SourceKind.Metric; - // Formulas (HDX-5080) only render on metric sources, and only through the - // composed multi-series metric query shapes (time series / table / number). - // Strip them elsewhere so a source or display-type switch can't persist a - // config the renderer would reject. The form state keeps them, so switching - // back restores the formula rows. + // Formulas (HDX-5080) render on metric and event (log/trace) sources, and + // only on the display types the formula query paths support (time series / + // table / number). Strip them elsewhere so a source or display-type switch + // can't persist a config the renderer would reject. The form state keeps + // them, so switching back restores the formula rows. const keepFormulas = - isMetricSource && + isFormulaSourceKind(source.kind) && (config.formulas?.length ?? 0) > 0 && isFormulaDisplayType(config.displayType); return { @@ -174,9 +174,9 @@ const isCustomOrderByDisplayType = ( displayType === DisplayType.Pie; /** - * Display types that can carry metric formulas (HDX-5080) — the shapes the - * composed multi-series metric query renders. Mirrors the "Add Formula" - * gating in ChartEditorControls. + * Display types that can carry formulas (HDX-5080) — the shapes the formula + * query paths render (composed multi-series for metrics, inline single-scan + * for events). Mirrors the "Add Formula" gating in ChartEditorControls. */ export const isFormulaDisplayType = ( displayType: DisplayType | undefined, @@ -190,6 +190,19 @@ export const isFormulaDisplayType = ( displayType === DisplayType.Table || displayType === DisplayType.Number; +/** + * Source kinds that can carry formulas: metric sources (rendered via the + * composed multi-series metric query) and event sources (log/trace, compiled + * inline in the single-scan SELECT — see renderSelectListWithFormulas in + * common-utils). + */ +export const isFormulaSourceKind = ( + kind: SourceKind | undefined, +): kind is SourceKind.Metric | SourceKind.Log | SourceKind.Trace => + kind === SourceKind.Metric || + kind === SourceKind.Log || + kind === SourceKind.Trace; + export function convertFormStateToSavedChartConfig( form: ChartEditorFormState, source: TSource | undefined, @@ -485,13 +498,13 @@ export const validateChartForm = ( }); } - // Validate metric formulas (HDX-5080) with the structured validator the - // query renderer uses, so a bad expression is caught here rather than at - // render time. Only applies where formulas survive normalization (metric - // source + formula-capable display type). + // Validate formulas (HDX-5080) with the structured validator the query + // renderer uses, so a bad expression is caught here rather than at render + // time. Only applies where formulas survive normalization (formula-capable + // source kind + display type). if ( !isRawSqlChart && - source?.kind === SourceKind.Metric && + isFormulaSourceKind(source?.kind) && isFormulaDisplayType(form.displayType) && Array.isArray(form.formulas) ) { @@ -578,7 +591,7 @@ export const validateChartForm = ( !isRawSqlChart && Array.isArray(form.series) && form.displayType === DisplayType.Number && - !(form.formulas?.length && source?.kind === SourceKind.Metric) && + !(form.formulas?.length && isFormulaSourceKind(source?.kind)) && form.series.length > (form.seriesReturnType === 'ratio' ? 2 : 1) ) { errors.push({ diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx index 2d327d1f4b..4f26493cf0 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx @@ -30,7 +30,10 @@ import { ChartEditorFormState, SavedChartConfigWithSelectArray, } from '@/components/ChartEditor/types'; -import { isFormulaDisplayType } from '@/components/ChartEditor/utils'; +import { + isFormulaDisplayType, + isFormulaSourceKind, +} from '@/components/ChartEditor/utils'; import MVOptimizationIndicator from '@/components/MaterializedViews/MVOptimizationIndicator'; import SearchWhereInput from '@/components/SearchInput/SearchWhereInput'; import SourceSchemaPreview, { @@ -110,10 +113,10 @@ export function ChartEditorControls({ openDisplaySettings, openHeatmapSettings, }: ChartEditorControlsProps) { - // Metric formulas (HDX-5080): derived series computed from the chart's - // series via letter-ref arithmetic expressions. Metric sources only, and - // only on display types the composed multi-series metric query renders - // (time series / table / number). + // Formulas (HDX-5080): derived series computed from the chart's series via + // letter-ref arithmetic expressions. Metric and event (log/trace) sources, + // and only on display types the formula query paths render (time series / + // table / number). const { fields: formulaFields, append: appendFormula, @@ -122,8 +125,7 @@ export function ChartEditorControls({ const hasFormulas = formulaFields.length > 0; const showOperandSeries = useWatch({ control, name: 'showOperandSeries' }); const sourceSupportsFormulas = - tableSource?.kind === SourceKind.Metric && - isFormulaDisplayType(displayType); + isFormulaSourceKind(tableSource?.kind) && isFormulaDisplayType(displayType); // Formulas and the ratio toggle are mutually exclusive (formulas supersede // ratio in the renderer, so the editor never lets both be set). Number // tiles display a single value, so they take exactly one formula. diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx index 1c88cc8a09..137806906b 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx @@ -42,6 +42,7 @@ import { ChartEditorFormState, SavedChartConfigWithSelectArray, } from '@/components/ChartEditor/types'; +import { isFormulaSourceKind } from '@/components/ChartEditor/utils'; import { CheckBoxControlled, TextInputControlled, @@ -258,9 +259,9 @@ export function ChartSeriesEditor({ {/* Formula series reference (HDX-5080): formulas address series positionally by letter (`A` = series 1, ...), so surface the - letter on each row. Metric sources only — formulas are only - supported there. */} - {tableSource?.kind === SourceKind.Metric && ( + letter on each row of formula-capable sources (metric and + log/trace events). */} + {isFormulaSourceKind(tableSource?.kind) && ( ({ default: () =>
Search Input
, })); +// The sample-events panel (rendered for event sources) reads Next router +// query state via nuqs, which isn't mounted in this test environment. +jest.mock('@/components/DBSqlRowTableWithSidebar', () => ({ + __esModule: true, + default: () =>
SQL Row Table
, +})); + jest.mock('../../MaterializedViews/MVOptimizationIndicator', () => ({ __esModule: true, default: () =>
MV Indicator
, @@ -907,7 +914,8 @@ describe('DBEditTimeChartForm - Metric formulas', () => { expect(onSave.mock.calls[0][0].showOperandSeries).toBe(false); }); - it('does not show formula controls for non-metric sources', () => { + it('shows formula controls and series letter badges for log event sources', async () => { + const onSave = jest.fn(); mockUseSourceData({ id: 'log-source', kind: SourceKind.Log, @@ -917,10 +925,55 @@ describe('DBEditTimeChartForm - Metric formulas', () => { timestampValueExpression: 'Timestamp', }); + const countSeries = { + aggFn: 'count' as const, + aggCondition: '', + aggConditionLanguage: 'lucene' as const, + valueExpression: '', + }; renderComponent({ chartConfig: { ...defaultChartConfig, source: 'log-source', + select: [ + { ...countSeries, aggCondition: 'SeverityText:error' }, + countSeries, + ], + }, + onSave, + }); + + const badges = screen.getAllByTestId('series-ref-badge'); + expect(badges.map(b => b.textContent)).toEqual(['A', 'B']); + + await userEvent.click(screen.getByTestId('add-formula-button')); + await userEvent.type( + screen.getByTestId('formula-expression-input'), + 'A / B * 100', + ); + await userEvent.click(screen.getByTestId('chart-save-button')); + + await waitFor(() => expect(onSave).toHaveBeenCalledTimes(1)); + expect(onSave.mock.calls[0][0].formulas).toEqual([ + { expression: 'A / B * 100', alias: '' }, + ]); + }); + + it('does not show formula controls for formula-incapable source kinds', () => { + mockUseSourceData({ + id: 'session-source', + kind: SourceKind.Session, + name: 'Sessions', + from: { databaseName: 'default', tableName: 'sessions' }, + connection: 'default', + timestampValueExpression: 'Timestamp', + traceSourceId: 'trace-source', + }); + + renderComponent({ + chartConfig: { + ...defaultChartConfig, + source: 'session-source', select: [ { aggFn: 'count', diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index 7e3995f019..db37076402 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -607,18 +607,16 @@ interface ResolvedNumberFormats { } /** - * The formula projection settings of a builder metric config, or undefined - * when formulas don't apply (non-builder configs, non-metric sources, or no - * formulas configured). + * The formula projection settings of a builder config, or undefined when + * formulas don't apply (non-builder configs or no formulas configured). + * Source-kind agnostic: metric formulas render through the composed + * multi-series query and event (log/trace) formulas compile inline, but both + * project the same column shape (operands unless hidden, then formulas). */ function getFormulaConfig( config: ChartConfig | ChartConfigWithOptDateRange, ): { formulas: MetricFormula[]; operandsHidden: boolean } | undefined { - if ( - !isBuilderChartConfig(config) || - config.metricTables == null || - !config.formulas?.length - ) { + if (!isBuilderChartConfig(config) || !config.formulas?.length) { return undefined; } return { diff --git a/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap index e23a2ed080..cb0158401c 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap @@ -1,5 +1,30 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing +exports[`renderBuilderConfigAsSqlTemplate compiles event formulas into the single-scan template projection 1`] = ` +"SELECT + countIf(ServiceName = 'api') AS "api", + count() AS "total", + ( + ( + coalesce(countIf(ServiceName = 'api'), 0) / nullif(coalesce(count(), 0), 0) + ) * 100 + ) AS "pct", + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters +GROUP BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + exports[`renderBuilderConfigAsSqlTemplate full SQL output per display type generates the full macro-based SQL for a line chart 1`] = ` "WITH \`__hdx_series_limit\` AS ( diff --git a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap index c8bb034ece..ddfa3fe466 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -32,6 +32,8 @@ exports[`renderChartConfig containing CTE clauses should render a ChSql CTE conf exports[`renderChartConfig containing CTE clauses should render a chart config CTE configuration correctly 1`] = `"WITH Parts AS (SELECT _part, _part_offset FROM default.some_table WHERE ((FieldA = 'test')) ORDER BY rand() DESC LIMIT 1000 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000) SELECT * FROM Parts WHERE ((FieldA = 'test') AND (indexHint((_part, _part_offset) IN (SELECT tuple(_part, _part_offset) FROM Parts)))) ORDER BY rand() DESC LIMIT 1000 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; +exports[`renderChartConfig event (log/trace) formula charts (inline single-query) appends the compiled formula column after the operand columns in one scan 1`] = `"SELECT countIf(SeverityText = 'error') AS "errors",count() AS "total",((coalesce(countIf(SeverityText = 'error'), 0) / nullif(coalesce(count(), 0), 0)) * 100) AS "Error rate",toStartOfInterval(toDateTime(timestamp), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM default.otel_logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY toStartOfInterval(toDateTime(timestamp), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(timestamp), INTERVAL 1 minute) AS \`__hdx_time_bucket\` SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; + exports[`renderChartConfig histogram metric queries count should generate a count query with grouping and time bucketing 1`] = ` "WITH source AS ( SELECT diff --git a/packages/common-utils/src/__tests__/builderToRawSql.test.ts b/packages/common-utils/src/__tests__/builderToRawSql.test.ts index b516531c84..60f9868dfb 100644 --- a/packages/common-utils/src/__tests__/builderToRawSql.test.ts +++ b/packages/common-utils/src/__tests__/builderToRawSql.test.ts @@ -208,6 +208,42 @@ describe('renderBuilderConfigAsSqlTemplate', () => { expect(sql).toMatchSnapshot(); }); + it('compiles event formulas into the single-scan template projection', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...baseLineConfig, + groupBy: undefined, + select: [ + { + aggFn: 'count', + aggCondition: "ServiceName = 'api'", + aggConditionLanguage: 'sql', + valueExpression: '', + alias: 'api', + }, + { + aggFn: 'count', + aggCondition: '', + valueExpression: '', + alias: 'total', + }, + ], + formulas: [{ expression: 'A / B * 100', alias: 'pct' }], + }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('$__sourceTable'); + expect(sql).toContain('"pct"'); + // The compiled formula projects over the operand aggregates in the same + // single-scan statement — no composed UNION ALL machinery. + expect(sql).toContain('coalesce('); + expect(sql).toContain('nullif('); + expect(sql).not.toContain('UNION ALL'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + describe('metric charts', () => { const metricTables = { gauge: 'otel_metrics_gauge', diff --git a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts index d7c188434d..a3228e7d74 100644 --- a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts +++ b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts @@ -2381,4 +2381,202 @@ describe('queryChartConfig Integration Tests', () => { }); }); }); + + // Formulas on event (log/trace) sources compile inline in the single-scan + // SELECT (renderSelectListWithFormulas) rather than through the composed + // metric UNION ALL + pivot path. + describe('event (log/trace) formula charts (inline single-query)', () => { + const EVENTS_TABLE = 'events_formula_int_test'; + + const insertTs = (minute: number) => `2025-04-15 10:0${minute}:00`; + const bucket = (minute: number) => `2025-04-15T10:0${minute}:00Z`; + const DATE_RANGE: [Date, Date] = [ + new Date('2025-04-14'), + new Date('2025-04-16'), + ]; + + type Row = Record; + const col = (row: Row | undefined, name: string): unknown => + row == null ? undefined : new Map(Object.entries(row)).get(name); + const expectGap = (value: unknown) => { + expect(value == null || Number.isNaN(Number(value))).toBe(true); + }; + + const errorCountSelect = { + aggFn: 'count' as const, + valueExpression: '', + aggCondition: "SeverityText = 'error'", + aggConditionLanguage: 'sql' as const, + alias: 'errors', + }; + const totalCountSelect = { + aggFn: 'count' as const, + valueExpression: '', + aggCondition: '', + alias: 'total', + }; + + const baseConfig = ( + overrides: Partial, + ): ChartConfigWithOptDateRange => + ({ + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: EVENTS_TABLE }, + select: [errorCountSelect, totalCountSelect], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: DATE_RANGE, + granularity: '1 minute', + ...overrides, + }) as ChartConfigWithOptDateRange; + + const runConfig = (config: ChartConfigWithOptDateRange) => + hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + beforeAll(async () => { + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${EVENTS_TABLE} ( + Timestamp DateTime CODEC(Delta, ZSTD(1)), + ServiceName LowCardinality(String) CODEC(ZSTD(1)), + SeverityText LowCardinality(String) CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (ServiceName, Timestamp)`, + }); + + // bucket 0: svc-a 4 rows (1 error), svc-b 4 rows (3 errors) + // -> overall 4 errors / 8 total = 50% + // bucket 1: svc-a 2 rows (0 errors) -> 0% + // grouped over both buckets: svc-a 1/6, svc-b 3/4 + const row = (minute: number, service: string, severity: string) => ({ + Timestamp: insertTs(minute), + ServiceName: service, + SeverityText: severity, + }); + await client.insert({ + table: `${DATABASE}.${EVENTS_TABLE}`, + values: [ + row(0, 'svc-a', 'error'), + row(0, 'svc-a', 'info'), + row(0, 'svc-a', 'info'), + row(0, 'svc-a', 'info'), + row(0, 'svc-b', 'error'), + row(0, 'svc-b', 'error'), + row(0, 'svc-b', 'error'), + row(0, 'svc-b', 'info'), + row(1, 'svc-a', 'info'), + row(1, 'svc-a', 'info'), + ], + format: 'JSONEachRow', + }); + }); + + afterAll(async () => { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${EVENTS_TABLE}`, + }); + }); + + it('appends formula columns after the operands on a time chart, with ratio gap semantics', async () => { + const result = await runConfig( + baseConfig({ + formulas: [ + { expression: 'A / B * 100', alias: 'Error rate' }, + // 0 / nullif(0, 0) at bucket 1 — a gap, not 0 or an error. + { expression: 'A / A', alias: 'self ratio' }, + ], + }), + ); + + // Meta contract: operand value columns first, in select order, then + // the formula columns — ahead of the bucket column. + const metaNames = result.meta?.map(m => m.name) ?? []; + expect(metaNames.slice(0, 4)).toEqual([ + 'errors', + 'total', + 'Error rate', + 'self ratio', + ]); + expect(metaNames.indexOf('__hdx_time_bucket')).toBeGreaterThanOrEqual(4); + + const rows = new Map( + (result.data as Row[]).map(r => [ + String(col(r, '__hdx_time_bucket')), + r, + ]), + ); + expect([...rows.keys()].sort()).toEqual([bucket(0), bucket(1)]); + + expect(Number(col(rows.get(bucket(0)), 'errors'))).toBe(4); + expect(Number(col(rows.get(bucket(0)), 'total'))).toBe(8); + expect(Number(col(rows.get(bucket(0)), 'Error rate'))).toBeCloseTo(50, 5); + expect(Number(col(rows.get(bucket(0)), 'self ratio'))).toBe(1); + + expect(Number(col(rows.get(bucket(1)), 'Error rate'))).toBe(0); + expectGap(col(rows.get(bucket(1)), 'self ratio')); + }); + + it('drops the operand columns from meta and rows when showOperandSeries is false', async () => { + const result = await runConfig( + baseConfig({ + formulas: [{ expression: 'A / B * 100', alias: 'Error rate' }], + showOperandSeries: false, + }), + ); + + const metaNames = result.meta?.map(m => m.name) ?? []; + expect(metaNames[0]).toBe('Error rate'); + expect(metaNames).not.toContain('errors'); + expect(metaNames).not.toContain('total'); + + const rows = new Map( + (result.data as Row[]).map(r => [ + String(col(r, '__hdx_time_bucket')), + r, + ]), + ); + expect(Number(col(rows.get(bucket(0)), 'Error rate'))).toBeCloseTo(50, 5); + }); + + it('computes a grouped table formula and lets HAVING filter on it', async () => { + const result = await runConfig( + baseConfig({ + displayType: DisplayType.Table, + granularity: undefined, + groupBy: 'ServiceName', + formulas: [{ expression: 'A / B * 100', alias: 'Error rate' }], + having: '"Error rate" > 50', + havingLanguage: 'sql', + }), + ); + + // svc-a: 1/6 ≈ 16.7% (filtered out); svc-b: 3/4 = 75%. + const rows = result.data as Row[]; + expect(rows).toHaveLength(1); + expect(col(rows[0], 'ServiceName')).toBe('svc-b'); + expect(Number(col(rows[0], 'Error rate'))).toBeCloseTo(75, 5); + }); + + it('collapses a number-shape formula to a single derived value', async () => { + const result = await runConfig( + baseConfig({ + displayType: DisplayType.Number, + granularity: undefined, + formulas: [{ expression: 'A / B * 100', alias: 'Error rate' }], + showOperandSeries: false, + }), + ); + + // 4 errors / 10 total across the whole range. + const rows = result.data as Row[]; + expect(rows).toHaveLength(1); + const metaNames = result.meta?.map(m => m.name) ?? []; + expect(metaNames).toEqual(['Error rate']); + expect(Number(col(rows[0], 'Error rate'))).toBeCloseTo(40, 5); + }); + }); }); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index f452c2c6ad..995a337342 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -4431,4 +4431,197 @@ describe('renderChartConfig', () => { }); }); }); + + // Formulas on event (log/trace) sources compile inline in the single-scan + // SELECT (renderSelectListWithFormulas) — no UNION ALL / pivot involved. + describe('event (log/trace) formula charts (inline single-query)', () => { + const baseEventFormulaConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: [ + { + aggFn: 'count' as const, + valueExpression: '', + aggCondition: "SeverityText = 'error'", + aggConditionLanguage: 'sql' as const, + alias: 'errors', + }, + { + aggFn: 'count' as const, + valueExpression: '', + aggCondition: '', + alias: 'total', + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + }; + + it('appends the compiled formula column after the operand columns in one scan', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + formulas: [{ expression: 'A / B * 100', alias: 'Error rate' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toMatchSnapshot(); + + // Single scan — no composed UNION ALL / pivot machinery. + expect(sql).not.toContain('UNION ALL'); + expect(sql).not.toContain('__hdx_series_idx'); + // Operand columns first (select order), then the formula column. + expect(sql).toContain('AS "errors"'); + expect(sql).toContain('AS "total"'); + expect(sql).toContain('AS "Error rate"'); + expect(sql.indexOf('AS "errors"')).toBeLessThan( + sql.indexOf('AS "total"'), + ); + expect(sql.indexOf('AS "total"')).toBeLessThan( + sql.indexOf('AS "Error rate"'), + ); + // Ratio-consistent missing-data semantics: refs coalesce to 0 and + // division denominators nullif to a gap. + expect(sql).toContain('coalesce('); + expect(sql).toContain('nullif('); + }); + + it('emits only the formula column when showOperandSeries is false', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + formulas: [{ expression: 'A / B', alias: 'ratio' }], + showOperandSeries: false, + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).not.toContain('AS "errors"'); + expect(sql).not.toContain('AS "total"'); + expect(sql).toContain('AS "ratio"'); + // The operand aggregates still evaluate inside the formula. + expect(sql).toContain('countIf('); + }); + + it('names an alias-less formula column by its expression text', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + formulas: [{ expression: 'A + B' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "A + B"'); + }); + + it('takes precedence over seriesReturnType ratio', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + seriesReturnType: 'ratio', + formulas: [{ expression: 'B / A', alias: 'inverse' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "inverse"'); + expect(sql).not.toContain('divide('); + }); + + it('suffixes a formula name colliding with an operand alias', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + formulas: [{ expression: 'A + B', alias: 'errors' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "errors"'); + // Formula column index continues after the select entries (2). + expect(sql).toContain('AS "errors__2"'); + }); + + it('escapes double quotes in the formula column name', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + formulas: [{ expression: 'A / B', alias: 'bad"name' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "bad""name"'); + expect(sql).not.toContain('AS "bad"name"'); + }); + + it('lets HAVING reference a formula output column on a table shape', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + displayType: DisplayType.Table, + granularity: undefined, + groupBy: 'ServiceName', + formulas: [{ expression: 'A / B', alias: 'err rate' }], + having: '"err rate" > 0.5', + havingLanguage: 'sql', + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql.match(/HAVING/g)).toHaveLength(1); + expect(sql.indexOf('HAVING')).toBeGreaterThan(sql.indexOf('GROUP BY')); + }); + + it('renders a single-series chart with a formula in one scan', async () => { + const generatedSql = await renderChartConfig( + { + ...baseEventFormulaConfig, + select: [ + { + aggFn: 'count' as const, + valueExpression: '', + aggCondition: '', + alias: 'total', + }, + ], + formulas: [{ expression: 'A * 100', alias: 'pct' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).not.toContain('UNION ALL'); + expect(sql).toContain('AS "total"'); + expect(sql).toContain('(coalesce(count(), 0) * 100) AS "pct"'); + }); + + it('throws a structured error for an invalid persisted formula', async () => { + await expect( + renderChartConfig( + { + ...baseEventFormulaConfig, + formulas: [{ expression: 'A / C' }], + }, + mockMetadata, + querySettings, + ), + ).rejects.toThrow( + 'Invalid formula "A / C": Unknown series "C" — this chart only has series A through B', + ); + }); + }); }); diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index fef4d0c7ce..75f973a181 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -1021,6 +1021,114 @@ export async function timeFilterExpr({ return concatChSql('AND', ...whereExprs); } +/** + * Value (and derived formula) columns for a builder chart whose formulas + * render inline in the single-scan SELECT — i.e. event (log/trace) sources. + * Metric formula configs never reach this path: they route through + * renderMultiSeriesMetricChartConfig, whose UNION ALL branches strip + * `formulas` before recursing. + * + * Operand columns render first in select order (dropped from the projection + * when `showOperandSeries` is false — the aggregates still evaluate inside + * the formula expressions), followed by one column per formula compiled over + * the bare operand aggregate expressions. All event series aggregate over + * the same scanned rows, so — unlike the composed metric path, which pivots + * UNION ALL branches — the formula embeds each referenced operand expression + * directly. Missing-data semantics follow the existing events ratio + * (`divide(a, b)`): a countIf with no matching rows reads 0, while an + * avg-class aggregate reads nan (a rendered gap) — there is no per-series + * UNION here, so the composed path's NULL pivot never comes into play. + * + * Formulas supersede `seriesReturnType: 'ratio'` (mutually exclusive in the + * editor; rendering stays deterministic if a hand-built config carries both), + * so ratio merging is skipped on this path. + */ +async function renderSelectListWithFormulas( + select: Exclude, + chartConfig: BuilderChartConfigWithOptDateRangeEx, + metadata: Metadata, +): Promise { + const formulas = chartConfig.formulas ?? []; + + // Bare operand expressions (no `AS "alias"` suffix) for embedding inside + // compiled formulas; mirrors the alias-stripping in renderSeriesLimitCte. + const operandExprs = await renderSelectList( + select.map(col => ({ ...col, alias: undefined })), + chartConfig, + metadata, + { mergeRatio: false }, + ); + const operandList = Array.isArray(operandExprs) + ? operandExprs + : [operandExprs]; + const operandAt = (index: number): ChSql => { + const operand = operandList.at(index); + if (operand == null) { + // Unreachable: validateFormula bounds every ref to select.length. + throw new Error(`Formula references unknown series index ${index}`); + } + return operand; + }; + + const columns: ChSql[] = []; + if (chartConfig.showOperandSeries !== false) { + const rendered = await renderSelectList(select, chartConfig, metadata, { + mergeRatio: false, + }); + columns.push(...(Array.isArray(rendered) ? rendered : [rendered])); + } + + // Formula columns are named by their alias, falling back to the raw + // expression text. Dedupe against operand aliases and each other with the + // same column-index suffix the composed metric path uses. + const seen = new Set( + select.flatMap(s => + s.alias != null && s.alias.trim() !== '' ? [s.alias] : [], + ), + ); + const uniqueName = (base: string, columnIdx: number) => { + const name = seen.has(base) ? `${base}__${columnIdx}` : base; + seen.add(name); + return name; + }; + + formulas.forEach((f, formulaIdx) => { + // Parse + validate against the chart's series before rendering any SQL. + // Persisted configs should already be valid (the editor validates on + // save); this is a render-time guard so a stale or hand-built config + // fails with a structured message instead of a ClickHouse error. + const parsed = validateFormula(f.expression, { + seriesCount: select.length, + }); + if (!parsed.ok) { + throw new Error( + `Invalid formula "${f.expression}": ${parsed.errors + .map(e => e.message) + .join('; ')}`, + ); + } + const name = uniqueName( + f.alias || f.expression, + select.length + formulaIdx, + ); + // The compiler only walks the validated AST; the referenced operand + // fragments are the same rendered expressions projected above. Params + // are keyed by value hash, so re-embedding a fragment (even repeatedly, + // e.g. `A / (A + B)`) merges to identical entries. + const sql = `${compileFormulaAst( + parsed.ast, + index => operandAt(index).sql, + )} AS ${quotedColumnName(name)}`; + const params = Object.assign( + {}, + ...parsed.referencedIndices.map(index => operandAt(index).params), + ); + columns.push({ sql, params }); + }); + + return columns; +} + async function renderSelect( chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, @@ -1034,12 +1142,25 @@ async function renderSelect( const isIncludingTimeBucket = isUsingGranularity(chartConfig); const isIncludingGroupBy = isUsingGroupBy(chartConfig); + // Formulas over an array select compile inline on this single-query path + // (event sources; see renderSelectListWithFormulas). Metric formula + // configs are intercepted upstream by the composed multi-series path. + const hasInlineFormulas = + Array.isArray(chartConfig.select) && + (chartConfig.formulas?.length ?? 0) > 0; + // TODO: clean up these await mess return concatChSql( ',', - await renderSelectList(chartConfig.select, chartConfig, metadata, { - mergeRatio: true, - }), + hasInlineFormulas && Array.isArray(chartConfig.select) + ? await renderSelectListWithFormulas( + chartConfig.select, + chartConfig, + metadata, + ) + : await renderSelectList(chartConfig.select, chartConfig, metadata, { + mergeRatio: true, + }), isIncludingGroupBy && chartConfig.selectGroupBy !== false ? await renderSelectList(chartConfig.groupBy, chartConfig, metadata, { mergeRatio: false,