diff --git a/.changeset/builder-to-sql-carryover.md b/.changeset/builder-to-sql-carryover.md new file mode 100644 index 0000000000..b4359cc7bc --- /dev/null +++ b/.changeset/builder-to-sql-carryover.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +--- + +feat: Convert current builder config to SQL during editor switch \ No newline at end of file diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index cb7bb2c59e..e8b7c18aaa 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -1,6 +1,5 @@ import { useMemo } from 'react'; import { add, differenceInSeconds } from 'date-fns'; -import { omit } from 'lodash'; import SqlString from 'sqlstring'; import { z } from 'zod'; import { @@ -15,6 +14,8 @@ import { convertDateRangeToGranularityString, convertGranularityToSeconds, convertToCategoricalChartConfig, + convertToNumberChartConfig, + convertToTableChartConfig, getAlignedDateRange, Granularity, } from '@hyperdx/common-utils/dist/core/utils'; @@ -22,7 +23,6 @@ import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; import { AggregateFunction as AggFnV2, BuilderChartConfigWithDateRange, - BuilderChartConfigWithOptTimestamp, BuilderSavedChartConfig, ChartConfigWithDateRange, ChartConfigWithOptDateRange, @@ -1140,37 +1140,11 @@ export function buildTableRowSearchUrl({ }); } -export function convertToNumberChartConfig( - config: BuilderChartConfigWithDateRange, -): BuilderChartConfigWithOptTimestamp { - return omit(config, ['granularity', 'groupBy']); -} - -// The pie/bar seriesLimit→LIMIT translation lives in common-utils -export { convertToCategoricalChartConfig }; - -export function convertToTableChartConfig( - config: BuilderChartConfigWithOptTimestamp, -): BuilderChartConfigWithOptTimestamp { - const convertedConfig = structuredClone(omit(config, ['granularity'])); - - // Set a default limit if not already set - if (!convertedConfig.limit) { - convertedConfig.limit = { limit: 200 }; - } - - // Set a default orderBy if groupBy is set but orderBy is not, - // so that the set of rows within the limit is stable. - if ( - convertedConfig.groupBy && - typeof convertedConfig.groupBy === 'string' && - !convertedConfig.orderBy - ) { - convertedConfig.orderBy = convertedConfig.groupBy; - } - - return convertedConfig; -} +export { + convertToCategoricalChartConfig, + convertToNumberChartConfig, + convertToTableChartConfig, +}; export function buildMVDateRangeIndicator({ mvOptimizationData, diff --git a/packages/app/src/ClickhousePage.tsx b/packages/app/src/ClickhousePage.tsx index 3a5aa4df1a..1ad6df4fad 100644 --- a/packages/app/src/ClickhousePage.tsx +++ b/packages/app/src/ClickhousePage.tsx @@ -811,7 +811,7 @@ function ClickhousePage() { ({ })); jest.mock('@/hooks/useMetadata', () => ({ + useMetadataWithSettings: jest.fn().mockReturnValue({}), useGetKeyValues: jest.fn().mockReturnValue({ data: [ { diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderToSqlTemplate.test.ts b/packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderToSqlTemplate.test.ts new file mode 100644 index 0000000000..ae2037fb36 --- /dev/null +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderToSqlTemplate.test.ts @@ -0,0 +1,96 @@ +import { classifyFormEdit } from '@/components/DBEditTimeChartForm/useBuilderToSqlConversion'; + +describe('classifyFormEdit', () => { + it('classifies a SQL editor change in SQL mode as a SQL edit', () => { + expect( + classifyFormEdit({ + name: 'sqlTemplate', + type: 'change', + configType: 'sql', + }), + ).toBe('sql'); + }); + + it('classifies a builder query change in builder mode as a builder edit', () => { + expect( + classifyFormEdit({ + name: 'series.0.aggFn', + type: 'change', + configType: 'builder', + }), + ).toBe('builder'); + expect( + classifyFormEdit({ + name: 'where', + type: 'change', + configType: 'builder', + }), + ).toBe('builder'); + expect( + classifyFormEdit({ + name: 'source', + type: 'change', + configType: 'builder', + }), + ).toBe('builder'); + }); + + it('ignores programmatic changes (undefined type)', () => { + // Mode-switch field resets and our own generated-SQL write arrive without a + // 'change' type, so they must never be counted as user edits. + expect( + classifyFormEdit({ + name: 'sqlTemplate', + type: undefined, + configType: 'sql', + }), + ).toBeNull(); + expect( + classifyFormEdit({ + name: 'select', + type: undefined, + configType: 'builder', + }), + ).toBeNull(); + }); + + it('ignores mode-agnostic fields (chart name, configType toggle)', () => { + expect( + classifyFormEdit({ name: 'name', type: 'change', configType: 'builder' }), + ).toBeNull(); + expect( + classifyFormEdit({ name: 'name', type: 'change', configType: 'sql' }), + ).toBeNull(); + expect( + classifyFormEdit({ + name: 'configType', + type: 'change', + configType: 'sql', + }), + ).toBeNull(); + }); + + it('does not count a builder field change while in SQL mode', () => { + // e.g. the source select is present in both modes; changing it in SQL mode + // is not a builder edit. + expect( + classifyFormEdit({ name: 'source', type: 'change', configType: 'sql' }), + ).toBeNull(); + }); + + it('does not count a SQL editor change while in builder mode', () => { + expect( + classifyFormEdit({ + name: 'sqlTemplate', + type: 'change', + configType: 'builder', + }), + ).toBeNull(); + }); + + it('ignores changes with no field name', () => { + expect( + classifyFormEdit({ name: undefined, type: 'change', configType: 'sql' }), + ).toBeNull(); + }); +}); diff --git a/packages/app/src/components/DBEditTimeChartForm/useBuilderToSqlConversion.ts b/packages/app/src/components/DBEditTimeChartForm/useBuilderToSqlConversion.ts new file mode 100644 index 0000000000..36eed01ea3 --- /dev/null +++ b/packages/app/src/components/DBEditTimeChartForm/useBuilderToSqlConversion.ts @@ -0,0 +1,184 @@ +import { useEffect, useRef } from 'react'; +import { + Control, + UseFormGetValues, + UseFormSetValue, + UseFormWatch, + useWatch, +} from 'react-hook-form'; +import { + renderBuilderConfigAsSqlTemplate, + RenderedSqlTemplate, +} from '@hyperdx/common-utils/dist/core/builderToRawSql'; +import { TSource } from '@hyperdx/common-utils/dist/types'; +import { usePrevious } from '@mantine/hooks'; +import { notifications } from '@mantine/notifications'; + +import { ChartEditorFormState } from '@/components/ChartEditor/types'; +import { convertFormStateToChartConfig } from '@/components/ChartEditor/utils'; +import { useMetadataWithSettings } from '@/hooks/useMetadata'; + +type ConfigType = ChartEditorFormState['configType']; + +/** The mode a chart edit belongs to for conversion purposes. */ +export type EditedMode = 'builder' | 'sql'; + +/** + * Fields that belong to neither the builder query nor the SQL editor (they're + * shared chart metadata), so editing them must not count as a "mode edit". + */ +const MODE_AGNOSTIC_FIELDS = new Set(['name', 'configType']); + +/** + * Classify a react-hook-form field change as a builder-mode edit, a SQL-mode + * edit, or neither. Only genuine user input (type='change') counts as an edit. + */ +export function classifyFormEdit({ + name, + type, + configType, +}: { + name?: string; + type?: string; + configType?: ConfigType; +}): EditedMode | null { + if (type !== 'change' || !name || MODE_AGNOSTIC_FIELDS.has(name)) { + return null; + } + // In SQL mode only the SQL editor is a query edit; in builder mode every + // query field except the SQL editor is (a denylist, so new builder fields + // are covered automatically). + if (configType === 'sql') { + return name === 'sqlTemplate' ? 'sql' : null; + } + if (configType === 'builder') { + return name === 'sqlTemplate' ? null : 'builder'; + } + return null; +} + +/** + * One-way Builder → SQL carry-over: when the chart editor switches from builder + * mode to SQL mode, generates a macro-based SQL template from the current + * builder config and populates the SQL editor with it. + * + * The SQL is regenerated only when the user edited the chart in builder mode + * more recently than in SQL mode. Configs that can't be converted surface a + * notification and leave the SQL editor untouched. + */ +export function useBuilderToSqlConversion({ + control, + getValues, + setValue, + watch, + tableSource, +}: { + control: Control; + getValues: UseFormGetValues; + setValue: UseFormSetValue; + watch: UseFormWatch; + tableSource: TSource | undefined; +}) { + const metadata = useMetadataWithSettings(); + + const configType = useWatch({ control, name: 'configType' }); + const prevConfigType = usePrevious(configType); + + // Which mode the user most recently edited in + const lastEditedModeRef = useRef( + getValues('configType') === 'sql' ? 'sql' : 'builder', + ); + + // A monotonically increasing request ID to ensure that only the latest + // generated SQL is written to the form. + const requestIdRef = useRef(0); + + // Track the most-recently-edited mode from genuine user input. + useEffect(() => { + const subscription = watch((_values, { name, type }) => { + const editMode = classifyFormEdit({ + name, + type, + configType: getValues('configType'), + }); + if (editMode) { + lastEditedModeRef.current = editMode; + } + }); + return () => subscription.unsubscribe(); + }, [watch, getValues]); + + useEffect(() => { + // Only generate a new SQL template when switching from builder mode to SQL mode. + if (prevConfigType !== 'builder' || configType !== 'sql') { + return; + } + + const showError = (message: string, err?: Error) => { + console.warn('Could not convert chart to SQL', { + form: getValues(), + err, + }); + notifications.show({ + id: 'builder-to-sql-error', + title: 'Could not auto-convert to SQL', + message, + color: 'red', + }); + }; + + // Only overwrite when the user edited in builder more recently than in SQL. + if (lastEditedModeRef.current !== 'builder') { + return; + } + + // A resolved source is required to build a ChartConfig from the form; + // every other reason a config can't be converted is reported by + // renderBuilderConfigAsSqlTemplate below. + if (!tableSource) { + showError('Auto-converting to SQL requires a source to be selected.'); + return; + } + + // Build a ChartConfig from the current form state + const form = getValues(); + const config = convertFormStateToChartConfig( + { ...form, configType: 'builder' }, + // dateRange is irrelevant here, since the SQL will contain date range macros + [new Date(0), new Date(0)], + tableSource, + ); + if (!config) return; + + const requestId = ++requestIdRef.current; + const applyResult = (result: RenderedSqlTemplate) => { + // Ignore results from generations that a newer switch has superseded. + if (requestId !== requestIdRef.current) { + return; + } + + if (result.isError) { + showError(result.error); + return; + } + + // Only write while still in SQL mode, and don't clobber a hand-edit the + // user made while generation was in flight. + if ( + getValues('configType') === 'sql' && + lastEditedModeRef.current === 'builder' + ) { + setValue('sqlTemplate', result.sql); + notifications.show({ + title: 'Chart converted to SQL', + message: 'The existing chart configuration has been converted to SQL', + color: 'green', + }); + } + }; + + renderBuilderConfigAsSqlTemplate(config, metadata) + .then(applyResult) + .catch(e => showError('Chart could not be auto-converted to SQL', e)); + }, [configType, tableSource, metadata, getValues, setValue, prevConfigType]); +} diff --git a/packages/app/tests/e2e/components/ChartEditorComponent.ts b/packages/app/tests/e2e/components/ChartEditorComponent.ts index be5f66f895..f60719f286 100644 --- a/packages/app/tests/e2e/components/ChartEditorComponent.ts +++ b/packages/app/tests/e2e/components/ChartEditorComponent.ts @@ -220,15 +220,57 @@ export class ChartEditorComponent { } /** - * Type a SQL query into the CodeMirror SQL editor. - * Call switchToSqlMode() first to make the SQL editor visible. + * Switch the chart editor from SQL back to Builder mode. + */ + async switchToBuilderMode() { + const builderLabel = this.page.locator( + '.mantine-SegmentedControl-label:has-text("Builder")', + ); + await builderLabel.waitFor({ state: 'visible', timeout: 5000 }); + await builderLabel.click(); + } + + /** + * Locator for the CodeMirror content of the SQL template editor. Scoped + * with .first() because the "Generated SQL" preview accordion further + * down the DOM renders another `.cm-editor` instance. + */ + sqlEditorContent(): Locator { + return this.page.locator('.cm-editor .cm-content').first(); + } + + /** + * Read the current text of the SQL template editor. + */ + async getSqlEditorText(): Promise { + return this.sqlEditorContent().innerText(); + } + + /** + * Type a SQL query into the CodeMirror SQL editor, replacing any existing + * contents first. Call switchToSqlMode() first to make the editor visible. + * + * Clearing before typing is important: switching Builder → SQL can + * auto-generate a template into the editor, and appending to it would + * corrupt the query. This always yields exactly `sql`. */ async typeSqlQuery(sql: string) { - // Target the cm-content (editable area) inside the SQL template editor. - // Use first() because the "Generated SQL" accordion may add another - // cm-editor further down the DOM. - const sqlContent = this.page.locator('.cm-editor .cm-content').first(); + await this.replaceSqlQuery(sql); + } + + /** + * Replace the entire contents of the SQL template editor with `sql`. + * Selects all existing text and deletes it before typing, so this fully + * replaces (rather than appends to) any auto-generated or hand-written SQL + * already in the editor. + */ + async replaceSqlQuery(sql: string) { + const sqlContent = this.sqlEditorContent(); await sqlContent.click(); + await this.page.keyboard.press( + process.platform === 'darwin' ? 'Meta+A' : 'Control+A', + ); + await this.page.keyboard.press('Delete'); await this.page.keyboard.type(sql); } diff --git a/packages/app/tests/e2e/features/chart-explorer.spec.ts b/packages/app/tests/e2e/features/chart-explorer.spec.ts index c85e008d14..acb05b582c 100644 --- a/packages/app/tests/e2e/features/chart-explorer.spec.ts +++ b/packages/app/tests/e2e/features/chart-explorer.spec.ts @@ -2,6 +2,7 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import { ChartExplorerPage } from '../page-objects/ChartExplorerPage'; import { expect, test } from '../utils/base-test'; +import { DEFAULT_LOGS_SOURCE_NAME, E2E_LOGS_TABLE } from '../utils/constants'; test.describe('Chart Explorer Functionality', { tag: ['@charts'] }, () => { let chartExplorerPage: ChartExplorerPage; @@ -223,4 +224,139 @@ test.describe('Chart Explorer Functionality', { tag: ['@charts'] }, () => { .toEqual(descendingLabels.slice(0, seriesLimit)); }); }); + test( + 'should carry the builder config over to a macro-based SQL template when switching to SQL mode', + { tag: '@full-stack' }, + async () => { + // (beforeEach already navigated to /chart.) Wait for the editor to load. + await test.step('Wait for the chart editor data to load', async () => { + await chartExplorerPage.chartEditor.waitForDataToLoad(); + }); + + // Select the source, keeping the default Line display type + count() series. + await test.step('Select the E2E Logs source', async () => { + await chartExplorerPage.chartEditor.selectSource( + DEFAULT_LOGS_SOURCE_NAME, + ); + }); + + await test.step('Switch to SQL mode', async () => { + await chartExplorerPage.chartEditor.switchToSqlMode(); + }); + + await test.step('Verify the SQL editor is populated with a macro-based template derived from the builder config', async () => { + const sqlEditor = chartExplorerPage.chartEditor.sqlEditorContent(); + await expect(sqlEditor).toContainText('$__sourceTable'); + await expect(sqlEditor).toContainText('$__fromTime_ms'); + await expect(sqlEditor).toContainText('$__toTime_ms'); + await expect(sqlEditor).toContainText('$__timeInterval('); + await expect(sqlEditor).toContainText('$__filters'); + await expect(sqlEditor).toContainText('count()'); + // Macros, not hardcoded values + await expect(sqlEditor).not.toContainText(E2E_LOGS_TABLE); + await expect(sqlEditor).not.toContainText('INTERVAL 1'); + }); + + await test.step('Run the generated SQL and verify a chart renders', async () => { + await chartExplorerPage.chartEditor.runQuery(); + await expect(chartExplorerPage.getFirstChart()).toBeVisible(); + }); + }, + ); + + test( + 'should not overwrite hand-edited SQL when toggling back to SQL mode', + { tag: '@full-stack' }, + async () => { + const handEditedSql = 'SELECT 1 AS hand_edited_marker'; + + await test.step('Wait for the chart editor data to load', async () => { + await chartExplorerPage.chartEditor.waitForDataToLoad(); + }); + + await test.step('Select the E2E Logs source', async () => { + await chartExplorerPage.chartEditor.selectSource( + DEFAULT_LOGS_SOURCE_NAME, + ); + }); + + await test.step('Switch to SQL mode and wait for the auto-generated template', async () => { + await chartExplorerPage.chartEditor.switchToSqlMode(); + await expect( + chartExplorerPage.chartEditor.sqlEditorContent(), + ).toContainText('$__sourceTable'); + }); + + await test.step('Replace the SQL with a hand-written sentinel query', async () => { + await chartExplorerPage.chartEditor.replaceSqlQuery(handEditedSql); + await expect( + chartExplorerPage.chartEditor.sqlEditorContent(), + ).toContainText('hand_edited_marker'); + }); + + await test.step('Switch back to Builder mode, then back to SQL mode', async () => { + await chartExplorerPage.chartEditor.switchToBuilderMode(); + // Settle on a builder-only control (the series aggregation select) + // to confirm the mode switch registered before toggling back. + await expect(chartExplorerPage.chartEditor.aggFn).toBeVisible(); + await chartExplorerPage.chartEditor.switchToSqlMode(); + }); + + await test.step('Verify the hand-edited SQL was not overwritten by regeneration', async () => { + const sqlEditor = chartExplorerPage.chartEditor.sqlEditorContent(); + await expect(sqlEditor).toContainText('hand_edited_marker'); + await expect(sqlEditor).not.toContainText('$__sourceTable'); + }); + }, + ); + + test( + 'should regenerate SQL when the builder is edited more recently than the SQL', + { tag: '@full-stack' }, + async () => { + const handEditedSql = 'SELECT 1 AS hand_edited_marker'; + + await test.step('Wait for the chart editor data to load', async () => { + await chartExplorerPage.chartEditor.waitForDataToLoad(); + }); + + await test.step('Select the E2E Logs source', async () => { + await chartExplorerPage.chartEditor.selectSource( + DEFAULT_LOGS_SOURCE_NAME, + ); + }); + + await test.step('Switch to SQL mode and wait for the auto-generated template', async () => { + await chartExplorerPage.chartEditor.switchToSqlMode(); + await expect( + chartExplorerPage.chartEditor.sqlEditorContent(), + ).toContainText('$__sourceTable'); + }); + + await test.step('Replace the SQL with a hand-written sentinel query', async () => { + await chartExplorerPage.chartEditor.replaceSqlQuery(handEditedSql); + await expect( + chartExplorerPage.chartEditor.sqlEditorContent(), + ).toContainText('hand_edited_marker'); + }); + + await test.step('Switch to Builder mode and make a builder edit (group by)', async () => { + await chartExplorerPage.chartEditor.switchToBuilderMode(); + await expect(chartExplorerPage.chartEditor.aggFn).toBeVisible(); + // A genuine builder edit — this is now the most recent edit, so the + // next switch to SQL should regenerate and discard the hand-edited SQL. + await chartExplorerPage.chartEditor.setGroupBy('ServiceName'); + }); + + await test.step('Switch back to SQL mode', async () => { + await chartExplorerPage.chartEditor.switchToSqlMode(); + }); + + await test.step('Verify the SQL was regenerated from the newer builder config', async () => { + const sqlEditor = chartExplorerPage.chartEditor.sqlEditorContent(); + await expect(sqlEditor).toContainText('$__sourceTable'); + await expect(sqlEditor).not.toContainText('hand_edited_marker'); + }); + }, + ); }); diff --git a/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap new file mode 100644 index 0000000000..1c12b85d49 --- /dev/null +++ b/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap @@ -0,0 +1,670 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`renderBuilderConfigAsSqlTemplate full SQL output per display type generates the full macro-based SQL for a line chart 1`] = ` +"WITH + \`__hdx_series_limit\` AS ( + SELECT + \`group\` + FROM + ( + SELECT + tuple(ServiceName) AS \`group\`, + count() AS \`__hdx_series_rank\` + FROM + $__sourceTable + WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters + AND ServiceName IS NOT NULL + GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` + ) + GROUP BY + \`group\` + ORDER BY + max(\`__hdx_series_rank\`) DESC, + \`group\` + LIMIT + 10 + ) +SELECT + count(), + AVG(toFloat64OrDefault(toString(Duration))), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters + AND tuple(ServiceName) IN ( + SELECT + \`group\` + FROM + \`__hdx_series_limit\` + ) +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate full SQL output per display type generates the full macro-based SQL for a number chart 1`] = ` +"SELECT + count() +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters" +`; + +exports[`renderBuilderConfigAsSqlTemplate full SQL output per display type generates the full macro-based SQL for a pie chart 1`] = ` +"SELECT + count(), + ServiceName +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters +GROUP BY + ServiceName" +`; + +exports[`renderBuilderConfigAsSqlTemplate full SQL output per display type generates the full macro-based SQL for a stacked_bar chart 1`] = ` +"WITH + \`__hdx_series_limit\` AS ( + SELECT + \`group\` + FROM + ( + SELECT + tuple(ServiceName) AS \`group\`, + count() AS \`__hdx_series_rank\` + FROM + $__sourceTable + WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters + AND ServiceName IS NOT NULL + GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` + ) + GROUP BY + \`group\` + ORDER BY + max(\`__hdx_series_rank\`) DESC, + \`group\` + LIMIT + 10 + ) +SELECT + count(), + AVG(toFloat64OrDefault(toString(Duration))), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters + AND tuple(ServiceName) IN ( + SELECT + \`group\` + FROM + \`__hdx_series_limit\` + ) +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate full SQL output per display type generates the full macro-based SQL for a table chart 1`] = ` +"SELECT + count(), + AVG(toFloat64OrDefault(toString(Duration))), + ServiceName +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters +GROUP BY + ServiceName +HAVING + count() > 5 +ORDER BY + count() DESC +LIMIT + 200" +`; + +exports[`renderBuilderConfigAsSqlTemplate generates a macro-based template for a line chart with group-by and lucene where 1`] = ` +"SELECT + count(), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate macro-izes the series-limit CTE and inlines its LIMIT 1`] = ` +"WITH + \`__hdx_series_limit\` AS ( + SELECT + \`group\` + FROM + ( + SELECT + tuple(ServiceName) AS \`group\`, + count() AS \`__hdx_series_rank\` + FROM + $__sourceTable + WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters + AND ServiceName IS NOT NULL + GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` + ) + GROUP BY + \`group\` + ORDER BY + max(\`__hdx_series_rank\`) DESC, + \`group\` + LIMIT + 5 + ) +SELECT + count(), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters + AND tuple(ServiceName) IN ( + SELECT + \`group\` + FROM + \`__hdx_series_limit\` + ) +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate metric charts (single-series only) generates a macro-based template for a single-series gauge metric line chart 1`] = ` +"WITH + Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM + $__sourceTable(gauge) + WHERE + ( + TimeUnix >= $__fromTime_ms + AND TimeUnix <= $__toTime_ms + ) + AND ((MetricName = 'my.metric')) + AND $__filters + ), + Bucketed AS ( + SELECT + $__timeInterval(TimeUnix) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM + Source + GROUP BY + AttributesHash, + __hdx_time_bucket2 + ORDER BY + AttributesHash, + __hdx_time_bucket2 + ) +SELECT + avg(toFloat64OrDefault(toString(LastValue))), + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` +FROM + \`Bucketed\` +WHERE + ( + __hdx_time_bucket2 >= $__fromTime_ms + AND __hdx_time_bucket2 <= $__toTime_ms + ) +GROUP BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` +SETTINGS + short_circuit_function_evaluation = 'force_enable'" +`; + +exports[`renderBuilderConfigAsSqlTemplate metric charts (single-series only) generates a macro-based template for a single-series histogram metric line chart 1`] = ` +"WITH + source AS ( + SELECT + MetricName, + ExplicitBounds, + $__timeInterval(TimeUnix) AS \`__hdx_time_bucket\`, + sumForEach(deltas) as rates + FROM + ( + SELECT + TimeUnix, + MetricName, + ResourceAttributes, + Attributes, + ExplicitBounds, + attr_hash, + any(attr_hash) OVER ( + ROWS BETWEEN 1 preceding + AND 1 preceding + ) AS prev_attr_hash, + any(bounds_hash) OVER ( + ROWS BETWEEN 1 preceding + AND 1 preceding + ) AS prev_bounds_hash, + any(counts) OVER ( + ROWS BETWEEN 1 preceding + AND 1 preceding + ) AS prev_counts, + counts, + IF( + AggregationTemporality = 1 + OR prev_attr_hash != attr_hash + OR bounds_hash != prev_bounds_hash + OR arrayExists((x) -> x .2 < x .1, arrayZip(prev_counts, counts)), + counts, + counts - prev_counts + ) AS deltas + FROM + ( + SELECT + TimeUnix, + MetricName, + AggregationTemporality, + ExplicitBounds, + ResourceAttributes, + Attributes, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS attr_hash, + cityHash64(ExplicitBounds) AS bounds_hash, + CAST(BucketCounts AS Array(Int64)) counts + FROM + $__sourceTable(histogram) + WHERE + ( + TimeUnix >= toStartOfInterval($__fromTime_ms, INTERVAL $__interval_s second) - INTERVAL $__interval_s second + AND TimeUnix <= toStartOfInterval($__toTime_ms, INTERVAL $__interval_s second) + INTERVAL $__interval_s second + ) + AND ((MetricName = 'my.metric')) + AND $__filters + ORDER BY + attr_hash, + TimeUnix ASC + ) + ) + GROUP BY + \`__hdx_time_bucket\`, + MetricName, + ExplicitBounds + ORDER BY + \`__hdx_time_bucket\` + ), + points AS ( + SELECT + \`__hdx_time_bucket\`, + MetricName, + arrayZipUnaligned(arrayCumSum(rates), ExplicitBounds) as point, + length(point) as n + FROM + source + ), + metrics AS ( + SELECT + \`__hdx_time_bucket\`, + MetricName, + point[n] .1 AS total, + 0.95 * total AS rank, + arrayFirstIndex(x -> if(x .1 > rank, 1, 0), point) AS upper_idx, + point[upper_idx] .1 AS upper_count, + ifNull(point[upper_idx] .2, inf) AS upper_bound, + CASE + WHEN upper_idx > 1 THEN point[upper_idx - 1] .2 + WHEN point[upper_idx] .2 > 0 THEN 0 + ELSE inf + END AS lower_bound, + if(lower_bound = 0, 0, point[upper_idx - 1] .1) AS lower_count, + CASE + WHEN upper_bound = inf THEN point[upper_idx - 1] .2 + WHEN lower_bound = inf THEN point[1] .2 + ELSE lower_bound + (upper_bound - lower_bound) * ( + (rank - lower_count) / (upper_count - lower_count) + ) + END AS "Value" + FROM + points + WHERE + length(point) > 1 + AND total > 0 + ) +SELECT + \`__hdx_time_bucket\`, + "Value" +FROM + \`metrics\` +WHERE + ( + \`__hdx_time_bucket\` >= $__fromTime_ms + AND \`__hdx_time_bucket\` <= $__toTime_ms + ) +SETTINGS + short_circuit_function_evaluation = 'force_enable'" +`; + +exports[`renderBuilderConfigAsSqlTemplate metric charts (single-series only) generates a macro-based template for a single-series sum metric line chart 1`] = ` +"WITH + Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash, + IF( + AggregationTemporality = 1, + Value, -- DELTA: Value is already the per-interval increase + greatest( + Value - lagInFrame(toNullable(Value), 1, NULL) OVER ( + PARTITION BY + AttributesHash + ORDER BY + TimeUnix + ), + 0 + ) + ) AS Rate, + IF( + AggregationTemporality = 1, + SUM(Value) OVER ( + PARTITION BY + AttributesHash + ORDER BY + TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING + AND CURRENT ROW + ), + Value + ) AS Sum + FROM + $__sourceTable(sum) + WHERE + ( + TimeUnix >= toStartOfInterval($__fromTime_ms, INTERVAL $__interval_s second) - INTERVAL $__interval_s second + AND TimeUnix <= toStartOfInterval($__toTime_ms, INTERVAL $__interval_s second) + INTERVAL $__interval_s second + ) + AND ((MetricName = 'my.metric')) + AND $__filters + ), + Bucketed AS ( + SELECT + \`__hdx_time_bucket2\`, + AttributesHash, + Rate, + Sum, + ResourceAttributes, + ResourceSchemaUrl, + ScopeName, + ScopeVersion, + ScopeAttributes, + ScopeDroppedAttrCount, + ScopeSchemaUrl, + ServiceName, + MetricName, + MetricDescription, + MetricUnit, + Attributes, + StartTimeUnix, + Flags, + AggregationTemporality, + IsMonotonic + FROM + ( + SELECT + $__timeInterval(TimeUnix) AS \`__hdx_time_bucket2\`, + AttributesHash, + -- Per-bucket increase: sum of raw per-row deltas. NULL + -- Source.Rate (first row of each partition) is ignored by sum(). + sum(Source.Rate) AS Rate, + -- Last cumulative reading in the bucket (by time), used by + -- the no-aggFn last_value(Sum) outer projection. argMax is + -- deterministic w.r.t. TimeUnix ordering unlike last_value + -- which in a GROUP BY context is anyLast (order-dependent). + argMax(Source.Sum, TimeUnix) AS Sum, + any(ResourceAttributes) AS ResourceAttributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeAttributes) AS ScopeAttributes, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricName) AS MetricName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(Attributes) AS Attributes, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags, + any(AggregationTemporality) AS AggregationTemporality, + any(IsMonotonic) AS IsMonotonic + FROM + Source + GROUP BY + AttributesHash, + \`__hdx_time_bucket2\` + ORDER BY + AttributesHash, + \`__hdx_time_bucket2\` + ) + ) +SELECT + avg(toFloat64OrDefault(toString(Rate))) AS "Value", + $__timeInterval(\`__hdx_time_bucket2\`) AS \`__hdx_time_bucket\` +FROM + \`Bucketed\` +WHERE + ( + \`__hdx_time_bucket2\` >= $__fromTime_ms + AND \`__hdx_time_bucket2\` <= $__toTime_ms + ) +GROUP BY + $__timeInterval(\`__hdx_time_bucket2\`) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(\`__hdx_time_bucket2\`) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate series-level WHERE (aggCondition → -If combinators) keeps -If per series but does not push to WHERE when only some series are filtered 1`] = ` +"SELECT + countIf((ServiceName ILIKE '%api%')), + AVG(toFloat64OrDefault(toString(value))), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND $__filters +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate series-level WHERE (aggCondition → -If combinators) renders a single filtered series as an -If combinator 1`] = ` +"SELECT + countIf((ServiceName ILIKE '%api%')), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate series-level WHERE (aggCondition → -If combinators) renders one -If per series and OR-pushes conditions when every series is filtered 1`] = ` +"SELECT + countIf((ServiceName ILIKE '%api%')), + avgIf( + toFloat64OrDefault(toString(value)), + (ServiceName ILIKE '%web%') + AND toFloat64OrDefault(toString(value)) IS NOT NULL + ), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ( + (ServiceName ILIKE '%api%') + OR (ServiceName ILIKE '%web%') + ) + AND $__filters +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate series-level WHERE (aggCondition → -If combinators) supports a sql-language series aggCondition 1`] = ` +"SELECT + countIf(ServiceName = 'api'), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND (ServiceName = 'api') + AND $__filters +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; + +exports[`renderBuilderConfigAsSqlTemplate series-level WHERE (aggCondition → -If combinators) tightens the two argument lists of a parametric combinator 1`] = ` +"SELECT + quantileIf(0.95)( + toFloat64OrDefault(toString(value)), + (ServiceName ILIKE '%api%') + AND toFloat64OrDefault(toString(value)) IS NOT NULL + ), + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +FROM + $__sourceTable +WHERE + ( + timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + ) + AND ((ServiceName ILIKE '%api%')) + AND $__filters +GROUP BY + ServiceName, + $__timeInterval(timestamp) AS \`__hdx_time_bucket\` +ORDER BY + $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" +`; diff --git a/packages/common-utils/src/__tests__/builderToRawSql.test.ts b/packages/common-utils/src/__tests__/builderToRawSql.test.ts new file mode 100644 index 0000000000..687e48278c --- /dev/null +++ b/packages/common-utils/src/__tests__/builderToRawSql.test.ts @@ -0,0 +1,810 @@ +import { renderBuilderConfigAsSqlTemplate as renderBuilderConfigAsSqlTemplateResult } from '@/core/builderToRawSql'; +import { Metadata } from '@/core/metadata'; +import { validateRawSqlForAlert } from '@/core/utils'; +import { replaceMacros } from '@/macros'; +import { + ChartConfigWithOptDateRange, + DisplayType, + MetricsDataType, + SelectList, +} from '@/types'; + +// Most tests here assert on the generated SQL. This wrapper unwraps the +// {sql}|{error} result to the SQL string (or null when the config was +// rejected), so the behavioral assertions stay focused; the "error results" +// block below covers the {error} messages via the raw function. +async function renderBuilderConfigAsSqlTemplate( + ...args: Parameters +): Promise { + const result = await renderBuilderConfigAsSqlTemplateResult(...args); + return result.isError ? null : result.sql; +} + +describe('renderBuilderConfigAsSqlTemplate', () => { + let mockMetadata: jest.Mocked; + + // Suppress expected console.warn noise from missing columns / optimization fallbacks + beforeAll(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); + }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + const columns = [ + { name: 'timestamp', type: 'DateTime' }, + { name: 'date', type: 'Date' }, + { name: 'value', type: 'Float64' }, + { name: 'ServiceName', type: 'String' }, + ]; + mockMetadata = { + getColumns: jest.fn().mockResolvedValue(columns), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(new Map()), + getColumn: jest + .fn() + .mockImplementation(async ({ column }) => + columns.find(col => col.name === column), + ), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'timestamp' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + } as unknown as jest.Mocked; + }); + + const baseLineConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: 'ServiceName:api', + whereLanguage: 'lucene', + timestampValueExpression: 'timestamp', + granularity: '1 minute', + }; + + it('generates a macro-based template for a line chart with group-by and lucene where', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + baseLineConfig, + mockMetadata, + ); + + expect(sql).not.toBeNull(); + expect(sql).toContain('$__fromTime_ms'); + expect(sql).toContain('$__toTime_ms'); + expect(sql).toContain('$__timeInterval(timestamp)'); + expect(sql).toContain('$__sourceTable'); + expect(sql).toContain('$__filters'); + expect(sql).toContain('`__hdx_time_bucket`'); + // No hardcoded table, interval, or unbound render params + expect(sql).not.toContain('otel_logs'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).not.toMatch(/INTERVAL 1 minute/i); + expect(sql).toMatchSnapshot(); + }); + + it('generates a template without interval macros for a table chart and inlines LIMIT', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...baseLineConfig, + displayType: DisplayType.Table, + granularity: undefined, + orderBy: 'count() DESC', + limit: { limit: 100 }, + }, + mockMetadata, + ); + + expect(sql).not.toBeNull(); + expect(sql).toContain('$__fromTime_ms'); + expect(sql).toContain('$__toTime_ms'); + expect(sql).toContain('$__sourceTable'); + // intervalSeconds is not bound for Table charts, so no interval macro + expect(sql).not.toContain('$__timeInterval'); + expect(sql).not.toContain('intervalSeconds'); + expect(sql).toMatch(/LIMIT\s+100/); + expect(sql).not.toContain('HYPERDX_PARAM_'); + }); + + it('strips granularity for a table chart even when the form carries one', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { ...baseLineConfig, displayType: DisplayType.Table }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toContain('$__timeInterval'); + }); + + it.each([DisplayType.Bar, DisplayType.Pie])( + 'carries seriesLimit into the SQL as a LIMIT (with default value-desc ORDER BY) for a %s chart', + async displayType => { + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...baseLineConfig, + displayType, + // Categorical charts carry a string groupBy (as the chart editor + // produces) and render seriesLimit as a plain SQL LIMIT. + groupBy: 'ServiceName', + seriesLimit: 2, + }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('$__fromTime_ms'); + expect(sql).toContain('$__toTime_ms'); + expect(sql).toContain('$__sourceTable'); + expect(sql).toContain('$__filters'); + expect(sql).toContain('GROUP BY'); + // Categorical (not time series) → no interval macro / bucketing. + expect(sql).not.toContain('$__timeInterval'); + expect(sql).not.toContain('intervalSeconds'); + // seriesLimit is carried over as a plain LIMIT... + expect(sql).toMatch(/LIMIT\s+2/); + // ...ordered by the first aggregated value descending so the limit + // deterministically keeps the largest bars/slices. + expect(sql).toMatch(/ORDER BY[\s\S]*DESC/i); + expect(sql).not.toContain('HYPERDX_PARAM_'); + }, + ); + + it('does not add a LIMIT to a categorical chart without a seriesLimit', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { ...baseLineConfig, displayType: DisplayType.Bar }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toMatch(/LIMIT/i); + }); + + it.each(['auto' as const, undefined])( + 'emits $__timeInterval for a line chart with granularity=%s', + async granularity => { + const sql = await renderBuilderConfigAsSqlTemplate( + { ...baseLineConfig, granularity }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('$__timeInterval(timestamp)'); + }, + ); + + it('wraps time-range macros for a Date-typed timestamp column', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { ...baseLineConfig, timestampValueExpression: 'date' }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('toDate($__fromTime_ms)'); + expect(sql).toContain('toDate($__toTime_ms)'); + }); + + it('macro-izes every condition of a multi-column timestamp expression', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { ...baseLineConfig, timestampValueExpression: 'date, timestamp' }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql?.match(/\$__fromTime_ms/g)?.length).toBeGreaterThanOrEqual(2); + expect(sql?.match(/\$__toTime_ms/g)?.length).toBeGreaterThanOrEqual(2); + expect(sql).not.toContain('HYPERDX_PARAM_'); + }); + + it('macro-izes the series-limit CTE and inlines its LIMIT', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { ...baseLineConfig, seriesLimit: 5 }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('__hdx_series_limit'); + expect(sql).toMatch(/LIMIT\s+5/); + // The CTE reuses the FROM/WHERE fragments, so macros must appear there too + expect(sql?.match(/\$__sourceTable/g)?.length).toBe(2); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + describe('metric charts (single-series only)', () => { + const metricTables = { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }; + + const metricLineConfig = ( + metricType: MetricsDataType, + aggFn = 'avg', + ): ChartConfigWithOptDateRange => ({ + displayType: DisplayType.Line, + connection: 'test-connection', + // Metric sources carry no single `from.tableName`; the table comes from + // metricTables per metric type. + from: { databaseName: 'default', tableName: '' }, + metricTables, + select: [ + { + aggFn: aggFn as any, + aggCondition: '', + valueExpression: 'Value', + metricType, + metricName: 'my.metric', + }, + ], + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'TimeUnix', + granularity: '1 minute', + }); + + it('generates a macro-based template for a single-series gauge metric line chart', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + metricLineConfig(MetricsDataType.Gauge), + mockMetadata, + ); + expect(sql).not.toBeNull(); + // The table is emitted as the typed source-table macro, not hardcoded. + expect(sql).toContain('$__sourceTable(gauge)'); + expect(sql).not.toContain('otel_metrics_gauge'); + // Still wired to the dashboard time range and granularity. + expect(sql).toContain('$__fromTime_ms'); + expect(sql).toContain('$__toTime_ms'); + expect(sql).toContain('$__timeInterval'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).not.toMatch(/INTERVAL 1 minute/i); + expect(sql).toMatchSnapshot(); + }); + + it('generates a macro-based template for a single-series sum metric line chart', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + metricLineConfig(MetricsDataType.Sum), + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('$__sourceTable(sum)'); + expect(sql).not.toContain('otel_metrics_sum'); + // Sum widens the source window by one bucket; that widening is expressed + // with the $__interval_s macro so it stays wired to the granularity. + expect(sql).toContain('$__interval_s'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('generates a macro-based template for a single-series histogram metric line chart', async () => { + const config = metricLineConfig(MetricsDataType.Histogram, 'quantile'); + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...config, + // Histograms support quantile (with a level) or count. + select: [{ ...((config as any).select as any[])[0], level: 0.95 }], + }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('$__sourceTable(histogram)'); + expect(sql).not.toContain('otel_metrics_histogram'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('resolves the metric source-table macro through replaceMacros', async () => { + const sqlTemplate = await renderBuilderConfigAsSqlTemplate( + metricLineConfig(MetricsDataType.Gauge), + mockMetadata, + ); + expect(sqlTemplate).not.toBeNull(); + + const expanded = replaceMacros({ + sqlTemplate: sqlTemplate!, + from: { databaseName: 'default', tableName: '' }, + metricTables, + }); + expect(expanded).not.toContain('$__'); + // $__sourceTable(gauge) resolves against metricTables. + expect(expanded).toContain('`default`.`otel_metrics_gauge`'); + }); + + // $__filters expands to dashboard filters that reference the raw source + // columns. Metric charts wrap the source in CTEs and the outer query reads + // a CTE that only projects a fixed column set, so emitting $__filters there + // would expand to unknown identifiers and break the query. It must appear + // once — inside the source-table CTE — never in the outer query. + // The outer query reads the intermediate CTE (backtick-quoted by the + // formatter, and only ever referenced this way — the CTE definitions use + // ` AS (`), so this token uniquely marks the start of the outer query. + it.each([ + [MetricsDataType.Gauge, 'avg', undefined, '`Bucketed`'], + [MetricsDataType.Sum, 'sum', undefined, '`Bucketed`'], + [MetricsDataType.Histogram, 'quantile', 0.95, '`metrics`'], + ] as const)( + 'emits $__filters only in the source CTE for a %s metric chart', + async (metricType, aggFn, level, outerFrom) => { + const config = metricLineConfig(metricType, aggFn); + const sql = await renderBuilderConfigAsSqlTemplate( + level != null + ? { + ...config, + select: [{ ...((config as any).select as any[])[0], level }], + } + : config, + mockMetadata, + ); + expect(sql).not.toBeNull(); + + // Exactly one $__filters — the duplicate in the outer query is the bug. + expect(sql!.match(/\$__filters/g)).toHaveLength(1); + + // ...and it sits inside a source CTE, ahead of the outer FROM that + // reads the (column-limited) intermediate CTE. + expect(sql).toContain(outerFrom); + expect(sql!.indexOf('$__filters')).toBeLessThan( + sql!.indexOf(outerFrom), + ); + }, + ); + + it('returns null for a multi-series metric chart', async () => { + const config = metricLineConfig(MetricsDataType.Gauge); + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...config, + select: [ + ...((config as any).select as any[]), + { + aggFn: 'sum', + aggCondition: '', + valueExpression: 'Value', + metricType: MetricsDataType.Gauge, + metricName: 'my.other.metric', + }, + ], + }, + mockMetadata, + ); + expect(sql).toBeNull(); + }); + + it('returns null for a non-time-series metric chart', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...metricLineConfig(MetricsDataType.Gauge), + displayType: DisplayType.Number, + granularity: undefined, + }, + mockMetadata, + ); + expect(sql).toBeNull(); + }); + }); + + it('returns null for string selects and unsupported display types', async () => { + await expect( + renderBuilderConfigAsSqlTemplate( + { + ...baseLineConfig, + displayType: DisplayType.Search, + select: 'timestamp, ServiceName', + }, + mockMetadata, + ), + ).resolves.toBeNull(); + + await expect( + renderBuilderConfigAsSqlTemplate( + { ...baseLineConfig, displayType: DisplayType.Heatmap }, + mockMetadata, + ), + ).resolves.toBeNull(); + }); + + it('returns null for raw SQL and PromQL configs', async () => { + await expect( + renderBuilderConfigAsSqlTemplate( + { + configType: 'sql', + sqlTemplate: 'SELECT 1', + connection: 'test-connection', + }, + mockMetadata, + ), + ).resolves.toBeNull(); + + await expect( + renderBuilderConfigAsSqlTemplate( + { + configType: 'promql', + promqlExpression: 'up', + connection: 'test-connection', + }, + mockMetadata, + ), + ).resolves.toBeNull(); + }); + + describe('error results', () => { + const metricConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: '' }, + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + select: [ + { + aggFn: 'avg', + aggCondition: '', + valueExpression: 'Value', + metricType: MetricsDataType.Gauge, + metricName: 'my.metric', + }, + ], + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'TimeUnix', + granularity: '1 minute', + }; + + it('reports a specific reason for a multi-series metric chart', async () => { + const result = await renderBuilderConfigAsSqlTemplateResult( + { + ...metricConfig, + select: [ + ...(metricConfig.select as any[]), + { + aggFn: 'sum', + aggCondition: '', + valueExpression: 'Value', + metricType: MetricsDataType.Gauge, + metricName: 'my.other.metric', + }, + ], + }, + mockMetadata, + ); + expect(result).toEqual({ + isError: true, + error: 'Multi-series metric charts cannot be auto-converted to SQL.', + }); + }); + + it('reports a specific reason for a non-time-series metric chart', async () => { + const result = await renderBuilderConfigAsSqlTemplateResult( + { + ...metricConfig, + displayType: DisplayType.Table, + granularity: undefined, + }, + mockMetadata, + ); + expect(result).toEqual({ + isError: true, + error: + 'Metric charts can only be auto-converted to SQL for time series display types.', + }); + }); + + it('reports a missing-source reason when the source table is unset', async () => { + const result = await renderBuilderConfigAsSqlTemplateResult( + { + ...baseLineConfig, + from: { databaseName: '', tableName: '' }, + }, + mockMetadata, + ); + expect(result).toEqual({ + isError: true, + error: 'Auto-converting to SQL requires a source to be selected.', + }); + }); + + it('reports an unconvertible-chart reason for unsupported display types', async () => { + const result = await renderBuilderConfigAsSqlTemplateResult( + { ...baseLineConfig, displayType: DisplayType.Heatmap }, + mockMetadata, + ); + expect(result).toEqual({ + isError: true, + error: 'This chart type cannot be auto-converted to SQL.', + }); + }); + + it('returns the generated SQL under the `sql` key on success', async () => { + const result = await renderBuilderConfigAsSqlTemplateResult( + baseLineConfig, + mockMetadata, + ); + expect(result).toHaveProperty('sql'); + expect((result as { sql: string }).sql).toContain('$__sourceTable'); + }); + }); + + it('round-trips through replaceMacros into an executable, alert-valid query', async () => { + const sqlTemplate = await renderBuilderConfigAsSqlTemplate( + baseLineConfig, + mockMetadata, + ); + expect(sqlTemplate).not.toBeNull(); + + const expanded = replaceMacros({ + sqlTemplate: sqlTemplate!, + from: { databaseName: 'default', tableName: 'otel_logs' }, + }); + // All macros resolved; only the documented raw-SQL query params remain + expect(expanded).not.toContain('$__'); + expect(expanded).toContain('{startDateMilliseconds:Int64}'); + expect(expanded).toContain('{endDateMilliseconds:Int64}'); + expect(expanded).toContain('{intervalSeconds:Int64}'); + expect(expanded).toContain('`default`.`otel_logs`'); + + const { errors } = validateRawSqlForAlert({ + configType: 'sql', + sqlTemplate: sqlTemplate!, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + displayType: DisplayType.Line, + }); + expect(errors).toEqual([]); + }); + + /** + * Full-output snapshots, one per raw-SQL-capable display type, each + * populated with every clause that display type exposes in the chart + * builder (see ChartEditorControls / ChartActionBar / ChartDisplaySettingsDrawer): + * + * - Line / StackedBar (time tab): multi-series, WHERE, GROUP BY, + * granularity (time bucket), series limit. + * - Table: multi-series, WHERE, GROUP BY, HAVING, ORDER BY. + * - Pie: single series, WHERE, GROUP BY. + * - Number: single series, WHERE. + * + * These lock in the exact generated SQL so regressions in how any clause + * is turned into macros are caught. + */ + describe('full SQL output per display type', () => { + // Shared source-level fields every builder config carries. + const sourceFields = { + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'timestamp', + where: 'ServiceName:api', + whereLanguage: 'lucene' as const, + }; + + const twoSeries = [ + { aggFn: 'count' as const, aggCondition: '', valueExpression: '' }, + { aggFn: 'avg' as const, aggCondition: '', valueExpression: 'Duration' }, + ]; + const groupBy = [{ aggCondition: '', valueExpression: 'ServiceName' }]; + + const configs: Record = { + // Time-series charts: multi-series + group by + granularity + series limit. + [DisplayType.Line]: { + ...sourceFields, + displayType: DisplayType.Line, + select: twoSeries, + groupBy, + granularity: '5 minute', + seriesLimit: 10, + }, + [DisplayType.StackedBar]: { + ...sourceFields, + displayType: DisplayType.StackedBar, + select: twoSeries, + groupBy, + granularity: '5 minute', + seriesLimit: 10, + }, + // Table: multi-series + group by + having + order by (no time bucket). + [DisplayType.Table]: { + ...sourceFields, + displayType: DisplayType.Table, + select: twoSeries, + groupBy, + having: 'count() > 5', + havingLanguage: 'sql', + orderBy: 'count() DESC', + }, + // Pie: single series + group by (max one series, no order/having). + [DisplayType.Pie]: { + ...sourceFields, + displayType: DisplayType.Pie, + select: [twoSeries[0]], + groupBy, + }, + // Number: single series + where only (no group by / granularity). + [DisplayType.Number]: { + ...sourceFields, + displayType: DisplayType.Number, + select: [twoSeries[0]], + }, + }; + + it.each(Object.keys(configs))( + 'generates the full macro-based SQL for a %s chart', + async (displayType: string) => { + const sql = await renderBuilderConfigAsSqlTemplate( + // eslint-disable-next-line security/detect-object-injection + configs[displayType], + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }, + ); + }); + + /** + * Series-level WHERE — the path the chart builder actually takes. + * + * In the chart editor the WHERE for a series is stored per-series on + * `select[].aggCondition` / `aggConditionLanguage` (see + * EditTimeChartForm's `series` field), NOT on the top-level `where`. + * renderChartConfig turns a series' aggCondition into an `-If` combinator + * at the aggregate function (countIf/avgIf/...), and — only when *every* + * series is filtered — also OR's those conditions into the WHERE clause so + * the primary index can still be used. These tests lock in that the + * builder-to-SQL conversion carries the per-series filters through as + * `-If` combinators (and macro-izes the surrounding query the same way). + */ + describe('series-level WHERE (aggCondition → -If combinators)', () => { + const seriesLevelConfig = ( + select: SelectList, + ): ChartConfigWithOptDateRange => ({ + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select, + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + // No top-level where — the filter lives on the series, as it does in + // the chart editor. + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'timestamp', + granularity: '1 minute', + }); + + it('renders a single filtered series as an -If combinator', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + seriesLevelConfig([ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, + ]), + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('renders one -If per series and OR-pushes conditions when every series is filtered', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + seriesLevelConfig([ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, + { + aggFn: 'avg', + aggCondition: 'ServiceName:web', + aggConditionLanguage: 'lucene', + valueExpression: 'value', + }, + ]), + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('keeps -If per series but does not push to WHERE when only some series are filtered', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + seriesLevelConfig([ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, + { aggFn: 'avg', aggCondition: '', valueExpression: 'value' }, + ]), + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('supports a sql-language series aggCondition', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + seriesLevelConfig([ + { + aggFn: 'count', + aggCondition: "ServiceName = 'api'", + aggConditionLanguage: 'sql', + valueExpression: '', + }, + ]), + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('tightens the two argument lists of a parametric combinator', async () => { + // A quantile series with a series-level filter renders as + // `quantileIf(level)(expr, condition)`; the formatter must not leave a + // space before either paren. + const sql = await renderBuilderConfigAsSqlTemplate( + seriesLevelConfig([ + { + aggFn: 'quantile', + level: 0.95, + aggCondition: 'ServiceName:api', + aggConditionLanguage: 'lucene', + valueExpression: 'value', + }, + ]), + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('round-trips a series-level -If template into an executable, alert-valid query', async () => { + const sqlTemplate = await renderBuilderConfigAsSqlTemplate( + seriesLevelConfig([ + { + aggFn: 'count', + aggCondition: 'ServiceName:api', + aggConditionLanguage: 'lucene', + valueExpression: '', + }, + { + aggFn: 'avg', + aggCondition: 'ServiceName:web', + aggConditionLanguage: 'lucene', + valueExpression: 'value', + }, + ]), + mockMetadata, + ); + expect(sqlTemplate).not.toBeNull(); + + const expanded = replaceMacros({ + sqlTemplate: sqlTemplate!, + from: { databaseName: 'default', tableName: 'otel_logs' }, + }); + expect(expanded).not.toContain('$__'); + expect(expanded).toMatch(/countIf\s*\(/); + expect(expanded).toContain('`default`.`otel_logs`'); + + const { errors } = validateRawSqlForAlert({ + configType: 'sql', + sqlTemplate: sqlTemplate!, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + displayType: DisplayType.Line, + }); + expect(errors).toEqual([]); + }); + }); +}); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 296cb9f4a3..f457f8f070 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -470,6 +470,32 @@ describe('renderChartConfig', () => { expect(actual).not.toContain('TopGroups'); }); + it('renders bound values (not template macros) when generateSqlTemplate is unset', async () => { + // Regression guard for the generateSqlTemplate flag threaded through + // timeFilterExpr / timeBucketExpr / renderFrom / renderWhere: real query + // paths never set it, so the output must keep bound params and literals. + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'logs' }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-13')], + granularity: '5 minute', + }; + const sql = parameterizedQueryToSql( + await renderChartConfig(config, mockMetadata, querySettings), + ); + expect(sql).not.toContain('$__'); + expect(sql).toContain('fromUnixTimestamp64Milli('); + expect(sql).toContain('INTERVAL 5 minute'); + // parameterizedQueryToSql inlines Identifier params without backticks + expect(sql).toContain('FROM default.logs'); + }); + describe('seriesLimit (group-by series cap)', () => { const baseLogsConfig: ChartConfigWithOptDateRange = { displayType: DisplayType.Line, diff --git a/packages/common-utils/src/__tests__/sqlFormatter.test.ts b/packages/common-utils/src/__tests__/sqlFormatter.test.ts index cbd10a3619..aec9242299 100644 --- a/packages/common-utils/src/__tests__/sqlFormatter.test.ts +++ b/packages/common-utils/src/__tests__/sqlFormatter.test.ts @@ -5,7 +5,7 @@ describe('sqlFormatter(clickhouse)', () => { const input = "SELECT countIf((ServiceName = 'hdx-oss-dev-api')),toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket` FROM default.otel_logs WHERE (TimestampTime >= fromUnixTimestamp64Milli(1741887731578) AND TimestampTime <= fromUnixTimestamp64Milli(1742492531585)) AND ((ServiceName = 'hdx-oss-dev-api')) GROUP BY toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket` ORDER BY toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket`"; const expected = `SELECT - countIf ((ServiceName = 'hdx-oss-dev-api')), + countIf((ServiceName = 'hdx-oss-dev-api')), toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS \`__hdx_time_bucket\` FROM default.otel_logs @@ -26,7 +26,7 @@ ORDER BY const input = "SELECT countIf(ResourceAttributes['telemetry.sdk.language'] = 'nodejs'),ResourceAttributes['telemetry.sdk.language'],toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket` FROM default.otel_logs WHERE (TimestampTime >= fromUnixTimestamp64Milli(1741887731578) AND TimestampTime <= fromUnixTimestamp64Milli(1742492531585)) AND (ResourceAttributes['telemetry.sdk.language'] = 'nodejs') GROUP BY ResourceAttributes['telemetry.sdk.language'],toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket` ORDER BY toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket`"; const expected = `SELECT - countIf ( + countIf( ResourceAttributes['telemetry.sdk.language'] = 'nodejs' ), ResourceAttributes['telemetry.sdk.language'], @@ -53,7 +53,7 @@ ORDER BY const input = "SELECT countIf((`ResourceAttributes`['telemetry.sdk.language'] = 'nodejs')),ResourceAttributes['telemetry.sdk.language'],toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket` FROM default.otel_logs WHERE (TimestampTime >= fromUnixTimestamp64Milli(1741887731578) AND TimestampTime <= fromUnixTimestamp64Milli(1742492531585)) AND ((`ResourceAttributes`['telemetry.sdk.language'] = 'nodejs')) GROUP BY ResourceAttributes['telemetry.sdk.language'],toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket` ORDER BY toStartOfInterval(toDateTime(TimestampTime), INTERVAL 6 hour) AS `__hdx_time_bucket`"; const expected = `SELECT - countIf ( + countIf( ( \`ResourceAttributes\` ['telemetry.sdk.language'] = 'nodejs' ) diff --git a/packages/common-utils/src/core/builderToRawSql.ts b/packages/common-utils/src/core/builderToRawSql.ts new file mode 100644 index 0000000000..b41ebf2e65 --- /dev/null +++ b/packages/common-utils/src/core/builderToRawSql.ts @@ -0,0 +1,176 @@ +import SqlString from 'sqlstring'; + +import { Metadata } from '@/core/metadata'; +import { renderChartConfig } from '@/core/renderChartConfig'; +import { + convertToCategoricalChartConfig, + convertToNumberChartConfig, + convertToTableChartConfig, +} from '@/core/utils'; +import { isBuilderChartConfig } from '@/guards'; +import { format } from '@/sqlFormatter'; +import { + ChartConfigWithDateRange, + ChartConfigWithOptDateRange, + DisplayType, +} from '@/types'; + +/** Display types that support raw-SQL chart configs. */ +const RAW_SQL_DISPLAY_TYPES = new Set([ + DisplayType.Table, + DisplayType.Line, + DisplayType.StackedBar, + DisplayType.Pie, + DisplayType.Bar, + DisplayType.Number, +]); + +/** + * Inlines any remaining ChSql query params (e.g. LIMIT/OFFSET values) into + * the SQL text with type-appropriate quoting. `parameterizedQueryToSql` is + * not suitable here — it splices values in raw, dropping quotes. + */ +function inlineParams(sql: string, params: Record): string { + return sql.replace( + /\{(HYPERDX_PARAM_\d+):(\w+)\}/g, + (match, key: string, type: string) => { + // eslint-disable-next-line security/detect-object-injection + const value = params[key]; + if (value === undefined) { + throw new Error(`Unbound query param ${match} in SQL template`); + } + if (type === 'Identifier') { + return `\`${String(value).replaceAll('`', '\\`')}\``; + } + if (type === 'String') { + return SqlString.escape(String(value)); + } + return String(value); + }, + ); +} + +function tryFormatSqlTemplate(sql: string): string { + try { + return format(sql); + } catch { + return sql; + } +} + +/** + * Result of {@link renderBuilderConfigAsSqlTemplate}: either the generated SQL + * template, or a user-facing reason the config can't be converted. + */ +export type RenderedSqlTemplate = + | { isError: false; sql: string } + | { isError: true; error: string }; + +/** + * Renders a builder chart config as a raw-SQL template string using the + * dynamic template macros ($__fromTime_ms, $__toTime_ms, $__timeInterval, + * $__sourceTable, $__filters) in place of bound date/interval/table values. + * + * On success returns `{ sql }`. When the config can't be represented as a + * single raw-SQL chart it returns `{ error }` with a user-facing reason — + * non-builder configs, multi-series or non-time-series metric charts, string + * selects (Search / EventPatterns), display types without raw-SQL support, or + * a missing source — so callers can surface the message without re-deriving it. + */ +export async function renderBuilderConfigAsSqlTemplate( + config: ChartConfigWithOptDateRange, + metadata: Metadata, +): Promise { + if (!isBuilderChartConfig(config) || typeof config.select === 'string') { + return { + isError: true, + error: 'This chart type cannot be auto-converted to SQL.', + }; + } + + // Metric charts render one query per series (see splitChartConfigs), so only + // single-series metric charts can be converted to a single raw-SQL query. + const isMetric = config.metricTables != null; + if (isMetric && Array.isArray(config.select) && config.select.length > 1) { + return { + isError: true, + error: 'Multi-series metric charts cannot be auto-converted to SQL.', + }; + } + + // A concrete source table is required for non-metric charts; metric charts + // resolve their table from metricTables via the $__sourceTable(metricType) + // macro, so they only need the database. + if (!config.from?.databaseName || (!isMetric && !config.from?.tableName)) { + return { + isError: true, + error: 'Auto-converting to SQL requires a source to be selected.', + }; + } + + const displayType = config.displayType ?? DisplayType.Line; + if (!RAW_SQL_DISPLAY_TYPES.has(displayType)) { + return { + isError: true, + error: 'This chart type cannot be auto-converted to SQL.', + }; + } + + // The intervalSeconds param behind $__timeInterval is only bound for + // time-series display types (QUERY_PARAMS_BY_DISPLAY_TYPE), so only those + // keep a granularity; its concrete value is resolved at query time. + const isTimeSeries = + displayType === DisplayType.Line || displayType === DisplayType.StackedBar; + + // Metric queries time-bucket inside their CTEs and therefore need the + // interval macros that only time-series display types bind, so metric + // conversion is restricted to time-series charts. + if (isMetric && !isTimeSeries) { + return { + isError: true, + error: + 'Metric charts can only be auto-converted to SQL for time series display types.', + }; + } + + // Apply the same per-display-type transform the chart itself runs so the + // template's query shape (LIMIT / ORDER BY / dropped group-by) matches the + // live query. Time-series (line/stacked bar) is intentionally handled inline via the + // granularity/date macros below rather than convertToTimeChartConfig, which + // resolves granularity and the date range to concrete values (and adds a + // live-query-only LIMIT) — the opposite of what a reusable template needs. + const templateConfig: ChartConfigWithDateRange = { + ...config, + dateRange: [new Date(0), new Date(0)], + }; + const renderConfig = + displayType === DisplayType.Pie || displayType === DisplayType.Bar + ? convertToCategoricalChartConfig(templateConfig) + : displayType === DisplayType.Number + ? convertToNumberChartConfig(templateConfig) + : displayType === DisplayType.Table + ? convertToTableChartConfig(templateConfig) + : config; + + const rendered = await renderChartConfig( + { + ...renderConfig, + displayType, + dateRange: [new Date(0), new Date(0)], + granularity: isTimeSeries ? (config.granularity ?? 'auto') : undefined, + isRenderingRawSqlTemplate: true, + }, + metadata, + undefined, + ); + + const sql = inlineParams(rendered.sql, rendered.params); + if (sql.includes('HYPERDX_PARAM_')) { + throw new Error('Unsubstituted query param in generated SQL template'); + } + + return { + isError: false, + sql: tryFormatSqlTemplate(sql), + }; +} diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index f8aa4c64a2..faa05f0003 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -804,6 +804,7 @@ function timeBucketExpr({ bucketTimestampValueExpression, dateRange, alias = FIXED_TIME_BUCKET_EXPR_ALIAS, + isRenderingRawSqlTemplate, }: { interval: SQLInterval | 'auto'; timestampValueExpression: string; @@ -816,12 +817,20 @@ function timeBucketExpr({ bucketTimestampValueExpression?: string; dateRange?: [Date, Date]; alias?: string; + isRenderingRawSqlTemplate?: boolean; }) { const unsafeTimestampValueExpression = { UNSAFE_RAW_SQL: bucketTimestampValueExpression ?? getFirstTimestampValueExpression(timestampValueExpression), }; + + if (isRenderingRawSqlTemplate) { + return chSql`$__timeInterval(${unsafeTimestampValueExpression}) AS \`${{ + UNSAFE_RAW_SQL: alias, + }}\``; + } + const unsafeInterval = { UNSAFE_RAW_SQL: interval === 'auto' && Array.isArray(dateRange) @@ -840,6 +849,7 @@ export async function timeFilterExpr({ dateRange, dateRangeEndInclusive, dateRangeStartInclusive, + isRenderingRawSqlTemplate, includedDataInterval, metadata, tableName, @@ -851,6 +861,7 @@ export async function timeFilterExpr({ dateRange: [Date, Date]; dateRangeEndInclusive: boolean; dateRangeStartInclusive: boolean; + isRenderingRawSqlTemplate?: boolean; includedDataInterval?: string; metadata: Metadata; tableName: string; @@ -923,13 +934,21 @@ export async function timeFilterExpr({ ); } - const rawStartBound = includedDataInterval - ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: startTime }}), INTERVAL ${includedDataInterval}) - INTERVAL ${includedDataInterval}` - : chSql`fromUnixTimestamp64Milli(${{ Int64: startTime }})`; - - const rawEndBound = includedDataInterval - ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: endTime }}), INTERVAL ${includedDataInterval}) + INTERVAL ${includedDataInterval}` - : chSql`fromUnixTimestamp64Milli(${{ Int64: endTime }})`; + const rawStartBound = isRenderingRawSqlTemplate + ? includedDataInterval + ? chSql`toStartOfInterval($__fromTime_ms, INTERVAL $__interval_s second) - INTERVAL $__interval_s second` + : chSql`$__fromTime_ms` + : includedDataInterval + ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: startTime }}), INTERVAL ${includedDataInterval}) - INTERVAL ${includedDataInterval}` + : chSql`fromUnixTimestamp64Milli(${{ Int64: startTime }})`; + + const rawEndBound = isRenderingRawSqlTemplate + ? includedDataInterval + ? chSql`toStartOfInterval($__toTime_ms, INTERVAL $__interval_s second) + INTERVAL $__interval_s second` + : chSql`$__toTime_ms` + : includedDataInterval + ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: endTime }}), INTERVAL ${includedDataInterval}) + INTERVAL ${includedDataInterval}` + : chSql`fromUnixTimestamp64Milli(${{ Int64: endTime }})`; const startTimeCond = toStartOf ? chSql`${toStartOf.function}(${rawStartBound}${toStartOf.formattedRemainingArgs})` @@ -985,6 +1004,7 @@ async function renderSelect( bucketTimestampValueExpression: chartConfig.bucketTimestampValueExpression, dateRange: chartConfig.dateRange, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, }) : [], ); @@ -992,9 +1012,24 @@ async function renderSelect( function renderFrom({ from, + isRenderingRawSqlTemplate, + metricType, }: { from: BuilderChartConfigWithDateRange['from']; + isRenderingRawSqlTemplate?: boolean; + /** Value passed to $__sourceTable(MetricType) when rendering a metric query as a SQL template */ + metricType?: MetricsDataType; }): ChSql { + if (isRenderingRawSqlTemplate) { + if (metricType != null) { + return chSql`$__sourceTable(${{ UNSAFE_RAW_SQL: metricType }})`; + } + // The $__sourceTable macro only stands in for the real source table. A + // FROM with no database is a CTE reference, so render it literally. + if (from.databaseName !== '') { + return chSql`$__sourceTable`; + } + } return concatChSql( '.', chSql`${from.databaseName === '' ? '' : { Identifier: from.databaseName }}`, @@ -1193,6 +1228,7 @@ async function renderWhere( dateRange: chartConfig.dateRange, dateRangeStartInclusive: chartConfig.dateRangeStartInclusive ?? true, dateRangeEndInclusive: chartConfig.dateRangeEndInclusive ?? true, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, metadata, connectionId: chartConfig.connection, databaseName: chartConfig.from.databaseName, @@ -1212,6 +1248,13 @@ async function renderWhere( '(', ')', ), + // $__filters expands (at query time) to the dashboard filters, which + // reference columns of the real source table. Only emit it when this WHERE + // targets that source table (indicated by a non-empty databaseName). + chartConfig.isRenderingRawSqlTemplate && + chartConfig.from.databaseName !== '' + ? chSql`$__filters` + : [], ); } @@ -1231,6 +1274,7 @@ async function renderGroupBy( bucketTimestampValueExpression: chartConfig.bucketTimestampValueExpression, dateRange: chartConfig.dateRange, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, }) : [], ); @@ -1385,6 +1429,7 @@ function renderOrderBy( bucketTimestampValueExpression: chartConfig.bucketTimestampValueExpression, dateRange: chartConfig.dateRange, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, }) : [], chartConfig.orderBy != null @@ -1436,6 +1481,13 @@ type InternalChartFields = { * partition pruning via the Date column continues to work. */ bucketTimestampValueExpression?: string; + /** + * Emit raw-SQL-template macros ($__fromTime_ms, $__toTime_ms, + * $__timeInterval, $__sourceTable, $__filters) instead of bound + * date/interval/table values, so the result can be used as an editable + * `sqlTemplate`. + */ + isRenderingRawSqlTemplate?: boolean; }; type BuilderChartConfigWithOptDateRangeEx = BuilderChartConfigWithOptDateRange & @@ -1635,6 +1687,7 @@ async function translateMetricChartConfig( chartConfig.bucketTimestampValueExpression, dateRange: chartConfig.dateRange, alias: timeBucketCol, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, }); const where = await renderWhere( @@ -1668,7 +1721,7 @@ async function translateMetricChartConfig( SELECT *, cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash - FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Gauge] } })} + FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Gauge] }, isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, metricType: MetricsDataType.Gauge })} WHERE ${where} `, }, @@ -1728,6 +1781,7 @@ async function translateMetricChartConfig( chartConfig.bucketTimestampValueExpression, dateRange: chartConfig.dateRange, alias: timeBucketCol, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, }); // Render the where clause to limit data selection on the source CTE but also search forward/back one @@ -1792,7 +1846,7 @@ async function translateMetricChartConfig( SUM(Value) OVER (PARTITION BY AttributesHash ORDER BY TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), Value ) AS Sum - FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Sum] } })} + FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Sum] }, isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, metricType: MetricsDataType.Sum })} WHERE ${where}`, }, { @@ -2006,6 +2060,7 @@ async function translateMetricChartConfig( interval: cteChartConfig.granularity, timestampValueExpression: cteChartConfig.timestampValueExpression, dateRange: cteChartConfig.dateRange, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, }) : chSql``; const where = await renderWhere(cteChartConfig, metadata); @@ -2033,6 +2088,8 @@ async function translateMetricChartConfig( ...from, tableName: metricTables[MetricsDataType.Histogram], }, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, + metricType: MetricsDataType.Histogram, }), where, valueAlias, @@ -2170,7 +2227,10 @@ export async function renderChartConfig( let withClauses = await renderWith(chartConfig, metadata, querySettings); const select = await renderSelect(chartConfig, metadata); - const from = renderFrom(chartConfig); + const from = renderFrom({ + from: chartConfig.from, + isRenderingRawSqlTemplate: chartConfig.isRenderingRawSqlTemplate, + }); let where = await renderWhere(chartConfig, metadata); const groupBy = await renderGroupBy(chartConfig, metadata); const having = await renderHaving(chartConfig, metadata); diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 7316c5fe9a..63050f4732 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -818,6 +818,44 @@ export function convertToCategoricalChartConfig( return convertedConfig; } +/** + * Number charts collapse to a single aggregate value, so drop the time bucket + * (granularity) and any group-by. + */ +export function convertToNumberChartConfig( + config: BuilderChartConfigWithOptTimestamp, +): BuilderChartConfigWithOptTimestamp { + return omit(config, ['granularity', 'groupBy']); +} + +/** + * Table charts drop the time bucket (granularity) and, so the set of rows kept + * within the limit is stable, default to a row limit and a group-by ordering + * when the user hasn't set them. + */ +export function convertToTableChartConfig( + config: BuilderChartConfigWithOptTimestamp, +): BuilderChartConfigWithOptTimestamp { + const convertedConfig = structuredClone(omit(config, ['granularity'])); + + // Set a default limit if not already set + if (!convertedConfig.limit) { + convertedConfig.limit = { limit: 200 }; + } + + // Set a default orderBy if groupBy is set but orderBy is not, + // so that the set of rows within the limit is stable. + if ( + convertedConfig.groupBy && + typeof convertedConfig.groupBy === 'string' && + !convertedConfig.orderBy + ) { + convertedConfig.orderBy = convertedConfig.groupBy; + } + + return convertedConfig; +} + export const getFirstOrderingItem = ( orderBy: BuilderChartConfigWithDateRange['orderBy'], ) => { diff --git a/packages/common-utils/src/sqlFormatter.ts b/packages/common-utils/src/sqlFormatter.ts index 034d85dff5..5b6995a156 100644 --- a/packages/common-utils/src/sqlFormatter.ts +++ b/packages/common-utils/src/sqlFormatter.ts @@ -1,5 +1,44 @@ import { clickhouse, formatDialect } from 'sql-formatter'; -export function format(query) { - return formatDialect(query, { dialect: clickhouse }); +/** Matches a call to an aggregate-combinator function (eg. `countIf(`,). */ +const COMBINATOR_FUNCTION_CALL_PATTERN = + /\b([A-Za-z_]\w*(?:If|Array|Map|State|Merge|ForEach|Distinct|OrDefault|OrNull|Resample))\s+\(/g; + +/** Matches a call to a parametric aggregate function with optional combinator (eg. `quantileIf(0.5)(`). */ +const PARAMETRIC_COMBINATOR_FUNCTION_CALL_PATTERN = + /\b([A-Za-z_]\w*(?:If|Array|Map|State|Merge|ForEach|Distinct|OrDefault|OrNull|Resample))\(([^()]*)\)\s+\(/g; + +/** + * Formats a ClickHouse SQL string (or raw-SQL macro template) for display. + * + * On top of sql-formatter this: + * - treats `$__…` template macros as params so the formatter doesn't choke on + * them (a no-op for already-resolved SQL that contains no macros), and + * tightens the spacing it inserts after a macro name, so `$__name (` — which + * `parseMacroArgs` would reject — stays `$__name(`. + * - tightens the space sql-formatter inserts after aggregate-combinator + * functions (`countIf (…)` → `countIf(…)`), which it treats as bare + * identifiers. + * + * Throws if the SQL can't be parsed (callers that want resilience should catch). + */ +export function format(query: string): string { + const formatted = formatDialect(query, { + dialect: clickhouse, + // sql-formatter can't parse $__macros, so treat them as params. + paramTypes: { custom: [{ regex: String.raw`\$__\w+` }] }, + }); + + return ( + formatted + // Remove any spaces inserted between a macro name and its opening paren, so `$__name (` → `$__name(`. + .replace(/\$__(\w+)\s+\(/g, (_m, name) => `$__${name}(`) + // Remove spaces between function names and their parentheses, eg. `countIf (...)` -> `countIf(...)` + .replace(COMBINATOR_FUNCTION_CALL_PATTERN, (_m, name) => `${name}(`) + // Remove spaces between parametric function names and their parentheses, eg. `countIf (...)` -> `countIf(...)` + .replace( + PARAMETRIC_COMBINATOR_FUNCTION_CALL_PATTERN, + (_m, name, params) => `${name}(${params})(`, + ) + ); }