diff --git a/.changeset/time-chart-series-limit.md b/.changeset/time-chart-series-limit.md new file mode 100644 index 0000000000..27aae22282 --- /dev/null +++ b/.changeset/time-chart-series-limit.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/app": patch +"@hyperdx/api": patch +--- + +feat(charts): cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys. The cap defaults to 100 (the number of series rendered) and is configurable per team via a new "Time Chart Series Limit" setting; series beyond the cap remain available in the series selector. diff --git a/packages/api/src/models/team.ts b/packages/api/src/models/team.ts index 4d6b620162..15e92ec72c 100644 --- a/packages/api/src/models/team.ts +++ b/packages/api/src/models/team.ts @@ -42,6 +42,7 @@ export default mongoose.model( fieldMetadataDisabled: Boolean, parallelizeWhenPossible: Boolean, filterKeysFetchLimit: Number, + seriesLimit: Number, }, { timestamps: true, diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index 544c6762d5..b284670182 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -37,6 +37,7 @@ import { notifications } from '@mantine/notifications'; import DateRangeIndicator from './components/charts/DateRangeIndicator'; import { MVOptimizationExplanationResult } from './hooks/useMVOptimizationExplanation'; +import { DEFAULT_SERIES_LIMIT } from './defaults'; import { getMetricNameSql } from './otelSemanticConventions'; import { AggFn, TableChartSeries, TimeChartSeries } from './types'; import { NumberFormat } from './types'; @@ -103,9 +104,14 @@ function getTimeChartDateRange( : getAlignedDateRange(dateRange, granularity); } +export const MAX_TIME_CHART_SERIES = DEFAULT_SERIES_LIMIT; + export function convertToTimeChartConfig( config: ChartConfigWithDateRange, + teamSeriesLimit?: number, ): ChartConfigWithDateRange { + const seriesLimit = Math.max(1, teamSeriesLimit ?? MAX_TIME_CHART_SERIES); + const granularity = getTimeChartGranularity( config.granularity, config.dateRange, @@ -133,6 +139,7 @@ export function convertToTimeChartConfig( dateRangeEndInclusive, granularity, limit: { limit: 100000 }, + seriesLimit, } : { ...config, diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index 1a08dc1785..d97285eda4 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -29,7 +29,11 @@ import { ChartTooltipContainer, ChartTooltipItem, } from './components/charts/ChartTooltip'; -import { LineData, toStartOfInterval } from './ChartUtils'; +import { + LineData, + MAX_TIME_CHART_SERIES, + toStartOfInterval, +} from './ChartUtils'; import { FormatTime, useFormatTime } from './useFormatTime'; import styles from '../styles/HDXLineChart.module.scss'; @@ -317,7 +321,7 @@ const LegendRenderer = memo<{ ); }); -export const HARD_LINES_LIMIT = 60; +export const HARD_LINES_LIMIT = MAX_TIME_CHART_SERIES; const StackedBarWithOverlap = (props: BarProps) => { const { x, y, width, height, fill } = props; diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index 2d3d5834ed..ab9acf58e7 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -11,6 +11,7 @@ import { formatResponseForPieChart, formatResponseForTimeChart, } from '@/ChartUtils'; +import { DEFAULT_SERIES_LIMIT } from '@/defaults'; import { COLORS } from '@/utils'; // Anchor info/error to concrete hexes rather than `getChartColorInfo()` / @@ -806,6 +807,36 @@ describe('ChartUtils', () => { expect(granularityFromFunction).toBe('5 minute'); }); + + const seriesLimitConfig = { + granularity: '5 minute', + dateRange: [ + new Date('2025-11-26T00:00:00Z'), + new Date('2025-11-27T00:00:00Z'), + ], + } as BuilderChartConfigWithDateRange; + + // seriesLimit lives on the builder member of the ChartConfigWithDateRange + // union, so narrow the result before reading it. + const seriesLimitOf = (teamSeriesLimit?: number) => + ( + convertToTimeChartConfig( + seriesLimitConfig, + teamSeriesLimit, + ) as BuilderChartConfigWithDateRange + ).seriesLimit; + + it('defaults seriesLimit to DEFAULT_SERIES_LIMIT when no team value is given', () => { + expect(seriesLimitOf()).toBe(DEFAULT_SERIES_LIMIT); + }); + + it('uses the team seriesLimit when provided', () => { + expect(seriesLimitOf(5)).toBe(5); + }); + + it('passes a large team seriesLimit through unbounded', () => { + expect(seriesLimitOf(100000)).toBe(100000); + }); }); describe('convertToNumberChartConfig', () => { diff --git a/packages/app/src/components/DBTimeChart.tsx b/packages/app/src/components/DBTimeChart.tsx index c0f2b26616..9ae6f1a54c 100644 --- a/packages/app/src/components/DBTimeChart.tsx +++ b/packages/app/src/components/DBTimeChart.tsx @@ -287,9 +287,11 @@ function DBTimeChartComponent({ fillNulls, } = useTimeChartSettings(config); + const { data: me, isLoading: isLoadingMe } = api.useMe(); + const queriedConfig = useMemo( - () => convertToTimeChartConfig(config), - [config], + () => convertToTimeChartConfig(config, me?.team?.seriesLimit), + [config, me?.team?.seriesLimit], ); // Determine whether the config can be optimized with an MV, to determine whether @@ -299,7 +301,6 @@ function DBTimeChartComponent({ const { data: mvOptimizationData } = useMVOptimizationExplanation(builderQueriedConfig); - const { data: me, isLoading: isLoadingMe } = api.useMe(); const { data, isLoading, isError, error, isPlaceholderData, isSuccess } = useQueriedChartConfig(queriedConfig, { placeholderData: (prev: any) => prev, diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index edceee9741..a6be03f89d 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -35,13 +35,13 @@ export function useSearchTotalCount( enableParallelQueries?: boolean; } = {}, ) { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + // queriedConfig, queryKey, and enableQueryChunking match DBTimeChart so that react query can de-dupe these queries. const queriedConfig = useMemo( - () => convertToTimeChartConfig(config), - [config], + () => convertToTimeChartConfig(config, me?.team?.seriesLimit), + [config, me?.team?.seriesLimit], ); - - const { data: me, isLoading: isLoadingMe } = api.useMe(); const { data: totalCountData, isLoading, diff --git a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx index de850d2d6f..2cfc66c9b4 100644 --- a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx +++ b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx @@ -24,6 +24,7 @@ import { DEFAULT_FILTER_KEYS_FETCH_LIMIT_WITH_MVS, DEFAULT_QUERY_TIMEOUT, DEFAULT_SEARCH_ROW_LIMIT, + DEFAULT_SERIES_LIMIT, } from '@/defaults'; import { useBrandDisplayName } from '@/theme/ThemeProvider'; @@ -343,6 +344,16 @@ export default function TeamQueryConfigSection() { type="boolean" displayValue={value => (value ? 'Enabled' : 'Disabled')} /> + diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 55bf3dafc2..c3537c5911 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -5,6 +5,7 @@ export const DEFAULT_SEARCH_ROW_LIMIT = 200; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 20; export const DEFAULT_FILTER_KEYS_FETCH_LIMIT_WITH_MVS = 100; +export const DEFAULT_SERIES_LIMIT = 100; export function searchChartConfigDefaults( team: any | undefined | null, diff --git a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts index 65e09e8d42..fc50abd854 100644 --- a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts +++ b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts @@ -172,4 +172,293 @@ describe('queryChartConfig Integration Tests', () => { expect(metaNames).toContain('__hdx_time_bucket'); expect(metaNames.indexOf('__hdx_time_bucket')).toBeGreaterThanOrEqual(3); }); + + // End-to-end: the cap CTE is valid SQL and restricts a high-cardinality + // group-by to the top N by max value in any bucket. + it('caps high-cardinality group-by series to the top N via seriesLimit', async () => { + const SERIES_TABLE = 'logs_series_limit_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${SERIES_TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + ServiceName String CODEC(ZSTD(1)), + Value Float64 CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (ServiceName, Timestamp)`, + }); + + // 50 distinct series; Value == index so the top 5 by value are svc-45..49. + const rows = Array.from({ length: 50 }, (_, i) => ({ + Timestamp: '2025-04-15 00:10:00', + ServiceName: `svc-${String(i).padStart(2, '0')}`, + Value: i, + })); + await client.insert({ + table: `${DATABASE}.${SERIES_TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: SERIES_TABLE }, + select: [ + { + aggFn: 'max', + aggCondition: '', + aggConditionLanguage: 'sql', + valueExpression: 'Value', + }, + ], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + seriesLimit: 5, + }; + + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + // Without the cap this would be 50 distinct services. + const services = new Set( + (result.data as Array<{ ServiceName: string }>).map(r => r.ServiceName), + ); + expect(services.size).toBeLessThanOrEqual(5); + expect([...services].sort()).toEqual([ + 'svc-45', + 'svc-46', + 'svc-47', + 'svc-48', + 'svc-49', + ]); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${SERIES_TABLE}`, + }); + } + }); + + // Regression: a comma-separated string group-by (with a Map access) must split + // per-column (not emit toString(col1, col2)); empty-string groups are kept. + it('handles a multi-column string group-by (with Map access) under seriesLimit', async () => { + const TABLE = 'logs_string_gb_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + LogAttributes Map(String, String) CODEC(ZSTD(1)), + ServiceName String CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (ServiceName, Timestamp)`, + }); + + const ts = '2025-04-15 00:10:00'; + const rows = [ + // capA/svc1 (5 rows), capB/svc2 (3 rows) — the two non-empty series. + ...Array.from({ length: 5 }, () => ({ + Timestamp: ts, + LogAttributes: { 'agentToServer.capabilities': 'capA' }, + ServiceName: 'svc1', + })), + ...Array.from({ length: 3 }, () => ({ + Timestamp: ts, + LogAttributes: { 'agentToServer.capabilities': 'capB' }, + ServiceName: 'svc2', + })), + // Missing capability key (Map access -> '') for svc3 — largest by count. + // Empty-string groups are kept, so this ranks #1 and survives the cap. + ...Array.from({ length: 10 }, () => ({ + Timestamp: ts, + LogAttributes: {}, + ServiceName: 'svc3', + })), + ]; + await client.insert({ + table: `${DATABASE}.${TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + // Comma-separated string group-by — the shape that previously errored. + groupBy: "LogAttributes['agentToServer.capabilities'],ServiceName", + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + // Cap to 2 so the ranking is observable: by count svc3 (10) > svc1 (5) + // > svc2 (3), so the top 2 are svc3 and svc1; svc2 is dropped. + seriesLimit: 2, + }; + + // The query must execute without a ClickHouse error (the original bug). + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + const services = new Set( + (result.data as Array<{ ServiceName: string }>).map(r => r.ServiceName), + ); + // svc3 (empty capability) is kept and ranks #1; the cap drops svc2. + expect([...services].sort()).toEqual(['svc1', 'svc3']); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE}`, + }); + } + }); + + // NULL group components are dropped from the ranking; otherwise a NULL group + // could take a slot the NULL-unsafe outer `tuple() IN (...)` can never fill. + it('excludes NULL group components from the series cap', async () => { + const TABLE = 'logs_nullable_gb_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + Region Nullable(String) CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (Timestamp)`, + }); + + const ts = '2025-04-15 00:10:00'; + const rows = [ + // 'us' has 5 rows; the NULL-region group has 10 (the largest by count). + ...Array.from({ length: 5 }, () => ({ Timestamp: ts, Region: 'us' })), + ...Array.from({ length: 10 }, () => ({ Timestamp: ts, Region: null })), + ]; + await client.insert({ + table: `${DATABASE}.${TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + groupBy: [{ aggCondition: '', valueExpression: 'Region' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + // Only one slot: without the NULL filter the (larger) NULL group would + // claim it and then match nothing, yielding an empty chart. + seriesLimit: 1, + }; + + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + const regions = (result.data as Array<{ Region: string | null }>).map( + r => r.Region, + ); + // 'us' (top non-null) takes the slot; the NULL group is excluded. + expect(new Set(regions)).toEqual(new Set(['us'])); + expect(regions).not.toContain(null); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE}`, + }); + } + }); + + // Multi-column array group-by with an alias: the 2-column tuple()/IN executes + // (alias stripped in the CTE — a leaked `AS "reg"` there is a syntax error) + // and the alias is preserved as the output column. + it('handles a multi-column array group-by with an alias under seriesLimit', async () => { + const TABLE = 'logs_array_alias_gb_int_test'; + await client.command({ + query: `CREATE OR REPLACE TABLE ${DATABASE}.${TABLE} ( + Timestamp DateTime CODEC(ZSTD(1)), + Region String CODEC(ZSTD(1)), + ServiceName String CODEC(ZSTD(1)), + Value Float64 CODEC(ZSTD(1)) + ) ENGINE = MergeTree ORDER BY (Timestamp)`, + }); + + const ts = '2025-04-15 00:10:00'; + const rows = [ + { Timestamp: ts, Region: 'us', ServiceName: 'svc1', Value: 10 }, + { Timestamp: ts, Region: 'eu', ServiceName: 'svc2', Value: 5 }, + { Timestamp: ts, Region: 'ap', ServiceName: 'svc3', Value: 1 }, + ]; + await client.insert({ + table: `${DATABASE}.${TABLE}`, + values: rows, + format: 'JSONEachRow', + }); + + try { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE }, + select: [ + { + aggFn: 'max', + aggCondition: '', + aggConditionLanguage: 'sql', + valueExpression: 'Value', + }, + ], + groupBy: [ + { aggCondition: '', valueExpression: 'Region', alias: 'reg' }, + { aggCondition: '', valueExpression: 'ServiceName' }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-04-15T00:00:00Z'), + new Date('2025-04-15T01:00:00Z'), + ], + granularity: '5 minute', + // Top 2 by max(Value): (us,svc1)=10 and (eu,svc2)=5; (ap,svc3)=1 dropped. + seriesLimit: 2, + }; + + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + const services = new Set( + (result.data as Array<{ ServiceName: string }>).map(r => r.ServiceName), + ); + expect(services).toEqual(new Set(['svc1', 'svc2'])); + // The alias survives in the output even though it is stripped in the CTE. + expect(result.meta?.some(m => m.name === 'reg')).toBe(true); + } finally { + await client.command({ + query: `DROP TABLE IF EXISTS ${DATABASE}.${TABLE}`, + }); + } + }); }); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 9816c0f935..13a588900c 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -469,6 +469,164 @@ describe('renderChartConfig', () => { expect(actual).not.toContain('TopGroups'); }); + describe('seriesLimit (group-by series cap)', () => { + const baseLogsConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'logs' }, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-13')], + granularity: '5 minute', + }; + + it('restricts to the top N group-by series via a CTE when seriesLimit is set', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { ...baseLogsConfig, seriesLimit: 60 }, + mockMetadata, + querySettings, + ), + ); + // A ranking CTE keeps the top N groups by max value in any bucket. + expect(sql).toContain('__hdx_series_limit'); + expect(sql).toMatch(/ORDER\s+BY\s+max\(`__hdx_series_rank`\)\s+DESC/); + expect(sql).toContain('LIMIT 60'); + // The outer query is restricted to those groups via an IN subquery. + expect(sql).toMatch( + /tuple\(ServiceName\)\s+IN\s*\(\s*SELECT\s+`group`\s+FROM\s+`__hdx_series_limit`\)/, + ); + // Groups with a NULL component are excluded; empty-string groups are kept + // (no `!= ''` check). + expect(sql).toMatch(/ServiceName\s+IS\s+NOT\s+NULL/); + expect(sql).not.toMatch(/toString\(ServiceName\)\s*!=\s*''/); + }); + + it('does not emit a series-limit CTE when seriesLimit is unset (e.g. alert evaluation)', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig(baseLogsConfig, mockMetadata, querySettings), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + + it('does not emit a series-limit CTE without a group-by', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { ...baseLogsConfig, groupBy: undefined, seriesLimit: 60 }, + mockMetadata, + querySettings, + ), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + + it('does not emit a series-limit CTE without granularity', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { ...baseLogsConfig, granularity: undefined, seriesLimit: 60 }, + mockMetadata, + querySettings, + ), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + + it('packs a multi-column group-by into a tuple for the series-limit CTE', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...baseLogsConfig, + groupBy: [ + { aggCondition: '', valueExpression: 'ServiceName' }, + { aggCondition: '', valueExpression: 'TraceId' }, + ], + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('__hdx_series_limit'); + expect(sql).toMatch(/tuple\(\s*ServiceName\s*,\s*TraceId\s*\)/); + }); + + it('strips group-by aliases inside the series-limit CTE tuple and null filter', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...baseLogsConfig, + groupBy: [ + { + aggCondition: '', + valueExpression: 'ServiceName', + alias: 'svc', + }, + ], + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('__hdx_series_limit'); + // tuple() and `IS NOT NULL` must use the bare expression, not `ServiceName + // AS "svc"` (which would be invalid SQL there). + expect(sql).toMatch( + /tuple\(ServiceName\)\s+IN\s*\(\s*SELECT\s+`group`\s+FROM\s+`__hdx_series_limit`\)/, + ); + expect(sql).not.toContain('tuple(ServiceName AS'); + expect(sql).not.toMatch(/ServiceName\s+AS\s+"svc"\s+IS\s+NOT\s+NULL/); + }); + + it('splits a comma-separated string group-by into per-column null checks', async () => { + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...baseLogsConfig, + groupBy: "LogAttributes['agentToServer.capabilities'],ServiceName", + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).toContain('__hdx_series_limit'); + // Each column gets its own NULL check, split on the top-level comma — not + // the comma inside Map['...']. + expect(sql).toMatch( + /LogAttributes\[['"]agentToServer\.capabilities['"]\]\s+IS\s+NOT\s+NULL/, + ); + expect(sql).toMatch(/ServiceName\s+IS\s+NOT\s+NULL/); + // Regression: must NOT emit a two-argument toString of both columns (the + // original bug that prompted the split). + expect(sql).not.toMatch(/toString\([^)]*,/); + // Both columns are packed into the tuple for the IN predicate. + expect(sql).toMatch( + /tuple\(\s*LogAttributes\[['"]agentToServer\.capabilities['"]\]\s*,\s*ServiceName\s*\)\s+IN\s*\(\s*SELECT\s+`group`\s+FROM\s+`__hdx_series_limit`\)/, + ); + }); + + it('does not emit a series-limit CTE for a metric source', async () => { + // Metric configs are rewritten to query a Bucketed CTE (no real source + // table to re-scan), so the cap is gated off even with seriesLimit set. + const sql = parameterizedQueryToSql( + await renderChartConfig( + { + ...gaugeConfiguration, + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + seriesLimit: 60, + }, + mockMetadata, + querySettings, + ), + ); + expect(sql).not.toContain('__hdx_series_limit'); + }); + }); + it('should throw when aggFn=increase is used on a non-Sum metric', async () => { const config: ChartConfigWithOptDateRange = { displayType: DisplayType.Line, diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 1ae4198c1b..c25e985890 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -1210,6 +1210,94 @@ async function renderGroupBy( ); } +async function renderSeriesLimitCte( + chartConfig: BuilderChartConfigWithOptDateRangeEx, + metadata: Metadata, + { + from, + where, + groupBy, + }: { from: ChSql; where: ChSql; groupBy: ChSql | undefined }, +): Promise<{ cte: ChSql; predicate: ChSql } | undefined> { + const { seriesLimit } = chartConfig; + if ( + seriesLimit == null || + !isUsingGroupBy(chartConfig) || + !isUsingGranularity(chartConfig) || + chartConfig.selectGroupBy === false || + // Skip CTE/metric sources (no real table to re-scan) and string selects. + !chartConfig.from?.databaseName || + !chartConfig.from?.tableName || + !Array.isArray(chartConfig.select) || + chartConfig.select.length === 0 || + groupBy == null + ) { + return undefined; + } + + // One ChSql per group-by column (groupBy may be an array or a comma-separated + // string). splitAndTrimWithBracket respects []/()/quotes so it won't split + // inside Map['a,b']; the per-column null filter below needs them separated. + let groupByCols: ChSql[]; + if (typeof chartConfig.groupBy === 'string') { + groupByCols = splitAndTrimWithBracket(chartConfig.groupBy).map( + col => chSql`${{ UNSAFE_RAW_SQL: col }}`, + ); + } else { + // Strip aliases: these go inside tuple(...)/`IS NOT NULL`, where an + // `AS "alias"` suffix is a syntax error (unlike the outer GROUP BY). + const rendered = await renderSelectList( + chartConfig.groupBy.map(col => ({ ...col, alias: undefined })), + chartConfig, + metadata, + ); + groupByCols = Array.isArray(rendered) ? rendered : [rendered]; + } + const groupByTuple = concatChSql(',', groupByCols); + + // Rank by the chart's first aggregate (alias stripped — we add our own). + const firstSelect = chartConfig.select[0]; + const rankSelectList = + typeof firstSelect === 'string' + ? firstSelect + : [{ ...firstSelect, alias: undefined }]; + const rankRendered = await renderSelectList( + rankSelectList, + chartConfig, + metadata, + ); + const rankValue = Array.isArray(rankRendered) + ? rankRendered[0] + : rankRendered; + + // Drop NULL components only (no-op on non-nullable columns). + const groupByNotNullFilter = concatChSql( + ' AND ', + groupByCols.map(g => chSql`${g} IS NOT NULL`), + ); + const innerWhere = where.sql + ? concatChSql(' AND ', where, groupByNotNullFilter) + : groupByNotNullFilter; + + // Per-(group, bucket) aggregate, then max per group, keeping the top N. + const cte = chSql`\`__hdx_series_limit\` AS ( + SELECT \`group\` + FROM ( + SELECT tuple(${groupByTuple}) AS \`group\`, ${rankValue} AS \`__hdx_series_rank\` + FROM ${from} + WHERE ${innerWhere} + GROUP BY ${groupBy} + ) + GROUP BY \`group\` + ORDER BY max(\`__hdx_series_rank\`) DESC, \`group\` + LIMIT ${{ Int32: seriesLimit }} + )`; + + const predicate = chSql`tuple(${groupByTuple}) IN (SELECT \`group\` FROM \`__hdx_series_limit\`)`; + + return { cte, predicate }; +} + async function renderHaving( chartConfig: BuilderChartConfigWithOptDateRangeEx, metadata: Metadata, @@ -2065,10 +2153,10 @@ export async function renderChartConfig( : undefined), }; - const withClauses = await renderWith(chartConfig, metadata, querySettings); + let withClauses = await renderWith(chartConfig, metadata, querySettings); const select = await renderSelect(chartConfig, metadata); const from = renderFrom(chartConfig); - const where = await renderWhere(chartConfig, metadata); + let where = await renderWhere(chartConfig, metadata); const groupBy = await renderGroupBy(chartConfig, metadata); const having = await renderHaving(chartConfig, metadata); const orderBy = renderOrderBy(chartConfig); @@ -2076,6 +2164,20 @@ export async function renderChartConfig( const limit = renderLimit(chartConfig); const settings = renderSettings(chartConfig, querySettings); + const seriesCap = await renderSeriesLimitCte(chartConfig, metadata, { + from, + where, + groupBy, + }); + if (seriesCap) { + withClauses = withClauses + ? concatChSql(',', withClauses, seriesCap.cte) + : seriesCap.cte; + where = where.sql + ? concatChSql(' AND ', where, seriesCap.predicate) + : seriesCap.predicate; + } + return concatChSql(' ', [ chSql`${withClauses?.sql ? chSql`WITH ${withClauses}` : ''}`, chSql`SELECT ${select}`, diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 95c0f157df..dc38e71b2f 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -275,6 +275,7 @@ export const SelectSQLStatementSchema = z.object({ havingLanguage: SearchConditionLanguageSchema.optional(), orderBy: SortSpecificationListSchema.optional(), limit: LimitSchema.optional(), + seriesLimit: z.number().int().positive().optional(), }); export type SQLInterval = z.infer; @@ -1522,6 +1523,7 @@ export const TeamClickHouseSettingsSchema = z.object({ metadataMaxRowsToRead: z.number().optional(), parallelizeWhenPossible: z.boolean().optional(), filterKeysFetchLimit: z.number().optional(), + seriesLimit: z.number().int().positive().optional(), }); /** Accepts null to unset (reset to default) a setting. */ @@ -1532,6 +1534,7 @@ export const TeamClickHouseSettingsUpdateSchema = z.object({ metadataMaxRowsToRead: z.number().nullish(), parallelizeWhenPossible: z.boolean().nullish(), filterKeysFetchLimit: z.number().nullish(), + seriesLimit: z.number().int().positive().nullish(), }); export type TeamClickHouseSettingsUpdate = z.infer< typeof TeamClickHouseSettingsUpdateSchema