From d41130844a2b34761dd6ce2c212643f468ccc424 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 19 Dec 2025 08:55:28 -0500 Subject: [PATCH] feat: Add materialized view support (Beta) --- .changeset/olive-snails-shake.md | 7 + packages/api/src/models/source.ts | 3 + packages/app/src/DBDashboardPage.tsx | 130 +- packages/app/src/DBSearchPage.tsx | 1 + packages/app/src/ServicesDashboardPage.tsx | 21 +- .../src/components/DBEditTimeChartForm.tsx | 80 +- .../MaterializedViews/MVConfigSummary.tsx | 79 + .../MVOptimizationIndicator.tsx | 99 + .../MaterializedViews/MVOptimizationModal.tsx | 122 ++ .../ServiceDashboardDbQuerySidePanel.tsx | 2 + ...rviceDashboardEndpointPerformanceChart.tsx | 1 + .../ServiceDashboardEndpointSidePanel.tsx | 4 +- .../ServiceDashboardSlowestEventsTile.tsx | 2 + packages/app/src/components/SourceForm.tsx | 409 ++++- .../src/components/SourceSchemaPreview.tsx | 12 +- packages/app/src/hooks/useChartConfig.tsx | 66 +- .../hooks/useMVOptimizationExplanation.tsx | 55 + .../app/src/hooks/useOffsetPaginatedQuery.tsx | 64 +- packages/app/src/utils/materializedViews.ts | 312 ++++ .../renderChartConfig.test.ts.snap | 8 + .../src/__tests__/renderChartConfig.test.ts | 130 +- .../__tests__/materializedViews.test.ts | 1603 +++++++++++++++++ packages/common-utils/src/clickhouse/index.ts | 46 + .../src/core/materializedViews.ts | 520 ++++++ packages/common-utils/src/core/metadata.ts | 28 + .../src/core/renderChartConfig.ts | 39 +- packages/common-utils/src/types.ts | 51 +- 27 files changed, 3777 insertions(+), 117 deletions(-) create mode 100644 .changeset/olive-snails-shake.md create mode 100644 packages/app/src/components/MaterializedViews/MVConfigSummary.tsx create mode 100644 packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx create mode 100644 packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx create mode 100644 packages/app/src/hooks/useMVOptimizationExplanation.tsx create mode 100644 packages/app/src/utils/materializedViews.ts create mode 100644 packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts create mode 100644 packages/common-utils/src/core/materializedViews.ts diff --git a/.changeset/olive-snails-shake.md b/.changeset/olive-snails-shake.md new file mode 100644 index 0000000000..998ecebf54 --- /dev/null +++ b/.changeset/olive-snails-shake.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +feat: Add materialized view support (Beta) diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 800f3aed06..68c6936a40 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -72,6 +72,9 @@ export const Source = mongoose.model( highlightedRowAttributeExpressions: { type: mongoose.Schema.Types.Array, }, + materializedViews: { + type: mongoose.Schema.Types.Array, + }, metricTables: { type: { diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 682417652a..79ad092af6 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -75,6 +75,7 @@ import { } from '@/dashboard'; import DBSqlRowTableWithSideBar from './components/DBSqlRowTableWithSidebar'; +import MVOptimizationIndicator from './components/MaterializedViews/MVOptimizationIndicator'; import OnboardingModal from './components/OnboardingModal'; import { Tags } from './components/Tags'; import useDashboardFilters from './hooks/useDashboardFilters'; @@ -256,66 +257,77 @@ const Tile = forwardRef( {chart.config.name} - {hovered ? ( - - {(chart.config.displayType === DisplayType.Line || - chart.config.displayType === DisplayType.StackedBar) && ( - +} - mr={4} - > - - - - - )} + + {hovered ? ( + e.stopPropagation()}> + {(chart.config.displayType === DisplayType.Line || + chart.config.displayType === DisplayType.StackedBar) && ( + +} + mr={4} + > + + + + + )} - - - - - ) : ( - - )} + + + + + ) : ( + + )} + {source?.materializedViews?.length && queriedConfig && ( + e.stopPropagation()}> + + + )} +
(bin.1 / ${expressions.durationDivisorForMillis}, bin.2 / ${expressions.durationDivisorForMillis}, bin.3), data_nanoseconds)`, }, ], filters: [ @@ -361,6 +370,7 @@ function HttpTab({ if (!source || !expressions) return null; if (reqChartType === 'overall') { return { + source: source.id, ...pick(source, ['timestampValueExpression', 'connection', 'from']), where: appliedConfig.where || '', whereLanguage: appliedConfig.whereLanguage || 'sql', @@ -392,6 +402,7 @@ function HttpTab({ return { timestampValueExpression: 'series_time_bucket', connection: source.connection, + source: source.id, with: [ { name: 'error_series', @@ -557,6 +568,7 @@ function HttpTab({ (undefined); + const [queriedSource, setQueriedSource] = useState( + undefined, + ); + + const setQueriedConfigAndSource = useCallback( + (config: ChartConfigWithDateRange, source: TSource) => { + setQueriedConfig(config); + setQueriedSource(source); + }, + [], + ); const [saveToDashboardModalOpen, setSaveToDashboardModalOpen] = useState(false); @@ -570,16 +582,23 @@ export default function EditTimeChartForm({ ? config.orderBy : undefined, }; - setQueriedConfig( + setQueriedConfigAndSource( // WARNING: DON'T JUST ASSIGN OBJECTS OR DO SPREAD OPERATOR STUFF WHEN // YOUR STATE IS AN OBJECT. YOU'RE COPYING BY REFERENCE WHICH MIGHT // ACCIDENTALLY CAUSE A useQuery SOMEWHERE TO FIRE A REQUEST EVERYTIME // AN INPUT CHANGES. USE structuredClone TO PERFORM A DEEP COPY INSTEAD structuredClone(newConfig), + tableSource, ); } })(); - }, [handleSubmit, setChartConfig, setQueriedConfig, tableSource, dateRange]); + }, [ + handleSubmit, + setChartConfig, + setQueriedConfigAndSource, + tableSource, + dateRange, + ]); const onTableSortingChange = useCallback( (sortState: SortingState | null) => { @@ -674,6 +693,29 @@ export default function EditTimeChartForm({ const queryReady = isQueryReady(queriedConfig); + // The chart config to use when explaining to to the user whether and why + // their query is or is not being executed against a materialized view. + const chartConfigForMvOptimizationExplanation: + | ChartConfigWithDateRange + | undefined = useMemo(() => { + // If the user has submitted a query, us the submitted query, unless they have changed sources + if (queriedConfig && queriedSource?.id === tableSource?.id) { + return queriedConfig; + } + + // If there is a chart config from the props (either a saved config or one from the URL params), use that, + // unless a different source has been selected. + return chartConfig && tableSource?.id === chartConfig.source + ? { + ...chartConfig, + dateRange, + timestampValueExpression: tableSource.timestampValueExpression, + from: tableSource.from, + connection: tableSource.connection, + } + : undefined; + }, [chartConfig, dateRange, tableSource, queriedConfig, queriedSource]); + const previousDateRange = getPreviousDateRange(dateRange); const sampleEventsConfig = useMemo( @@ -786,19 +828,27 @@ export default function EditTimeChartForm({
) : ( <> - - - Data Source - - - } - /> + + + + Data Source + + + } + /> + + {tableSource && activeTab !== 'search' && ( + + )} {displayType !== DisplayType.Search && Array.isArray(select) ? ( diff --git a/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx new file mode 100644 index 0000000000..a67b04dae1 --- /dev/null +++ b/packages/app/src/components/MaterializedViews/MVConfigSummary.tsx @@ -0,0 +1,79 @@ +import { useMemo } from 'react'; +import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { MaterializedViewConfiguration } from '@hyperdx/common-utils/dist/types'; +import { Group, Pill, Stack, Table, Text } from '@mantine/core'; + +export default function MVConfigSummary({ + config, +}: { + config: MaterializedViewConfiguration; +}) { + const dimensionColumnsSplit = useMemo( + () => splitAndTrimWithBracket(config.dimensionColumns), + [config.dimensionColumns], + ); + + const columnsAndAggFns = useMemo(() => { + const map = new Map(); + for (const { sourceColumn = '', aggFn } of config.aggregatedColumns) { + if (map.has(sourceColumn)) { + map.get(sourceColumn)?.push(aggFn); + } else { + map.set(sourceColumn, [aggFn]); + } + } + return Array.from(map.entries()).sort(([sourceColA], [sourceColB]) => + sourceColA.localeCompare(sourceColB), + ); + }, [config.aggregatedColumns]); + + return ( + +
+ + Minimum Granularity + + {config.minGranularity} +
+ +
+ + Available Group and Filter Columns + + + {dimensionColumnsSplit.map(col => ( + {col} + ))} + +
+ +
+ + Available Aggregated Columns + + + + + Column + Aggregation + + + + {columnsAndAggFns.map(([sourceColumn, aggFns]) => ( + + {sourceColumn} + + + {aggFns.map(aggFn => ( + {aggFn} + ))} + + + + ))} + +
+
+ + ); +} diff --git a/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx new file mode 100644 index 0000000000..022d333be2 --- /dev/null +++ b/packages/app/src/components/MaterializedViews/MVOptimizationIndicator.tsx @@ -0,0 +1,99 @@ +import { useState } from 'react'; +import { + ChartConfigWithOptDateRange, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { ActionIcon, Badge, Tooltip } from '@mantine/core'; +import { IconBolt, IconBoltOff } from '@tabler/icons-react'; + +import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanation'; + +import MVOptimizationModal from './MVOptimizationModal'; + +const WARNING_COLOR = 'var(--color-bg-warning)'; +const SUCCESS_COLOR = 'var(--color-bg-success)'; + +function MVOptimizationIcon({ + isInWarningState, + onClick, +}: { + isInWarningState: boolean; + onClick: () => void; +}) { + return isInWarningState ? ( + + + + + + ) : ( + + + + + + ); +} + +function MVOptimizationBadge({ + isInWarningState, + onClick, +}: { + isInWarningState: boolean; + onClick: () => void; +}) { + return ( + + {isInWarningState ? 'Not Accelerated' : 'Accelerated'} + + ); +} + +export default function MVOptimizationIndicator({ + source, + config, + variant = 'badge', +}: { + source: TSource; + config: ChartConfigWithOptDateRange | undefined; + variant?: 'badge' | 'icon'; +}) { + const [modalOpen, setModalOpen] = useState(false); + const { data } = useMVOptimizationExplanation(config); + + const mvConfigs = source.materializedViews ?? []; + if (!mvConfigs?.length) { + return null; + } + + const isInWarningState = !!config && !!data && !data?.optimizedConfig; + + return ( + <> + {variant === 'icon' ? ( + setModalOpen(true)} + /> + ) : ( + setModalOpen(true)} + /> + )} + + {data && ( + setModalOpen(false)} + /> + )} + + ); +} diff --git a/packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx b/packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx new file mode 100644 index 0000000000..f50f5797f8 --- /dev/null +++ b/packages/app/src/components/MaterializedViews/MVOptimizationModal.tsx @@ -0,0 +1,122 @@ +import { useMemo } from 'react'; +import { MVOptimizationExplanation } from '@hyperdx/common-utils/dist/core/materializedViews'; +import { MaterializedViewConfiguration } from '@hyperdx/common-utils/dist/types'; +import { + Accordion, + Alert, + Badge, + Group, + Modal, + Text, + Tooltip, +} from '@mantine/core'; + +import MVConfigSummary from './MVConfigSummary'; + +const WARNING_COLOR = 'var(--color-bg-warning)'; +const SUCCESS_COLOR = 'var(--color-bg-success)'; + +function mvConfigToKey(config: MaterializedViewConfiguration) { + return `mv-${config.databaseName}-${config.tableName}`; +} + +export default function MVOptimizationModal({ + mvConfigs, + explanations, + opened, + onClose, +}: { + mvConfigs: MaterializedViewConfiguration[]; + explanations: MVOptimizationExplanation[]; + opened: boolean; + onClose: () => void; +}) { + const hasMultipleMVs = mvConfigs.length > 1; + + const explanationsByKey = useMemo( + () => new Map(explanations.map(e => [mvConfigToKey(e.mvConfig), e])), + [explanations], + ); + + const firstUsedMv = explanations.find(e => e.success)?.mvConfig; + + return ( + + {hasMultipleMVs ? 'Materialized Views' : 'Materialized View'} + + Beta + + + } + opened={opened} + onClose={onClose} + size="lg" + > + + This source is configured with{' '} + {hasMultipleMVs ? 'materialized views' : 'a materialized view'} for + accelerating some aggregations. + + + + {mvConfigs.map(config => { + const key = mvConfigToKey(config); + const explanation = explanationsByKey.get(key); + const hasErrors = !!explanation?.errors.length; + const isBeingUsedByOptimizedConfig = explanation?.success; + const rowEstimate = + explanation?.rowEstimate?.toLocaleString() ?? 'N/A'; + + return ( + + + + {config.tableName} + {isBeingUsedByOptimizedConfig ? ( + + + Active + + + ) : hasErrors ? ( + + + Incompatible + + + ) : explanation ? ( + + + Skipped + + + ) : null} + + + + <> + + {hasErrors && ( + + + The query cannot be accelerated using this materialized + view for the following reason(s): + + {explanation.errors.map((error, idx) => ( + + {error} + + ))} + + )} + + + + ); + })} + + + ); +} diff --git a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx index 7e7bf49c37..7490fcd06c 100644 --- a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx +++ b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx @@ -98,6 +98,7 @@ export default function ServiceDashboardDbQuerySidePanel({ sourceId={sourceId} hiddenSeries={['total_duration_ns']} config={{ + source: source.id, ...pick(source, [ 'timestampValueExpression', 'connection', @@ -135,6 +136,7 @@ export default function ServiceDashboardDbQuerySidePanel({ ({ + value: fn, + label: fn, +})); + // TODO: maybe otel clickhouse export migrate the schema? const OTEL_CLICKHOUSE_EXPRESSIONS = { timestampValueExpression: 'TimeUnix', @@ -264,6 +285,388 @@ function HighlightedAttributeExpressionsFormRow({ ); } +/** Component for configuring one or more materialized views */ +function MaterializedViewsFormSection({ + control, + watch, + setValue, +}: TableModelProps) { + const databaseName = + useWatch({ control, name: `from.databaseName` }) || DEFAULT_DATABASE; + + const { + fields: materializedViews, + append: appendMaterializedView, + remove: removeMaterializedView, + } = useFieldArray({ + control, + name: 'materializedViews', + }); + + return ( + + + Materialized Views + + Beta + + + } + helpText="Configure materialized views for query optimization. These pre-aggregated views can significantly improve query performance on aggregation queries." + > + + {materializedViews.map((field, index) => ( + removeMaterializedView(index)} + /> + ))} + + + + + + ); +} + +/** Component for configuring a single materialized view */ +function MaterializedViewFormSection({ + watch, + control, + mvIndex, + onRemove, + setValue, +}: { mvIndex: number; onRemove: () => void } & TableModelProps) { + const connection = useWatch({ control, name: `connection` }); + const sourceDatabaseName = + useWatch({ control, name: `from.databaseName` }) || DEFAULT_DATABASE; + const mvDatabaseName = + useWatch({ control, name: `materializedViews.${mvIndex}.databaseName` }) || + sourceDatabaseName; + const mvTableName = + useWatch({ control, name: `materializedViews.${mvIndex}.tableName` }) || ''; + + return ( + + + + + + + + + + + + + + + + + + + Timestamp Column + + + + + + + Granularity + + + + + ( +