diff --git a/.changeset/source-vars-1.md b/.changeset/source-vars-1.md new file mode 100644 index 0000000000..eeacef4d13 --- /dev/null +++ b/.changeset/source-vars-1.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': patch +--- + +feat: add data source and connection variable types to dashboards diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index f1c69d0ee5..f49cf85a6f 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -9,6 +9,7 @@ import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; +import { configUsesReferenceVariable } from '@hyperdx/common-utils/dist/referenceVariables'; import { isRangeThresholdType } from '@hyperdx/common-utils/dist/types'; import { groupBy } from 'lodash'; import { type FilterQuery, Types } from 'mongoose'; @@ -97,6 +98,11 @@ export const validateAlertInput = async ( // The alert checker cannot evaluate these tiles, so the alert would never run. if (tile.config != null && !savedChartConfigSupportsAlerts(tile.config)) { + if (configUsesReferenceVariable(tile.config)) { + throw new Api400Error( + 'Alerts are not supported on tiles that use a data source or connection variable', + ); + } if (isPromqlSavedChartConfig(tile.config)) { throw new Api400Error('Alerts are not supported on PromQL tiles'); } diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index 2abf2baf64..b0d983335b 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -1518,6 +1518,130 @@ describe('dashboard router', () => { ); }); + describe('data source variables', () => { + const dataSourceFilter = { + id: new Types.ObjectId().toString(), + type: 'DATA_SOURCE' as const, + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }; + + it('persists a tile and a filter that reference the variable', async () => { + const tile = makeTile({ sourceId: '$logs' }); + const queriedFilter = { + id: new Types.ObjectId().toString(), + type: 'QUERY_EXPRESSION' as const, + name: 'Service', + expression: 'ServiceName', + source: '$logs', + }; + + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [tile], + filters: [dataSourceFilter, queriedFilter], + }) + .expect(200); + + expect(created.body.tiles[0].config.source).toBe('$logs'); + expect(created.body.filters).toEqual([dataSourceFilter, queriedFilter]); + }); + + it('rejects a reference to an unknown variable', async () => { + const res = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, tiles: [makeTile({ sourceId: '$nope' })] }) + .expect(400); + + expect(JSON.stringify(res.body)).toMatch( + /does not name a data source variable/, + ); + }); + + it('rejects a variable offering kinds the reference cannot use', async () => { + const res = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [ + { + ...dataSourceFilter, + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }, + ], + }) + .expect(400); + + expect(JSON.stringify(res.body)).toMatch(/promql/); + }); + + it('validates a PATCH of tiles against the stored filters', async () => { + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [dataSourceFilter] }) + .expect(200); + + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ tiles: [makeTile({ sourceId: '$logs' })] }) + .expect(200); + await agent + .patch(`/dashboards/${created.body.id}`) + .send({ filters: [] }) + .expect(400); + }); + + it('drops an alert on a tile that uses the variable', async () => { + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [ + makeTile({ + sourceId: '$logs', + alert: makeMockAlert(webhook._id.toString()), + }), + ], + filters: [dataSourceFilter], + }) + .expect(200); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('rejects a tile alert on a tile that uses the variable', async () => { + const created = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + tiles: [makeTile({ sourceId: '$logs' })], + filters: [dataSourceFilter], + }) + .expect(200); + + const res = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: created.body.id, + tileId: created.body.tiles[0].id, + webhookId: webhook._id.toString(), + }), + ) + .expect(400); + + expect(res.body.message).toMatch(/data source or connection variable/); + }); + }); + describe('promql-label filters', () => { const createPromqlSource = () => Source.create({ diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 544679ef92..8ec73d04b6 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -2,12 +2,14 @@ import { validateDashboardFilterModes, validateDashboardFilterOptionUniqueness, validateDashboardFilterVariableNames, + validateDashboardVariableReferences, } from '@hyperdx/common-utils/dist/dashboardValidation'; import { getAdhocFilterSourceKinds, isAdhocFilter, isPrometheusLabelFilter, } from '@hyperdx/common-utils/dist/filters'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { DashboardFilter, DashboardSchema, @@ -39,6 +41,7 @@ import { import { getSources } from '@/controllers/sources'; import { getNonNullUserWithTeam } from '@/middleware/auth'; import type { ObjectId } from '@/models'; +import { getVariableReferenceError } from '@/utils/dashboardVariableReferences'; import { objectIdSchema } from '@/utils/zod'; // create routes that will get and update dashboards @@ -101,9 +104,12 @@ async function validateFilterSources( new Set(getFilterSourceRefs(filter).map(refKey)), ]), ); + // Variable references are checked by validateDashboardVariableReferences const newRefs = filters.flatMap(filter => getFilterSourceRefs(filter).filter( - ref => !existingRefsById.get(filter.id)?.has(refKey(ref)), + ref => + parseVariableReference(ref.id) == null && + !existingRefsById.get(filter.id)?.has(refKey(ref)), ), ); if (newRefs.length === 0) return null; @@ -164,7 +170,14 @@ router.post( '/', migrateLegacyDashboardTileColors, validateRequest({ - body: DashboardWithoutIdSchema.superRefine(addFilterIssues), + body: DashboardWithoutIdSchema.superRefine(addFilterIssues).superRefine( + (data, ctx) => + validateDashboardVariableReferences( + data.tiles, + data.filters ?? [], + ctx, + ), + ), }), async (req, res, next) => { try { @@ -223,11 +236,16 @@ router.patch( // null still signals field removal downstream. const updates = DashboardWithoutIdSchema.partial().parse(req.body); - const sourceError = await validateFilterSources( - teamId, - req.body.filters, - dashboard.filters, - ); + const sourceError = + getVariableReferenceError( + req.body.tiles ?? dashboard.tiles, + req.body.filters ?? dashboard.filters, + ) ?? + (await validateFilterSources( + teamId, + req.body.filters, + dashboard.filters, + )); if (sourceError != null) { return res.status(400).json({ message: sourceError }); } diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts index 3e17594813..c91e79b5b2 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts @@ -143,6 +143,41 @@ describe('provisionDashboards', () => { // rejects the legacy enum and `readDashboardFiles` skips the file // outright, so a provisioned dashboard authored against #2265 // wouldn't ship at all. + it('parses a tile that references a declared data source variable', () => { + fs.writeFileSync( + path.join(tmpDir, 'source-variable.json'), + JSON.stringify({ + name: 'Source Variable', + tiles: [makeTile({ sourceId: '$logs' })], + filters: [ + { + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: ['log'], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + }), + ); + expect(readDashboardFiles(tmpDir)).toHaveLength(1); + }); + + it('skips a tile that references an undeclared data source variable', () => { + fs.writeFileSync( + path.join(tmpDir, 'unknown-variable.json'), + JSON.stringify({ + name: 'Unknown Variable', + tiles: [makeTile({ sourceId: '$logs' })], + }), + ); + expect(readDashboardFiles(tmpDir)).toEqual([]); + }); + it('migrates legacy chart-N tile colors before schema validation', () => { const tile = makeTile(); (tile.config as any).color = 'chart-1'; diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index cf10427c72..4199d96182 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -1,4 +1,7 @@ -import { validateDashboardFilterOptionUniqueness } from '@hyperdx/common-utils/dist/dashboardValidation'; +import { + validateDashboardFilterOptionUniqueness, + validateDashboardVariableReferences, +} from '@hyperdx/common-utils/dist/dashboardValidation'; import { DashboardWithoutId, DashboardWithoutIdSchema, @@ -30,8 +33,10 @@ function migrateLegacyDashboardTileColorsRaw(raw: unknown): unknown { } const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine( - (data, ctx) => - validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx), + (data, ctx) => { + validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx); + validateDashboardVariableReferences(data.tiles, data.filters ?? [], ctx); + }, ); export function readDashboardFiles(dir: string): DashboardWithoutId[] { diff --git a/packages/api/src/utils/dashboardVariableReferences.ts b/packages/api/src/utils/dashboardVariableReferences.ts new file mode 100644 index 0000000000..9e1eea3442 --- /dev/null +++ b/packages/api/src/utils/dashboardVariableReferences.ts @@ -0,0 +1,24 @@ +import { + collectRefinementIssues, + validateDashboardVariableReferences, +} from '@hyperdx/common-utils/dist/dashboardValidation'; +import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; + +/** + * Rejects data source and connection variable references (`$name` in source + * and connection fields) that name no variable of the right type, or a data + * source variable offering kinds the field cannot use. Takes internal-shape + * tiles and filters, as they will be stored, so a partial update can be + * checked against the stored remainder. Returns an error message, or null. + */ +export function getVariableReferenceError( + tiles: { config: object }[] = [], + filters: DashboardFilter[] = [], +): string | null { + const issues = collectRefinementIssues(ctx => + validateDashboardVariableReferences(tiles, filters, ctx), + ); + return issues.length > 0 + ? issues.map(issue => issue.message).join('; ') + : null; +} diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 187abd7a24..cb63a42da8 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -5,6 +5,7 @@ import { isFilterBroadcastEnabled, isFilterRequired, isFilterVariableEnabled, + isReferenceVariableFilter, } from '@hyperdx/common-utils/dist/filters'; import { AlertErrorType, @@ -241,7 +242,10 @@ export function translateExternalChartToTileConfig( } /** Filter types the external API and MCP don't expose yet. */ -type InternalOnlyDashboardFilter = Extract; +type InternalOnlyDashboardFilter = Extract< + DashboardFilter, + { type: 'ADHOC' | 'DATA_SOURCE' | 'CONNECTION' } +>; export type ExternalCompatibleDashboardFilter = Exclude< DashboardFilter, InternalOnlyDashboardFilter @@ -253,7 +257,7 @@ export function getExternalCompatibleFilters( ): ExternalCompatibleDashboardFilter[] { return (filters ?? []).filter( (filter): filter is ExternalCompatibleDashboardFilter => - !isAdhocFilter(filter), + !isAdhocFilter(filter) && !isReferenceVariableFilter(filter), ); } diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx index 22e99756ed..2703363e6a 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFiltersList.tsx @@ -5,6 +5,7 @@ import { isFilterRequired, isFilterVariableEnabled, } from '@hyperdx/common-utils/dist/filters'; +import { getObjectName } from '@hyperdx/common-utils/dist/referenceVariables'; import { DashboardFilter } from '@hyperdx/common-utils/dist/types'; import { Button, @@ -17,15 +18,19 @@ import { import { IconAsterisk, IconBuildingBroadcastTower, + IconDatabase, IconFilter, IconLabel, IconList, IconPencil, IconRefresh, IconSearch, + IconServer, IconTrash, } from '@tabler/icons-react'; +import { SOURCE_KIND_LABELS } from '@/components/Sources/SourceForm/constants'; +import { useConnections } from '@/connection'; import { useSources } from '@/source'; import { MODAL_SIZE } from './constants'; @@ -53,7 +58,7 @@ function getSourceNames( sources?: { id: string; name: string }[], ) { return ids - .map(id => sources?.find(s => s.id === id)?.name) + .map(id => getObjectName(id, sources)) .filter((name): name is string => !!name) .join(', '); } @@ -62,19 +67,20 @@ function getSourceNames( function getValuesAttribute( filter: DashboardFilter, sources?: { id: string; name: string }[], + connections?: { id: string; name: string }[], ): DashboardFilterAttribute { switch (filter.type) { case 'QUERY_EXPRESSION': return { icon: , tooltip: 'Source the dropdown values are queried from', - label: sources?.find(s => s.id === filter.source)?.name ?? '', + label: getObjectName(filter.source, sources) ?? '', }; case 'PROMETHEUS_LABEL': return { icon: , tooltip: 'PromQL source and label the dropdown values come from', - label: [sources?.find(s => s.id === filter.source)?.name, filter.label] + label: [getObjectName(filter.source, sources), filter.label] .filter(Boolean) .join(' · '), }; @@ -91,6 +97,25 @@ function getValuesAttribute( tooltip: 'Sources the condition keys and values come from', label: getSourceNames(filter.sources, sources), }; + case 'DATA_SOURCE': + return { + icon: , + tooltip: + 'Kinds of sources offered, and the connection they are limited to', + label: [ + filter.allowedSourceKinds + .map(kind => SOURCE_KIND_LABELS[kind]) + .join(', '), + filter.connection && getObjectName(filter.connection, connections), + ] + .filter(Boolean) + .join(' · '), + }; + case 'CONNECTION': + return { + icon: , + label: 'Connections', + }; default: filter satisfies never; // exhaustive check return { icon: , label: '' }; @@ -122,6 +147,7 @@ export const DashboardFiltersList = ({ onAddNew, }: DashboardFiltersListProps) => { const { data: sources } = useSources(); + const { data: connections } = useConnections(); return ( {filters.map(filter => { - const attributes = [getValuesAttribute(filter, sources)]; + const attributes = [getValuesAttribute(filter, sources, connections)]; const broadcastTarget = getBroadcastTargetDisplay(filter, sources); if (!hideAppliesTo && broadcastTarget != null) { diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx index 91d55296f3..04b1d99c62 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/DashboardFiltersModal.test.tsx @@ -12,6 +12,7 @@ import { FilterFormControl } from '@/components/DashboardFiltersModal/filterForm jest.mock('@/useConfirm', () => ({ useConfirm: () => jest.fn() })); jest.mock('@/source', () => ({ useSources: () => ({ data: [] }) })); +jest.mock('@/connection', () => ({ useConnections: () => ({ data: [] }) })); // Mantine's Combobox calls scrollIntoView when its dropdown opens; jsdom lacks it. window.HTMLElement.prototype.scrollIntoView = jest.fn(); diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index d0e3131fbe..9f2ea9c142 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -326,7 +326,7 @@ describe('toSavedFilter', () => { /** The minimum a form needs to save, per filter type the form supports. */ const byType: Record< - Exclude, + Exclude, Partial > = { QUERY_EXPRESSION: { diff --git a/packages/common-utils/src/__tests__/referenceVariables.test.ts b/packages/common-utils/src/__tests__/referenceVariables.test.ts new file mode 100644 index 0000000000..feb32581dd --- /dev/null +++ b/packages/common-utils/src/__tests__/referenceVariables.test.ts @@ -0,0 +1,657 @@ +import { + convertToDashboardTemplate, + savedChartConfigSupportsAlerts, +} from '@/core/utils'; +import { getBlockingRequiredFilterNames } from '@/dashboardFilterValues'; +import { + collectRefinementIssues, + validateDashboardVariableReferences, +} from '@/dashboardValidation'; +import { + getDashboardVariableDeclarations, + isReferenceVariableFilterType, + validateVariableName, +} from '@/filters'; +import { + configUsesReferenceVariable, + getConfigReferenceVariables, + getConnectionVariableNames, + getObjectName, + getSourceVariableDeclarations, + isSourceOfferedByVariable, + isSourceVariableReferenceAccepted, + parseVariableReference, + resolveConfigReferences, + resolveFilterReferences, + resolveVariableReference, + restoreConfigReferences, +} from '@/referenceVariables'; +import { + AdhocDashboardFilter, + ConnectionDashboardFilter, + DashboardFilter, + DashboardFilterSchema, + DataSourceDashboardFilter, + DisplayType, + QueryExpressionDashboardFilter, + RawSqlSavedChartConfig, + SavedChartConfig, + SourceKind, +} from '@/types'; + +const dataSourceFilter = ( + overrides: Partial = {}, +): DataSourceDashboardFilter => ({ + id: 'ds', + type: 'DATA_SOURCE', + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + +const queriedFilter = ( + overrides: Partial = {}, +): QueryExpressionDashboardFilter => ({ + id: 'q', + type: 'QUERY_EXPRESSION', + name: 'Service', + expression: 'ServiceName', + source: '$logs', + ...overrides, +}); + +const connectionFilter = ( + overrides: Partial = {}, +): ConnectionDashboardFilter => ({ + id: 'conn-filter', + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + +const rawSqlConfig = ( + overrides: Partial = {}, +): RawSqlSavedChartConfig => ({ + configType: 'sql', + displayType: DisplayType.Table, + sqlTemplate: 'SELECT 1', + connection: 'conn-1', + ...overrides, +}); + +const lineConfig = (source: string): SavedChartConfig => ({ + source, + displayType: DisplayType.Line, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + whereLanguage: 'lucene', +}); + +describe('DataSourceDashboardFilterSchema', () => { + it('accepts a data source filter', () => { + expect(DashboardFilterSchema.safeParse(dataSourceFilter()).success).toBe( + true, + ); + }); + + it('requires at least one allowed kind', () => { + expect( + DashboardFilterSchema.safeParse( + dataSourceFilter({ allowedSourceKinds: [] }), + ).success, + ).toBe(false); + }); + + it('is always single-select and required', () => { + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + maxSelections: undefined, + }).success, + ).toBe(false); + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + minSelections: 0, + }).success, + ).toBe(false); + }); + + it('rejects unknown keys', () => { + expect( + DashboardFilterSchema.safeParse({ + ...dataSourceFilter(), + isGlobalRequirement: true, + }).success, + ).toBe(false); + }); +}); + +describe('parseVariableReference', () => { + it.each([ + ['$logs', 'logs'], + ['$Logs_2', 'Logs_2'], + ['65f1c0ffee0000000000abcd', undefined], + ['$', undefined], + ['$1abc', undefined], + ['x$logs', undefined], + ['$logs ', undefined], + [undefined, undefined], + ])('parses %p as %p', (source, expected) => { + expect(parseVariableReference(source)).toBe(expected); + }); +}); + +describe('resolveVariableReference', () => { + it('passes source IDs through and resolves references', () => { + expect(resolveVariableReference('abc', new Map([['logs', 'x']]))).toBe( + 'abc', + ); + expect(resolveVariableReference('$logs', new Map([['logs', 'x']]))).toBe( + 'x', + ); + expect(resolveVariableReference('$logs', new Map())).toBeUndefined(); + expect(resolveVariableReference(undefined, new Map())).toBeUndefined(); + }); +}); + +describe('resolveConfigReferences', () => { + const values = new Map([ + ['logs', 'abc'], + ['conn', 'conn-1'], + ]); + + it('replaces a source reference with the selected source', () => { + expect(resolveConfigReferences(lineConfig('$logs'), values)).toEqual( + lineConfig('abc'), + ); + }); + + it('replaces a connection reference with the selected connection', () => { + expect( + resolveConfigReferences(rawSqlConfig({ connection: '$conn' }), values), + ).toEqual(rawSqlConfig({ connection: 'conn-1' })); + }); + + it('leaves unselected references and IDs as written', () => { + expect(resolveConfigReferences(lineConfig('$logs'), new Map())).toEqual( + lineConfig('$logs'), + ); + expect(resolveConfigReferences(lineConfig('abc'), values)).toEqual( + lineConfig('abc'), + ); + }); +}); + +describe('getConfigReferenceVariables', () => { + it('names the variables a config source and connection reference', () => { + expect( + getConfigReferenceVariables( + rawSqlConfig({ connection: '$conn', source: '$logs' }), + ), + ).toEqual(['logs', 'conn']); + expect(getConfigReferenceVariables(lineConfig('abc'))).toEqual([]); + }); +}); + +describe('isSourceVariableReferenceAccepted', () => { + const declarations = [ + { name: 'logs', allowedSourceKinds: [SourceKind.Log] }, + { name: 'any', allowedSourceKinds: [SourceKind.Log, SourceKind.Promql] }, + ]; + + it('accepts a variable whose every kind the field accepts', () => { + expect( + isSourceVariableReferenceAccepted('$logs', declarations, [ + SourceKind.Log, + SourceKind.Trace, + ]), + ).toBe(true); + }); + + it('rejects a variable offering a kind the field does not accept', () => { + expect( + isSourceVariableReferenceAccepted('$any', declarations, [SourceKind.Log]), + ).toBe(false); + }); + + it('rejects an unknown variable and a source ID', () => { + expect( + isSourceVariableReferenceAccepted('$nope', declarations, [ + SourceKind.Log, + ]), + ).toBe(false); + expect( + isSourceVariableReferenceAccepted('abc', declarations, [SourceKind.Log]), + ).toBe(false); + }); +}); + +describe('resolveFilterReferences', () => { + it('resolves source and appliesToSourceIds', () => { + const { filter, unresolvedVariables } = resolveFilterReferences( + queriedFilter({ appliesToSourceIds: ['$logs', 'other'] }), + new Map([['logs', 'abc']]), + ); + expect(filter.source).toBe('abc'); + expect(filter.appliesToSourceIds).toEqual(['abc', 'other']); + expect(unresolvedVariables).toEqual([]); + }); + + it('leaves unselected references as written so they match no tile', () => { + const adhoc: AdhocDashboardFilter = { + id: 'a', + type: 'ADHOC', + name: 'Conditions', + sourceType: 'sql', + sources: ['$logs'], + isVariableEnabled: true, + }; + const { filter, unresolvedVariables } = resolveFilterReferences( + adhoc, + new Map(), + ); + expect(filter.sources).toEqual(['$logs']); + expect(unresolvedVariables).toEqual(['logs']); + }); +}); + +describe('dashboard variable declarations', () => { + it('keeps the first of two variables sharing a name', () => { + expect( + getSourceVariableDeclarations([ + dataSourceFilter(), + dataSourceFilter({ + id: 'ds2', + allowedSourceKinds: [SourceKind.Trace], + }), + ]), + ).toEqual([ + { + name: 'logs', + allowedSourceKinds: [SourceKind.Log], + connection: undefined, + }, + ]); + }); + + it('keeps reference variables out of text variables', () => { + const filters: DashboardFilter[] = [dataSourceFilter(), connectionFilter()]; + expect(getDashboardVariableDeclarations(filters)).toEqual([]); + expect(getSourceVariableDeclarations(filters)).toEqual([ + { name: 'logs', allowedSourceKinds: [SourceKind.Log] }, + ]); + expect(getConnectionVariableNames(filters)).toEqual(['conn']); + }); + + it('shares the variable name namespace with other filters', () => { + expect( + validateVariableName({ + value: 'logs', + otherFilters: [dataSourceFilter()], + }), + ).toMatch(/used by another filter/); + }); +}); + +describe('alerts', () => { + it('are not supported on tiles that use a connection variable', () => { + expect( + savedChartConfigSupportsAlerts( + rawSqlConfig({ displayType: DisplayType.Line, connection: '$conn' }), + ), + ).toBe(false); + }); + + it('are not supported on tiles that use a data source variable', () => { + expect(configUsesReferenceVariable(lineConfig('$logs'))).toBe(true); + expect(savedChartConfigSupportsAlerts(lineConfig('$logs'))).toBe(false); + expect(savedChartConfigSupportsAlerts(lineConfig('abc'))).toBe(true); + }); +}); + +describe('getBlockingRequiredFilterNames (connections)', () => { + it('blocks a tile until its connection variable has a selection', () => { + expect( + getBlockingRequiredFilterNames({ + config: rawSqlConfig({ connection: '$conn' }), + sourceId: undefined, + unsatisfiedRequiredFilters: [connectionFilter()], + referencedVariables: [], + referenceVariableNames: ['conn'], + }), + ).toEqual(['Connection']); + }); +}); + +describe('getBlockingRequiredFilterNames', () => { + it('blocks a tile until its data source variable has a selection', () => { + const unsatisfied = [dataSourceFilter()]; + expect( + getBlockingRequiredFilterNames({ + config: lineConfig('$logs'), + sourceId: undefined, + unsatisfiedRequiredFilters: unsatisfied, + referencedVariables: [], + referenceVariableNames: ['logs'], + }), + ).toEqual(['Logs source']); + expect( + getBlockingRequiredFilterNames({ + config: lineConfig('abc'), + sourceId: 'abc', + unsatisfiedRequiredFilters: unsatisfied, + referencedVariables: [], + }), + ).toEqual([]); + }); +}); + +describe('validateDashboardVariableReferences', () => { + const collectIssues = ( + tiles: { config: SavedChartConfig }[], + filters: DashboardFilter[], + ) => + collectRefinementIssues(ctx => + validateDashboardVariableReferences(tiles, filters, ctx), + ); + + it('accepts references to declared variables of compatible kinds', () => { + expect( + collectIssues( + [{ config: lineConfig('$logs') }], + [dataSourceFilter(), queriedFilter()], + ), + ).toEqual([]); + }); + + it('accepts connection references to a declared connection variable', () => { + expect( + collectIssues( + [{ config: rawSqlConfig({ connection: '$conn' }) }], + [connectionFilter(), dataSourceFilter({ connection: '$conn' })], + ), + ).toEqual([]); + }); + + it('rejects a connection reference that names no connection variable', () => { + const issues = collectIssues( + [{ config: rawSqlConfig({ connection: '$logs' }) }], + [dataSourceFilter({ connection: '$nope' })], + ); + expect(issues.map(issue => issue.path)).toEqual([ + ['tiles', 0, 'config', 'connection'], + ['filters', 0, 'connection'], + ]); + }); + + it('rejects a source reference to a connection variable', () => { + expect( + collectIssues([{ config: lineConfig('$conn') }], [connectionFilter()]), + ).toHaveLength(1); + }); + + it('rejects a reference to an unknown variable', () => { + const issues = collectIssues([{ config: lineConfig('$nope') }], []); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['tiles', 0, 'config', 'source']); + }); + + it.each<[string, { config: SavedChartConfig }[], DashboardFilter[]]>([ + [ + 'a PromQL tile', + [ + { + config: { + configType: 'promql', + source: '$logs', + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + }, + }, + ], + [], + ], + [ + 'a PromQL label filter', + [], + [ + { + id: 'p', + type: 'PROMETHEUS_LABEL', + name: 'Job', + source: '$logs', + label: 'job', + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + ], + [ + 'an ad hoc filter of another source type', + [], + [ + { + id: 'a', + type: 'ADHOC', + name: 'Conditions', + sourceType: 'promql', + sources: ['$logs'], + isVariableEnabled: true, + }, + ], + ], + [ + "a queried filter's applies-to list", + [], + [ + queriedFilter({ + source: 'abc', + appliesToSourceIds: ['$logs'], + }), + ], + ], + ])( + 'rejects a logs-or-PromQL variable used by %s', + (_label, tiles, filters) => { + const issues = collectIssues(tiles, [ + dataSourceFilter({ + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }), + ...filters, + ]); + expect(issues.length).toBeGreaterThan(0); + }, + ); + + it('rejects a variable offering kinds the field does not accept', () => { + const issues = collectIssues( + [], + [ + dataSourceFilter({ + allowedSourceKinds: [SourceKind.Log, SourceKind.Promql], + }), + queriedFilter(), + ], + ); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['filters', 1, 'source']); + expect(issues[0].message).toMatch(/promql/); + }); +}); + +describe('convertToDashboardTemplate (connections)', () => { + it('keeps connection references and names a data source connection', () => { + const template = convertToDashboardTemplate( + { + id: 'd', + name: 'Dashboard', + tags: [], + tiles: [ + { + id: 't', + x: 0, + y: 0, + w: 6, + h: 4, + config: rawSqlConfig({ connection: '$conn' }), + }, + ], + filters: [ + connectionFilter(), + dataSourceFilter({ connection: 'conn-1' }), + ], + savedFilterValues: [{ type: 'variable', name: 'conn', values: ['c'] }], + }, + [], + [{ id: 'conn-1', name: 'Primary', host: '', username: '' }], + ); + expect(template.tiles[0].config).toMatchObject({ connection: '$conn' }); + expect(template.filters?.[1]).toMatchObject({ connection: 'Primary' }); + expect(template.savedFilterValues).toBeUndefined(); + }); +}); + +describe('convertToDashboardTemplate', () => { + it('keeps references and drops the saved data source selection', () => { + const template = convertToDashboardTemplate( + { + id: 'd', + name: 'Dashboard', + tags: [], + tiles: [ + { id: 't', x: 0, y: 0, w: 6, h: 4, config: lineConfig('$logs') }, + ], + filters: [dataSourceFilter(), queriedFilter()], + savedFilterValues: [ + { type: 'variable', name: 'logs', values: ['abc'] }, + { type: 'variable', name: 'env', values: ['prod'] }, + ], + }, + [], + ); + expect(template.tiles[0].config).toMatchObject({ source: '$logs' }); + expect(template.filters?.[1]).toMatchObject({ source: '$logs' }); + expect(template.savedFilterValues).toEqual([ + { type: 'variable', name: 'env', values: ['prod'] }, + ]); + }); +}); + +describe('resolveFilterReferences (connections)', () => { + it("resolves a data source variable's connection reference", () => { + expect( + resolveFilterReferences( + dataSourceFilter({ connection: '$conn' }), + new Map([['conn', 'conn-1']]), + ), + ).toEqual({ + filter: dataSourceFilter({ connection: 'conn-1' }), + unresolvedVariables: [], + }); + expect( + resolveFilterReferences( + dataSourceFilter({ connection: '$conn' }), + new Map(), + ).unresolvedVariables, + ).toEqual(['conn']); + }); +}); + +describe('restoreConfigReferences', () => { + it('puts stored source and connection references back', () => { + const stored = rawSqlConfig({ connection: '$conn', source: '$logs' }); + const resolved = resolveConfigReferences( + stored, + new Map([ + ['conn', 'conn-1'], + ['logs', 'abc'], + ]), + ); + expect( + restoreConfigReferences( + { ...resolved, displayType: DisplayType.Number }, + stored, + ), + ).toEqual({ ...stored, displayType: DisplayType.Number }); + }); + + it('keeps fields the stored config holds as IDs', () => { + expect( + restoreConfigReferences( + rawSqlConfig({ connection: 'conn-2' }), + rawSqlConfig({ connection: 'conn-1' }), + ), + ).toEqual(rawSqlConfig({ connection: 'conn-2' })); + }); +}); + +describe('isSourceOfferedByVariable', () => { + const source = { kind: SourceKind.Log, connection: 'conn-1' }; + + it('offers an enabled source of an allowed kind on the variable connection', () => { + expect( + isSourceOfferedByVariable(source, { + allowedSourceKinds: [SourceKind.Log], + connection: 'conn-1', + }), + ).toBe(true); + expect( + isSourceOfferedByVariable(source, { + allowedSourceKinds: [SourceKind.Log], + }), + ).toBe(true); + }); + + it.each([ + ['another kind', source, { allowedSourceKinds: [SourceKind.Trace] }], + [ + 'another connection', + source, + { allowedSourceKinds: [SourceKind.Log], connection: 'conn-2' }, + ], + [ + 'an unresolved connection reference', + source, + { allowedSourceKinds: [SourceKind.Log], connection: '$conn' }, + ], + [ + 'a disabled source', + { ...source, disabled: true }, + { allowedSourceKinds: [SourceKind.Log] }, + ], + ])('does not offer %s', (_label, candidate, variable) => { + expect(isSourceOfferedByVariable(candidate, variable)).toBe(false); + }); +}); + +describe('getObjectName', () => { + const objects = [{ id: 'abc', name: 'Logs' }]; + + it('names an object, keeps a reference, and is undefined for a missing ID', () => { + expect(getObjectName('abc', objects)).toBe('Logs'); + expect(getObjectName('$logs', objects)).toBe('$logs'); + expect(getObjectName('nope', objects)).toBeUndefined(); + }); +}); + +describe('isReferenceVariableFilterType', () => { + it.each([ + ['DATA_SOURCE', true], + ['CONNECTION', true], + ['STATIC_LIST', false], + [undefined, false], + ] as const)('%p is %p', (type, expected) => { + expect(isReferenceVariableFilterType(type)).toBe(expected); + }); +}); diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 7c93078458..422eecb1d0 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -8,6 +8,8 @@ export { default as objectHash } from 'object-hash'; import { isAdhocFilter, + isConnectionFilter, + isDataSourceFilter, isQueryExpressionFilter, isStaticListFilter, } from '@/filters'; @@ -26,6 +28,11 @@ import { replaceMacros, } from '@/macros'; import { QUERY_PARAMS, RawSqlQueryParam } from '@/rawSqlParams'; +import { + configUsesReferenceVariable, + getObjectName, + getReferenceVariableFilters, +} from '@/referenceVariables'; import { BuilderChartConfig, BuilderChartConfigWithDateRange, @@ -914,44 +921,40 @@ export function convertToDashboardTemplate( }; }; + // A variable reference is portable as written; '' marks a missing object + const getSourceName = (id: string) => getObjectName(id, sources) ?? ''; + const getConnectionName = (id: string) => + getObjectName(id, connections) ?? ''; + const convertToTileTemplate = ( input: Dashboard['tiles'][0], - sources: TSource[], - connections: Connection[], ): TileTemplate => { const tile = TileTemplateSchema.strip().parse(structuredClone(input)); // Extract name from source/connection or default to '' if not found const tileConfig = tile.config; if (isBuilderSavedChartConfig(tileConfig)) { - tileConfig.source = ( - sources.find(source => source.id === tileConfig.source) ?? { name: '' } - ).name; + tileConfig.source = getSourceName(tileConfig.source); } else if (isRawSqlSavedChartConfig(tileConfig)) { - tileConfig.connection = ( - connections.find(conn => conn.id === tileConfig.connection) ?? { - name: '', - } - ).name; + tileConfig.connection = getConnectionName(tileConfig.connection); if (tileConfig.source) { - tileConfig.source = - sources.find(source => source.id === tileConfig.source)?.name ?? ''; + tileConfig.source = getSourceName(tileConfig.source); } } convertOnClickTargetIdToName(tileConfig); return tile; }; - const convertToFilterTemplate = ( - input: DashboardFilter, - sources: TSource[], - ): DashboardFilter => { + const convertToFilterTemplate = (input: DashboardFilter): DashboardFilter => { const filter = DashboardFilterSchema.parse(structuredClone(input)); - // A static filter references nothing in the workspace - if (isStaticListFilter(filter)) return filter; - - const getSourceName = (id: string) => - sources.find(source => source.id === id)?.name ?? ''; + // These reference nothing in the workspace + if (isStaticListFilter(filter) || isConnectionFilter(filter)) return filter; + if (isDataSourceFilter(filter)) { + if (filter.connection) { + filter.connection = getConnectionName(filter.connection) || undefined; + } + return filter; + } if (isAdhocFilter(filter)) { filter.sources = filter.sources.map(getSourceName); @@ -969,13 +972,13 @@ export function convertToDashboardTemplate( }; for (const tile of input.tiles) { - output.tiles.push(convertToTileTemplate(tile, sources, connections)); + output.tiles.push(convertToTileTemplate(tile)); } if (input.filters) { output.filters = []; for (const filter of input.filters ?? []) { - output.filters.push(convertToFilterTemplate(filter, sources)); + output.filters.push(convertToFilterTemplate(filter)); } } @@ -988,8 +991,16 @@ export function convertToDashboardTemplate( output.savedQueryLanguage = input.savedQueryLanguage ?? null; } - if (input.savedFilterValues && input.savedFilterValues.length > 0) { - output.savedFilterValues = structuredClone(input.savedFilterValues); + // A reference variable's saved value is an ID from this workspace + const referenceVariableNames = new Set( + getReferenceVariableFilters(input.filters).map(({ name }) => name), + ); + const savedFilterValues = (input.savedFilterValues ?? []).filter( + value => + !(value.type === 'variable' && referenceVariableNames.has(value.name)), + ); + if (savedFilterValues.length > 0) { + output.savedFilterValues = structuredClone(savedFilterValues); } return output; @@ -1728,6 +1739,8 @@ export function displayTypeSupportsPromQLAlerts( export function savedChartConfigSupportsAlerts( config: SavedChartConfig, ): boolean { + // An alert must query one fixed source, not the viewer's current selection. + if (configUsesReferenceVariable(config)) return false; if (isPromqlSavedChartConfig(config)) { return displayTypeSupportsPromQLAlerts(config.displayType); } diff --git a/packages/common-utils/src/dashboardFilterValues.ts b/packages/common-utils/src/dashboardFilterValues.ts index 2ea039b02b..a0e7922d12 100644 --- a/packages/common-utils/src/dashboardFilterValues.ts +++ b/packages/common-utils/src/dashboardFilterValues.ts @@ -14,6 +14,7 @@ import { } from '@/filters'; import { isPromqlSavedChartConfig, isRawSqlSavedChartConfig } from '@/guards'; import { hasMacro, isMissingFiltersMacro } from '@/macros'; +import { getReferenceVariableFilters } from '@/referenceVariables'; import { AdhocDashboardFilter, AdhocFilterValue, @@ -269,6 +270,8 @@ export function getAdhocFilterTargetsForTile( * 2. The tile references the filter's variable. * 3. The filter applies via broadcast to the tile's source, and the tile * applies broadcast conditions (`consumesBroadcastFilters`). + * 4. The filter is a data source or connection variable the tile's stored + * config references. */ export function getBlockingRequiredFilters( unsatisfiedRequiredFilters: DashboardFilter[], @@ -277,8 +280,16 @@ export function getBlockingRequiredFilters( referencedVariableNames?: readonly string[]; /** Whether the tile's query applies broadcast filter conditions. */ consumesBroadcastFilters: boolean; + /** The reference variables the tile's stored config references. */ + referenceVariableNames?: readonly string[]; }, ): DashboardFilter[] { + const referenceVariableNames = new Set(tile.referenceVariableNames ?? []); + const referenceVariableFilterIds = new Set( + getReferenceVariableFilters(unsatisfiedRequiredFilters) + .filter(({ name }) => referenceVariableNames.has(name)) + .map(({ filter }) => filter.id), + ); const referencedNamesSet = new Set(tile.referencedVariableNames ?? []); const referencedFilterIds = new Set( getDashboardVariableFilters(unsatisfiedRequiredFilters) @@ -290,6 +301,7 @@ export function getBlockingRequiredFilters( filter => isFilterGlobalRequirement(filter) || referencedFilterIds.has(filter.id) || + referenceVariableFilterIds.has(filter.id) || (tile.consumesBroadcastFilters && doesFilterApplyToSource(filter, tile.sourceId)), ); @@ -301,15 +313,19 @@ export function getBlockingRequiredFilterNames({ sourceId, unsatisfiedRequiredFilters, referencedVariables, + referenceVariableNames, }: { config: SavedChartConfig | ChartConfigWithOptDateRange; sourceId: string | undefined; unsatisfiedRequiredFilters: DashboardFilter[] | undefined; /** The tile's variables, already narrowed to the ones it references. */ referencedVariables: readonly { name: string }[] | undefined; + /** The reference variables the tile's stored config references. */ + referenceVariableNames?: readonly string[]; }): string[] { return getBlockingRequiredFilters(unsatisfiedRequiredFilters ?? [], { sourceId, + referenceVariableNames, referencedVariableNames: referencedVariables?.map( variable => variable.name, ), diff --git a/packages/common-utils/src/dashboardValidation.ts b/packages/common-utils/src/dashboardValidation.ts index 8a88bf999e..f08a52584e 100644 --- a/packages/common-utils/src/dashboardValidation.ts +++ b/packages/common-utils/src/dashboardValidation.ts @@ -7,7 +7,22 @@ import { isFilterBroadcastEnabled, isFilterVariableEnabled, } from './filters'; -import { DashboardContainerSchema, DisplayType } from './types'; +import { + getConfigReferenceFields, + getConnectionVariableNames, + getFilterReferenceFields, + getReferenceFieldValues, + getSourceVariableFilters, + getStringField, + getUnacceptedSourceKinds, + parseVariableReference, + ReferenceField, +} from './referenceVariables'; +import { + DashboardContainerSchema, + DashboardFilter, + DisplayType, +} from './types'; // Inputs shared by the internal `DashboardSchema` refinement and the // external API body schema: the validation only depends on the @@ -374,3 +389,115 @@ export function validateChartConfigFormulas( }); } } + +/** The issues a refinement raises, run on its own outside a schema. */ +export function collectRefinementIssues( + refine: (ctx: z.RefinementCtx) => void, +): z.ZodIssue[] { + const result = z + .object({}) + .superRefine((_value, ctx) => refine(ctx)) + .safeParse({}); + return result.success ? [] : result.error.issues; +} + +/** + * Every reference variable reference must name a variable of the right type + * on the dashboard: `$name` in a source field names a data source variable + * offering only kinds the field accepts; `$name` in a connection field names a + * connection variable. The fields are those of `getConfigReferenceFields` and + * `getFilterReferenceFields`. + * + * Issues raised: + * - An unknown variable, or one of the wrong type (path `[i].config.` or `[i].[j]`). + * - A data source variable offering kinds the field does not accept (same paths). + */ +export function validateDashboardVariableReferences( + // Structural so stored documents (whose alerts carry server fields) fit + tiles: { config: object }[], + filters: DashboardFilter[], + ctx: z.RefinementCtx, + paths?: { + tilesPath?: (string | number)[]; + filtersPath?: (string | number)[]; + }, +): void { + const tilesPath = paths?.tilesPath ?? ['tiles']; + const filtersPath = paths?.filtersPath ?? ['filters']; + const kindsByName = new Map( + getSourceVariableFilters(filters).map(({ filter, name }) => [ + name, + filter.allowedSourceKinds, + ]), + ); + const connectionVariableNames = new Set(getConnectionVariableNames(filters)); + + const check = ( + value: string, + field: ReferenceField, + path: (string | number)[], + ) => { + const name = parseVariableReference(value); + if (name == null) return; + if (field.target === 'connection') { + if (connectionVariableNames.has(name)) return; + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `"${value}" does not name a connection variable on this dashboard`, + path, + }); + return; + } + const kinds = kindsByName.get(name); + if (kinds == null) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `"${value}" does not name a data source variable on this dashboard`, + path, + }); + return; + } + const unaccepted = getUnacceptedSourceKinds( + { allowedSourceKinds: kinds }, + field.acceptedKinds, + ); + if (unaccepted.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Data source variable "${value}" offers ${unaccepted.join(', ')} sources, which cannot be used here`, + path, + }); + } + }; + + const checkFields = ( + object: object, + fields: ReferenceField[], + path: (string | number)[], + ) => { + for (const field of fields) { + const value = getStringField(object, field.key); + if (value != null) { + check(value, field, [...path, field.key]); + } else { + getReferenceFieldValues(object, field.key).forEach((entry, i) => + check(entry, field, [...path, field.key, i]), + ); + } + } + }; + + tiles.forEach((tile, tileIdx) => + checkFields(tile.config, getConfigReferenceFields(tile.config), [ + ...tilesPath, + tileIdx, + 'config', + ]), + ); + filters.forEach((filter, filterIdx) => + checkFields(filter, getFilterReferenceFields(filter), [ + ...filtersPath, + filterIdx, + ]), + ); +} diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 3412e16d8c..5115216b59 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -938,6 +938,37 @@ export function isAdhocFilter( return filter.type === 'ADHOC'; } +/** Type guard for DATA_SOURCE type filters. */ +export function isDataSourceFilter( + filter: T, +): filter is Extract { + return filter.type === 'DATA_SOURCE'; +} + +/** Type guard for CONNECTION type filters. */ +export function isConnectionFilter( + filter: T, +): filter is Extract { + return filter.type === 'CONNECTION'; +} + +/** + * Whether a filter type is a reference variable: a data source or connection + * variable, whose value is an ID that tiles and filters use in place of one. + */ +export function isReferenceVariableFilterType( + type: DashboardFilterKind | undefined, +): type is 'DATA_SOURCE' | 'CONNECTION' { + return type === 'DATA_SOURCE' || type === 'CONNECTION'; +} + +/** Type guard for reference variable filters; see `isReferenceVariableFilterType`. */ +export function isReferenceVariableFilter< + T extends { type: DashboardFilterKind }, +>(filter: T): filter is Extract { + return isReferenceVariableFilterType(filter.type); +} + /** * The sources an ADHOC filter broadcasts its conditions to: its * `appliesToSourceIds`, or its own `sources` when that is unset or empty. @@ -1211,6 +1242,7 @@ export type DashboardVariableDeclaration = Pick< /** Minimal projection of fields necessary to extract the variables a dashboard declares. */ export type FilterForVariableDeclaration = { + type?: DashboardFilterKind; name: string; expression?: string; variableName?: string; @@ -1220,17 +1252,33 @@ export type FilterForVariableDeclaration = { /** * The variable-enabled filters a dashboard declares, paired with the name each - * one answers to, in filter order. + * one answers to, in filter order. Data source variables are left out: their + * value selects a tile's source or connection and is never substituted into + * query text. */ export function getDashboardVariableFilters< T extends FilterForVariableDeclaration, >(filters: T[] | undefined): { filter: T; name: string }[] { + return pairWithVariableNames( + (filters ?? []).filter( + filter => + isFilterVariableEnabled(filter) && + !isReferenceVariableFilterType(filter.type), + ), + ); +} + +/** + * Each filter paired with the name it answers to, skipping filters with no + * usable name. + */ +export function pairWithVariableNames< + T extends { name: string; variableName?: string }, +>(filters: T[]): { filter: T; name: string }[] { const results: { filter: T; name: string }[] = []; const takenNames = new Set(); - for (const filter of filters ?? []) { - if (!isFilterVariableEnabled(filter)) continue; - + for (const filter of filters) { // There shouldn't be any duplicate names, but if there are then the first one wins. const name = getFilterVariableName(filter); if (!name || takenNames.has(name)) continue; diff --git a/packages/common-utils/src/referenceVariables.ts b/packages/common-utils/src/referenceVariables.ts new file mode 100644 index 0000000000..7dd476b3da --- /dev/null +++ b/packages/common-utils/src/referenceVariables.ts @@ -0,0 +1,366 @@ +/** + * Reference variables are dashboard variables whose value is the ID of a + * workspace object: a data source variable selects a source, a connection + * variable selects a connection. Tiles and filters reference them as `$name` + * in the fields that otherwise hold that ID. + */ +import { + getAdhocFilterSourceKinds, + isConnectionFilter, + isDataSourceFilter, + isReferenceVariableFilter, + pairWithVariableNames, + QUERY_EXPRESSION_FILTER_SOURCE_KINDS, +} from '@/filters'; +import { + ConnectionDashboardFilter, + DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED, + DashboardFilter, + DataSourceDashboardFilter, + NON_PROMQL_SOURCE_KINDS, + SourceKind, +} from '@/types'; + +/** + * The variable name a field references, if it holds a `$name` reference + * rather than an ID. The whole field is the token. + */ +export function parseVariableReference( + value: string | undefined | null, +): string | undefined { + if (!value?.startsWith('$')) return undefined; + const name = value.slice(1); + return DASHBOARD_VARIABLE_NAME_PATTERN_ANCHORED.test(name) ? name : undefined; +} + +export function toVariableReference(name: string): string { + return `$${name}`; +} + +/** A field's value, read without assuming the object's shape. */ +function getField(object: object, field: string): unknown { + return field in object ? Reflect.get(object, field) : undefined; +} + +/** A field's value when it is a string; undefined when absent or not a string. */ +export function getStringField( + object: object, + field: string, +): string | undefined { + const value = getField(object, field); + return typeof value === 'string' ? value : undefined; +} + +/** + * An object's name, or the reference itself for a variable reference. + * Undefined when no object has the ID. + */ +export function getObjectName( + id: string, + objects: readonly { id: string; name: string }[] | undefined, +): string | undefined { + if (parseVariableReference(id) != null) return id; + return objects?.find(object => object.id === id)?.name; +} + +/** Selected ID per reference variable name; undefined when unselected. */ +export type ReferenceVariableValues = ReadonlyMap; + +/** A data source variable's identity, as offered by source pickers. */ +export type SourceVariableDeclaration = { + name: string; + allowedSourceKinds: SourceKind[]; + /** A connection ID or connection variable reference the sources are limited to. */ + connection?: string; +}; + +/** + * The kinds a data source variable can resolve to that a field accepting + * `acceptedKinds` cannot use. + */ +export function getUnacceptedSourceKinds( + variable: Pick, + acceptedKinds: readonly SourceKind[], +): SourceKind[] { + return variable.allowedSourceKinds.filter( + kind => !acceptedKinds.includes(kind), + ); +} + +/** + * Whether a field accepting `acceptedKinds` can use a data source variable, + * whatever source it selects. + */ +export function isSourceVariableAccepted( + variable: Pick, + acceptedKinds: readonly SourceKind[], +): boolean { + return getUnacceptedSourceKinds(variable, acceptedKinds).length === 0; +} + +/** `isSourceVariableAccepted` for the variable that `source` references. */ +export function isSourceVariableReferenceAccepted( + source: string | undefined, + declarations: SourceVariableDeclaration[], + acceptedKinds: readonly SourceKind[], +): boolean { + const name = parseVariableReference(source); + const variable = declarations.find(v => v.name === name); + return variable != null && isSourceVariableAccepted(variable, acceptedKinds); +} + +/** + * Whether a data source variable can select `source`. `connection` is the + * variable's connection already resolved; a still unresolved reference + * matches no source. + */ +export function isSourceOfferedByVariable( + source: { kind: SourceKind; connection: string; disabled?: boolean }, + variable: Pick< + SourceVariableDeclaration, + 'allowedSourceKinds' | 'connection' + >, +): boolean { + return ( + !source.disabled && + variable.allowedSourceKinds.includes(source.kind) && + (!variable.connection || source.connection === variable.connection) + ); +} + +/** The data source variables a dashboard declares, in filter order. */ +export function getSourceVariableFilters( + filters: DashboardFilter[] | undefined, +): { filter: DataSourceDashboardFilter; name: string }[] { + return pairWithVariableNames((filters ?? []).filter(isDataSourceFilter)); +} + +/** The connection variables a dashboard declares, in filter order. */ +export function getConnectionVariableFilters( + filters: DashboardFilter[] | undefined, +): { filter: ConnectionDashboardFilter; name: string }[] { + return pairWithVariableNames((filters ?? []).filter(isConnectionFilter)); +} + +/** + * The data source and connection variables a dashboard declares, in filter + * order. Structural so the external API's filter shape fits too. + */ +export function getReferenceVariableFilters< + T extends { + type: DashboardFilter['type']; + name: string; + variableName?: string; + }, +>( + filters: T[] | undefined, +): { + filter: Extract; + name: string; +}[] { + return pairWithVariableNames( + (filters ?? []).filter(isReferenceVariableFilter), + ); +} + +export function getSourceVariableDeclarations( + filters: DashboardFilter[] | undefined, +): SourceVariableDeclaration[] { + return getSourceVariableFilters(filters).map(({ filter, name }) => ({ + name, + allowedSourceKinds: filter.allowedSourceKinds, + connection: filter.connection, + })); +} + +export function getConnectionVariableNames( + filters: DashboardFilter[] | undefined, +): string[] { + return getConnectionVariableFilters(filters).map(({ name }) => name); +} + +/** + * Resolve a field: an ID passes through, a reference resolves to the + * variable's selection (undefined while unselected). + */ +export function resolveVariableReference( + value: string | undefined, + values: ReferenceVariableValues | undefined, +): string | undefined { + const name = parseVariableReference(value); + if (name == null) return value; + return values?.get(name); +} + +/** + * A field that can hold a reference variable reference: what it may reference, + * and, for a source field, the source kinds it accepts. + */ +export type ReferenceField = + | { key: K; target: 'source'; acceptedKinds: readonly SourceKind[] } + | { key: K; target: 'connection' }; + +/** The fields of a chart config that can hold a reference. */ +export function getConfigReferenceFields( + config: object, +): ReferenceField<'source' | 'connection'>[] { + const isPromql = getStringField(config, 'configType') === 'promql'; + return [ + { + key: 'source', + target: 'source', + acceptedKinds: isPromql ? [SourceKind.Promql] : NON_PROMQL_SOURCE_KINDS, + }, + { key: 'connection', target: 'connection' }, + ]; +} + +/** The fields of a filter that can hold a reference. */ +export function getFilterReferenceFields( + filter: DashboardFilter, +): ReferenceField< + 'source' | 'sources' | 'appliesToSourceIds' | 'connection' +>[] { + switch (filter.type) { + case 'QUERY_EXPRESSION': + return (['source', 'appliesToSourceIds'] as const).map(key => ({ + key, + target: 'source', + acceptedKinds: QUERY_EXPRESSION_FILTER_SOURCE_KINDS, + })); + case 'PROMETHEUS_LABEL': + return [ + { key: 'source', target: 'source', acceptedKinds: [SourceKind.Promql] }, + ]; + case 'ADHOC': + return (['sources', 'appliesToSourceIds'] as const).map(key => ({ + key, + target: 'source', + acceptedKinds: getAdhocFilterSourceKinds(filter.sourceType), + })); + case 'DATA_SOURCE': + return [{ key: 'connection', target: 'connection' }]; + case 'STATIC_LIST': + case 'CONNECTION': + return []; + default: + filter satisfies never; + return []; + } +} + +/** A field's references and IDs: one for a string field, each entry of a list. */ +export function getReferenceFieldValues(object: object, key: string): string[] { + const value = getField(object, key); + if (typeof value === 'string') return [value]; + return Array.isArray(value) + ? value.filter((v): v is string => typeof v === 'string') + : []; +} + +/** Each field's value with `resolve` applied, keeping strings and lists as such. */ +function mapReferenceFields( + object: T, + keys: readonly string[], + resolve: (value: string) => string, +): T { + let mapped = object; + for (const key of keys) { + const value = getField(object, key); + if (typeof value === 'string') { + const next = resolve(value); + if (next !== value) mapped = { ...mapped, [key]: next }; + } else if (Array.isArray(value)) { + const next = value.map(v => (typeof v === 'string' ? resolve(v) : v)); + if (next.some((v, i) => v !== value.at(i))) { + mapped = { ...mapped, [key]: next }; + } + } + } + return mapped; +} + +/** The variables named by references in the given fields, deduplicated. */ +function getReferencedVariables( + object: object, + keys: readonly string[], +): string[] { + return Array.from( + new Set( + keys + .flatMap(key => getReferenceFieldValues(object, key)) + .map(parseVariableReference) + .filter((name): name is string => name != null), + ), + ); +} + +/** The reference variables a chart config's `source` and `connection` name. */ +export function getConfigReferenceVariables(config: object): string[] { + return getReferencedVariables( + config, + getConfigReferenceFields(config).map(({ key }) => key), + ); +} + +/** Whether a chart config's source or connection is a variable reference. */ +export function configUsesReferenceVariable(config: object): boolean { + return getConfigReferenceVariables(config).length > 0; +} + +/** + * Replace the references in a config's `source` and `connection` with the + * selected IDs. An unselected reference is left as written. Returns `config` + * itself when nothing changes. + */ +export function resolveConfigReferences( + config: T, + values: ReferenceVariableValues | undefined, +): T { + return mapReferenceFields( + config, + getConfigReferenceFields(config).map(({ key }) => key), + value => resolveVariableReference(value, values) || value, + ); +} + +/** + * The inverse of `resolveConfigReferences`: put the references `stored` holds + * back on a config derived from its resolved form. + */ +export function restoreConfigReferences( + config: T, + stored: object, +): T { + let restored = config; + for (const { key } of getConfigReferenceFields(stored)) { + const value = getStringField(stored, key); + if (parseVariableReference(value) != null && key in config) { + restored = { ...restored, [key]: value }; + } + } + return restored; +} + +/** + * Replace the references in a filter's source and connection fields with the + * selected IDs. An unselected reference is left as written, so it matches no + * tile's source rather than widening a list to every tile, and is reported in + * `unresolvedVariables`. + */ +export function resolveFilterReferences( + filter: T, + values: ReferenceVariableValues | undefined, +): { filter: T; unresolvedVariables: string[] } { + const keys = getFilterReferenceFields(filter).map(({ key }) => key); + return { + filter: mapReferenceFields( + filter, + keys, + value => resolveVariableReference(value, values) || value, + ), + unresolvedVariables: getReferencedVariables(filter, keys).filter( + name => !values?.get(name), + ), + }; +} diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dba9c7854e..30a4c90f6a 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2295,12 +2295,27 @@ export const DashboardContainerSchema = z.object({ export type DashboardContainer = z.infer; +export enum SourceKind { + Log = 'log', + Trace = 'trace', + Session = 'session', + Metric = 'metric', + Promql = 'promql', +} + +/** The source kinds builder and raw SQL charts can query. */ +export const NON_PROMQL_SOURCE_KINDS: readonly SourceKind[] = Object.values( + SourceKind, +).filter(kind => kind !== SourceKind.Promql); + /** Type of dashboard filter, determining how its dropdown values are populated. */ export const DashboardFilterType = z.enum([ 'QUERY_EXPRESSION', 'STATIC_LIST', 'PROMETHEUS_LABEL', 'ADHOC', + 'DATA_SOURCE', + 'CONNECTION', ]); /** Allowed variable names for dashboard filters. Alphanumeric + underscore, must start with a letter. */ @@ -2440,11 +2455,52 @@ export const AdhocDashboardFilterSchema = dashboardFilterBaseSchema }) .strict(); +/** + * A variable whose single selected value is a source ID. Tiles and other + * filters reference it as `$variableName` in place of a source ID. Always + * single-select and required, since a tile cannot query without a source. + */ +export const DataSourceDashboardFilterSchema = dashboardFilterBaseSchema + .omit({ isGlobalRequirement: true }) + .extend({ + type: z.literal(DashboardFilterType.enum.DATA_SOURCE), + /** The kinds of sources offered as values. */ + allowedSourceKinds: z.array(z.nativeEnum(SourceKind)).min(1), + /** + * Limits the offered sources to one connection: a connection ID, or a + * `$name` reference to a connection variable. + */ + connection: z.string().min(1).optional(), + minSelections: z.literal(1), + maxSelections: z.literal(1), + isBroadcastEnabled: z.literal(false), + isVariableEnabled: z.literal(true), + }) + .strict(); + +/** + * A variable whose single selected value is a connection ID. Raw SQL tiles + * and data source variables reference it as `$variableName` in place of a + * connection ID. + */ +export const ConnectionDashboardFilterSchema = dashboardFilterBaseSchema + .omit({ isGlobalRequirement: true }) + .extend({ + type: z.literal(DashboardFilterType.enum.CONNECTION), + minSelections: z.literal(1), + maxSelections: z.literal(1), + isBroadcastEnabled: z.literal(false), + isVariableEnabled: z.literal(true), + }) + .strict(); + export const DashboardFilterSchema = z.discriminatedUnion('type', [ QueryExpressionDashboardFilterSchema, StaticListDashboardFilterSchema, PromqlLabelDashboardFilterSchema, AdhocDashboardFilterSchema, + DataSourceDashboardFilterSchema, + ConnectionDashboardFilterSchema, ]); export type QueryExpressionDashboardFilter = z.infer< @@ -2457,6 +2513,12 @@ export type PromqlLabelDashboardFilter = z.infer< typeof PromqlLabelDashboardFilterSchema >; export type AdhocDashboardFilter = z.infer; +export type DataSourceDashboardFilter = z.infer< + typeof DataSourceDashboardFilterSchema +>; +export type ConnectionDashboardFilter = z.infer< + typeof ConnectionDashboardFilterSchema +>; export type DashboardFilter = z.infer; export enum PresetDashboard { @@ -2612,13 +2674,6 @@ export type Team = z.infer; // -------------------------- // TABLE SOURCES // -------------------------- -export enum SourceKind { - Log = 'log', - Trace = 'trace', - Session = 'session', - Metric = 'metric', - Promql = 'promql', -} // -------------------------- // TABLE SOURCE FORM VALIDATION