feat: add data source and connection variable types to dashboards

.changeset/source-vars-1.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/common-utils': minor
+'@hyperdx/api': minor
+'@hyperdx/app': patch
+---
+
+feat: add data source and connection variable types to dashboards

packages/api/src/controllers/alerts.ts

@@ -9,6 +9,7 @@ import {
   isPromqlSavedChartConfig,
   isRawSqlSavedChartConfig,
 } from '@hyperdx/common-utils/dist/guards';
+import { configUsesReferenceVariable } from '@hyperdx/common-utils/dist/referenceVariables';
 import { isRangeThresholdType } from '@hyperdx/common-utils/dist/types';
 import { groupBy } from 'lodash';
 import { type FilterQuery, Types } from 'mongoose';
@@ -97,6 +98,11 @@ export const validateAlertInput = async (
 
     // The alert checker cannot evaluate these tiles, so the alert would never run.
     if (tile.config != null && !savedChartConfigSupportsAlerts(tile.config)) {
+      if (configUsesReferenceVariable(tile.config)) {
+        throw new Api400Error(
+          'Alerts are not supported on tiles that use a data source or connection variable',
+        );
+      }
       if (isPromqlSavedChartConfig(tile.config)) {
         throw new Api400Error('Alerts are not supported on PromQL tiles');
       }

packages/api/src/routers/api/__tests__/dashboard.int.test.ts

@@ -1518,6 +1518,130 @@ describe('dashboard router', () => {
     );
   });
 
+  describe('data source variables', () => {
+    const dataSourceFilter = {
+      id: new Types.ObjectId().toString(),
+      type: 'DATA_SOURCE' as const,
+      name: 'Logs source',
+      variableName: 'logs',
+      allowedSourceKinds: [SourceKind.Log],
+      minSelections: 1,
+      maxSelections: 1,
+      isBroadcastEnabled: false,
+      isVariableEnabled: true,
+    };
+
+    it('persists a tile and a filter that reference the variable', async () => {
+      const tile = makeTile({ sourceId: '$logs' });
+      const queriedFilter = {
+        id: new Types.ObjectId().toString(),
+        type: 'QUERY_EXPRESSION' as const,
+        name: 'Service',
+        expression: 'ServiceName',
+        source: '$logs',
+      };
+
+      const created = await agent
+        .post('/dashboards')
+        .send({
+          ...MOCK_DASHBOARD,
+          tiles: [tile],
+          filters: [dataSourceFilter, queriedFilter],
+        })
+        .expect(200);
+
+      expect(created.body.tiles[0].config.source).toBe('$logs');
+      expect(created.body.filters).toEqual([dataSourceFilter, queriedFilter]);
+    });
+
+    it('rejects a reference to an unknown variable', async () => {
+      const res = await agent
+        .post('/dashboards')
+        .send({ ...MOCK_DASHBOARD, tiles: [makeTile({ sourceId: '$nope' })] })
+        .expect(400);
+
+      expect(JSON.stringify(res.body)).toMatch(
+        /does not name a data source variable/,
+      );
+    });
+
+    it('rejects a variable offering kinds the reference cannot use', async () => {
+      const res = await agent
+        .post('/dashboards')
+        .send({
+          ...MOCK_DASHBOARD,
+          tiles: [makeTile({ sourceId: '$logs' })],
+          filters: [
+            {
+              ...dataSourceFilter,
+              allowedSourceKinds: [SourceKind.Log, SourceKind.Promql],
+            },
+          ],
+        })
+        .expect(400);
+
+      expect(JSON.stringify(res.body)).toMatch(/promql/);
+    });
+
+    it('validates a PATCH of tiles against the stored filters', async () => {
+      const created = await agent
+        .post('/dashboards')
+        .send({ ...MOCK_DASHBOARD, filters: [dataSourceFilter] })
+        .expect(200);
+
+      await agent
+        .patch(`/dashboards/${created.body.id}`)
+        .send({ tiles: [makeTile({ sourceId: '$logs' })] })
+        .expect(200);
+      await agent
+        .patch(`/dashboards/${created.body.id}`)
+        .send({ filters: [] })
+        .expect(400);
+    });
+
+    it('drops an alert on a tile that uses the variable', async () => {
+      await agent
+        .post('/dashboards')
+        .send({
+          ...MOCK_DASHBOARD,
+          tiles: [
+            makeTile({
+              sourceId: '$logs',
+              alert: makeMockAlert(webhook._id.toString()),
+            }),
+          ],
+          filters: [dataSourceFilter],
+        })
+        .expect(200);
+
+      expect(await Alert.countDocuments({ team: team._id })).toBe(0);
+    });
+
+    it('rejects a tile alert on a tile that uses the variable', async () => {
+      const created = await agent
+        .post('/dashboards')
+        .send({
+          ...MOCK_DASHBOARD,
+          tiles: [makeTile({ sourceId: '$logs' })],
+          filters: [dataSourceFilter],
+        })
+        .expect(200);
+
+      const res = await agent
+        .post('/alerts')
+        .send(
+          makeAlertInput({
+            dashboardId: created.body.id,
+            tileId: created.body.tiles[0].id,
+            webhookId: webhook._id.toString(),
+          }),
+        )
+        .expect(400);
+
+      expect(res.body.message).toMatch(/data source or connection variable/);
+    });
+  });
+
   describe('promql-label filters', () => {
     const createPromqlSource = () =>
       Source.create({

packages/api/src/routers/api/dashboards.ts

@@ -2,12 +2,14 @@ import {
   validateDashboardFilterModes,
   validateDashboardFilterOptionUniqueness,
   validateDashboardFilterVariableNames,
+  validateDashboardVariableReferences,
 } from '@hyperdx/common-utils/dist/dashboardValidation';
 import {
   getAdhocFilterSourceKinds,
   isAdhocFilter,
   isPrometheusLabelFilter,
 } from '@hyperdx/common-utils/dist/filters';
+import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables';
 import {
   DashboardFilter,
   DashboardSchema,
@@ -39,6 +41,7 @@ import {
 import { getSources } from '@/controllers/sources';
 import { getNonNullUserWithTeam } from '@/middleware/auth';
 import type { ObjectId } from '@/models';
+import { getVariableReferenceError } from '@/utils/dashboardVariableReferences';
 import { objectIdSchema } from '@/utils/zod';
 
 // create routes that will get and update dashboards
@@ -101,9 +104,12 @@ async function validateFilterSources(
       new Set(getFilterSourceRefs(filter).map(refKey)),
     ]),
   );
+  // Variable references are checked by validateDashboardVariableReferences
   const newRefs = filters.flatMap(filter =>
     getFilterSourceRefs(filter).filter(
-      ref => !existingRefsById.get(filter.id)?.has(refKey(ref)),
+      ref =>
+        parseVariableReference(ref.id) == null &&
+        !existingRefsById.get(filter.id)?.has(refKey(ref)),
     ),
   );
   if (newRefs.length === 0) return null;
@@ -164,7 +170,14 @@ router.post(
   '/',
   migrateLegacyDashboardTileColors,
   validateRequest({
-    body: DashboardWithoutIdSchema.superRefine(addFilterIssues),
+    body: DashboardWithoutIdSchema.superRefine(addFilterIssues).superRefine(
+      (data, ctx) =>
+        validateDashboardVariableReferences(
+          data.tiles,
+          data.filters ?? [],
+          ctx,
+        ),
+    ),
   }),
   async (req, res, next) => {
     try {
@@ -223,11 +236,16 @@ router.patch(
       // null still signals field removal downstream.
       const updates = DashboardWithoutIdSchema.partial().parse(req.body);
 
-      const sourceError = await validateFilterSources(
-        teamId,
-        req.body.filters,
-        dashboard.filters,
-      );
+      const sourceError =
+        getVariableReferenceError(
+          req.body.tiles ?? dashboard.tiles,
+          req.body.filters ?? dashboard.filters,
+        ) ??
+        (await validateFilterSources(
+          teamId,
+          req.body.filters,
+          dashboard.filters,
+        ));
       if (sourceError != null) {
         return res.status(400).json({ message: sourceError });
       }

packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts

@@ -143,6 +143,41 @@ describe('provisionDashboards', () => {
     // rejects the legacy enum and `readDashboardFiles` skips the file
     // outright, so a provisioned dashboard authored against #2265
     // wouldn't ship at all.
+    it('parses a tile that references a declared data source variable', () => {
+      fs.writeFileSync(
+        path.join(tmpDir, 'source-variable.json'),
+        JSON.stringify({
+          name: 'Source Variable',
+          tiles: [makeTile({ sourceId: '$logs' })],
+          filters: [
+            {
+              id: 'ds',
+              type: 'DATA_SOURCE',
+              name: 'Logs source',
+              variableName: 'logs',
+              allowedSourceKinds: ['log'],
+              minSelections: 1,
+              maxSelections: 1,
+              isBroadcastEnabled: false,
+              isVariableEnabled: true,
+            },
+          ],
+        }),
+      );
+      expect(readDashboardFiles(tmpDir)).toHaveLength(1);
+    });
+
+    it('skips a tile that references an undeclared data source variable', () => {
+      fs.writeFileSync(
+        path.join(tmpDir, 'unknown-variable.json'),
+        JSON.stringify({
+          name: 'Unknown Variable',
+          tiles: [makeTile({ sourceId: '$logs' })],
+        }),
+      );
+      expect(readDashboardFiles(tmpDir)).toEqual([]);
+    });
+
     it('migrates legacy chart-N tile colors before schema validation', () => {
       const tile = makeTile();
       (tile.config as any).color = 'chart-1';

packages/api/src/tasks/provisionDashboards/index.ts

@@ -1,4 +1,7 @@
-import { validateDashboardFilterOptionUniqueness } from '@hyperdx/common-utils/dist/dashboardValidation';
+import {
+  validateDashboardFilterOptionUniqueness,
+  validateDashboardVariableReferences,
+} from '@hyperdx/common-utils/dist/dashboardValidation';
 import {
   DashboardWithoutId,
   DashboardWithoutIdSchema,
@@ -30,8 +33,10 @@ function migrateLegacyDashboardTileColorsRaw(raw: unknown): unknown {
 }
 
 const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine(
-  (data, ctx) =>
-    validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx),
+  (data, ctx) => {
+    validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx);
+    validateDashboardVariableReferences(data.tiles, data.filters ?? [], ctx);
+  },
 );
 
 export function readDashboardFiles(dir: string): DashboardWithoutId[] {

packages/api/src/utils/dashboardVariableReferences.ts

@@ -0,0 +1,24 @@
+import {
+  collectRefinementIssues,
+  validateDashboardVariableReferences,
+} from '@hyperdx/common-utils/dist/dashboardValidation';
+import { DashboardFilter } from '@hyperdx/common-utils/dist/types';
+
+/**
+ * Rejects data source and connection variable references (`$name` in source
+ * and connection fields) that name no variable of the right type, or a data
+ * source variable offering kinds the field cannot use. Takes internal-shape
+ * tiles and filters, as they will be stored, so a partial update can be
+ * checked against the stored remainder. Returns an error message, or null.
+ */
+export function getVariableReferenceError(
+  tiles: { config: object }[] = [],
+  filters: DashboardFilter[] = [],
+): string | null {
+  const issues = collectRefinementIssues(ctx =>
+    validateDashboardVariableReferences(tiles, filters, ctx),
+  );
+  return issues.length > 0
+    ? issues.map(issue => issue.message).join('; ')
+    : null;
+}

packages/api/src/utils/externalApi.ts

@@ -5,6 +5,7 @@ import {
   isFilterBroadcastEnabled,
   isFilterRequired,
   isFilterVariableEnabled,
+  isReferenceVariableFilter,
 } from '@hyperdx/common-utils/dist/filters';
 import {
   AlertErrorType,
@@ -241,7 +242,10 @@ export function translateExternalChartToTileConfig(
 }
 
 /** Filter types the external API and MCP don't expose yet. */
-type InternalOnlyDashboardFilter = Extract<DashboardFilter, { type: 'ADHOC' }>;
+type InternalOnlyDashboardFilter = Extract<
+  DashboardFilter,
+  { type: 'ADHOC' | 'DATA_SOURCE' | 'CONNECTION' }
+>;
 export type ExternalCompatibleDashboardFilter = Exclude<
   DashboardFilter,
   InternalOnlyDashboardFilter
@@ -253,7 +257,7 @@ export function getExternalCompatibleFilters(
 ): ExternalCompatibleDashboardFilter[] {
   return (filters ?? []).filter(
     (filter): filter is ExternalCompatibleDashboardFilter =>
-      !isAdhocFilter(filter),
+      !isAdhocFilter(filter) && !isReferenceVariableFilter(filter),
   );
 }