feat(alerts): hand an alert's investigation to a Claude agent

.changeset/agent-alert-channel.md

@@ -0,0 +1,14 @@
+---
+'@hyperdx/common-utils': minor
+'@hyperdx/api': minor
+---
+
+feat: hand an alert's investigation to a Claude agent
+
+An alert's notification channel can now be an agent instead of a webhook. When the alert fires, HyperDX starts an Anthropic session with a structured payload — status, comparator, threshold and its upper bound, current value, group key, source query, time range, and the alert's note as a runbook — and the agent investigates from there through the ClickStack MCP server. The result lives in that session; nothing is delivered from here, so the agent's own configuration decides where findings go.
+
+Channels are additive, so a webhook still pages you immediately while the agent works the same incident. Sessions are deduped per alert, per agent, within a cooldown window, and the key is claimed before the Anthropic calls — so a grouped alert breaching on a thousand group keys starts one investigation rather than a thousand, and two concurrent firings cannot each pay for a session. Only the firing edge dispatches: the prompt says an alert fired, so investigating a resolution would be wrong as well as wasteful. A dispatch that fails is recorded against the alert as a distinct error naming the agent, and cannot consume another channel's notification slot.
+
+Existing alerts are unaffected — the channel shape gates only what can be written, and nothing that used to validate stops validating. Agent channels require `HDX_MANAGED_AGENTS_ENABLED`, enforced once in shared validation so the internal API, the external API v2 and the MCP `saveAlert` tool cannot drift on what they accept.
+
+Also fixes `{{note}}` in webhook templates, which always rendered empty because nothing ever set it.

packages/api/openapi.json

@@ -173,19 +173,13 @@
         ],
         "description": "Current alert state."
       },
-      "AlertChannelType": {
-        "type": "string",
-        "enum": [
-          "webhook"
-        ],
-        "description": "Channel type."
-      },
       "AlertErrorType": {
         "type": "string",
         "enum": [
           "QUERY_ERROR",
           "QUERY_TIMEOUT",
           "WEBHOOK_ERROR",
+          "AGENT_ERROR",
           "INVALID_ALERT",
           "UNKNOWN"
         ],
@@ -250,8 +244,12 @@
         ],
         "properties": {
           "type": {
-            "$ref": "#/components/schemas/AlertChannelType",
-            "description": "Channel type. Must be \"webhook\" for webhook alerts."
+            "type": "string",
+            "enum": [
+              "webhook"
+            ],
+            "description": "Channel type. Must be \"webhook\" for webhook alerts.",
+            "example": "webhook"
           },
           "webhookId": {
             "type": "string",
@@ -260,14 +258,44 @@
           }
         }
       },
+      "AlertChannelAgent": {
+        "type": "object",
+        "description": "AI agent notification target. A firing alert starts an investigation session on the referenced managed agent. Only available on deployments with managed agents enabled.",
+        "required": [
+          "type",
+          "agentId"
+        ],
+        "properties": {
+          "type": {
+            "type": "string",
+            "enum": [
+              "agent"
+            ],
+            "description": "Channel type. Must be \"agent\" for agent alerts.",
+            "example": "agent"
+          },
+          "agentId": {
+            "type": "string",
+            "description": "Managed agent ID.",
+            "example": "65f5e4a3b9e77c001a789014"
+          }
+        }
+      },
       "AlertChannel": {
         "oneOf": [
           {
             "$ref": "#/components/schemas/AlertChannelWebhook"
+          },
+          {
+            "$ref": "#/components/schemas/AlertChannelAgent"
           }
         ],
         "discriminator": {
-          "propertyName": "type"
+          "propertyName": "type",
+          "mapping": {
+            "webhook": "#/components/schemas/AlertChannelWebhook",
+            "agent": "#/components/schemas/AlertChannelAgent"
+          }
         }
       },
       "AlertChannels": {

packages/api/src/controllers/alerts.ts

@@ -10,6 +10,7 @@ import { groupBy } from 'lodash';
 import { Types } from 'mongoose';
 import { z } from 'zod';
 
+import { IS_MANAGED_AGENTS_ENABLED } from '@/config';
 import type { ObjectId } from '@/models';
 import Alert, {
   AlertChannel,
@@ -19,6 +20,7 @@ import Alert, {
 } from '@/models/alert';
 import Connection from '@/models/connection';
 import Dashboard, { IDashboard } from '@/models/dashboard';
+import ManagedAgent from '@/models/managedAgent';
 import { ISavedSearch, SavedSearch } from '@/models/savedSearch';
 import { Source } from '@/models/source';
 import { IUser } from '@/models/user';
@@ -217,19 +219,46 @@ export const validateAlertInput = async (
     throw new Api400Error('At least one notification channel is required');
   }
 
-  const webhookIds = channels
-    .filter(c => c.type === 'webhook')
-    .map(c => c.webhookId);
-  for (const webhookId of webhookIds) {
-    validateObjectId(webhookId, 'Invalid webhook ID');
-  }
-  const uniqueIds = [...new Set(webhookIds)];
-  const found = await Webhook.countDocuments({
-    _id: { $in: uniqueIds },
-    team: teamId,
-  });
-  if (found !== uniqueIds.length) {
-    throw new Api400Error('Webhook not found');
+  // Each channel kind names a row in a different collection; the check is the
+  // same either way — the ids parse, and every distinct one belongs to this
+  // team. Kept as one pass so a third channel kind cannot arrive with a
+  // subtly different rule.
+  const assertChannelRefsExist = async (
+    ids: string[],
+    model: { countDocuments: (f: object) => PromiseLike<number> },
+    label: string,
+  ) => {
+    for (const id of ids) {
+      validateObjectId(id, `Invalid ${label} ID`);
+    }
+    const unique = [...new Set(ids)];
+    const found = await model.countDocuments({
+      _id: { $in: unique },
+      team: teamId,
+    });
+    if (found !== unique.length) {
+      throw new Api400Error(
+        `${label[0].toUpperCase()}${label.slice(1)} not found`,
+      );
+    }
+  };
+
+  await assertChannelRefsExist(
+    channels.filter(c => c.type === 'webhook').map(c => c.webhookId),
+    Webhook,
+    'webhook',
+  );
+
+  const agentIds = channels.filter(c => c.type === 'agent').map(c => c.agentId);
+  if (agentIds.length > 0) {
+    // The alert editor is flag-gated too; this rejects an agent channel saved
+    // via the API on a deployment where dispatch would silently do nothing.
+    if (!IS_MANAGED_AGENTS_ENABLED) {
+      throw new Api400Error(
+        'AI agent notification channels are not enabled on this deployment (set HDX_MANAGED_AGENTS_ENABLED)',
+      );
+    }
+    await assertChannelRefsExist(agentIds, ManagedAgent, 'agent');
   }
 
   return refs;

packages/api/src/mcp/tools/alerts/__tests__/schemas.test.ts

@@ -209,3 +209,40 @@ describe('mcpSaveAlertSchema isDelta dialect bridge', () => {
     },
   );
 });
+
+describe('validateSaveAlertInput per-type channel id pairing', () => {
+  const ag = (id: string) => ({ type: 'agent' as const, agentId: id });
+
+  it('accepts a webhook channel with a webhookId and an agent channel with an agentId', () => {
+    expect(
+      validateSaveAlertInput({ ...baseInput, channels: [wh('a'), ag('b')] }),
+    ).toBeNull();
+  });
+
+  it('rejects a webhook channel missing webhookId', () => {
+    expect(
+      validateSaveAlertInput({
+        ...baseInput,
+        channel: foreignChannel({ type: 'webhook' }),
+      }),
+    ).toBe('webhookId is required for a webhook channel');
+  });
+
+  it('rejects an agent channel missing agentId', () => {
+    expect(
+      validateSaveAlertInput({
+        ...baseInput,
+        channel: foreignChannel({ type: 'agent' }),
+      }),
+    ).toBe('agentId is required for an agent channel');
+  });
+
+  it('rejects when any entry in channels misses its per-type id', () => {
+    expect(
+      validateSaveAlertInput({
+        ...baseInput,
+        channels: [wh('a'), foreignChannel({ type: 'agent' })],
+      }),
+    ).toBe('agentId is required for an agent channel');
+  });
+});

packages/api/src/mcp/tools/alerts/saveAlert.ts

@@ -15,7 +15,7 @@ import {
   mcpUserError,
   validateObjectId,
 } from '@/mcp/utils/errors';
-import { AlertSource } from '@/models/alert';
+import { AlertChannel, AlertSource } from '@/models/alert';
 import {
   convertExternalAlertChartConfigToInternal,
   translateAlertDocumentToExternalAlertWithChartConfig,
@@ -48,11 +48,13 @@ export function registerSaveAlert({
         'Alerts monitor a saved search, a dashboard tile, or an inline chart ' +
         'config (source "inline" + chartConfig — no saved search or dashboard ' +
         'needed) and fire when the metric crosses a threshold. At least one ' +
-        'webhook notification channel ' +
-        'is required: pass "channels" for 1-10 targets, or the legacy singular ' +
-        '"channel" for one. Updates replace the alert configuration rather than ' +
-        'merging it, so read the alert first and resend its full "channels" ' +
-        'array to avoid dropping channels you did not mean to remove.',
+        'notification channel is required — a webhook ' +
+        '({type:"webhook", webhookId}) or, on deployments with managed agents ' +
+        'enabled, an AI agent investigation ({type:"agent", agentId}). Pass ' +
+        '"channels" for 1-10 targets, or the legacy singular "channel" for ' +
+        'one. Updates replace the alert configuration rather than merging it, ' +
+        'so read the alert first and resend its full "channels" array to ' +
+        'avoid dropping channels you did not mean to remove.',
       inputSchema: mcpSaveAlertSchema,
     },
     async input => {
@@ -95,10 +97,24 @@ export function registerSaveAlert({
 
       // Build the alert input matching the shape expected by controllers.
       const source = MCP_SOURCE_TO_ALERT_SOURCE[input.source];
+      // The MCP schema keeps webhookId/agentId both optional on one object
+      // shape (the per-type pairing is enforced in validateSaveAlertInput), so
+      // narrow into the discriminated AlertChannel here.
+      const toAlertChannel = (c: {
+        type: 'webhook' | 'agent';
+        webhookId?: string;
+        agentId?: string;
+      }): AlertChannel =>
+        c.type === 'agent'
+          ? { type: 'agent', agentId: c.agentId ?? '' }
+          : { type: 'webhook', webhookId: c.webhookId ?? '' };
+
       const alertInput: AlertInput = {
         source,
         // `channel` is omitted; makeAlert mirrors it from channels[0].
-        channels: input.channels ?? (input.channel ? [input.channel] : []),
+        channels: (
+          input.channels ?? (input.channel ? [input.channel] : [])
+        ).map(toAlertChannel),
         interval: input.interval,
         threshold: input.threshold,
         thresholdType: input.thresholdType as AlertThresholdType,

packages/api/src/mcp/tools/alerts/schemas.ts

@@ -106,11 +106,18 @@ const mcpAlertChartConfigSchema = z
 const mcpAlertChannelSchema = z
   .object({
     type: z
-      .literal('webhook')
+      .enum(['webhook', 'agent'])
       .describe('Channel type for alert notifications.'),
     webhookId: z
       .string()
+      .optional()
       .describe('Webhook destination ID (required for webhook channel).'),
+    agentId: z
+      .string()
+      .optional()
+      .describe(
+        'Managed agent ID (required for agent channel; a firing alert starts an AI investigation on this agent).',
+      ),
   })
   .describe('Alert notification channel configuration.');
 
@@ -245,6 +252,18 @@ export function validateSaveAlertInput(data: McpSaveAlertInput): string | null {
     }
   }
 
+  // Per-type required channel id (the Zod schema keeps both ids optional so
+  // one object shape serves both types; enforce the pairing here).
+  for (const channel of [data.channel, ...(data.channels ?? [])]) {
+    if (!channel) continue;
+    if (channel.type === 'webhook' && !channel.webhookId) {
+      return 'webhookId is required for a webhook channel';
+    }
+    if (channel.type === 'agent' && !channel.agentId) {
+      return 'agentId is required for an agent channel';
+    }
+  }
+
   // Source-specific required fields
   if (data.source === 'tile') {
     if (!data.dashboardId) {

packages/api/src/models/alert.ts

@@ -47,6 +47,12 @@ export type AlertChannel =
       type: 'webhook';
       webhookId: string;
     }
+  | {
+      // A firing alert starts an investigation session on the referenced
+      // managed agent (see models/managedAgent.ts) instead of POSTing a payload.
+      type: 'agent';
+      agentId: string;
+    }
   | {
       type: null;
     };