feat(api): provision and manage Claude agents for a team

.changeset/managed-agent-provisioning.md

@@ -0,0 +1,12 @@
+---
+'@hyperdx/common-utils': minor
+'@hyperdx/api': minor
+---
+
+feat: provision and manage Claude agents for a team
+
+A team can own Claude managed agents through `/api/managed-agents`. Provisioning creates the Anthropic pieces a session needs — an environment, a vault holding the creating user's ClickStack access key, and the agent itself — and rolls back whatever it made if a later step fails. An agent written by hand on Anthropic can be imported by its ID instead, and is rejected if it points at a different MCP server or none at all, since the credential provisioned for it would go unused and the agent would run with no ClickStack access. Deleting one tears down the remote resources before the local record and keeps the record if any of that fails, so a partial teardown can be retried; it is refused while an alert still targets the agent.
+
+The agent's tool policy is written out rather than inherited. Anthropic's built-in toolset auto-approves every tool by default, so it is configured tool by tool: no shell, no web search, and a runbook fetch evaluated per call. Only the read-only ClickStack MCP tools are auto-approved, which matters because nothing answers an approval prompt in an unattended session — anything left asking simply never runs.
+
+Everything is behind `HDX_MANAGED_AGENTS_ENABLED`, which 404s the whole surface when unset. Provisioning needs a second opt-in, `HDX_MANAGED_AGENTS_ALLOW_CREATE`, since it spends on the deployment's Anthropic account; importing works with the first flag alone. `HDX_MCP_ALLOWED_HOSTS` allowlists extra hostnames for the MCP endpoint, for a deployment that serves the API on a host of its own.

docker-compose.yml

@@ -79,6 +79,19 @@ services:
       # NEXT_PUBLIC_ENABLE_PROMQL: 'true'
       # Uncomment to enable the alert detail page (in development)
       # NEXT_PUBLIC_ENABLE_ALERT_DETAILS: 'true'
+      # Extra hostnames the MCP endpoint answers to, comma-separated. Needed
+      # when clients reach /api/mcp on a host other than HYPERDX_APP_URL's.
+      HDX_MCP_ALLOWED_HOSTS: ${HDX_MCP_ALLOWED_HOSTS:-}
+      # Uncomment the next four lines to enable AI agent alert investigations.
+      # Provisioning spends on your Anthropic account, so it needs the second
+      # flag as well; importing an existing agent needs only the first.
+      # HDX_MANAGED_AGENTS_ENABLED: 'true'
+      # NEXT_PUBLIC_HDX_MANAGED_AGENTS_ENABLED: 'true'
+      # HDX_MANAGED_AGENTS_ALLOW_CREATE: 'true'
+      # NEXT_PUBLIC_HDX_MANAGED_AGENTS_ALLOW_CREATE: 'true'
+      # Public HTTPS URL of this instance's MCP server, if it is not
+      # HYPERDX_APP_URL + /api/mcp. Anthropic's sandbox reaches it directly.
+      # HDX_MANAGED_AGENTS_MCP_URL: 'https://example.com/api/mcp'
       DEFAULT_CONNECTIONS:
         '[{"name":"Local
         ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]'

knip.json

@@ -52,5 +52,6 @@
     "@stryker-mutator/core"
   ],
   "include": ["nsExports"],
-  "exclude": ["enumMembers", "duplicates"]
+  "exclude": ["enumMembers", "duplicates"],
+  "tags": ["-public"]
 }

packages/api/.env.development

@@ -5,6 +5,9 @@ FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}"
 HDX_NODE_ADVANCED_NETWORK_CAPTURE=1
 HDX_NODE_BETA_MODE=1
 HDX_NODE_CONSOLE_CAPTURE=1
+HDX_MANAGED_AGENTS_ENABLED=true
+# Provisioning is a separate opt-in; without it only importing works.
+HDX_MANAGED_AGENTS_ALLOW_CREATE=true
 MONGO_URI="mongodb://localhost:${HDX_DEV_MONGO_PORT}/hyperdx"
 NODE_ENV=development
 OTEL_SERVICE_NAME="hdx-oss-dev-api"
@@ -25,4 +28,10 @@ DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kin
 INGESTION_API_KEY="super-secure-ingestion-api-key"
 HYPERDX_API_KEY=$INGESTION_API_KEY
 ANTHROPIC_API_KEY="your-anthropic-api-key-here"
+# Anthropic's sandbox connects to the MCP server directly, so this must be a
+# public HTTPS tunnel to your local /api/mcp.
+# HDX_MANAGED_AGENTS_MCP_URL="https://<subdomain>.ngrok-free.app/api/mcp"
+# Extra hostnames the MCP endpoint answers to, comma-separated. Needed when the
+# API is served on a hostname of its own rather than through the app origin.
+# HDX_MCP_ALLOWED_HOSTS="api.example.com"
 ENABLE_PROMQL=true

packages/api/.env.test

@@ -16,4 +16,8 @@ OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320}
 # Note: the logger module is mocked in jest.setup.ts to suppress expected
 # operational noise (validation errors, MCP tool errors, etc.) during tests.
 HYPERDX_LOG_LEVEL=error
-ENABLE_PROMQL=true
+ENABLE_PROMQL=true
+# Managed-agent code paths are feature-gated on this flag; enable it so the
+# agent service / alert-dispatch tests exercise the real behaviour.
+HDX_MANAGED_AGENTS_ENABLED=true
+HDX_MANAGED_AGENTS_ALLOW_CREATE=true

packages/api/src/api-app.ts

@@ -103,6 +103,7 @@ app.use('/dashboards', isUserAuthenticated, routers.dashboardRouter);
 app.use('/me', isUserAuthenticated, routers.meRouter);
 app.use('/team', isUserAuthenticated, routers.teamRouter);
 app.use('/webhooks', isUserAuthenticated, routers.webhooksRouter);
+app.use('/managed-agents', isUserAuthenticated, routers.managedAgentsRouter);
 app.use('/connections', isUserAuthenticated, connectionsRouter);
 app.use('/sources', isUserAuthenticated, sourcesRouter);
 app.use('/saved-search', isUserAuthenticated, savedSearchRouter);

packages/api/src/config.ts

@@ -69,11 +69,48 @@ export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD as string;
 
 // AI Assistant
 // Provider-agnostic configuration (preferred)
-export const AI_PROVIDER = env.AI_PROVIDER as string; // 'anthropic' | 'openai'
+
+// Supported AI providers. Values are the accepted `AI_PROVIDER` env strings —
+// compare against these members rather than bare string literals.
+export enum AIProvider {
+  Anthropic = 'anthropic',
+  OpenAI = 'openai',
+}
+
+// Raw env string (validated against AIProvider at the point of use so an
+// unrecognised value fails loudly rather than being silently cast).
+export const AI_PROVIDER = env.AI_PROVIDER as string;
 export const AI_API_KEY = env.AI_API_KEY as string;
 export const AI_BASE_URL = env.AI_BASE_URL as string;
 export const AI_MODEL_NAME = env.AI_MODEL_NAME as string;
 export const AI_REQUEST_HEADERS = env.AI_REQUEST_HEADERS as string;
 
 // Legacy Anthropic-specific configuration (backward compatibility)
 export const ANTHROPIC_API_KEY = env.ANTHROPIC_API_KEY as string;
+
+// AI agent alert investigations (in-product Claude managed-agent provisioning
+// and the `agent` alert notification channel). Off by default. The Anthropic
+// key is read from the environment (AI_API_KEY / ANTHROPIC_API_KEY — see
+// anthropicAgents.getTeamAnthropicKey); a downstream distribution can supply a
+// per-team key via the resolveAnthropicKey extension seam.
+export const IS_MANAGED_AGENTS_ENABLED =
+  env.HDX_MANAGED_AGENTS_ENABLED === 'true';
+
+// Provisioning specifically: creating an agent spends money on the deployment's
+// Anthropic account and writes the creating user's ClickStack key into a vault,
+// so it needs its own opt-in on top of the feature flag. Off by default —
+// importing an agent someone else created still works with only the flag above.
+export const IS_MANAGED_AGENT_CREATE_ENABLED =
+  IS_MANAGED_AGENTS_ENABLED && env.HDX_MANAGED_AGENTS_ALLOW_CREATE === 'true';
+
+// Public HTTPS URL of the ClickStack MCP server for managed agents (falls back
+// to FRONTEND_URL + /api/mcp at the call sites). Read at call time, not module
+// load, so test suites can set it per-suite without mocking this module.
+export const getManagedAgentsMcpUrl = (): string | undefined =>
+  process.env.HDX_MANAGED_AGENTS_MCP_URL;
+
+// Extra hostnames the MCP endpoint will answer to, comma-separated. The
+// endpoint is mounted on the API app, so a deployment that serves the API on
+// its own hostname needs that host allowlisted — FRONTEND_URL alone does not
+// cover it, and the SDK's default is localhost only.
+export const MCP_ALLOWED_HOSTS = (env.HDX_MCP_ALLOWED_HOSTS as string) || '';

packages/api/src/extensions/index.ts

@@ -0,0 +1,15 @@
+// Downstream extension registration point.
+//
+// This file is intentionally empty in the open-source distribution.
+// Downstream distributions REPLACE this file's body to register their
+// extensions, for example:
+//
+//   import { registerAgentRunExtension } from '@/services/agentRunExtensions';
+//   registerAgentRunExtension(myExtension);
+//
+// CONTRACT: upstream never edits this file after its creation, so downstream
+// replacements never conflict on upstream merges. Add new registration
+// surfaces via new register* calls in downstream copies — not by editing
+// core modules.
+
+export {};

packages/api/src/mcp/__tests__/agentToolPolicy.test.ts

@@ -0,0 +1,110 @@
+import {
+  AGENT_TOOLSET,
+  AUTO_ALLOWED_MCP_TOOLS,
+} from '@hyperdx/common-utils/dist/managedAgents';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+import alertsTools from '@/mcp/tools/alerts/index';
+import dashboardsTools from '@/mcp/tools/dashboards/index';
+import queryTools from '@/mcp/tools/query/index';
+import savedSearchesTools from '@/mcp/tools/savedSearches/index';
+import sourcesTools from '@/mcp/tools/sources/index';
+import traceTools from '@/mcp/tools/trace/index';
+import type { ToolRegistrar } from '@/mcp/tools/types';
+
+// Registration only touches registrar.registerTool, so a capturing stub reads
+// the real registry without standing up a transport or a ClickHouse connection.
+const registeredTools = () => {
+  const tools = new Map<string, { destructive: boolean }>();
+  const registrar: ToolRegistrar = {
+    server: new McpServer({ name: 'test', version: '0' }),
+    context: { teamId: 'team', userId: 'user' },
+    registerTool: (name, config) => {
+      tools.set(name, {
+        destructive: config.annotations?.destructiveHint === true,
+      });
+    },
+  };
+
+  for (const register of [
+    sourcesTools,
+    alertsTools,
+    dashboardsTools,
+    queryTools,
+    savedSearchesTools,
+    traceTools,
+  ]) {
+    register(registrar);
+  }
+  return tools;
+};
+
+describe('AUTO_ALLOWED_MCP_TOOLS', () => {
+  const tools = registeredTools();
+
+  it('names only tools the MCP server actually registers', () => {
+    const unknown = AUTO_ALLOWED_MCP_TOOLS.filter(name => !tools.has(name));
+    expect(unknown).toEqual([]);
+  });
+
+  // The allowlist is what an unattended alert investigation may call without
+  // an approval prompt, and no one is there to answer one. Keyed off the
+  // registry's own destructiveHint rather than a name pattern, so a mutating
+  // tool named with a new verb cannot slip in unnoticed.
+  it('auto-approves no tool the registry marks destructive', () => {
+    const destructive = AUTO_ALLOWED_MCP_TOOLS.filter(
+      name => tools.get(name)?.destructive,
+    );
+    expect(destructive).toEqual([]);
+  });
+
+  it('covers every non-destructive tool, so new read tools are a deliberate choice', () => {
+    const missing = [...tools.entries()]
+      .filter(
+        ([name, t]) => !t.destructive && !AUTO_ALLOWED_MCP_TOOLS.includes(name),
+      )
+      .map(([name]) => name);
+    expect(missing).toEqual([]);
+  });
+});
+
+// See the AGENT_TOOLSET docblock for why each value is what it is. The whole
+// surface is pinned rather than spot-checked: a tool dropped from configs
+// inherits the toolset default, which is the failure this policy exists to
+// prevent, and a spot-check would not see it.
+describe('AGENT_TOOLSET', () => {
+  it('fails closed by default', () => {
+    expect(AGENT_TOOLSET.default_config.permission_policy.type).toBe(
+      'always_ask',
+    );
+  });
+
+  it('pins every tool to its intended policy', () => {
+    const surface = AGENT_TOOLSET.configs.map(c => [
+      c.name,
+      c.enabled ?? true,
+      c.permission_policy?.type ??
+        AGENT_TOOLSET.default_config.permission_policy.type,
+    ]);
+    expect(surface).toEqual([
+      ['read', true, 'always_allow'],
+      ['write', true, 'always_allow'],
+      ['edit', true, 'always_allow'],
+      ['glob', true, 'always_allow'],
+      ['grep', true, 'always_allow'],
+      // Gated, not open: the agent is told to follow a runbook link.
+      ['web_fetch', true, 'auto'],
+      ['web_search', false, 'always_ask'],
+      // No unattended shell — see the docblock.
+      ['bash', false, 'always_ask'],
+    ]);
+  });
+
+  it('is frozen, so a consumer cannot widen it at runtime', () => {
+    expect(Object.isFrozen(AGENT_TOOLSET.configs)).toBe(true);
+    expect(() => {
+      // @ts-expect-error -- readonly at compile time; this asserts runtime too
+      AGENT_TOOLSET.configs[0].enabled = false;
+    }).toThrow();
+  });
+});

packages/api/src/mcp/__tests__/app.test.ts

@@ -1,7 +1,7 @@
 import express from 'express';
 import request from 'supertest';
 
-import mcpRouter from '@/mcp/app';
+import mcpRouter, { buildAllowedHosts } from '@/mcp/app';
 
 // The MCP transport is stateless, so GET (standalone SSE stream) and DELETE
 // (session termination) are not offered and must return 405 so spec-compliant
@@ -22,3 +22,41 @@ describe('mcp app transport methods', () => {
     expect(res.headers.allow).toBe('POST');
   });
 });
+
+describe('buildAllowedHosts', () => {
+  it('always includes the localhost defaults', () => {
+    expect(buildAllowedHosts([])).toEqual(['localhost', '127.0.0.1', '[::1]']);
+  });
+
+  it('adds the bare hostname of a configured URL (drops path and port)', () => {
+    const hosts = buildAllowedHosts([
+      'https://78fc-1-2-3.ngrok-free.app/mcp',
+      'http://localhost:30287',
+    ]);
+    expect(hosts).toContain('78fc-1-2-3.ngrok-free.app');
+    expect(hosts).toContain('localhost');
+    // No port, no path leaked into the allowlist.
+    expect(hosts).not.toContain('localhost:30287');
+  });
+
+  it('adds bare hostnames from the extra-hosts list', () => {
+    // The MCP endpoint is mounted on the API app, so a deployment serving the
+    // API on its own hostname has to be able to name it.
+    const hosts = buildAllowedHosts(
+      ['https://app.example.com'],
+      'api.example.com, mcp.example.com',
+    );
+    expect(hosts).toContain('api.example.com');
+    expect(hosts).toContain('mcp.example.com');
+    expect(hosts).toContain('app.example.com');
+  });
+
+  it('ignores undefined and malformed URLs rather than throwing', () => {
+    expect(() => buildAllowedHosts([undefined, 'not a url', ''])).not.toThrow();
+    expect(buildAllowedHosts([undefined, 'not a url'])).toEqual([
+      'localhost',
+      '127.0.0.1',
+      '[::1]',
+    ]);
+  });
+});

packages/api/src/mcp/app.ts

@@ -3,6 +3,7 @@ import { createMcpExpressApp } from '@modelcontextprotocol/sdk/server/express.js
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 import express from 'express';
 
+import * as config from '@/config';
 import { validateUserAccessKey } from '@/middleware/auth';
 import logger from '@/utils/logger';
 import rateLimiter, { rateLimiterKeyGenerator } from '@/utils/rateLimiter';
@@ -11,7 +12,48 @@ import { createServer } from './mcpServer';
 import { McpContext } from './tools/types';
 import { userAgentClientInfo } from './utils/mcpClient';
 
-const app = createMcpExpressApp();
+// The SDK applies localhost-only DNS-rebinding protection by default, which
+// rejects any request whose Host header isn't localhost. Managed agents (and
+// any remote MCP client) reach us over a public tunnel/domain, so allowlist the
+// configured public hosts alongside localhost. Matching is port-agnostic, so we
+// pass bare hostnames.
+export const buildAllowedHosts = (
+  urls: (string | undefined)[],
+  extraHosts = '',
+): string[] => {
+  const hosts = ['localhost', '127.0.0.1', '[::1]'];
+  for (const url of urls) {
+    if (!url) continue;
+    try {
+      hosts.push(new URL(url).hostname);
+    } catch {
+      // ignore a malformed URL — it just won't be allowlisted
+    }
+  }
+  // Bare hostnames rather than URLs, so an operator can name the API's own
+  // host without inventing a scheme for it.
+  for (const host of extraHosts.split(',')) {
+    const trimmed = host.trim();
+    if (trimmed) hosts.push(trimmed);
+  }
+  return hosts;
+};
+
+const app = createMcpExpressApp({
+  // FRONTEND_URL is always allowlisted — the endpoint is reachable through the
+  // app origin in every deployment. The managed-agents URL is added only when
+  // that feature is on, and HDX_MCP_ALLOWED_HOSTS covers a deployment that
+  // serves the API on a hostname of its own.
+  allowedHosts: buildAllowedHosts(
+    [
+      config.IS_MANAGED_AGENTS_ENABLED
+        ? config.getManagedAgentsMcpUrl()
+        : undefined,
+      config.FRONTEND_URL,
+    ],
+    config.MCP_ALLOWED_HOSTS,
+  ),
+});
 
 const mcpRateLimiter = rateLimiter({
   windowMs: 60 * 1000, // 1 minute