From ad3247165cdc3a192e51aafab7163588d94e32c6 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Wed, 2 Sep 2026 11:45:45 +1000 Subject: [PATCH] feat(alerts): enrich Generic and incident.io webhook template variables A webhook receiver could only see the rendered title and body, so routing or deduping a HyperDX alert meant parsing prose. Body templates now also carry the alert's identity and its condition: alertId, status, alertType, comparator, threshold, value, groupKey, sourceQuery, teamId, note, and ISO-8601 variants of the existing Unix-ms start and end times. Every variable is additive and renders empty when an alert doesn't carry it, so existing templates are unaffected. --- .../alert-webhook-template-variables.md | 13 ++++ docs/alert-webhook-template-variables.md | 50 +++++++++++++++ .../__tests__/renderAlertTemplate.int.test.ts | 63 +++++++++++++++++++ .../api/src/tasks/checkAlerts/template.ts | 42 +++++++++++++ .../transports/__tests__/generic.test.ts | 46 ++++++++++++++ .../tasks/checkAlerts/transports/generic.ts | 47 +++++++++++--- .../src/tasks/checkAlerts/transports/types.ts | 12 ++++ 7 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 .changeset/alert-webhook-template-variables.md create mode 100644 docs/alert-webhook-template-variables.md diff --git a/.changeset/alert-webhook-template-variables.md b/.changeset/alert-webhook-template-variables.md new file mode 100644 index 0000000000..241aa413c7 --- /dev/null +++ b/.changeset/alert-webhook-template-variables.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/api': minor +--- + +Add enriched template variables to Generic and incident.io webhook bodies: +`{{alertId}}`, `{{status}}`, `{{alertType}}`, `{{comparator}}`, `{{threshold}}`, +`{{value}}`, `{{groupKey}}`, `{{sourceQuery}}`, `{{teamId}}`, `{{note}}`, and +ISO-8601 `{{startTimeISO}}` / `{{endTimeISO}}` alongside the existing Unix-ms +`{{startTime}}` / `{{endTime}}`. + +Receivers can now route, filter and dedupe on an alert's identity and condition +without parsing the rendered message body. Existing templates are unaffected — +every new variable is additive and renders empty when an alert doesn't carry it. diff --git a/docs/alert-webhook-template-variables.md b/docs/alert-webhook-template-variables.md new file mode 100644 index 0000000000..f9fcbece9b --- /dev/null +++ b/docs/alert-webhook-template-variables.md @@ -0,0 +1,50 @@ +# Alert webhook template variables + +**Generic** and **incident.io** webhook bodies are Handlebars templates. These +variables are available inside them, so a receiver can route, filter or dedupe +a HyperDX alert without parsing the human-readable message body. + +| Variable | Type | Notes | +| --- | --- | --- | +| `{{title}}` | string | Alert title. | +| `{{body}}` | string | Rendered message body (markdown). | +| `{{link}}` | string | Deep link back into HyperDX. | +| `{{state}}` | string | Raw internal alert state. | +| `{{eventId}}` | string | Unique id for this firing. | +| `{{startTime}}` / `{{endTime}}` | number | Evaluation window, Unix ms. | +| `{{startTimeISO}}` / `{{endTimeISO}}` | string | The same window, ISO-8601. | +| `{{alertId}}` | string | Stable id of the alert itself — the key to dedupe on. | +| `{{status}}` | string | `firing`, `resolved`, `no_data`, `pending` or `error`. | +| `{{alertType}}` | string | `search`, `dashboard_chart` or `inline_query`. | +| `{{comparator}}` | string | `>=`, `>`, `<`, `<=`, `=`, `!=`, `between`, `outside`. | +| `{{threshold}}` | number | The configured threshold. | +| `{{value}}` | number | The value that triggered or resolved the alert. | +| `{{groupKey}}` | string | The breaching group, for a grouped alert. | +| `{{sourceQuery}}` | string | The search expression or SQL behind the alert. | +| `{{teamId}}` | string | Team the alert belongs to. | +| `{{note}}` | string | The alert's freeform note — commonly a runbook link. | + +Strings are JSON-escaped, so they are safe to drop into a quoted slot. +Numbers (`startTime`, `endTime`, `threshold`, `value`) are emitted raw for +unquoted slots. Every enriched variable is optional and renders as an empty +string when the alert doesn't carry it — an alert with no group has an empty +`{{groupKey}}`, and a dashboard-tile alert has an empty `{{sourceQuery}}`. + +## Example + +Routing by severity and deduping on the alert rather than the firing: + +```json +{ + "alert_id": "{{alertId}}", + "dedup_key": "{{alertId}}-{{groupKey}}", + "status": "{{status}}", + "summary": "{{title}}", + "urgency": "{{#if (eq alertType \"dashboard_chart\")}}low{{else}}high{{/if}}", + "value": {{value}}, + "threshold": {{threshold}}, + "window": { "start": "{{startTimeISO}}", "end": "{{endTimeISO}}" }, + "runbook": "{{note}}", + "link": "{{link}}" +} +``` diff --git a/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.int.test.ts index 2c67db351c..d19ec269c9 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/renderAlertTemplate.int.test.ts @@ -430,6 +430,69 @@ describe('renderAlertTemplate', () => { }); }); +// The enriched fields are what a receiver routes and dedupes on, so they have +// to survive the render, not just the variable builder in isolation. +describe('enriched message fields', () => { + const renderWithWebhook = async ( + state: AlertState, + viewOverrides: Parameters[0] = {}, + ) => { + const webhook = castWebhook({ + _id: new mongoose.Types.ObjectId(), + team: new mongoose.Types.ObjectId(), + service: 'slack', + name: 'enriched-hook', + url: 'https://hooks.slack.com/services/x', + }); + const { dispatcher, dispatched } = makeRecordingDispatcher(); + const base = makeSearchView(viewOverrides); + + const result = await renderAlertTemplate({ + alertProvider, + clickhouseClient: mockClickhouseClient, + metadata: mockMetadata, + state, + template: null, + title: 'Test Alert Title', + view: { + ...base, + alert: { + ...base.alert, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + channels: [{ type: 'webhook', webhookId: webhook._id.toString() }], + }, + }, + teamId: TEST_TEAM_ID, + teamWebhooksById: new Map([[webhook._id.toString(), webhook]]), + dispatcher, + }); + return { dispatched, result }; + }; + + it('carries the alert identity and condition onto the dispatched job', async () => { + const { dispatched, result } = await renderWithWebhook(AlertState.ALERT, { + group: 'http', + }); + + expect(result.failures).toEqual([]); + expect(dispatched[0].message).toMatchObject({ + status: 'firing', + alertType: 'search', + comparator: '>=', + threshold: 5, + groupKey: 'http', + sourceQuery: 'Body: "error"', + teamId: TEST_TEAM_ID, + }); + }); + + it('reports a resolve as resolved', async () => { + const { dispatched } = await renderWithWebhook(AlertState.OK); + + expect(dispatched[0].message).toMatchObject({ status: 'resolved' }); + }); +}); + describe('buildAlertMessageTemplateTitle', () => { describe('saved search alerts', () => { describe('ALERT state', () => { diff --git a/packages/api/src/tasks/checkAlerts/template.ts b/packages/api/src/tasks/checkAlerts/template.ts index 80cd778b60..598c45791f 100644 --- a/packages/api/src/tasks/checkAlerts/template.ts +++ b/packages/api/src/tasks/checkAlerts/template.ts @@ -105,6 +105,37 @@ const describeThreshold = (alert: AlertInput): string => { : `${alert.threshold}`; }; +// Mappings for the enriched webhook template variables. These turn internal +// enums into stable, consumer-friendly strings a receiver can branch on +// without knowing HyperDX's internals. +const ALERT_STATUS_BY_STATE: Record = { + [AlertState.ALERT]: 'firing', + [AlertState.OK]: 'resolved', + [AlertState.INSUFFICIENT_DATA]: 'no_data', + [AlertState.DISABLED]: 'no_data', + [AlertState.PENDING]: 'pending', + [AlertState.ERROR]: 'error', +}; + +const COMPARATOR_BY_THRESHOLD_TYPE: Record = { + [AlertThresholdType.ABOVE]: '>=', + [AlertThresholdType.ABOVE_EXCLUSIVE]: '>', + [AlertThresholdType.BELOW]: '<', + [AlertThresholdType.BELOW_OR_EQUAL]: '<=', + [AlertThresholdType.EQUAL]: '=', + [AlertThresholdType.NOT_EQUAL]: '!=', + [AlertThresholdType.BETWEEN]: 'between', + [AlertThresholdType.NOT_BETWEEN]: 'outside', +}; + +const ALERT_TYPE_BY_SOURCE: Record = { + [AlertSource.SAVED_SEARCH]: 'search', + [AlertSource.TILE]: 'dashboard_chart', + // Detached alert: the chart config lives on the alert itself, so there is + // no saved search or tile behind it to open. + [AlertSource.INLINE]: 'inline_query', +}; + const MAX_MESSAGE_LENGTH = 500; const NOTIFY_FN_NAME = '__hdx_notify_channel__'; const IS_MATCH_FN_NAME = 'is_match'; @@ -519,6 +550,17 @@ export const renderAlertTemplate = async ({ startTime: view.startTime.getTime(), endTime: view.endTime.getTime(), eventId, + // Enriched fields, exposed to Generic/incident.io body templates. + alertId: alert.id ?? '', + status: ALERT_STATUS_BY_STATE[state], + alertType: alert.source ? ALERT_TYPE_BY_SOURCE[alert.source] : '', + comparator: COMPARATOR_BY_THRESHOLD_TYPE[alert.thresholdType], + threshold: alert.threshold, + value, + groupKey: group ?? '', + sourceQuery: savedSearch?.where ?? '', + teamId, + note: alert.note ?? '', }, }); return true; diff --git a/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts b/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts index be540fade6..54c75f932f 100644 --- a/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts +++ b/packages/api/src/tasks/checkAlerts/transports/__tests__/generic.test.ts @@ -3,6 +3,7 @@ import { ObjectId } from 'mongodb'; import { AlertState } from '@/models/alert'; import { + buildWebhookTemplateVariables, getWebhookFetchTimeoutMs, handleSendGenericWebhook, } from '@/tasks/checkAlerts/transports/generic'; @@ -85,3 +86,48 @@ describe('handleSendGenericWebhook — per-attempt timeout', () => { expect(fetchMock).toHaveBeenCalledTimes(1); }, 2000); // short test-level timeout: a regression here should fail fast, not stall on Jest's default 30s }); + +describe('buildWebhookTemplateVariables', () => { + it('exposes the enriched variables alongside the original set', () => { + const vars = buildWebhookTemplateVariables({ + ...message, + startTime: 1700000000000, + endTime: 1700000300000, + alertId: 'alert-1', + status: 'firing', + alertType: 'search', + comparator: '>=', + threshold: 5, + value: 42, + groupKey: 'checkout', + sourceQuery: 'Body: "error"', + teamId: 'team-1', + note: 'Runbook: https://wiki.example/runbook', + }); + + expect(vars).toMatchObject({ + eventId: 'evt-1', + state: AlertState.ALERT, + alertId: 'alert-1', + status: 'firing', + alertType: 'search', + comparator: '>=', + threshold: 5, + value: 42, + groupKey: 'checkout', + teamId: 'team-1', + startTimeISO: new Date(1700000000000).toISOString(), + endTimeISO: new Date(1700000300000).toISOString(), + }); + // Strings destined for JSON template slots are escaped. + expect(vars.sourceQuery).toBe('Body: \\"error\\"'); + }); + + it('renders enriched fields as empty strings (never "undefined") when absent', () => { + const vars = buildWebhookTemplateVariables(message); + expect(vars.alertId).toBe(''); + expect(vars.status).toBe(''); + expect(vars.note).toBe(''); + expect(vars.startTimeISO).toBe(new Date(0).toISOString()); + }); +}); diff --git a/packages/api/src/tasks/checkAlerts/transports/generic.ts b/packages/api/src/tasks/checkAlerts/transports/generic.ts index 14d7f8e97e..1d82ce7713 100644 --- a/packages/api/src/tasks/checkAlerts/transports/generic.ts +++ b/packages/api/src/tasks/checkAlerts/transports/generic.ts @@ -66,6 +66,43 @@ export const logBlockedWebhookDelivery = ( const DEFAULT_GENERIC_WEBHOOK_BODY_TEMPLATE = '{"text": "{{title}} | {{body}} | {{link}} | {{state}} | {{startTime}} | {{endTime}} | {{eventId}}"}'; +// Renders a Unix-ms timestamp as an ISO-8601 string, or '' if it isn't a valid +// time (so a template slot never becomes the literal "Invalid Date"). +const toIsoTimestamp = (t: number): string => { + const d = new Date(t); + return Number.isNaN(d.getTime()) ? '' : d.toISOString(); +}; + +/** + * The variable set exposed to user-editable webhook body templates. Strings + * are JSON-escaped; numbers are emitted raw for unquoted JSON slots. + * `startTime`/`endTime` are raw Unix ms; `startTimeISO`/`endTimeISO` are ISO + * strings for human-readable time ranges. The enriched variables (status, + * comparator, sourceQuery, …) enable routing/dedup in receivers without + * parsing the message body. + */ +export const buildWebhookTemplateVariables = (message: Message) => ({ + body: escapeJsonString(message.body), + endTime: message.endTime, + endTimeISO: escapeJsonString(toIsoTimestamp(message.endTime)), + eventId: message.eventId, + link: escapeJsonString(message.hdxLink), + startTime: message.startTime, + startTimeISO: escapeJsonString(toIsoTimestamp(message.startTime)), + state: message.state, + title: escapeJsonString(message.title), + alertId: escapeJsonString(message.alertId ?? ''), + alertType: escapeJsonString(message.alertType ?? ''), + comparator: escapeJsonString(message.comparator ?? ''), + groupKey: escapeJsonString(message.groupKey ?? ''), + note: escapeJsonString(message.note ?? ''), + sourceQuery: escapeJsonString(message.sourceQuery ?? ''), + status: escapeJsonString(message.status ?? ''), + teamId: escapeJsonString(message.teamId ?? ''), + threshold: message.threshold, + value: message.value, +}); + /** * Creates a Handlebars instance with common helpers registered. * Use this to ensure consistent helper availability across all template rendering. @@ -183,15 +220,7 @@ const sendGenericWebhook = async ( body = handlebars.compile(bodyTemplate, { noEscape: true, - })({ - body: escapeJsonString(message.body), - endTime: message.endTime, - eventId: message.eventId, - link: escapeJsonString(message.hdxLink), - startTime: message.startTime, - state: message.state, - title: escapeJsonString(message.title), - }); + })(buildWebhookTemplateVariables(message)); } catch (e) { logger.error( { diff --git a/packages/api/src/tasks/checkAlerts/transports/types.ts b/packages/api/src/tasks/checkAlerts/transports/types.ts index 5b75b3932c..4fc161ef4c 100644 --- a/packages/api/src/tasks/checkAlerts/transports/types.ts +++ b/packages/api/src/tasks/checkAlerts/transports/types.ts @@ -12,6 +12,18 @@ export interface Message { startTime: number; endTime: number; eventId: string; + // Enriched fields exposed as Generic/incident.io template variables. + // Optional so existing callers and non-enriched templates are unaffected. + alertId?: string; + status?: string; // firing | resolved | no_data | pending + alertType?: string; // search | dashboard_chart + comparator?: string; // >=, >, <=, <, =, !=, between, outside + threshold?: number; + value?: number; // the value that triggered/resolved the alert + groupKey?: string; + sourceQuery?: string; // the search expr / SQL that defines the alert + teamId?: string; + note?: string; // freeform alert note (markdown); commonly holds a runbook link } export type WebhookChannel = Extract<