From c32b78e5d4039f24eec9dad394a154ec2a626dc3 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:33:04 -0700 Subject: [PATCH] =?UTF-8?q?feat(alerts):=20evaluations=20read=20model=20?= =?UTF-8?q?=E2=80=94=20ERROR=20rows,=20analytics,=20per-group=20windows=20?= =?UTF-8?q?(HDX-4997)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AlertHistory read-side support for the alert detail page: - Types (common-utils) for evaluation errors (AlertError/AlertErrorType incl. QUERY_TIMEOUT), per-window evaluations with per-group breakdown (capped at ALERT_EVALUATION_GROUPS_LIMIT, firing-first), and evaluation analytics (queryDurationMs, webhookDurationMs, backfilledBuckets). - AlertHistory schema gains optional errors + analytics fields, and AlertState gains ERROR (only ever used on history rows). - GET /alerts/:id/evaluations: per-window evaluation history scoped to a startTime/endTime range (clamped to the 31d retention window), grouped across group-by groups newest-first with a hard-bounded scan of at most ~(limit+1) intervals per request and a server-provided nextBefore cursor that always advances past the scanned slice, so paging progresses across gaps instead of stalling. - Windows with ERROR rows surface their errors (deduped, newest-first) and rank as ERROR; firing-transition annotations exclude ERROR rows. Nothing writes ERROR rows or analytics yet — the alert task's write side lands separately. --- .changeset/alert-evaluations-read-model.md | 12 + .../__tests__/alertHistory.int.test.ts | 540 ++++++++++++++++++ packages/api/src/controllers/alertHistory.ts | 382 ++++++++++++- packages/api/src/models/alert.ts | 7 + packages/api/src/models/alertHistory.ts | 61 +- .../routers/api/__tests__/alerts.int.test.ts | 226 ++++++++ packages/api/src/routers/api/alerts.ts | 91 ++- .../components/alerts/AlertHistoryCards.tsx | 1 + packages/common-utils/src/types.ts | 79 +++ 9 files changed, 1370 insertions(+), 29 deletions(-) create mode 100644 .changeset/alert-evaluations-read-model.md diff --git a/.changeset/alert-evaluations-read-model.md b/.changeset/alert-evaluations-read-model.md new file mode 100644 index 0000000000..3b5ce79845 --- /dev/null +++ b/.changeset/alert-evaluations-read-model.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': patch +--- + +Add the AlertHistory evaluations read model and GET /alerts/:id/evaluations +endpoint: per-window evaluation history scoped to a time range (clamped to the +retention window) with per-group breakdown for group-by alerts, evaluation +analytics fields, deduped error surfacing for ERROR-state windows, and +cursor-based pagination that always advances across gaps. Adds read-side +schema/type support for ERROR-state AlertHistory rows and evaluation analytics. diff --git a/packages/api/src/controllers/__tests__/alertHistory.int.test.ts b/packages/api/src/controllers/__tests__/alertHistory.int.test.ts index 1539410493..71a1a31ade 100644 --- a/packages/api/src/controllers/__tests__/alertHistory.int.test.ts +++ b/packages/api/src/controllers/__tests__/alertHistory.int.test.ts @@ -1,6 +1,8 @@ import { ObjectId } from 'mongodb'; import { + ALERT_EVALUATION_GROUPS_LIMIT, + getAlertEvaluations, getAlertTransitionsInRange, getRecentAlertHistories, getRecentAlertHistoriesBatch, @@ -368,6 +370,520 @@ describe('alertHistory controller', () => { expect(histories[0].state).toBe(AlertState.ALERT); expect(histories[0].counts).toBe(5); }); + + it('surfaces ERROR windows with their recorded errors', async () => { + const team = await Team.create({ name: 'Test Team' }); + const alert = await Alert.create({ + team: team._id, + threshold: 100, + interval: '5m', + channel: { type: null }, + }); + + const errorWindow = new Date(Date.now() - 60000); + const okWindow = new Date(Date.now() - 120000); + const errorTimestamp = new Date(Date.now() - 55000); + + await AlertHistory.create({ + alert: alert._id, + createdAt: errorWindow, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: errorTimestamp, + type: 'QUERY_TIMEOUT', + message: 'Alert query did not complete within the 300s timeout', + }, + ], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: okWindow, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: okWindow, count: 0 }], + }); + + const histories = await getRecentAlertHistories({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + }); + + expect(histories).toHaveLength(2); + expect(histories[0].state).toBe(AlertState.ERROR); + expect(histories[0].errors).toHaveLength(1); + expect(histories[0].errors![0].type).toBe('QUERY_TIMEOUT'); + expect(histories[0].errors![0].message).toContain('300s timeout'); + expect(histories[1].state).toBe(AlertState.OK); + expect(histories[1].errors).toBeUndefined(); + }); + + it('lets ALERT/PENDING outrank ERROR within a grouped window, but ERROR outrank OK', async () => { + const team = await Team.create({ name: 'Test Team' }); + const alert = await Alert.create({ + team: team._id, + threshold: 100, + interval: '5m', + channel: { type: null }, + }); + + const alertWindow = new Date(Date.now() - 60000); + const okWindow = new Date(Date.now() - 120000); + const makeError = () => ({ + timestamp: new Date(), + type: 'WEBHOOK_ERROR', + message: 'Failed to send webhook notification.', + }); + + // Window that fired AND recorded a notification error → shows ALERT + await AlertHistory.create({ + alert: alert._id, + createdAt: alertWindow, + state: AlertState.ALERT, + counts: 2, + lastValues: [{ startTime: alertWindow, count: 2 }], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: alertWindow, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [makeError()], + }); + + // Window that was OK but the resolve notification failed → shows ERROR + await AlertHistory.create({ + alert: alert._id, + createdAt: okWindow, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: okWindow, count: 0 }], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: okWindow, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [makeError()], + }); + + const histories = await getRecentAlertHistories({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + }); + + expect(histories).toHaveLength(2); + expect(histories[0].state).toBe(AlertState.ALERT); + // Errors from the ERROR row are still surfaced on the merged window + expect(histories[0].errors).toHaveLength(1); + expect(histories[1].state).toBe(AlertState.ERROR); + expect(histories[1].errors).toHaveLength(1); + }); + }); + + describe('getAlertEvaluations', () => { + const createAlert = async () => { + const team = await Team.create({ name: 'Test Team' }); + return Alert.create({ + team: team._id, + threshold: 100, + interval: '5m', + channel: { type: null }, + }); + }; + + const createOkWindow = (alertId: any, createdAt: Date) => + AlertHistory.create({ + alert: alertId, + createdAt, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: createdAt, count: 0 }], + }); + + it('only returns windows within [startTime, endTime]', async () => { + const alert = await createAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await createOkWindow(alert._id, at(5)); // after endTime + await createOkWindow(alert._id, at(15)); // in range + await createOkWindow(alert._id, at(20)); // in range + await createOkWindow(alert._id, at(40)); // before startTime + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: at(30), + endTime: at(10), + }); + + expect(page.data).toHaveLength(2); + expect(page.data[0].createdAt).toEqual(at(15)); + expect(page.data[1].createdAt).toEqual(at(20)); + expect(page.hasMore).toBe(false); + expect(page.nextBefore).toBeUndefined(); + }); + + it('paginates older windows via the nextBefore cursor when the page fills up', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windows = [1, 2, 3, 4].map(i => new Date(now - i * 5 * 60_000)); + for (const createdAt of windows) { + await createOkWindow(alert._id, createdAt); + } + + const startTime = new Date(now - 60 * 60_000); + const endTime = new Date(now); + + const firstPage = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 2, + startTime, + endTime, + }); + expect(firstPage.data).toHaveLength(2); + expect(firstPage.data[0].createdAt).toEqual(windows[0]); + expect(firstPage.data[1].createdAt).toEqual(windows[1]); + expect(firstPage.hasMore).toBe(true); + expect(firstPage.nextBefore).toEqual(windows[1]); + + const secondPage = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 2, + startTime, + endTime, + before: firstPage.nextBefore, + }); + expect(secondPage.data).toHaveLength(2); + expect(secondPage.data[0].createdAt).toEqual(windows[2]); + expect(secondPage.data[1].createdAt).toEqual(windows[3]); + }); + + it('keeps paging across gaps: the scan bound truncates but nextBefore advances', async () => { + const alert = await createAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + // One recent window, then a gap far wider than the per-request scan + // bound ((limit + 1) × interval = 3 minutes for limit=2 / 1m interval), + // then an old window still inside the requested range. + await createOkWindow(alert._id, at(1)); + await createOkWindow(alert._id, at(20)); + + const startTime = at(30); + const endTime = at(0); + + // First page: finds the recent window; the scan bound stops long + // before startTime, so more may exist. + const firstPage = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '1m', + limit: 2, + startTime, + endTime, + }); + expect(firstPage.data).toHaveLength(1); + expect(firstPage.data[0].createdAt).toEqual(at(1)); + expect(firstPage.hasMore).toBe(true); + expect(firstPage.nextBefore).toBeDefined(); + + // Follow the cursor until the old window is found or the range is + // exhausted. Every hop advances by at least one scan bound, so this + // terminates. + let before = firstPage.nextBefore; + let found: Date | undefined; + for (let i = 0; i < 20 && before != null; i++) { + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '1m', + limit: 2, + startTime, + endTime, + before, + }); + if (page.data.length > 0) { + found = page.data[0].createdAt; + break; + } + expect(page.hasMore).toBe(true); + // Empty pages still advance the cursor + expect(page.nextBefore!.getTime()).toBeLessThan(before.getTime()); + before = page.nextBefore; + } + expect(found).toEqual(at(20)); + }); + + it('reports hasMore=false once the scan reaches startTime', async () => { + const alert = await createAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await createOkWindow(alert._id, at(5)); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: at(20), + endTime: at(0), + }); + + expect(page.data).toHaveLength(1); + expect(page.hasMore).toBe(false); + expect(page.nextBefore).toBeUndefined(); + // Non-grouped rows carry no per-group breakdown + expect(page.data[0].groups).toBeUndefined(); + expect(page.data[0].groupsTotal).toBeUndefined(); + }); + + it('breaks grouped windows down per group, firing-first', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + const createGroupRow = ( + group: string, + state: AlertState, + count: number, + fired?: boolean, + ) => + AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state, + counts: state === AlertState.OK ? 0 : 1, + lastValues: [{ startTime: bucket, count }], + group, + ...(fired != null && { fired }), + }); + + await createGroupRow('ServiceName:web', AlertState.OK, 3); + await createGroupRow('ServiceName:api', AlertState.ALERT, 14, true); + await createGroupRow('ServiceName:worker', AlertState.PENDING, 9); + // A notification failure recorded for the window: contributes errors, + // never a group entry. + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: 'WEBHOOK_ERROR', + message: 'Failed to send webhook notification.', + }, + ], + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + const window = page.data[0]; + // Overall window state merges the per-group states + expect(window.state).toBe(AlertState.ALERT); + expect(window.groupsTotal).toBe(3); + expect(window.groups).toHaveLength(3); + // Firing-first ordering: ALERT > PENDING > OK + expect(window.groups!.map(g => g.group)).toEqual([ + 'ServiceName:api', + 'ServiceName:worker', + 'ServiceName:web', + ]); + expect(window.groups![0]).toMatchObject({ + group: 'ServiceName:api', + state: AlertState.ALERT, + counts: 1, + fired: true, + }); + expect(window.groups![0].lastValue).toMatchObject({ count: 14 }); + expect(window.groups![2]).toMatchObject({ + group: 'ServiceName:web', + state: AlertState.OK, + counts: 0, + }); + // The ERROR row surfaces as window errors, not as a group + expect(window.errors).toHaveLength(1); + expect(window.groups!.every(g => g.state !== AlertState.ERROR)).toBe( + true, + ); + }); + + it('surfaces evaluation analytics, preferring the successful evaluation over a failed attempt', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + // Failed first attempt for this window (its query ran 300s) + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: 'QUERY_TIMEOUT', + message: 'timed out', + }, + ], + analytics: { queryDurationMs: 300_000 }, + }); + // Successful retry + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: bucket, count: 1 }], + analytics: { + queryDurationMs: 1_200, + webhookDurationMs: 340, + backfilledBuckets: 0, + }, + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + expect(page.data[0].analytics).toMatchObject({ + queryDurationMs: 1_200, + webhookDurationMs: 340, + backfilledBuckets: 0, + }); + }); + + it('falls back to the failed attempt analytics when no successful row exists', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: 'QUERY_TIMEOUT', + message: 'timed out', + }, + ], + analytics: { queryDurationMs: 300_000 }, + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + expect(page.data[0].analytics).toMatchObject({ + queryDurationMs: 300_000, + // Derived from lastValues (none) for rows lacking the field + backfilledBuckets: 0, + }); + }); + + it('derives backfilled buckets from lastValues for rows written before analytics existed', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucketAt = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + // Legacy backfill row: one evaluation covering three buckets, no + // analytics field. + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.OK, + counts: 0, + lastValues: [ + { startTime: bucketAt(20), count: 0 }, + { startTime: bucketAt(15), count: 0 }, + { startTime: bucketAt(10), count: 0 }, + ], + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + expect(page.data[0].analytics).toEqual({ backfilledBuckets: 2 }); + }); + + it('caps the per-group breakdown and reports the pre-cap total', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + const total = ALERT_EVALUATION_GROUPS_LIMIT + 5; + // One firing group buried among many OK groups: firing-first ordering + // must keep it visible after the cap. + const rows = Array.from({ length: total }, (_, i) => ({ + alert: alert._id, + createdAt: windowStart, + state: i === total - 1 ? AlertState.ALERT : AlertState.OK, + counts: i === total - 1 ? 1 : 0, + lastValues: [{ startTime: bucket, count: i }], + group: `ServiceName:svc-${String(i).padStart(3, '0')}`, + })); + await AlertHistory.insertMany(rows); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + const window = page.data[0]; + expect(window.groupsTotal).toBe(total); + expect(window.groups).toHaveLength(ALERT_EVALUATION_GROUPS_LIMIT); + expect(window.groups![0].state).toBe(AlertState.ALERT); + expect(window.groups![0].group).toBe( + `ServiceName:svc-${String(total - 1).padStart(3, '0')}`, + ); + }); }); describe('getRecentAlertHistoriesBatch', () => { @@ -703,6 +1219,30 @@ describe('alertHistory controller', () => { expect(transitions[1].createdAt).toBe(t(15).toISOString()); }); + it('ignores ERROR windows so a failed evaluation mid-firing is not a recovery', async () => { + const alert = await createAlert(); + await createHistory(alert._id, t(30), AlertState.OK, 0); + await createHistory(alert._id, t(25), AlertState.ALERT, 3); + // Evaluation failed mid-episode — must not read as a recovery + refire + await createHistory(alert._id, t(20), AlertState.ERROR, 0); + await createHistory(alert._id, t(15), AlertState.ALERT, 4); + await createHistory(alert._id, t(10), AlertState.OK, 0); + + const transitions = await getAlertTransitionsInRange({ + alertId: new ObjectId(alert._id), + interval: '5m', + startTime: t(40), + endTime: t(0), + }); + + expect(transitions.map(tr => tr.state)).toEqual([ + AlertState.ALERT, + AlertState.OK, + ]); + expect(transitions[0].createdAt).toBe(t(25).toISOString()); + expect(transitions[1].createdAt).toBe(t(10).toISOString()); + }); + it('only considers history for the specified alert', async () => { const alert1 = await createAlert(); const alert2 = await createAlert(); diff --git a/packages/api/src/controllers/alertHistory.ts b/packages/api/src/controllers/alertHistory.ts index 68caff4749..6ff1e9381b 100644 --- a/packages/api/src/controllers/alertHistory.ts +++ b/packages/api/src/controllers/alertHistory.ts @@ -1,22 +1,35 @@ import PQueue from '@esm2cjs/p-queue'; import { + ALERT_EVALUATION_GROUPS_LIMIT, ALERT_INTERVAL_TO_MINUTES, AlertInterval, AlertTransition, } from '@hyperdx/common-utils/dist/types'; import { ObjectId } from 'mongodb'; -import { AlertState } from '@/models/alert'; -import AlertHistory, { IAlertHistory } from '@/models/alertHistory'; +import { AlertState, IAlertError } from '@/models/alert'; +import AlertHistory, { + IAlertHistory, + IAlertHistoryAnalytics, +} from '@/models/alertHistory'; + +// Re-exported for API-side consumers/tests; the app imports it from +// common-utils to explain the cap in the UI. +export { ALERT_EVALUATION_GROUPS_LIMIT }; // Max parallel per-alert queries to avoid overwhelming the DB connection pool export const ALERT_HISTORY_QUERY_CONCURRENCY = 20; +/** Alert evaluation interval in milliseconds. */ +const intervalToMs = (interval: AlertInterval): number => + ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000; + type GroupedAlertHistory = { _id: Date; states: string[]; counts: number; lastValues: IAlertHistory['lastValues'][]; + errors: IAlertError[][]; }; function groupStateToOverallState(states: string[]): AlertState { @@ -28,25 +41,103 @@ function groupStateToOverallState(states: string[]): AlertState { return AlertState.PENDING; } + // An evaluation window that neither fired nor was pending, but recorded an + // error (query failure, or a notification failure on an OK window), is + // surfaced as ERROR. + if (states.includes(AlertState.ERROR)) { + return AlertState.ERROR; + } + return AlertState.OK; } +/** Dedupe errors by type+message, keeping the most recent occurrence. */ +function dedupeErrors(errors: IAlertError[]): IAlertError[] { + const map = new Map(); + for (const error of errors) { + const key = `${error.type}||${error.message}`; + const existing = map.get(key); + if (!existing || error.timestamp > existing.timestamp) { + map.set(key, error); + } + } + return Array.from(map.values()).sort( + (a, b) => b.timestamp.getTime() - a.timestamp.getTime(), + ); +} + function mapGroupedHistories( groupedHistories: GroupedAlertHistory[], ): Omit[] { - return groupedHistories.map(group => ({ - createdAt: group._id, - state: groupStateToOverallState(group.states), - counts: group.counts, - lastValues: group.lastValues - .flat() - .sort((a, b) => a.startTime.getTime() - b.startTime.getTime()), - })); + return groupedHistories.map(group => { + // $push skips documents where the field is missing, but be defensive + // about nulls in case of engine differences (e.g. DocumentDB). + const errors = dedupeErrors( + (group.errors ?? []).flat().filter((e): e is IAlertError => e != null), + ); + return { + createdAt: group._id, + state: groupStateToOverallState(group.states), + counts: group.counts, + lastValues: group.lastValues + .flat() + .sort((a, b) => a.startTime.getTime() - b.startTime.getTime()), + ...(errors.length > 0 && { errors }), + }; + }); +} + +/** + * Fetch grouped evaluation windows (one entry per createdAt, newest first) + * for the given alert within the createdAt bounds, capped at `limit` groups. + */ +async function fetchGroupedWindows( + alertId: ObjectId, + createdAt: Record, + limit: number, +): Promise[]> { + const groupedHistories = await AlertHistory.aggregate([ + { + $match: { + alert: new ObjectId(alertId), + createdAt, + }, + }, + { $sort: { createdAt: -1 } }, + { + $group: { + _id: '$createdAt', + states: { + $push: '$state', + }, + counts: { + $sum: '$counts', + }, + lastValues: { + $push: '$lastValues', + }, + errors: { + $push: '$errors', + }, + }, + }, + { + $sort: { + _id: -1, + }, + }, + { + $limit: limit, + }, + ]); + + return mapGroupedHistories(groupedHistories); } /** * Gets the most recent alert histories for a given alert ID, - * limiting to the given number of entries. + * limiting to the given number of entries. Results are one entry per + * evaluation window (grouped by createdAt), newest first. */ export async function getRecentAlertHistories({ alertId, @@ -57,27 +148,189 @@ export async function getRecentAlertHistories({ interval: AlertInterval; limit: number; }): Promise[]> { - const lookbackMs = limit * ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000; + // One extra interval of slack so a window sitting exactly `limit` intervals + // back (the newest window is up to one interval old) isn't cut off by the + // lookback bound. + const lookbackMs = (limit + 1) * intervalToMs(interval); + return fetchGroupedWindows( + alertId, + { $gte: new Date(Date.now() - lookbackMs) }, + limit, + ); +} - const groupedHistories = await AlertHistory.aggregate([ +type AlertEvaluationGroupEntry = { + group: string; + state: AlertState; + counts: number; + /** The group's most recent bucket value in this window, if any. */ + lastValue?: { startTime: Date; count: number }; + /** True when a notification was actually sent for this group. */ + fired?: boolean; +}; + +type AlertEvaluationEntry = Omit & { + /** + * Per-group breakdown for group-by alerts, firing-first, capped at + * ALERT_EVALUATION_GROUPS_LIMIT. Absent for non-grouped alerts. + */ + groups?: AlertEvaluationGroupEntry[]; + /** Total number of groups evaluated in this window (before the cap). */ + groupsTotal?: number; +}; + +export type AlertEvaluationsPage = { + data: AlertEvaluationEntry[]; + /** True when older windows may exist within [startTime, ...). */ + hasMore: boolean; + /** Cursor for the next-older page (pass as `before`). Set when hasMore. */ + nextBefore?: Date; +}; + +// One AlertHistory row's fields, as pushed into a per-window sub-array by the +// evaluations aggregation (unlike the alerts-page pipeline, group identity is +// preserved so windows can be broken down per group). +type EvaluationWindowRow = { + group?: string; + state: AlertState; + counts?: number; + lastValues?: IAlertHistory['lastValues']; + fired?: boolean; + errors?: IAlertError[]; + analytics?: IAlertHistoryAnalytics; +}; + +type StructuredWindow = { + _id: Date; + rows: EvaluationWindowRow[]; +}; + +// Firing-first ordering for the per-group breakdown. +const groupStatePriority = (state: AlertState): number => { + switch (state) { + case AlertState.ALERT: + return 0; + case AlertState.PENDING: + return 1; + default: + return 2; + } +}; + +/** + * Pick the evaluation analytics for a window. A window's rows all carry the + * same evaluation-level analytics, except when it contains rows from two + * evaluations (a failed attempt's ERROR row + the successful retry's rows) — + * prefer the successful evaluation's. For rows written before analytics + * existed, `backfilledBuckets` is derived from the distinct bucket times so + * the "Backfilled Buckets" column works on historical data. + */ +function resolveWindowAnalytics( + rows: EvaluationWindowRow[], + lastValues: IAlertHistory['lastValues'], +): IAlertHistoryAnalytics | undefined { + const analytics = + rows.find(r => r.state !== AlertState.ERROR && r.analytics != null) + ?.analytics ?? rows.find(r => r.analytics != null)?.analytics; + + if (analytics?.backfilledBuckets != null) { + return analytics; + } + + const distinctBuckets = new Set(lastValues.map(v => v.startTime.getTime())) + .size; + const derivedBackfilled = Math.max(0, distinctBuckets - 1); + if (analytics == null && derivedBackfilled === 0) { + return undefined; + } + return { ...analytics, backfilledBuckets: derivedBackfilled }; +} + +function mapStructuredWindow(window: StructuredWindow): AlertEvaluationEntry { + const rows = window.rows ?? []; + const errors = dedupeErrors( + rows.flatMap(r => r.errors ?? []).filter(e => e != null), + ); + const lastValues = rows + .flatMap(r => r.lastValues ?? []) + .sort((a, b) => a.startTime.getTime() - b.startTime.getTime()); + const analytics = resolveWindowAnalytics(rows, lastValues); + + // Per-group breakdown: rows carrying a group identity. ERROR rows never + // carry one (they record the evaluation failure, not a group result). + const groupRows = rows.filter( + r => r.group != null && r.group !== '' && r.state !== AlertState.ERROR, + ); + const groups = groupRows + .map(r => { + const groupLastValues = r.lastValues ?? []; + const lastValue = + groupLastValues.length > 0 + ? groupLastValues.reduce((latest, v) => + v.startTime.getTime() > latest.startTime.getTime() ? v : latest, + ) + : undefined; + return { + group: r.group as string, + state: r.state, + counts: r.counts ?? 0, + ...(lastValue != null && { lastValue }), + ...(r.fired != null && { fired: r.fired }), + }; + }) + .sort( + (a, b) => + groupStatePriority(a.state) - groupStatePriority(b.state) || + (b.lastValue?.count ?? Number.NEGATIVE_INFINITY) - + (a.lastValue?.count ?? Number.NEGATIVE_INFINITY) || + a.group.localeCompare(b.group), + ); + + return { + createdAt: window._id, + state: groupStateToOverallState(rows.map(r => r.state)), + counts: rows.reduce((sum, r) => sum + (r.counts ?? 0), 0), + lastValues, + ...(errors.length > 0 && { errors }), + ...(analytics != null && { analytics }), + ...(groups.length > 0 && { + groups: groups.slice(0, ALERT_EVALUATION_GROUPS_LIMIT), + groupsTotal: groups.length, + }), + }; +} + +/** + * Fetch evaluation windows (one entry per createdAt, newest first) within the + * createdAt bounds, capped at `limit` windows, preserving per-group rows so + * grouped alerts can be broken down per group. + */ +async function fetchStructuredWindows( + alertId: ObjectId, + createdAt: Record, + limit: number, +): Promise { + const windows = await AlertHistory.aggregate([ { $match: { alert: new ObjectId(alertId), - createdAt: { $gte: new Date(Date.now() - lookbackMs) }, + createdAt, }, }, { $sort: { createdAt: -1 } }, { $group: { _id: '$createdAt', - states: { - $push: '$state', - }, - counts: { - $sum: '$counts', - }, - lastValues: { - $push: '$lastValues', + rows: { + $push: { + group: '$group', + state: '$state', + counts: '$counts', + lastValues: '$lastValues', + fired: '$fired', + errors: '$errors', + analytics: '$analytics', + }, }, }, }, @@ -91,7 +344,83 @@ export async function getRecentAlertHistories({ }, ]); - return mapGroupedHistories(groupedHistories); + return windows.map(mapStructuredWindow); +} + +/** + * Paginated evaluation windows for the alert detail page, newest first, + * scoped to [startTime, endTime]. + * + * Every request scans a hard-bounded slice of at most ~(limit + 1) intervals + * of history (anchored at `before ?? endTime`), so a wide time range can + * never force an unbounded scan — group-by alerts can have many rows per + * window, and the $group stage processes every matched row. + * + * Because of that bound, a page may end before reaching `startTime` even if + * fewer than `limit` windows were returned (a gap with no evaluations). The + * returned `nextBefore` cursor always advances past the scanned slice, so + * callers can keep paging across gaps: pass it as `before` on the next call. + */ +export async function getAlertEvaluations({ + alertId, + interval, + limit, + startTime, + endTime, + before, +}: { + alertId: ObjectId; + interval: AlertInterval; + limit: number; + startTime: Date; + endTime: Date; + before?: Date; +}): Promise { + const intervalMs = intervalToMs(interval); + // One extra interval of slack so a window sitting exactly `limit` intervals + // back isn't cut off by the scan bound. + const scanMs = (limit + 1) * intervalMs; + + // Upper bound: the page cursor when provided (exclusive), else the range + // end (inclusive). A cursor past the range end is ignored. + const usableBefore = + before != null && before.getTime() <= endTime.getTime() + ? before + : undefined; + const pageEndMs = usableBefore?.getTime() ?? endTime.getTime(); + + // Lower bound: the scan bound, clamped to the requested range start. + const scanFloorMs = Math.max(startTime.getTime(), pageEndMs - scanMs); + + const createdAt: Record = { + $gte: new Date(scanFloorMs), + }; + if (usableBefore != null) { + createdAt.$lt = usableBefore; + } else { + createdAt.$lte = endTime; + } + + // Fetch one extra window to detect count truncation. + const windows = await fetchStructuredWindows(alertId, createdAt, limit + 1); + + const truncatedByCount = windows.length > limit; + const data = truncatedByCount ? windows.slice(0, limit) : windows; + // More windows may exist when the page filled up, or when the scan bound + // stopped before reaching the range start. + const truncatedByScanBound = scanFloorMs > startTime.getTime(); + const hasMore = truncatedByCount || truncatedByScanBound; + + // Count truncation: resume strictly before the last returned window. + // Scan-bound truncation: the slice down to scanFloor (inclusive) is fully + // covered, so resume strictly before it. + const nextBefore = !hasMore + ? undefined + : truncatedByCount + ? data[data.length - 1].createdAt + : new Date(scanFloorMs); + + return { data, hasMore, nextBefore }; } /** @@ -150,16 +479,19 @@ export async function getAlertTransitionsInRange({ startTime: Date; endTime: Date; }): Promise { - const intervalMs = ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000; + const intervalMs = intervalToMs(interval); const lookbackStart = new Date(startTime.getTime() - intervalMs); - // Only the per-window state is needed to detect crossings. + // Only the per-window state is needed to detect crossings. ERROR rows are + // failed evaluations, not state observations — excluding them prevents a + // query failure mid-firing from drawing a false recovery annotation. const windows = await AlertHistory.aggregate<{ _id: Date; states: string[] }>( [ { $match: { alert: new ObjectId(alertId), createdAt: { $gte: lookbackStart, $lte: endTime }, + state: { $ne: AlertState.ERROR }, }, }, { $group: { _id: '$createdAt', states: { $push: '$state' } } }, diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index 93dfbc42c5..fd5e1df9e2 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -12,6 +12,13 @@ import Team from './team'; export enum AlertState { ALERT = 'ALERT', DISABLED = 'DISABLED', + /** + * Only used on AlertHistory records (never on the alert itself): marks an + * evaluation window whose evaluation or notification failed. ERROR history + * rows are excluded from alert scheduling/backfill computations so the + * failed window is still retried. + */ + ERROR = 'ERROR', INSUFFICIENT_DATA = 'INSUFFICIENT_DATA', OK = 'OK', PENDING = 'PENDING', diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 5cf8630f81..78b968208c 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -1,10 +1,35 @@ +import { AlertErrorType } from '@hyperdx/common-utils/dist/types'; import mongoose, { Schema } from 'mongoose'; import ms from 'ms'; -import { AlertState } from '@/models/alert'; +import { AlertState, IAlertError } from '@/models/alert'; import type { ObjectId } from '.'; +/** + * Diagnostics for the evaluation that wrote a history record. + * Evaluation-level: identical on every row one evaluation writes (including + * per-group rows). + */ +export interface IAlertHistoryAnalytics { + /** + * ClickHouse query duration for the evaluation (ms). On query-failure + * ERROR records, the time until the query failed — for QUERY_TIMEOUT this + * is approximately the configured evaluation timeout. + */ + queryDurationMs?: number; + /** + * Total wall time delivering webhook notifications in the evaluation, + * including retries (ms). + */ + webhookDurationMs?: number; + /** + * Earlier buckets backfilled in this run after missed ticks + * (expected buckets − 1). 0 in steady state. + */ + backfilledBuckets?: number; +} + export interface IAlertHistory { alert: ObjectId; counts: number; @@ -13,6 +38,14 @@ export interface IAlertHistory { lastValues: { startTime: Date; count: number }[]; group?: string; // For group-by alerts, stores the group identifier fired?: boolean; + /** + * Errors recorded for this evaluation window. Present on ERROR-state rows + * (query/processing failures where no normal history is written) and on + * the ERROR row created alongside normal rows when notifications fail. + */ + errors?: IAlertError[]; + /** Diagnostics for the evaluation that wrote this record. */ + analytics?: IAlertHistoryAnalytics; } const AlertHistorySchema = new Schema({ @@ -50,6 +83,32 @@ const AlertHistorySchema = new Schema({ type: Boolean, required: false, }, + errors: { + type: [ + { + _id: false, + timestamp: { type: Date, required: true }, + type: { + type: String, + enum: AlertErrorType, + required: true, + }, + message: { type: String, required: true }, + }, + ], + required: false, + default: undefined, + }, + analytics: { + type: { + _id: false, + queryDurationMs: { type: Number, required: false }, + webhookDurationMs: { type: Number, required: false }, + backfilledBuckets: { type: Number, required: false }, + }, + required: false, + default: undefined, + }, }); AlertHistorySchema.index( diff --git a/packages/api/src/routers/api/__tests__/alerts.int.test.ts b/packages/api/src/routers/api/__tests__/alerts.int.test.ts index 01dce61b1b..6250c0203a 100644 --- a/packages/api/src/routers/api/__tests__/alerts.int.test.ts +++ b/packages/api/src/routers/api/__tests__/alerts.int.test.ts @@ -1132,6 +1132,232 @@ describe('alerts router', () => { }); }); + describe('GET /alerts/:id/evaluations', () => { + const createTileAlert = async (): Promise => { + const dashboard = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: dashboard.body.tiles[0].id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + return String(alert.body.data._id); + }; + + it('returns evaluation windows newest-first, including error windows', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await AlertHistory.create({ + alert: alertId, + createdAt: at(10), + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: at(10), count: 0 }], + }); + await AlertHistory.create({ + alert: alertId, + createdAt: at(5), + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: at(4), + type: AlertErrorType.QUERY_TIMEOUT, + message: 'Alert query did not complete within the 300s timeout', + }, + ], + }); + + const res = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: at(30).getTime(), endTime: now }) + .expect(200); + + expect(res.body.hasMore).toBe(false); + expect(res.body.data).toHaveLength(2); + expect(res.body.data[0].state).toBe(AlertState.ERROR); + expect(res.body.data[0].createdAt).toBe(at(5).toISOString()); + expect(res.body.data[0].errors).toHaveLength(1); + expect(res.body.data[0].errors[0].type).toBe( + AlertErrorType.QUERY_TIMEOUT, + ); + expect(res.body.data[1].state).toBe(AlertState.OK); + }); + + it('paginates with limit + the nextBefore cursor and reports hasMore', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + // Windows aligned to the alert interval cadence (5m apart) + const windows = [5, 10, 15].map( + minsAgo => new Date(now - minsAgo * 60_000), + ); + for (const createdAt of windows) { + await AlertHistory.create({ + alert: alertId, + createdAt, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: createdAt, count: 0 }], + }); + } + + // Range chosen so the second page's bounded scan reaches startTime + // exactly (limit=2 → each page scans (2+1)×5m = 15m of history). + const startTime = now - 20 * 60_000; + const endTime = now; + + const firstPage = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ limit: 2, startTime, endTime }) + .expect(200); + expect(firstPage.body.data).toHaveLength(2); + expect(firstPage.body.hasMore).toBe(true); + expect(firstPage.body.nextBefore).toBe(windows[1].getTime()); + expect(firstPage.body.data[0].createdAt).toBe(windows[0].toISOString()); + + const secondPage = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ + limit: 2, + startTime, + endTime, + before: firstPage.body.nextBefore, + }) + .expect(200); + expect(secondPage.body.data).toHaveLength(1); + expect(secondPage.body.hasMore).toBe(false); + expect(secondPage.body.nextBefore).toBeUndefined(); + expect(secondPage.body.data[0].createdAt).toBe(windows[2].toISOString()); + }); + + it('returns the per-group breakdown for grouped windows', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + await AlertHistory.create({ + alert: alertId, + createdAt: windowStart, + state: AlertState.ALERT, + counts: 2, + lastValues: [{ startTime: bucket, count: 12 }], + group: 'ServiceName:api', + fired: true, + }); + await AlertHistory.create({ + alert: alertId, + createdAt: windowStart, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: bucket, count: 1 }], + group: 'ServiceName:web', + }); + + const res = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: now - 30 * 60_000, endTime: now }) + .expect(200); + + expect(res.body.data).toHaveLength(1); + const window = res.body.data[0]; + expect(window.state).toBe(AlertState.ALERT); + expect(window.groupsTotal).toBe(2); + expect(window.groups).toHaveLength(2); + // Firing group first + expect(window.groups[0]).toMatchObject({ + group: 'ServiceName:api', + state: AlertState.ALERT, + counts: 2, + fired: true, + }); + expect(window.groups[0].lastValue.count).toBe(12); + expect(window.groups[1]).toMatchObject({ + group: 'ServiceName:web', + state: AlertState.OK, + }); + }); + + it('scopes results to the requested time range', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await AlertHistory.create({ + alert: alertId, + createdAt: at(5), + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: at(5), count: 0 }], + }); + await AlertHistory.create({ + alert: alertId, + createdAt: at(45), + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: at(45), count: 0 }], + }); + + const res = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: at(30).getTime(), endTime: now }) + .expect(200); + + expect(res.body.data).toHaveLength(1); + expect(res.body.data[0].createdAt).toBe(at(5).toISOString()); + expect(res.body.hasMore).toBe(false); + }); + + it('accepts a very wide range (span is clamped, not rejected)', async () => { + const alertId = await createTileAlert(); + await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: 1, endTime: Date.now() }) + .expect(200); + }); + + it('rejects an out-of-range limit', async () => { + const alertId = await createTileAlert(); + await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ limit: 10_000 }) + .expect(400); + }); + + it('rejects startTime >= endTime', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: now, endTime: now - 60_000 }) + .expect(400); + }); + + it('returns 404 for an unknown alert id', async () => { + await agent.get(`/alerts/${randomMongoId()}/evaluations`).expect(404); + }); + + it("returns 404 for another team's alert", async () => { + const otherTeamAlert = await Alert.create({ + team: randomMongoId(), + threshold: 1, + interval: '5m', + channel: { type: null }, + }); + await agent.get(`/alerts/${otherTeamAlert._id}/evaluations`).expect(404); + }); + }); + describe('errors propagation', () => { it('returns the errors field on a single alert response', async () => { const dashboard = await agent diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index 915be23272..a44603c5a0 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -1,5 +1,6 @@ import type { AlertApiResponse, + AlertEvaluationsApiResponse, AlertHistoryRangeApiResponse, AlertsApiResponse, AlertsPageItem, @@ -11,6 +12,7 @@ import { z } from 'zod'; import { processRequest, validateRequest } from 'zod-express-middleware'; import { + getAlertEvaluations, getAlertTransitionsInRange, getRecentAlertHistories, getRecentAlertHistoriesBatch, @@ -88,6 +90,7 @@ const formatAlertResponse = ( 'numConsecutiveWindows', ]), tileId: alert.tileId ?? undefined, + groupBy: alert.groupBy ?? undefined, }; }; @@ -155,11 +158,93 @@ router.get( }, ); +// Alert history has a ~30-day TTL, so cap queried spans to bound the +// aggregations regardless of how small a startTime the caller sends. +const MAX_HISTORY_SPAN_MS = 31 * 24 * 60 * 60 * 1000; + +// Paginated evaluation history for the alert detail page: one entry per +// evaluation window (grouped across group-by groups), newest first, including +// any errors recorded for the window. Scoped to [startTime, endTime] (epoch +// ms; endTime defaults to now, startTime is clamped to the history retention +// span). `before` (epoch ms, from the previous response's `nextBefore`) pages +// to older windows within the range. Note: /:id/history (below) returns +// firing transitions for chart annotations, which is a different shape. +const EVALUATIONS_LIMIT = 200; +type AlertEvaluationsExpRes = express.Response; +router.get( + '/:id/evaluations', + processRequest({ + params: z.object({ id: objectIdSchema }), + query: z + .object({ + limit: z.coerce + .number() + .int() + .min(1) + .max(EVALUATIONS_LIMIT) + .default(EVALUATIONS_LIMIT), + before: z.coerce.number().int().positive().optional(), + startTime: z.coerce.number().int().positive().optional(), + endTime: z.coerce.number().int().positive().optional(), + }) + .refine( + q => + q.startTime == null || q.endTime == null || q.startTime < q.endTime, + { message: 'startTime must be less than endTime' }, + ), + }), + async (req, res: AlertEvaluationsExpRes, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + // Scope to the caller's team (404 for alerts they can't see). + const alert = await getAlertById(req.params.id, teamId); + if (!alert) { + return res.sendStatus(404); + } + + // zod applies the default at runtime, but the middleware types the + // parsed query with the input (pre-default) shape. + const limit = req.query.limit ?? EVALUATIONS_LIMIT; + const { before } = req.query; + const endTime = + req.query.endTime != null ? new Date(req.query.endTime) : new Date(); + // Clamp the span so a tiny/zero startTime can't page beyond the history + // retention window (same cap as the /history transitions endpoint). + const startTime = new Date( + Math.max( + req.query.startTime ?? endTime.getTime() - MAX_HISTORY_SPAN_MS, + endTime.getTime() - MAX_HISTORY_SPAN_MS, + ), + ); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: alert.interval, + limit, + startTime, + endTime, + before: before != null ? new Date(before) : undefined, + }); + + sendJson(res, { + data: page.data, + hasMore: page.hasMore, + ...(page.nextBefore != null && { + nextBefore: page.nextBefore.getTime(), + }), + }); + } catch (e) { + next(e); + } + }, +); + // Alert firing/recovery transitions within a time range, used to draw // annotations on dashboard charts (startTime/endTime are epoch milliseconds). -// Alert history has a ~30-day TTL, so cap the queried span to bound the -// aggregation regardless of how small a startTime the caller sends. -const MAX_HISTORY_SPAN_MS = 31 * 24 * 60 * 60 * 1000; type AlertHistoryRangeExpRes = express.Response; router.get( '/:id/history', diff --git a/packages/app/src/components/alerts/AlertHistoryCards.tsx b/packages/app/src/components/alerts/AlertHistoryCards.tsx index a20537308f..331ac16e1c 100644 --- a/packages/app/src/components/alerts/AlertHistoryCards.tsx +++ b/packages/app/src/components/alerts/AlertHistoryCards.tsx @@ -97,6 +97,7 @@ function AlertHistoryCard({ const ALERT_ERROR_TYPE_LABELS: Record = { [AlertErrorType.INVALID_ALERT]: 'Invalid Configuration', [AlertErrorType.QUERY_ERROR]: 'Query Error', + [AlertErrorType.QUERY_TIMEOUT]: 'Query Timeout', [AlertErrorType.WEBHOOK_ERROR]: 'Webhook Error', [AlertErrorType.UNKNOWN]: 'Unknown Error', }; diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 610cbeb9cd..e171fa9d3e 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -599,6 +599,13 @@ export const isRangeThresholdType = (type: string): boolean => export enum AlertState { ALERT = 'ALERT', DISABLED = 'DISABLED', + /** + * Only used on AlertHistory records (never on the alert itself): marks an + * evaluation window whose evaluation or notification failed. ERROR history + * rows are excluded from alert scheduling/backfill computations so the + * failed window is still retried. + */ + ERROR = 'ERROR', INSUFFICIENT_DATA = 'INSUFFICIENT_DATA', OK = 'OK', PENDING = 'PENDING', @@ -606,6 +613,8 @@ export enum AlertState { export enum AlertErrorType { QUERY_ERROR = 'QUERY_ERROR', + /** The alert query did not complete within the evaluation timeout. */ + QUERY_TIMEOUT = 'QUERY_TIMEOUT', WEBHOOK_ERROR = 'WEBHOOK_ERROR', INVALID_ALERT = 'INVALID_ALERT', UNKNOWN = 'UNKNOWN', @@ -801,15 +810,40 @@ export const AlertSchema = z.union([ export type Alert = z.infer; +// Diagnostics for the evaluation that wrote a history record. Evaluation- +// level: identical on every row one evaluation writes (incl. per-group rows). +export const AlertHistoryAnalyticsSchema = z.object({ + /** ClickHouse query duration for the evaluation (ms). On query-failure ERROR records, the time until the query failed. */ + queryDurationMs: z.number().optional(), + /** Total wall time delivering webhook notifications in the evaluation, including retries (ms). */ + webhookDurationMs: z.number().optional(), + /** Earlier buckets backfilled in this run after missed ticks (expected buckets − 1). */ + backfilledBuckets: z.number().optional(), +}); + +export type AlertHistoryAnalytics = z.infer; + export const AlertHistorySchema = z.object({ counts: z.number(), createdAt: z.string(), lastValues: z.array(z.object({ startTime: z.string(), count: z.number() })), state: z.nativeEnum(AlertState), + /** Errors recorded for this evaluation window (query/webhook failures). */ + errors: z.array(AlertErrorSchema).optional(), + /** Diagnostics for the evaluation that wrote this record. */ + analytics: AlertHistoryAnalyticsSchema.optional(), }); export type AlertHistory = z.infer; +/** + * Max per-group entries returned per evaluation window on the alert detail + * page. Groups are sorted firing-first before the cap, so firing groups are + * always visible. Shared between the API (enforces the cap) and the app + * (explains it in the UI). + */ +export const ALERT_EVALUATION_GROUPS_LIMIT = 50; + // A single alert state transition within a time range, used to draw // firing/recovery annotations on dashboard charts. Only boundary crossings are // emitted: ALERT = fired, OK = recovered. @@ -2119,6 +2153,7 @@ export const AlertsPageItemSchema = z.object({ dashboardId: z.string().optional(), savedSearchId: z.string().optional(), tileId: z.string().optional(), + groupBy: z.string().optional(), name: z.string().nullish(), message: z.string().nullish(), note: alertNoteSchema, @@ -2187,6 +2222,50 @@ export type AlertHistoryRangeApiResponse = z.infer< typeof AlertHistoryRangeApiResponseSchema >; +// Per-group result of one evaluation window for a group-by alert. +export const AlertEvaluationGroupSchema = z.object({ + group: z.string(), + state: z.nativeEnum(AlertState), + counts: z.number(), + /** The group's most recent bucket value in this window, if any. */ + lastValue: z.object({ startTime: z.string(), count: z.number() }).optional(), + /** True when a notification was actually sent for this group. */ + fired: z.boolean().optional(), +}); + +export type AlertEvaluationGroup = z.infer; + +// One evaluation window on the alert detail page. For group-by alerts, +// carries the per-group breakdown (firing-first, capped server-side). +export const AlertEvaluationSchema = AlertHistorySchema.extend({ + groups: z.array(AlertEvaluationGroupSchema).optional(), + /** Total number of groups evaluated in this window (before the cap). */ + groupsTotal: z.number().optional(), +}); + +export type AlertEvaluation = z.infer; + +// Paginated evaluation history for the alert detail page. Each entry is one +// evaluation window (newest first), including any errors recorded for it. +export const AlertEvaluationsApiResponseSchema = z.object({ + data: z.array(AlertEvaluationSchema), + /** + * True when older evaluation windows may exist within the requested time + * range beyond the returned page. + */ + hasMore: z.boolean(), + /** + * Cursor (epoch ms) for the next-older page: pass as `before` on the next + * request. Present when hasMore is true. Cursor-based (not offset-based) + * so pages advance even across gaps with no evaluations. + */ + nextBefore: z.number().optional(), +}); + +export type AlertEvaluationsApiResponse = z.infer< + typeof AlertEvaluationsApiResponseSchema +>; + // Webhooks export const WebhooksApiResponseSchema = z.object({ data: z.array(WebhookSchema),