diff --git a/.changeset/optimize-alert-history-queries.md b/.changeset/optimize-alert-history-queries.md new file mode 100644 index 0000000000..0c3dd61cb8 --- /dev/null +++ b/.changeset/optimize-alert-history-queries.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": patch +--- + +perf: optimize AlertHistory aggregation queries with time-window filters and compound index diff --git a/packages/api/src/controllers/__tests__/alertHistory.test.ts b/packages/api/src/controllers/__tests__/alertHistory.test.ts index 8d0922a00a..766df0cfdd 100644 --- a/packages/api/src/controllers/__tests__/alertHistory.test.ts +++ b/packages/api/src/controllers/__tests__/alertHistory.test.ts @@ -24,6 +24,7 @@ describe('alertHistory controller', () => { const alertId = new ObjectId(); const histories = await getRecentAlertHistories({ alertId, + interval: '5m', limit: 10, }); @@ -39,8 +40,8 @@ describe('alertHistory controller', () => { channel: { type: null }, }); - const now = new Date('2024-01-15T12:00:00Z'); - const earlier = new Date('2024-01-15T11:00:00Z'); + const now = new Date(Date.now() - 60000); + const earlier = new Date(Date.now() - 120000); await AlertHistory.create({ alert: alert._id, @@ -60,6 +61,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: '5m', limit: 10, }); @@ -96,6 +98,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: '5m', limit: 3, }); @@ -111,7 +114,7 @@ describe('alertHistory controller', () => { channel: { type: null }, }); - const timestamp = new Date('2024-01-15T12:00:00Z'); + const timestamp = new Date(Date.now() - 60000); // Create multiple histories with the same timestamp await AlertHistory.create({ @@ -132,6 +135,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: '5m', limit: 10, }); @@ -150,7 +154,7 @@ describe('alertHistory controller', () => { channel: { type: null }, }); - const timestamp = new Date('2024-01-15T12:00:00Z'); + const timestamp = new Date(Date.now() - 60000); // Create histories with mixed states at the same timestamp await AlertHistory.create({ @@ -179,6 +183,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: '5m', limit: 10, }); @@ -196,7 +201,7 @@ describe('alertHistory controller', () => { channel: { type: null }, }); - const timestamp = new Date('2024-01-15T12:00:00Z'); + const timestamp = new Date(Date.now() - 60000); await AlertHistory.create({ alert: alert._id, @@ -216,6 +221,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: '5m', limit: 10, }); @@ -232,9 +238,9 @@ describe('alertHistory controller', () => { channel: { type: null }, }); - const oldest = new Date('2024-01-15T10:00:00Z'); - const middle = new Date('2024-01-15T11:00:00Z'); - const newest = new Date('2024-01-15T12:00:00Z'); + const oldest = new Date(Date.now() - 180000); + const middle = new Date(Date.now() - 120000); + const newest = new Date(Date.now() - 60000); // Create in random order await AlertHistory.create({ @@ -263,6 +269,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: '5m', limit: 10, }); @@ -281,9 +288,9 @@ describe('alertHistory controller', () => { channel: { type: null }, }); - const timestamp = new Date('2024-01-15T12:00:00Z'); - const older = new Date('2024-01-15T11:00:00Z'); - const newer = new Date('2024-01-15T13:00:00Z'); + const timestamp = new Date(Date.now() - 60000); + const older = new Date(Date.now() - 120000); + const newer = new Date(Date.now() - 30000); await AlertHistory.create({ alert: alert._id, @@ -303,6 +310,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: '5m', limit: 10, }); @@ -328,7 +336,7 @@ describe('alertHistory controller', () => { channel: { type: null }, }); - const timestamp = new Date('2024-01-15T12:00:00Z'); + const timestamp = new Date(Date.now() - 60000); await AlertHistory.create({ alert: alert1._id, @@ -348,6 +356,7 @@ describe('alertHistory controller', () => { const histories = await getRecentAlertHistories({ alertId: new ObjectId(alert1._id), + interval: '5m', limit: 10, }); diff --git a/packages/api/src/controllers/alertHistory.ts b/packages/api/src/controllers/alertHistory.ts index 9b06ca262f..99d31d4882 100644 --- a/packages/api/src/controllers/alertHistory.ts +++ b/packages/api/src/controllers/alertHistory.ts @@ -1,3 +1,7 @@ +import { + ALERT_INTERVAL_TO_MINUTES, + AlertInterval, +} from '@hyperdx/common-utils/dist/types'; import { ObjectId } from 'mongodb'; import { AlertState } from '@/models/alert'; @@ -16,19 +20,23 @@ type GroupedAlertHistory = { */ export async function getRecentAlertHistories({ alertId, + interval, limit, }: { alertId: ObjectId; + interval: AlertInterval; limit: number; }): Promise[]> { + const lookbackMs = limit * ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000; + const groupedHistories = await AlertHistory.aggregate([ - // Filter for the specific alert { $match: { alert: new ObjectId(alertId), + createdAt: { $gte: new Date(Date.now() - lookbackMs) }, }, }, - // Group documents by createdAt + { $sort: { createdAt: -1 } }, { $group: { _id: '$createdAt', @@ -43,7 +51,6 @@ export async function getRecentAlertHistories({ }, }, }, - // Take the `createdAtLimit` most recent groups { $sort: { _id: -1, diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 47c0f1b3da..93cd4ca254 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -52,11 +52,12 @@ AlertHistorySchema.index( { expireAfterSeconds: ms('30d') / 1000 }, ); -// Compound index for querying alert histories by alert and time -// Used by getPreviousAlertHistories and alerts router -// Supports queries like: { alert: id, createdAt: { $lte: date } } with sort { createdAt: -1 } +// Used by getRecentAlertHistories (matches on alert, sorts by createdAt) AlertHistorySchema.index({ alert: 1, createdAt: -1 }); +// Used by getPreviousAlertHistories (groups by {alert, group}, sorts by createdAt) +AlertHistorySchema.index({ alert: 1, group: 1, createdAt: -1 }); + export default mongoose.model( 'AlertHistory', AlertHistorySchema, diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index 670d6e36c6..e32034fcc8 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -31,6 +31,7 @@ router.get('/', async (req, res: AlertsExpRes, next) => { alerts.map(async alert => { const history = await getRecentAlertHistories({ alertId: new ObjectId(alert._id), + interval: alert.interval, limit: 20, }); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 8e32224d7b..779a43f909 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -966,21 +966,19 @@ export const getPreviousAlertHistories = async ( 50, ); + const lookbackDate = new Date(now.getTime() - ms('7d')); + const resultChunks = await Promise.all( chunkedIds.map(async ids => AlertHistory.aggregate([ - // Filter for the given alerts, and only entries created before "now" - // This uses the compound index { alert: 1, createdAt: -1 } { $match: { alert: { $in: ids }, - createdAt: { $lte: now }, + createdAt: { $lte: now, $gte: lookbackDate }, }, }, - // Sort by alert and createdAt to leverage the index - // This ensures we can use the compound index efficiently { - $sort: { alert: 1, createdAt: -1 }, + $sort: { alert: 1, group: 1, createdAt: -1 }, }, // Group by alert ID AND group (if present), taking the first (latest) document for each combination {