perf: optimize AlertHistory queries (HDX-3706)

.changeset/optimize-alert-history-queries.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+perf: optimize AlertHistory aggregation queries with time-window filters and compound index

packages/api/src/controllers/__tests__/alertHistory.test.ts

@@ -24,6 +24,7 @@ describe('alertHistory controller', () => {
       const alertId = new ObjectId();
       const histories = await getRecentAlertHistories({
         alertId,
+        interval: '5m',
         limit: 10,
       });
 
@@ -39,8 +40,8 @@ describe('alertHistory controller', () => {
         channel: { type: null },
       });
 
-      const now = new Date('2024-01-15T12:00:00Z');
-      const earlier = new Date('2024-01-15T11:00:00Z');
+      const now = new Date(Date.now() - 60000);
+      const earlier = new Date(Date.now() - 120000);
 
       await AlertHistory.create({
         alert: alert._id,
@@ -60,6 +61,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert._id),
+        interval: '5m',
         limit: 10,
       });
 
@@ -96,6 +98,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert._id),
+        interval: '5m',
         limit: 3,
       });
 
@@ -111,7 +114,7 @@ describe('alertHistory controller', () => {
         channel: { type: null },
       });
 
-      const timestamp = new Date('2024-01-15T12:00:00Z');
+      const timestamp = new Date(Date.now() - 60000);
 
       // Create multiple histories with the same timestamp
       await AlertHistory.create({
@@ -132,6 +135,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert._id),
+        interval: '5m',
         limit: 10,
       });
 
@@ -150,7 +154,7 @@ describe('alertHistory controller', () => {
         channel: { type: null },
       });
 
-      const timestamp = new Date('2024-01-15T12:00:00Z');
+      const timestamp = new Date(Date.now() - 60000);
 
       // Create histories with mixed states at the same timestamp
       await AlertHistory.create({
@@ -179,6 +183,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert._id),
+        interval: '5m',
         limit: 10,
       });
 
@@ -196,7 +201,7 @@ describe('alertHistory controller', () => {
         channel: { type: null },
       });
 
-      const timestamp = new Date('2024-01-15T12:00:00Z');
+      const timestamp = new Date(Date.now() - 60000);
 
       await AlertHistory.create({
         alert: alert._id,
@@ -216,6 +221,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert._id),
+        interval: '5m',
         limit: 10,
       });
 
@@ -232,9 +238,9 @@ describe('alertHistory controller', () => {
         channel: { type: null },
       });
 
-      const oldest = new Date('2024-01-15T10:00:00Z');
-      const middle = new Date('2024-01-15T11:00:00Z');
-      const newest = new Date('2024-01-15T12:00:00Z');
+      const oldest = new Date(Date.now() - 180000);
+      const middle = new Date(Date.now() - 120000);
+      const newest = new Date(Date.now() - 60000);
 
       // Create in random order
       await AlertHistory.create({
@@ -263,6 +269,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert._id),
+        interval: '5m',
         limit: 10,
       });
 
@@ -281,9 +288,9 @@ describe('alertHistory controller', () => {
         channel: { type: null },
       });
 
-      const timestamp = new Date('2024-01-15T12:00:00Z');
-      const older = new Date('2024-01-15T11:00:00Z');
-      const newer = new Date('2024-01-15T13:00:00Z');
+      const timestamp = new Date(Date.now() - 60000);
+      const older = new Date(Date.now() - 120000);
+      const newer = new Date(Date.now() - 30000);
 
       await AlertHistory.create({
         alert: alert._id,
@@ -303,6 +310,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert._id),
+        interval: '5m',
         limit: 10,
       });
 
@@ -328,7 +336,7 @@ describe('alertHistory controller', () => {
         channel: { type: null },
       });
 
-      const timestamp = new Date('2024-01-15T12:00:00Z');
+      const timestamp = new Date(Date.now() - 60000);
 
       await AlertHistory.create({
         alert: alert1._id,
@@ -348,6 +356,7 @@ describe('alertHistory controller', () => {
 
       const histories = await getRecentAlertHistories({
         alertId: new ObjectId(alert1._id),
+        interval: '5m',
         limit: 10,
       });
 

packages/api/src/controllers/alertHistory.ts

@@ -1,3 +1,7 @@
+import {
+  ALERT_INTERVAL_TO_MINUTES,
+  AlertInterval,
+} from '@hyperdx/common-utils/dist/types';
 import { ObjectId } from 'mongodb';
 
 import { AlertState } from '@/models/alert';
@@ -16,19 +20,23 @@ type GroupedAlertHistory = {
  */
 export async function getRecentAlertHistories({
   alertId,
+  interval,
   limit,
 }: {
   alertId: ObjectId;
+  interval: AlertInterval;
   limit: number;
 }): Promise<Omit<IAlertHistory, 'alert'>[]> {
+  const lookbackMs = limit * ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000;
+
   const groupedHistories = await AlertHistory.aggregate<GroupedAlertHistory>([
-    // Filter for the specific alert
     {
       $match: {
         alert: new ObjectId(alertId),
+        createdAt: { $gte: new Date(Date.now() - lookbackMs) },
       },
     },
-    // Group documents by createdAt
+    { $sort: { createdAt: -1 } },
     {
       $group: {
         _id: '$createdAt',
@@ -43,7 +51,6 @@ export async function getRecentAlertHistories({
         },
       },
     },
-    // Take the `createdAtLimit` most recent groups
     {
       $sort: {
         _id: -1,

packages/api/src/models/alertHistory.ts

@@ -52,11 +52,12 @@ AlertHistorySchema.index(
   { expireAfterSeconds: ms('30d') / 1000 },
 );
 
-// Compound index for querying alert histories by alert and time
-// Used by getPreviousAlertHistories and alerts router
-// Supports queries like: { alert: id, createdAt: { $lte: date } } with sort { createdAt: -1 }
+// Used by getRecentAlertHistories (matches on alert, sorts by createdAt)
 AlertHistorySchema.index({ alert: 1, createdAt: -1 });
 
+// Used by getPreviousAlertHistories (groups by {alert, group}, sorts by createdAt)
+AlertHistorySchema.index({ alert: 1, group: 1, createdAt: -1 });
+
 export default mongoose.model<IAlertHistory>(
   'AlertHistory',
   AlertHistorySchema,

packages/api/src/routers/api/alerts.ts

@@ -31,6 +31,7 @@ router.get('/', async (req, res: AlertsExpRes, next) => {
       alerts.map(async alert => {
         const history = await getRecentAlertHistories({
           alertId: new ObjectId(alert._id),
+          interval: alert.interval,
           limit: 20,
         });
 

packages/api/src/tasks/checkAlerts/index.ts

@@ -966,21 +966,19 @@ export const getPreviousAlertHistories = async (
     50,
   );
 
+  const lookbackDate = new Date(now.getTime() - ms('7d'));
+
   const resultChunks = await Promise.all(
     chunkedIds.map(async ids =>
       AlertHistory.aggregate<AggregatedAlertHistory>([
-        // Filter for the given alerts, and only entries created before "now"
-        // This uses the compound index { alert: 1, createdAt: -1 }
         {
           $match: {
             alert: { $in: ids },
-            createdAt: { $lte: now },
+            createdAt: { $lte: now, $gte: lookbackDate },
           },
         },
-        // Sort by alert and createdAt to leverage the index
-        // This ensures we can use the compound index efficiently
         {
-          $sort: { alert: 1, createdAt: -1 },
+          $sort: { alert: 1, group: 1, createdAt: -1 },
         },
         // Group by alert ID AND group (if present), taking the first (latest) document for each combination
         {