feat: allow applying session settings to queries

.changeset/hip-icons-build.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+---
+
+feat: allow applying session settings to queries

packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap

@@ -473,6 +473,17 @@ Array [
 ]
 `;
 
+exports[`renderChartConfig Query settings handles the the query settings 1`] = `
+Array [
+  Object {
+    "Body": "Oh no! Something went wrong!",
+  },
+  Object {
+    "Body": "This is a test message.",
+  },
+]
+`;
+
 exports[`renderChartConfig aggFn numeric agg functions should handle numeric values as strings 1`] = `
 Array [
   Object {

packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts

@@ -1,13 +1,12 @@
 // TODO: we might want to move this test file to common-utils package
 
-import { ChSql } from '@hyperdx/common-utils/dist/clickhouse';
+import { ChSql, chSql } from '@hyperdx/common-utils/dist/clickhouse';
 import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
 import {
-  AggregateFunctionSchema,
-  DerivedColumn,
   MetricsDataType,
+  QuerySettings,
 } from '@hyperdx/common-utils/dist/types';
 import _ from 'lodash';
 import ms from 'ms';
@@ -36,6 +35,13 @@ const TEST_METRIC_TABLES = {
   'exponential histogram': DEFAULT_METRICS_TABLE.EXPONENTIAL_HISTOGRAM,
 };
 
+const querySettings: QuerySettings = [
+  { setting: 'optimize_read_in_order', value: '0' },
+  { setting: 'cast_keep_nullable', value: '1' },
+  { setting: 'count_distinct_implementation', value: 'uniqCombined64' },
+  { setting: 'async_insert_busy_timeout_min_ms', value: '20000' },
+];
+
 describe('renderChartConfig', () => {
   const server = getServer();
 
@@ -152,6 +158,7 @@ describe('renderChartConfig', () => {
           timestampValueExpression: 'ts',
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -190,6 +197,7 @@ describe('renderChartConfig', () => {
           timestampValueExpression: 'ts',
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -226,16 +234,10 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
-      const resp = await clickhouseClient
-        .query<'JSON'>({
-          query: query.sql,
-          query_params: query.params,
-          format: 'JSON',
-        })
-        .then(res => res.json() as any);
-      expect(resp.data).toMatchSnapshot();
+      expect(await queryData(query)).toMatchSnapshot();
     });
 
     it('simple select + group by query logs', async () => {
@@ -272,6 +274,7 @@ describe('renderChartConfig', () => {
           groupBy: 'ServiceName',
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -342,6 +345,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(avgQuery)).toMatchSnapshot();
       const maxQuery = await renderChartConfig(
@@ -363,6 +367,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(maxQuery)).toMatchSnapshot();
       const sumQuery = await renderChartConfig(
@@ -384,6 +389,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(sumQuery)).toMatchSnapshot();
     });
@@ -409,6 +415,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -434,6 +441,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -459,6 +467,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -485,6 +494,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -657,6 +667,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -681,6 +692,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -705,6 +717,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(query)).toMatchSnapshot();
     });
@@ -749,6 +762,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(minQuery)).toMatchSnapshot('minSum');
 
@@ -771,6 +785,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       expect(await queryData(maxQuery)).toMatchSnapshot('maxSum');
     });
@@ -1151,6 +1166,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1200,6 +1216,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1250,6 +1267,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1294,6 +1312,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1320,6 +1339,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1347,6 +1367,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1374,6 +1395,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
       const res = await queryData(query);
       expect(res).toMatchSnapshot();
@@ -1444,6 +1466,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
       const res = await queryData(query);
@@ -1480,6 +1503,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
       const res = await queryData(query);
@@ -1509,6 +1533,7 @@ describe('renderChartConfig', () => {
           connection: connection.id,
         },
         metadata,
+        querySettings,
       );
 
       const res = await queryData(query);
@@ -1520,4 +1545,43 @@ describe('renderChartConfig', () => {
       expect(query.sql).not.toMatch(/MetricName IN /);
     });
   });
+
+  describe('Query settings', () => {
+    it('handles the the query settings', async () => {
+      const now = new Date('2023-11-16T22:12:00.000Z');
+
+      await bulkInsertLogs([
+        {
+          ServiceName: 'api',
+          Timestamp: now,
+          SeverityText: 'error',
+          Body: 'Oh no! Something went wrong!',
+        },
+        {
+          ServiceName: 'api',
+          Timestamp: now,
+          SeverityText: 'info',
+          Body: 'This is a test message.',
+        },
+      ]);
+
+      const query = await renderChartConfig(
+        {
+          select: [{ valueExpression: 'Body' }],
+          from: logSource.from,
+          where: '',
+          timestampValueExpression: 'Timestamp',
+          connection: connection.id,
+          settings: chSql`max_result_rows = 1`,
+        },
+        metadata,
+        [...querySettings, { setting: 'result_overflow_mode', value: 'break' }],
+      );
+
+      const res = await queryData(query);
+      // ensures `result_overflow_mode = break` is applied, otherwise query would error.
+      expect(res).toHaveLength(2);
+      expect(res).toMatchSnapshot();
+    });
+  });
 });

packages/api/src/models/source.ts

@@ -86,6 +86,17 @@ export const Source = mongoose.model<ISource>(
         },
         default: undefined,
       },
+
+      querySettings: {
+        type: [
+          {
+            setting: { type: String, required: true },
+            value: { type: String, required: true },
+          },
+        ],
+        default: undefined,
+        maxlength: 10,
+      },
     },
     {
       toJSON: { virtuals: true },

packages/api/src/routers/api/ai.ts

@@ -274,6 +274,7 @@ router.post(
       const keyValues = await metadata.getKeyValues({
         chartConfig: cc,
         keys: keysToFetch.map(f => f.key),
+        source,
       });
 
       const anthropic = createAnthropic({

packages/api/src/routers/external-api/v2/charts.ts

@@ -588,6 +588,7 @@ router.post(
             const result = await clickhouseClient.queryChartConfig({
               config: chartConfig,
               metadata,
+              querySettings: source.querySettings,
             });
 
             return {

packages/api/src/tasks/checkAlerts/index.ts

@@ -448,6 +448,7 @@ export const processAlert = async (
     const checksData = await clickhouseClient.queryChartConfig({
       config: optimizedChartConfig,
       metadata,
+      querySettings: source.querySettings,
     });
 
     logger.info(

packages/api/src/tasks/checkAlerts/template.ts

@@ -602,7 +602,11 @@ ${targetTemplate}`;
 
     let truncatedResults = '';
     try {
-      const query = await renderChartConfig(chartConfig, metadata);
+      const query = await renderChartConfig(
+        chartConfig,
+        metadata,
+        source.querySettings,
+      );
       const raw = await clickhouseClient
         .query<'CSV'>({
           query: query.sql,

packages/api/src/tasks/pingPongTask.ts

@@ -4,7 +4,6 @@ import logger from '@/utils/logger';
 export default class PingPongTask implements HdxTask<PingTaskArgs> {
   constructor(private args: PingTaskArgs) {}
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async execute(): Promise<void> {
     logger.info(`
                    O .

packages/app/src/TeamPage.tsx

@@ -1,9 +1,9 @@
 import { useCallback, useState } from 'react';
 import Head from 'next/head';
 import { CopyToClipboard } from 'react-copy-to-clipboard';
-import { SubmitHandler, useForm, useWatch } from 'react-hook-form';
+import { SubmitHandler, useForm } from 'react-hook-form';
 import { DEFAULT_METADATA_MAX_ROWS_TO_READ } from '@hyperdx/common-utils/dist/core/metadata';
-import { TeamClickHouseSettings } from '@hyperdx/common-utils/dist/types';
+import { type TeamClickHouseSettings } from '@hyperdx/common-utils/dist/types';
 import {
   Box,
   Button,