perf(app): scope the sessions trace scan to RUM services

.changeset/quick-rum-scope.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/app': patch
+---
+
+speed up the sessions query by scoping the trace scan to RUM services

packages/app/src/__tests__/sessions.test.ts

@@ -1,4 +1,25 @@
-import { buildRRWebStreamChartConfig } from '@/sessions';
+import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser';
+import {
+  Metadata,
+  MetadataCache,
+} from '@hyperdx/common-utils/dist/core/metadata';
+import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
+import {
+  SourceKind,
+  TSessionSource,
+  TTraceSource,
+} from '@hyperdx/common-utils/dist/types';
+
+import {
+  buildRRWebStreamChartConfig,
+  buildSessionsQueryFilters,
+  fetchServiceScopeFilters,
+} from '@/sessions';
+
+jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({
+  ...jest.requireActual('@hyperdx/common-utils/dist/core/renderChartConfig'),
+  renderChartConfig: jest.fn(async () => ({ sql: 'SQL', params: { p: 1 } })),
+}));
 
 describe('buildRRWebStreamChartConfig', () => {
   // Mirrors useFieldExpressionGenerator's output for Map columns
@@ -51,3 +72,207 @@ describe('buildRRWebStreamChartConfig', () => {
     expect(config.limit).toEqual({ limit: 1000000, offset: 0 });
   });
 });
+
+describe('fetchServiceScopeFilters', () => {
+  const traceSource: TTraceSource = {
+    id: 'traces',
+    name: 'Traces',
+    kind: SourceKind.Trace,
+    from: { databaseName: 'default', tableName: 'otel_traces' },
+    connection: 'trace-conn',
+    timestampValueExpression: 'Timestamp',
+    defaultTableSelectExpression: 'Timestamp',
+    durationExpression: 'Duration',
+    durationPrecision: 9,
+    traceIdExpression: 'TraceId',
+    spanIdExpression: 'SpanId',
+    parentSpanIdExpression: 'ParentSpanId',
+    spanNameExpression: 'SpanName',
+    spanKindExpression: 'SpanKind',
+    // Differs from the session table's ServiceName column so the tests catch
+    // the two being swapped.
+    serviceNameExpression: "ResourceAttributes['service.name']",
+  };
+  const sessionSource: TSessionSource = {
+    id: 'sessions',
+    name: 'Sessions',
+    kind: SourceKind.Session,
+    from: { databaseName: 'default', tableName: 'hyperdx_sessions' },
+    connection: 'conn',
+    timestampValueExpression: 'TimestampTime',
+    traceSourceId: 'traces',
+  };
+  const dateRange: [Date, Date] = [
+    new Date('2026-01-01T01:00:00Z'),
+    new Date('2026-01-01T02:00:00Z'),
+  ];
+
+  function jsonResultSet(data: unknown[]) {
+    const text = JSON.stringify({ data });
+    return {
+      text: async () => text,
+      // JSON.parse returns any, which satisfies json<T>() without a cast.
+      json: async () => JSON.parse(text),
+      stream: () => {
+        throw new Error('not implemented');
+      },
+      close: () => {},
+      query_id: 'query-id',
+      response_headers: {},
+    };
+  }
+
+  function setup({
+    columns = ['ServiceName', 'TimestampTime'],
+    rows = [{ serviceName: 'web' }],
+  }: { columns?: string[]; rows?: unknown[] } = {}) {
+    const clickhouseClient = new ClickhouseClient({ host: '' });
+    const metadata = new Metadata(clickhouseClient, new MetadataCache());
+    const getColumns = jest.spyOn(metadata, 'getColumns').mockResolvedValue(
+      columns.map(name => ({
+        name,
+        type: 'String',
+        codec_expression: '',
+        comment: '',
+        default_expression: '',
+        default_type: '',
+        ttl_expression: '',
+      })),
+    );
+    const query = jest
+      .spyOn(clickhouseClient, 'query')
+      .mockImplementation(async () => jsonResultSet(rows));
+    const run = (overrides: { traceSource?: TTraceSource } = {}) =>
+      fetchServiceScopeFilters({
+        traceSource,
+        sessionSource,
+        dateRange,
+        metadata,
+        clickhouseClient,
+        ...overrides,
+      });
+    return { run, metadata, getColumns, query };
+  }
+
+  it('queries the session table services over the padded range', async () => {
+    const { run, metadata, getColumns, query } = setup();
+
+    expect(await run()).toEqual([
+      {
+        type: 'sql',
+        condition: "ResourceAttributes['service.name'] IN ('web')",
+      },
+    ]);
+    expect(getColumns).toHaveBeenCalledWith({
+      databaseName: 'default',
+      tableName: 'hyperdx_sessions',
+      connectionId: 'conn',
+    });
+    expect(renderChartConfig).toHaveBeenCalledWith(
+      {
+        select: [
+          { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' },
+        ],
+        from: { databaseName: 'default', tableName: 'hyperdx_sessions' },
+        dateRange: [
+          new Date('2026-01-01T00:00:00Z'),
+          new Date('2026-01-01T03:00:00Z'),
+        ],
+        where: '',
+        timestampValueExpression: 'TimestampTime',
+        connection: 'conn',
+      },
+      metadata,
+      undefined,
+    );
+    expect(query).toHaveBeenCalledWith({
+      query: 'SQL',
+      query_params: { p: 1 },
+      connectionId: 'conn',
+      format: 'JSON',
+    });
+  });
+
+  it('escapes and keeps empty service names', async () => {
+    const { run } = setup({
+      rows: [{ serviceName: "x') OR 1=1 --" }, { serviceName: '' }],
+    });
+    expect(await run()).toEqual([
+      {
+        type: 'sql',
+        condition:
+          "ResourceAttributes['service.name'] IN ('x'') OR 1=1 --', '')",
+      },
+    ]);
+  });
+
+  it('drops null service names', async () => {
+    const { run } = setup({
+      rows: [{ serviceName: 'web' }, { serviceName: null }],
+    });
+    expect(await run()).toEqual([
+      {
+        type: 'sql',
+        condition: "ResourceAttributes['service.name'] IN ('web')",
+      },
+    ]);
+  });
+
+  it('returns no filter when the lookup finds no services', async () => {
+    const { run } = setup({ rows: [] });
+    expect(await run()).toEqual([]);
+  });
+
+  it('returns no filter and warns when the lookup throws', async () => {
+    const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    const { run, query } = setup();
+    query.mockRejectedValue(new Error('boom'));
+    expect(await run()).toEqual([]);
+    expect(warn).toHaveBeenCalled();
+  });
+
+  it('skips the lookup when the session table has no ServiceName column', async () => {
+    const { run, query } = setup({ columns: ['TimestampTime'] });
+    expect(await run()).toEqual([]);
+    expect(query).not.toHaveBeenCalled();
+  });
+
+  it('skips the lookup when the trace source has no service name expression', async () => {
+    const { run, getColumns, query } = setup();
+    expect(
+      await run({
+        traceSource: { ...traceSource, serviceNameExpression: undefined },
+      }),
+    ).toEqual([]);
+    expect(getColumns).not.toHaveBeenCalled();
+    expect(query).not.toHaveBeenCalled();
+  });
+});
+
+describe('buildSessionsQueryFilters', () => {
+  const scope = [{ type: 'sql', condition: "ServiceName IN ('web')" }] as const;
+
+  it('puts the scope before a lucene query', () => {
+    expect(buildSessionsQueryFilters([...scope], 'foo', 'lucene')).toEqual([
+      ...scope,
+      { type: 'lucene', condition: 'foo' },
+    ]);
+  });
+
+  it('keeps a SQL query as SQL', () => {
+    expect(buildSessionsQueryFilters([...scope], 'a = 1', 'sql')).toEqual([
+      ...scope,
+      { type: 'sql', condition: 'a = 1' },
+    ]);
+  });
+
+  it('treats a missing language as lucene', () => {
+    expect(buildSessionsQueryFilters([], 'foo', undefined)).toEqual([
+      { type: 'lucene', condition: 'foo' },
+    ]);
+  });
+
+  it('returns no filters with no scope and no query', () => {
+    expect(buildSessionsQueryFilters([], '', 'lucene')).toEqual([]);
+  });
+});

packages/app/src/sessions.ts

@@ -5,9 +5,13 @@ import {
   chSql,
   streamToAsyncIterator,
 } from '@hyperdx/common-utils/dist/clickhouse';
+import type { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser';
+import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
+import { filtersToQuery } from '@hyperdx/common-utils/dist/filters';
 import {
   DateRange,
+  Filter,
   pickSampleWeightExpressionProps,
   SearchCondition,
   SearchConditionLanguage,
@@ -41,6 +45,96 @@ export type Session = {
   userName: string;
 };
 
+// Recording rows are written throughout a session, so the range only needs
+// padding for skew between recording and span timestamps.
+const SERVICE_SCOPE_PADDING_MS = 60 * 60 * 1000;
+
+// Trace tables sorted by ServiceName first can't prune on time alone, so the
+// sessions query is scoped to the services in the session table. Returns []
+// (unscoped) when that isn't possible or the lookup fails.
+export async function fetchServiceScopeFilters({
+  traceSource,
+  sessionSource,
+  dateRange,
+  metadata,
+  clickhouseClient,
+}: {
+  traceSource: TTraceSource;
+  sessionSource: TSessionSource;
+  dateRange: [Date, Date];
+  metadata: Metadata;
+  clickhouseClient: ClickhouseClient;
+}): Promise<Filter[]> {
+  if (!traceSource.serviceNameExpression) {
+    return [];
+  }
+  try {
+    const columns = await metadata.getColumns({
+      databaseName: sessionSource.from.databaseName,
+      tableName: sessionSource.from.tableName,
+      connectionId: sessionSource.connection,
+    });
+    if (!columns.some(c => c.name === 'ServiceName')) {
+      return [];
+    }
+    const query = await renderChartConfig(
+      {
+        select: [
+          { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' },
+        ],
+        from: sessionSource.from,
+        dateRange: [
+          new Date(dateRange[0].getTime() - SERVICE_SCOPE_PADDING_MS),
+          new Date(dateRange[1].getTime() + SERVICE_SCOPE_PADDING_MS),
+        ],
+        where: '',
+        timestampValueExpression: sessionSource.timestampValueExpression,
+        connection: sessionSource.connection,
+      },
+      metadata,
+      sessionSource.querySettings,
+    );
+    const json = await clickhouseClient
+      .query({
+        query: query.sql,
+        query_params: query.params,
+        connectionId: sessionSource.connection,
+        format: 'JSON',
+      })
+      .then(res => res.json<{ serviceName: string }>());
+    const serviceNames = (json.data ?? [])
+      .map(row => row.serviceName)
+      .filter(name => name != null);
+    return filtersToQuery({
+      [traceSource.serviceNameExpression]: {
+        included: new Set(serviceNames),
+        excluded: new Set(),
+      },
+    });
+  } catch (e) {
+    console.warn('Sessions service scope lookup failed', e);
+    return [];
+  }
+}
+
+export function buildSessionsQueryFilters(
+  serviceScopeFilters: Filter[],
+  where: SearchCondition | undefined,
+  whereLanguage: SearchConditionLanguage | undefined,
+): Filter[] {
+  return [
+    ...serviceScopeFilters,
+    ...(where
+      ? [
+          {
+            type: whereLanguage === 'sql' ? 'sql' : 'lucene',
+            condition: where,
+          } satisfies Filter,
+        ]
+      : []),
+  ];
+}
+
 export function useSessions(
   {
     traceSource,
@@ -97,6 +191,20 @@ export function useSessions(
         'rum.sessionId',
       );
 
+      const serviceScopeFilters = await fetchServiceScopeFilters({
+        traceSource,
+        sessionSource,
+        dateRange,
+        metadata,
+        clickhouseClient,
+      });
+
+      const sessionsQueryFilters = buildSessionsQueryFilters(
+        serviceScopeFilters,
+        where,
+        whereLanguage,
+      );
+
       const [
         sessionsQuery,
         sessionIdsWithRecordingsQuery,
@@ -157,15 +265,8 @@ export function useSessions(
             dateRange,
             where: `${traceSource.resourceAttributesExpression}.rum.sessionId:*`,
             whereLanguage: 'lucene',
-            ...(where && {
-              filters: [
-                {
-                  type:
-                    (whereLanguage === 'promql' ? 'lucene' : whereLanguage) ??
-                    'lucene',
-                  condition: where,
-                },
-              ],
+            ...(sessionsQueryFilters.length > 0 && {
+              filters: sessionsQueryFilters,
             }),
             timestampValueExpression: traceSource.timestampValueExpression,
             implicitColumnExpression: traceSource.implicitColumnExpression,