From 29e985b6488a9b0fb719aeb8ee3e5a529635dec8 Mon Sep 17 00:00:00 2001 From: vinzee Date: Mon, 12 Oct 2026 16:49:57 -0700 Subject: [PATCH 1/2] perf(app): scope the sessions trace scan to RUM services Trace tables sorted by ServiceName first (the default OTel schema) can't prune the sessions aggregation on time alone, so it scans every mark to evaluate the rum.sessionId skip index. Read the distinct ServiceName values from the session table and add a ServiceName IN (...) filter, turning the scan into a small primary-key range. - fetchServiceScopeFilters reads the session table's ServiceName column over the selected range padded by 1h, and builds the filter with filtersToQuery. Null names are dropped; empty names are kept. - Skip the scope when the session table has no ServiceName column or the trace source has no service-name expression; on lookup failure (logged) or no results, run the unscoped query. - buildSessionsQueryFilters combines the scope with the user's where. - Tests cover the lookup's success, empty, null, error, and skip paths, and the filter composition. --- .changeset/quick-rum-scope.md | 5 + packages/app/src/__tests__/sessions.test.ts | 224 +++++++++++++++++++- packages/app/src/sessions.ts | 119 ++++++++++- 3 files changed, 338 insertions(+), 10 deletions(-) create mode 100644 .changeset/quick-rum-scope.md diff --git a/.changeset/quick-rum-scope.md b/.changeset/quick-rum-scope.md new file mode 100644 index 0000000000..5d949e8dbc --- /dev/null +++ b/.changeset/quick-rum-scope.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +speed up the sessions query by scoping the trace scan to RUM services diff --git a/packages/app/src/__tests__/sessions.test.ts b/packages/app/src/__tests__/sessions.test.ts index 6d165e86dc..5683ae3350 100644 --- a/packages/app/src/__tests__/sessions.test.ts +++ b/packages/app/src/__tests__/sessions.test.ts @@ -1,4 +1,22 @@ -import { buildRRWebStreamChartConfig } from '@/sessions'; +import type { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { + SourceKind, + TSessionSource, + TTraceSource, +} from '@hyperdx/common-utils/dist/types'; + +import { + buildRRWebStreamChartConfig, + buildSessionsQueryFilters, + fetchServiceScopeFilters, +} from '@/sessions'; + +jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({ + ...jest.requireActual('@hyperdx/common-utils/dist/core/renderChartConfig'), + renderChartConfig: jest.fn(async () => ({ sql: 'SQL', params: {} })), +})); describe('buildRRWebStreamChartConfig', () => { // Mirrors useFieldExpressionGenerator's output for Map columns @@ -51,3 +69,207 @@ describe('buildRRWebStreamChartConfig', () => { expect(config.limit).toEqual({ limit: 1000000, offset: 0 }); }); }); + +describe('fetchServiceScopeFilters', () => { + const traceSource: TTraceSource = { + id: 'traces', + name: 'Traces', + kind: SourceKind.Trace, + from: { databaseName: 'default', tableName: 'otel_traces' }, + connection: 'conn', + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp', + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + serviceNameExpression: 'ServiceName', + }; + const sessionSource: TSessionSource = { + id: 'sessions', + name: 'Sessions', + kind: SourceKind.Session, + from: { databaseName: 'default', tableName: 'hyperdx_sessions' }, + connection: 'conn', + timestampValueExpression: 'TimestampTime', + traceSourceId: 'traces', + }; + const dateRange: [Date, Date] = [ + new Date('2026-01-01T01:00:00Z'), + new Date('2026-01-01T02:00:00Z'), + ]; + + function setup({ + columns = ['ServiceName', 'TimestampTime'], + query = jest.fn(async () => ({ + json: async () => ({ data: [{ serviceName: 'web' }] }), + })), + }: { columns?: string[]; query?: jest.Mock } = {}) { + // Partial fakes: only the methods fetchServiceScopeFilters calls. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + const metadata = { + getColumns: jest.fn(async () => columns.map(name => ({ name }))), + } as unknown as Metadata; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + const clickhouseClient = { query } as unknown as ClickhouseClient; + return { metadata, clickhouseClient, query }; + } + + it('scopes to the session table services over the padded range', async () => { + const { metadata, clickhouseClient } = setup(); + const filters = await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }); + + expect(filters).toEqual([ + { type: 'sql', condition: "ServiceName IN ('web')" }, + ]); + expect(jest.mocked(renderChartConfig).mock.lastCall?.[0]).toMatchObject({ + select: [ + { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' }, + ], + dateRange: [ + new Date('2026-01-01T00:00:00Z'), + new Date('2026-01-01T03:00:00Z'), + ], + }); + }); + + it('escapes and keeps empty service names', async () => { + const { metadata, clickhouseClient } = setup({ + query: jest.fn(async () => ({ + json: async () => ({ + data: [{ serviceName: "x') OR 1=1 --" }, { serviceName: '' }], + }), + })), + }); + expect( + await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }), + ).toEqual([ + { type: 'sql', condition: "ServiceName IN ('x'') OR 1=1 --', '')" }, + ]); + }); + + it('drops null service names', async () => { + const { metadata, clickhouseClient } = setup({ + query: jest.fn(async () => ({ + json: async () => ({ + data: [{ serviceName: 'web' }, { serviceName: null }], + }), + })), + }); + expect( + await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }), + ).toEqual([{ type: 'sql', condition: "ServiceName IN ('web')" }]); + }); + + it('returns no filter when the lookup finds no services', async () => { + const { metadata, clickhouseClient } = setup({ + query: jest.fn(async () => ({ json: async () => ({ data: [] }) })), + }); + expect( + await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }), + ).toEqual([]); + }); + + it('returns no filter when the lookup throws', async () => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + const { metadata, clickhouseClient } = setup({ + query: jest.fn(async () => { + throw new Error('boom'); + }), + }); + expect( + await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }), + ).toEqual([]); + }); + + it('skips the lookup when the session table has no ServiceName column', async () => { + const { metadata, clickhouseClient, query } = setup({ + columns: ['TimestampTime'], + }); + expect( + await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }), + ).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); + + it('skips the lookup when the trace source has no service name expression', async () => { + const { metadata, clickhouseClient, query } = setup(); + expect( + await fetchServiceScopeFilters({ + traceSource: { ...traceSource, serviceNameExpression: undefined }, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }), + ).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('buildSessionsQueryFilters', () => { + const scope = [{ type: 'sql', condition: "ServiceName IN ('web')" }] as const; + + it('puts the scope before a lucene query', () => { + expect(buildSessionsQueryFilters([...scope], 'foo', 'lucene')).toEqual([ + ...scope, + { type: 'lucene', condition: 'foo' }, + ]); + }); + + it('keeps a SQL query as SQL', () => { + expect(buildSessionsQueryFilters([...scope], 'a = 1', 'sql')).toEqual([ + ...scope, + { type: 'sql', condition: 'a = 1' }, + ]); + }); + + it('treats a missing language as lucene', () => { + expect(buildSessionsQueryFilters([], 'foo', undefined)).toEqual([ + { type: 'lucene', condition: 'foo' }, + ]); + }); + + it('returns no filters with no scope and no query', () => { + expect(buildSessionsQueryFilters([], '', 'lucene')).toEqual([]); + }); +}); diff --git a/packages/app/src/sessions.ts b/packages/app/src/sessions.ts index f7b229452e..825e2fc036 100644 --- a/packages/app/src/sessions.ts +++ b/packages/app/src/sessions.ts @@ -5,9 +5,13 @@ import { chSql, streamToAsyncIterator, } from '@hyperdx/common-utils/dist/clickhouse'; +import type { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { filtersToQuery } from '@hyperdx/common-utils/dist/filters'; import { DateRange, + Filter, pickSampleWeightExpressionProps, SearchCondition, SearchConditionLanguage, @@ -41,6 +45,96 @@ export type Session = { userName: string; }; +// Recording rows are written throughout a session, so the range only needs +// padding for skew between recording and span timestamps. +const SERVICE_SCOPE_PADDING_MS = 60 * 60 * 1000; + +// Trace tables sorted by ServiceName first can't prune on time alone, so the +// sessions query is scoped to the services in the session table. Returns [] +// (unscoped) when that isn't possible or the lookup fails. +export async function fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, +}: { + traceSource: TTraceSource; + sessionSource: TSessionSource; + dateRange: [Date, Date]; + metadata: Metadata; + clickhouseClient: ClickhouseClient; +}): Promise { + if (!traceSource.serviceNameExpression) { + return []; + } + try { + const columns = await metadata.getColumns({ + databaseName: sessionSource.from.databaseName, + tableName: sessionSource.from.tableName, + connectionId: sessionSource.connection, + }); + if (!columns.some(c => c.name === 'ServiceName')) { + return []; + } + const query = await renderChartConfig( + { + select: [ + { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' }, + ], + from: sessionSource.from, + dateRange: [ + new Date(dateRange[0].getTime() - SERVICE_SCOPE_PADDING_MS), + new Date(dateRange[1].getTime() + SERVICE_SCOPE_PADDING_MS), + ], + where: '', + timestampValueExpression: sessionSource.timestampValueExpression, + connection: sessionSource.connection, + }, + metadata, + sessionSource.querySettings, + ); + const json = await clickhouseClient + .query({ + query: query.sql, + query_params: query.params, + connectionId: sessionSource.connection, + format: 'JSON', + }) + .then(res => res.json<{ serviceName: string }>()); + const serviceNames = (json.data ?? []) + .map(row => row.serviceName) + .filter(name => name != null); + return filtersToQuery({ + [traceSource.serviceNameExpression]: { + included: new Set(serviceNames), + excluded: new Set(), + }, + }); + } catch (e) { + console.warn('Sessions service scope lookup failed', e); + return []; + } +} + +export function buildSessionsQueryFilters( + serviceScopeFilters: Filter[], + where: SearchCondition | undefined, + whereLanguage: SearchConditionLanguage | undefined, +): Filter[] { + return [ + ...serviceScopeFilters, + ...(where + ? [ + { + type: whereLanguage === 'sql' ? 'sql' : 'lucene', + condition: where, + } satisfies Filter, + ] + : []), + ]; +} + export function useSessions( { traceSource, @@ -97,6 +191,20 @@ export function useSessions( 'rum.sessionId', ); + const serviceScopeFilters = await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }); + + const sessionsQueryFilters = buildSessionsQueryFilters( + serviceScopeFilters, + where, + whereLanguage, + ); + const [ sessionsQuery, sessionIdsWithRecordingsQuery, @@ -157,15 +265,8 @@ export function useSessions( dateRange, where: `${traceSource.resourceAttributesExpression}.rum.sessionId:*`, whereLanguage: 'lucene', - ...(where && { - filters: [ - { - type: - (whereLanguage === 'promql' ? 'lucene' : whereLanguage) ?? - 'lucene', - condition: where, - }, - ], + ...(sessionsQueryFilters.length > 0 && { + filters: sessionsQueryFilters, }), timestampValueExpression: traceSource.timestampValueExpression, implicitColumnExpression: traceSource.implicitColumnExpression, From 75fc48079e04d31a700f75c47e20545749b7568b Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 15 Oct 2026 08:40:04 -0400 Subject: [PATCH 2/2] chore: Fix lint warnings in test --- packages/app/src/__tests__/sessions.test.ts | 229 ++++++++++---------- 1 file changed, 116 insertions(+), 113 deletions(-) diff --git a/packages/app/src/__tests__/sessions.test.ts b/packages/app/src/__tests__/sessions.test.ts index 5683ae3350..fcb2a9738e 100644 --- a/packages/app/src/__tests__/sessions.test.ts +++ b/packages/app/src/__tests__/sessions.test.ts @@ -1,5 +1,8 @@ -import type { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; -import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import { + Metadata, + MetadataCache, +} from '@hyperdx/common-utils/dist/core/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { SourceKind, @@ -15,7 +18,7 @@ import { jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({ ...jest.requireActual('@hyperdx/common-utils/dist/core/renderChartConfig'), - renderChartConfig: jest.fn(async () => ({ sql: 'SQL', params: {} })), + renderChartConfig: jest.fn(async () => ({ sql: 'SQL', params: { p: 1 } })), })); describe('buildRRWebStreamChartConfig', () => { @@ -76,7 +79,7 @@ describe('fetchServiceScopeFilters', () => { name: 'Traces', kind: SourceKind.Trace, from: { databaseName: 'default', tableName: 'otel_traces' }, - connection: 'conn', + connection: 'trace-conn', timestampValueExpression: 'Timestamp', defaultTableSelectExpression: 'Timestamp', durationExpression: 'Duration', @@ -86,7 +89,9 @@ describe('fetchServiceScopeFilters', () => { parentSpanIdExpression: 'ParentSpanId', spanNameExpression: 'SpanName', spanKindExpression: 'SpanKind', - serviceNameExpression: 'ServiceName', + // Differs from the session table's ServiceName column so the tests catch + // the two being swapped. + serviceNameExpression: "ResourceAttributes['service.name']", }; const sessionSource: TSessionSource = { id: 'sessions', @@ -102,146 +107,144 @@ describe('fetchServiceScopeFilters', () => { new Date('2026-01-01T02:00:00Z'), ]; + function jsonResultSet(data: unknown[]) { + const text = JSON.stringify({ data }); + return { + text: async () => text, + // JSON.parse returns any, which satisfies json() without a cast. + json: async () => JSON.parse(text), + stream: () => { + throw new Error('not implemented'); + }, + close: () => {}, + query_id: 'query-id', + response_headers: {}, + }; + } + function setup({ columns = ['ServiceName', 'TimestampTime'], - query = jest.fn(async () => ({ - json: async () => ({ data: [{ serviceName: 'web' }] }), - })), - }: { columns?: string[]; query?: jest.Mock } = {}) { - // Partial fakes: only the methods fetchServiceScopeFilters calls. - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - const metadata = { - getColumns: jest.fn(async () => columns.map(name => ({ name }))), - } as unknown as Metadata; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - const clickhouseClient = { query } as unknown as ClickhouseClient; - return { metadata, clickhouseClient, query }; + rows = [{ serviceName: 'web' }], + }: { columns?: string[]; rows?: unknown[] } = {}) { + const clickhouseClient = new ClickhouseClient({ host: '' }); + const metadata = new Metadata(clickhouseClient, new MetadataCache()); + const getColumns = jest.spyOn(metadata, 'getColumns').mockResolvedValue( + columns.map(name => ({ + name, + type: 'String', + codec_expression: '', + comment: '', + default_expression: '', + default_type: '', + ttl_expression: '', + })), + ); + const query = jest + .spyOn(clickhouseClient, 'query') + .mockImplementation(async () => jsonResultSet(rows)); + const run = (overrides: { traceSource?: TTraceSource } = {}) => + fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + ...overrides, + }); + return { run, metadata, getColumns, query }; } - it('scopes to the session table services over the padded range', async () => { - const { metadata, clickhouseClient } = setup(); - const filters = await fetchServiceScopeFilters({ - traceSource, - sessionSource, - dateRange, - metadata, - clickhouseClient, - }); + it('queries the session table services over the padded range', async () => { + const { run, metadata, getColumns, query } = setup(); - expect(filters).toEqual([ - { type: 'sql', condition: "ServiceName IN ('web')" }, + expect(await run()).toEqual([ + { + type: 'sql', + condition: "ResourceAttributes['service.name'] IN ('web')", + }, ]); - expect(jest.mocked(renderChartConfig).mock.lastCall?.[0]).toMatchObject({ - select: [ - { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' }, - ], - dateRange: [ - new Date('2026-01-01T00:00:00Z'), - new Date('2026-01-01T03:00:00Z'), - ], + expect(getColumns).toHaveBeenCalledWith({ + databaseName: 'default', + tableName: 'hyperdx_sessions', + connectionId: 'conn', + }); + expect(renderChartConfig).toHaveBeenCalledWith( + { + select: [ + { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' }, + ], + from: { databaseName: 'default', tableName: 'hyperdx_sessions' }, + dateRange: [ + new Date('2026-01-01T00:00:00Z'), + new Date('2026-01-01T03:00:00Z'), + ], + where: '', + timestampValueExpression: 'TimestampTime', + connection: 'conn', + }, + metadata, + undefined, + ); + expect(query).toHaveBeenCalledWith({ + query: 'SQL', + query_params: { p: 1 }, + connectionId: 'conn', + format: 'JSON', }); }); it('escapes and keeps empty service names', async () => { - const { metadata, clickhouseClient } = setup({ - query: jest.fn(async () => ({ - json: async () => ({ - data: [{ serviceName: "x') OR 1=1 --" }, { serviceName: '' }], - }), - })), + const { run } = setup({ + rows: [{ serviceName: "x') OR 1=1 --" }, { serviceName: '' }], }); - expect( - await fetchServiceScopeFilters({ - traceSource, - sessionSource, - dateRange, - metadata, - clickhouseClient, - }), - ).toEqual([ - { type: 'sql', condition: "ServiceName IN ('x'') OR 1=1 --', '')" }, + expect(await run()).toEqual([ + { + type: 'sql', + condition: + "ResourceAttributes['service.name'] IN ('x'') OR 1=1 --', '')", + }, ]); }); it('drops null service names', async () => { - const { metadata, clickhouseClient } = setup({ - query: jest.fn(async () => ({ - json: async () => ({ - data: [{ serviceName: 'web' }, { serviceName: null }], - }), - })), + const { run } = setup({ + rows: [{ serviceName: 'web' }, { serviceName: null }], }); - expect( - await fetchServiceScopeFilters({ - traceSource, - sessionSource, - dateRange, - metadata, - clickhouseClient, - }), - ).toEqual([{ type: 'sql', condition: "ServiceName IN ('web')" }]); + expect(await run()).toEqual([ + { + type: 'sql', + condition: "ResourceAttributes['service.name'] IN ('web')", + }, + ]); }); it('returns no filter when the lookup finds no services', async () => { - const { metadata, clickhouseClient } = setup({ - query: jest.fn(async () => ({ json: async () => ({ data: [] }) })), - }); - expect( - await fetchServiceScopeFilters({ - traceSource, - sessionSource, - dateRange, - metadata, - clickhouseClient, - }), - ).toEqual([]); + const { run } = setup({ rows: [] }); + expect(await run()).toEqual([]); }); - it('returns no filter when the lookup throws', async () => { - jest.spyOn(console, 'warn').mockImplementation(() => {}); - const { metadata, clickhouseClient } = setup({ - query: jest.fn(async () => { - throw new Error('boom'); - }), - }); - expect( - await fetchServiceScopeFilters({ - traceSource, - sessionSource, - dateRange, - metadata, - clickhouseClient, - }), - ).toEqual([]); + it('returns no filter and warns when the lookup throws', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const { run, query } = setup(); + query.mockRejectedValue(new Error('boom')); + expect(await run()).toEqual([]); + expect(warn).toHaveBeenCalled(); }); it('skips the lookup when the session table has no ServiceName column', async () => { - const { metadata, clickhouseClient, query } = setup({ - columns: ['TimestampTime'], - }); - expect( - await fetchServiceScopeFilters({ - traceSource, - sessionSource, - dateRange, - metadata, - clickhouseClient, - }), - ).toEqual([]); + const { run, query } = setup({ columns: ['TimestampTime'] }); + expect(await run()).toEqual([]); expect(query).not.toHaveBeenCalled(); }); it('skips the lookup when the trace source has no service name expression', async () => { - const { metadata, clickhouseClient, query } = setup(); + const { run, getColumns, query } = setup(); expect( - await fetchServiceScopeFilters({ + await run({ traceSource: { ...traceSource, serviceNameExpression: undefined }, - sessionSource, - dateRange, - metadata, - clickhouseClient, }), ).toEqual([]); + expect(getColumns).not.toHaveBeenCalled(); expect(query).not.toHaveBeenCalled(); }); });