diff --git a/.changeset/quick-rum-scope.md b/.changeset/quick-rum-scope.md new file mode 100644 index 0000000000..5d949e8dbc --- /dev/null +++ b/.changeset/quick-rum-scope.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +speed up the sessions query by scoping the trace scan to RUM services diff --git a/packages/app/src/__tests__/sessions.test.ts b/packages/app/src/__tests__/sessions.test.ts index 6d165e86dc..fcb2a9738e 100644 --- a/packages/app/src/__tests__/sessions.test.ts +++ b/packages/app/src/__tests__/sessions.test.ts @@ -1,4 +1,25 @@ -import { buildRRWebStreamChartConfig } from '@/sessions'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import { + Metadata, + MetadataCache, +} from '@hyperdx/common-utils/dist/core/metadata'; +import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { + SourceKind, + TSessionSource, + TTraceSource, +} from '@hyperdx/common-utils/dist/types'; + +import { + buildRRWebStreamChartConfig, + buildSessionsQueryFilters, + fetchServiceScopeFilters, +} from '@/sessions'; + +jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({ + ...jest.requireActual('@hyperdx/common-utils/dist/core/renderChartConfig'), + renderChartConfig: jest.fn(async () => ({ sql: 'SQL', params: { p: 1 } })), +})); describe('buildRRWebStreamChartConfig', () => { // Mirrors useFieldExpressionGenerator's output for Map columns @@ -51,3 +72,207 @@ describe('buildRRWebStreamChartConfig', () => { expect(config.limit).toEqual({ limit: 1000000, offset: 0 }); }); }); + +describe('fetchServiceScopeFilters', () => { + const traceSource: TTraceSource = { + id: 'traces', + name: 'Traces', + kind: SourceKind.Trace, + from: { databaseName: 'default', tableName: 'otel_traces' }, + connection: 'trace-conn', + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp', + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + // Differs from the session table's ServiceName column so the tests catch + // the two being swapped. + serviceNameExpression: "ResourceAttributes['service.name']", + }; + const sessionSource: TSessionSource = { + id: 'sessions', + name: 'Sessions', + kind: SourceKind.Session, + from: { databaseName: 'default', tableName: 'hyperdx_sessions' }, + connection: 'conn', + timestampValueExpression: 'TimestampTime', + traceSourceId: 'traces', + }; + const dateRange: [Date, Date] = [ + new Date('2026-01-01T01:00:00Z'), + new Date('2026-01-01T02:00:00Z'), + ]; + + function jsonResultSet(data: unknown[]) { + const text = JSON.stringify({ data }); + return { + text: async () => text, + // JSON.parse returns any, which satisfies json() without a cast. + json: async () => JSON.parse(text), + stream: () => { + throw new Error('not implemented'); + }, + close: () => {}, + query_id: 'query-id', + response_headers: {}, + }; + } + + function setup({ + columns = ['ServiceName', 'TimestampTime'], + rows = [{ serviceName: 'web' }], + }: { columns?: string[]; rows?: unknown[] } = {}) { + const clickhouseClient = new ClickhouseClient({ host: '' }); + const metadata = new Metadata(clickhouseClient, new MetadataCache()); + const getColumns = jest.spyOn(metadata, 'getColumns').mockResolvedValue( + columns.map(name => ({ + name, + type: 'String', + codec_expression: '', + comment: '', + default_expression: '', + default_type: '', + ttl_expression: '', + })), + ); + const query = jest + .spyOn(clickhouseClient, 'query') + .mockImplementation(async () => jsonResultSet(rows)); + const run = (overrides: { traceSource?: TTraceSource } = {}) => + fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + ...overrides, + }); + return { run, metadata, getColumns, query }; + } + + it('queries the session table services over the padded range', async () => { + const { run, metadata, getColumns, query } = setup(); + + expect(await run()).toEqual([ + { + type: 'sql', + condition: "ResourceAttributes['service.name'] IN ('web')", + }, + ]); + expect(getColumns).toHaveBeenCalledWith({ + databaseName: 'default', + tableName: 'hyperdx_sessions', + connectionId: 'conn', + }); + expect(renderChartConfig).toHaveBeenCalledWith( + { + select: [ + { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' }, + ], + from: { databaseName: 'default', tableName: 'hyperdx_sessions' }, + dateRange: [ + new Date('2026-01-01T00:00:00Z'), + new Date('2026-01-01T03:00:00Z'), + ], + where: '', + timestampValueExpression: 'TimestampTime', + connection: 'conn', + }, + metadata, + undefined, + ); + expect(query).toHaveBeenCalledWith({ + query: 'SQL', + query_params: { p: 1 }, + connectionId: 'conn', + format: 'JSON', + }); + }); + + it('escapes and keeps empty service names', async () => { + const { run } = setup({ + rows: [{ serviceName: "x') OR 1=1 --" }, { serviceName: '' }], + }); + expect(await run()).toEqual([ + { + type: 'sql', + condition: + "ResourceAttributes['service.name'] IN ('x'') OR 1=1 --', '')", + }, + ]); + }); + + it('drops null service names', async () => { + const { run } = setup({ + rows: [{ serviceName: 'web' }, { serviceName: null }], + }); + expect(await run()).toEqual([ + { + type: 'sql', + condition: "ResourceAttributes['service.name'] IN ('web')", + }, + ]); + }); + + it('returns no filter when the lookup finds no services', async () => { + const { run } = setup({ rows: [] }); + expect(await run()).toEqual([]); + }); + + it('returns no filter and warns when the lookup throws', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const { run, query } = setup(); + query.mockRejectedValue(new Error('boom')); + expect(await run()).toEqual([]); + expect(warn).toHaveBeenCalled(); + }); + + it('skips the lookup when the session table has no ServiceName column', async () => { + const { run, query } = setup({ columns: ['TimestampTime'] }); + expect(await run()).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); + + it('skips the lookup when the trace source has no service name expression', async () => { + const { run, getColumns, query } = setup(); + expect( + await run({ + traceSource: { ...traceSource, serviceNameExpression: undefined }, + }), + ).toEqual([]); + expect(getColumns).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('buildSessionsQueryFilters', () => { + const scope = [{ type: 'sql', condition: "ServiceName IN ('web')" }] as const; + + it('puts the scope before a lucene query', () => { + expect(buildSessionsQueryFilters([...scope], 'foo', 'lucene')).toEqual([ + ...scope, + { type: 'lucene', condition: 'foo' }, + ]); + }); + + it('keeps a SQL query as SQL', () => { + expect(buildSessionsQueryFilters([...scope], 'a = 1', 'sql')).toEqual([ + ...scope, + { type: 'sql', condition: 'a = 1' }, + ]); + }); + + it('treats a missing language as lucene', () => { + expect(buildSessionsQueryFilters([], 'foo', undefined)).toEqual([ + { type: 'lucene', condition: 'foo' }, + ]); + }); + + it('returns no filters with no scope and no query', () => { + expect(buildSessionsQueryFilters([], '', 'lucene')).toEqual([]); + }); +}); diff --git a/packages/app/src/sessions.ts b/packages/app/src/sessions.ts index f7b229452e..825e2fc036 100644 --- a/packages/app/src/sessions.ts +++ b/packages/app/src/sessions.ts @@ -5,9 +5,13 @@ import { chSql, streamToAsyncIterator, } from '@hyperdx/common-utils/dist/clickhouse'; +import type { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { filtersToQuery } from '@hyperdx/common-utils/dist/filters'; import { DateRange, + Filter, pickSampleWeightExpressionProps, SearchCondition, SearchConditionLanguage, @@ -41,6 +45,96 @@ export type Session = { userName: string; }; +// Recording rows are written throughout a session, so the range only needs +// padding for skew between recording and span timestamps. +const SERVICE_SCOPE_PADDING_MS = 60 * 60 * 1000; + +// Trace tables sorted by ServiceName first can't prune on time alone, so the +// sessions query is scoped to the services in the session table. Returns [] +// (unscoped) when that isn't possible or the lookup fails. +export async function fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, +}: { + traceSource: TTraceSource; + sessionSource: TSessionSource; + dateRange: [Date, Date]; + metadata: Metadata; + clickhouseClient: ClickhouseClient; +}): Promise { + if (!traceSource.serviceNameExpression) { + return []; + } + try { + const columns = await metadata.getColumns({ + databaseName: sessionSource.from.databaseName, + tableName: sessionSource.from.tableName, + connectionId: sessionSource.connection, + }); + if (!columns.some(c => c.name === 'ServiceName')) { + return []; + } + const query = await renderChartConfig( + { + select: [ + { valueExpression: 'DISTINCT ServiceName', alias: 'serviceName' }, + ], + from: sessionSource.from, + dateRange: [ + new Date(dateRange[0].getTime() - SERVICE_SCOPE_PADDING_MS), + new Date(dateRange[1].getTime() + SERVICE_SCOPE_PADDING_MS), + ], + where: '', + timestampValueExpression: sessionSource.timestampValueExpression, + connection: sessionSource.connection, + }, + metadata, + sessionSource.querySettings, + ); + const json = await clickhouseClient + .query({ + query: query.sql, + query_params: query.params, + connectionId: sessionSource.connection, + format: 'JSON', + }) + .then(res => res.json<{ serviceName: string }>()); + const serviceNames = (json.data ?? []) + .map(row => row.serviceName) + .filter(name => name != null); + return filtersToQuery({ + [traceSource.serviceNameExpression]: { + included: new Set(serviceNames), + excluded: new Set(), + }, + }); + } catch (e) { + console.warn('Sessions service scope lookup failed', e); + return []; + } +} + +export function buildSessionsQueryFilters( + serviceScopeFilters: Filter[], + where: SearchCondition | undefined, + whereLanguage: SearchConditionLanguage | undefined, +): Filter[] { + return [ + ...serviceScopeFilters, + ...(where + ? [ + { + type: whereLanguage === 'sql' ? 'sql' : 'lucene', + condition: where, + } satisfies Filter, + ] + : []), + ]; +} + export function useSessions( { traceSource, @@ -97,6 +191,20 @@ export function useSessions( 'rum.sessionId', ); + const serviceScopeFilters = await fetchServiceScopeFilters({ + traceSource, + sessionSource, + dateRange, + metadata, + clickhouseClient, + }); + + const sessionsQueryFilters = buildSessionsQueryFilters( + serviceScopeFilters, + where, + whereLanguage, + ); + const [ sessionsQuery, sessionIdsWithRecordingsQuery, @@ -157,15 +265,8 @@ export function useSessions( dateRange, where: `${traceSource.resourceAttributesExpression}.rum.sessionId:*`, whereLanguage: 'lucene', - ...(where && { - filters: [ - { - type: - (whereLanguage === 'promql' ? 'lucene' : whereLanguage) ?? - 'lucene', - condition: where, - }, - ], + ...(sessionsQueryFilters.length > 0 && { + filters: sessionsQueryFilters, }), timestampValueExpression: traceSource.timestampValueExpression, implicitColumnExpression: traceSource.implicitColumnExpression,