From ca7d9b2cddc5e743929526df76dc980c8550ea64 Mon Sep 17 00:00:00 2001 From: vinzee Date: Wed, 14 Oct 2026 22:02:01 -0700 Subject: [PATCH] fix: v2 search drops rows after a sub-second startTime; make maxResults cap configurable A DateTime column holds second-truncated times, so comparing it to a millisecond start bound (TimestampTime >= 22:03:21.827) drops every row in that second even when the precise Timestamp column would match. Floor the start bound to the second for DateTime columns and keep it inclusive, mirroring the existing Date and toStartOf* handling. Add EXTERNAL_API_SEARCH_MAX_RESULTS (default 2000, the previous hardcoded cap) so deployments with bulk readers can raise the /api/v2/search page size. Invalid values fall back to the default, like EXTERNAL_API_RATE_LIMIT_MAX. --- .changeset/configurable-search-max-results.md | 5 ++ .changeset/datetime-subsecond-start.md | 5 ++ packages/api/openapi.json | 3 +- packages/api/src/__tests__/config.test.ts | 39 +++++++++++++ packages/api/src/config.ts | 5 ++ .../api/src/routers/external-api/v2/search.ts | 12 ++-- .../src/__tests__/renderChartConfig.test.ts | 57 ++++++++++++++++++- .../src/core/renderChartConfig.ts | 20 +++++-- 8 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 .changeset/configurable-search-max-results.md create mode 100644 .changeset/datetime-subsecond-start.md diff --git a/.changeset/configurable-search-max-results.md b/.changeset/configurable-search-max-results.md new file mode 100644 index 0000000000..310c587df9 --- /dev/null +++ b/.changeset/configurable-search-max-results.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': patch +--- + +feat: Make the external API v2 search `maxResults` cap configurable via `EXTERNAL_API_SEARCH_MAX_RESULTS`. Defaults to 2000, matching the previous hardcoded value. diff --git a/.changeset/datetime-subsecond-start.md b/.changeset/datetime-subsecond-start.md new file mode 100644 index 0000000000..346f2850a8 --- /dev/null +++ b/.changeset/datetime-subsecond-start.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/common-utils': patch +--- + +fix: Floor the time filter start on second-precision DateTime columns so a sub-second start no longer drops the rest of that second diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 30ff03cf14..d38a45cc72 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4239,9 +4239,8 @@ "maxResults": { "type": "integer", "minimum": 1, - "maximum": 2000, "default": 100, - "description": "Maximum number of rows to return. Default is 100, max is 2000." + "description": "Maximum number of rows to return. Default is 100, max is 2000\nunless the server sets EXTERNAL_API_SEARCH_MAX_RESULTS.\n" }, "offset": { "type": "integer", diff --git a/packages/api/src/__tests__/config.test.ts b/packages/api/src/__tests__/config.test.ts index 88f17a9d71..1f96dd87d6 100644 --- a/packages/api/src/__tests__/config.test.ts +++ b/packages/api/src/__tests__/config.test.ts @@ -124,6 +124,45 @@ describe('config', () => { ); }); + describe('EXTERNAL_API_SEARCH_MAX_RESULTS', () => { + const ORIGINAL = process.env.EXTERNAL_API_SEARCH_MAX_RESULTS; + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.EXTERNAL_API_SEARCH_MAX_RESULTS; + } else { + process.env.EXTERNAL_API_SEARCH_MAX_RESULTS = ORIGINAL; + } + jest.resetModules(); + }); + + const loadMaxResults = () => { + let loaded!: typeof import('@/config'); + jest.isolateModules(() => { + loaded = jest.requireActual('@/config'); + }); + return loaded.EXTERNAL_API_SEARCH_MAX_RESULTS; + }; + + it('defaults to 2000 when unset', () => { + delete process.env.EXTERNAL_API_SEARCH_MAX_RESULTS; + expect(loadMaxResults()).toBe(2000); + }); + + it('accepts a valid positive integer', () => { + process.env.EXTERNAL_API_SEARCH_MAX_RESULTS = '10000'; + expect(loadMaxResults()).toBe(10_000); + }); + + it.each(['abc', '0', '-5', '100rows', '', '0.5', 'Infinity'])( + 'falls back to 2000 for %p', + raw => { + process.env.EXTERNAL_API_SEARCH_MAX_RESULTS = raw; + expect(loadMaxResults()).toBe(2000); + }, + ); + }); + describe('EXPRESS_SESSION_SECRET', () => { const ORIGINAL_SECRET = process.env.EXPRESS_SESSION_SECRET; const ORIGINAL_LOCAL_APP_MODE = process.env.IS_LOCAL_APP_MODE; diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index fb920c9159..a3270e89a2 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -76,6 +76,11 @@ export const EXTERNAL_API_RATE_LIMIT_MAX = (() => { return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : 100; })(); +export const EXTERNAL_API_SEARCH_MAX_RESULTS = (() => { + const parsed = Number(env.EXTERNAL_API_SEARCH_MAX_RESULTS); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : 2000; +})(); + // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; diff --git a/packages/api/src/routers/external-api/v2/search.ts b/packages/api/src/routers/external-api/v2/search.ts index dfc98b1196..08dc8ea065 100644 --- a/packages/api/src/routers/external-api/v2/search.ts +++ b/packages/api/src/routers/external-api/v2/search.ts @@ -3,6 +3,7 @@ import express from 'express'; import { ObjectId } from 'mongodb'; import { z } from 'zod'; +import { EXTERNAL_API_SEARCH_MAX_RESULTS } from '@/config'; import { parseTimeRange } from '@/mcp/tools/query/helpers'; import { getNonNullUserWithTeam } from '@/middleware/auth'; import { processRequestWithEnhancedErrors as validateRequest } from '@/utils/enhancedErrors'; @@ -117,9 +118,10 @@ const CH_USER_INPUT_ERRORS = new Set([ * maxResults: * type: integer * minimum: 1 - * maximum: 2000 * default: 100 - * description: Maximum number of rows to return. Default is 100, max is 2000. + * description: | + * Maximum number of rows to return. Default is 100, max is 2000 + * unless the server sets EXTERNAL_API_SEARCH_MAX_RESULTS. * offset: * type: integer * minimum: 0 @@ -223,10 +225,12 @@ const searchRequestSchema = z.object({ .number() .int() .min(1) - .max(2000) + .max(EXTERNAL_API_SEARCH_MAX_RESULTS) .optional() .default(100) - .describe('Maximum rows to return (1-2000). Default: 100.'), + .describe( + `Maximum rows to return (1-${EXTERNAL_API_SEARCH_MAX_RESULTS}). Default: 100.`, + ), offset: z .number() .int() diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 8e54b666cf..d798002927 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -2650,7 +2650,9 @@ describe('renderChartConfig', () => { mockMetadata.getColumn.mockImplementation(async ({ column }) => column === 'date' ? ({ type: 'Date' } as ColumnMeta) - : ({ type: 'DateTime' } as ColumnMeta), + : column === 'TimestampTime' + ? ({ type: 'DateTime' } as ColumnMeta) + : ({ type: 'DateTime64(3)' } as ColumnMeta), ); }); @@ -2759,6 +2761,59 @@ describe('renderChartConfig', () => { ); }); + it.each([ + ['inclusive', true], + ['exclusive', false], + ])( + 'floors a sub-second %s start on a DateTime column and keeps it inclusive', + async (_, dateRangeStartInclusive) => { + const start = new Date('2026-10-13T22:03:21.827Z'); + const end = new Date('2026-10-13T22:05:00.500Z'); + + const actual = await timeFilterExpr({ + timestampValueExpression: 'TimestampTime, Timestamp', + dateRangeEndInclusive: false, + dateRangeStartInclusive, + dateRange: [start, end], + connectionId: 'test-connection', + databaseName: 'default', + tableName: 'target_table', + metadata: mockMetadata, + }); + + const precise = dateRangeStartInclusive ? '>=' : '>'; + expect(parameterizedQueryToSql(actual)).toBe( + `(TimestampTime >= fromUnixTimestamp64Milli(${new Date('2026-10-13T22:03:21Z').getTime()}) AND TimestampTime < fromUnixTimestamp64Milli(${end.getTime()}))AND(Timestamp ${precise} fromUnixTimestamp64Milli(${start.getTime()}) AND Timestamp < fromUnixTimestamp64Milli(${end.getTime()}))`, + ); + }, + ); + + it.each(['DateTime', "DateTime('UTC')", 'Nullable(DateTime)'])( + 'floors the start bound of a %s column', + async type => { + mockMetadata.getColumn.mockImplementation( + async () => ({ type }) as ColumnMeta, + ); + const start = new Date('2026-10-13T22:03:21.827Z'); + const end = new Date('2026-10-13T22:05:00Z'); + + const actual = await timeFilterExpr({ + timestampValueExpression: 'ts', + dateRangeEndInclusive: true, + dateRangeStartInclusive: true, + dateRange: [start, end], + connectionId: 'test-connection', + databaseName: 'default', + tableName: 'target_table', + metadata: mockMetadata, + }); + + expect(parameterizedQueryToSql(actual)).toBe( + `(ts >= fromUnixTimestamp64Milli(${new Date('2026-10-13T22:03:21Z').getTime()}) AND ts <= fromUnixTimestamp64Milli(${end.getTime()}))`, + ); + }, + ); + it.each(['Date32', 'Nullable(Date)', 'LowCardinality(Date)'])( 'wraps a %s column in toDate() and keeps the bounds inclusive', async type => { diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 70dab887c4..4bd8f04078 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -992,13 +992,21 @@ export async function timeFilterExpr({ ); } + const columnType = stripTypeWrappers(columnMeta?.type ?? ''); + // DateTime truncates to the second, so a sub-second or exclusive start + // compared as-is drops the rest of that second. + const isSecondPrecision = /^DateTime(?:\(.*\))?$/i.test(columnType); + const columnStartTime = isSecondPrecision + ? Math.floor(startTime / 1000) * 1000 + : startTime; + const rawStartBound = isRenderingRawSqlTemplate ? includedDataInterval ? chSql`toStartOfInterval($__fromTime_ms, INTERVAL $__interval_s second) - INTERVAL $__interval_s second` : chSql`$__fromTime_ms` : includedDataInterval - ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: startTime }}), INTERVAL ${includedDataInterval}) - INTERVAL ${includedDataInterval}` - : chSql`fromUnixTimestamp64Milli(${{ Int64: startTime }})`; + ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: columnStartTime }}), INTERVAL ${includedDataInterval}) - INTERVAL ${includedDataInterval}` + : chSql`fromUnixTimestamp64Milli(${{ Int64: columnStartTime }})`; const rawEndBound = isRenderingRawSqlTemplate ? includedDataInterval @@ -1016,13 +1024,13 @@ export async function timeFilterExpr({ ? chSql`${toStartOf.function}(${rawEndBound}${toStartOf.formattedRemainingArgs})` : rawEndBound; - const isDateType = - /^Date(?:32)?$/i.test(stripTypeWrappers(columnMeta?.type ?? '')) || - isToDateExpr; + const isDateType = /^Date(?:32)?$/i.test(columnType) || isToDateExpr; // toStartOf* and Date filters must stay inclusive — strict < on a rounded value drops a whole interval const startOp = - dateRangeStartInclusive || toStartOf || isDateType ? '>=' : '>'; + dateRangeStartInclusive || toStartOf || isDateType || isSecondPrecision + ? '>=' + : '>'; const endOp = dateRangeEndInclusive || toStartOf || isDateType ? '<=' : '<';