{suggestionsHeader}
+ {isLoadingValues && (
+
+ )}
{suggestedProperties.length > suggestionsLimit && (
diff --git a/packages/app/src/components/SearchInput/SearchInputV2.tsx b/packages/app/src/components/SearchInput/SearchInputV2.tsx
index 54cad034aa..7ff4545608 100644
--- a/packages/app/src/components/SearchInput/SearchInputV2.tsx
+++ b/packages/app/src/components/SearchInput/SearchInputV2.tsx
@@ -44,6 +44,7 @@ export default function SearchInputV2({
onSubmit,
additionalSuggestions,
queryHistoryType,
+ dateRange,
'data-testid': dataTestId,
...props
}: {
@@ -56,6 +57,7 @@ export default function SearchInputV2({
onSubmit?: () => void;
additionalSuggestions?: string[];
queryHistoryType?: string;
+ dateRange?: [Date, Date];
'data-testid'?: string;
} & UseControllerProps
&
TableConnectionChoice) {
@@ -67,12 +69,18 @@ export default function SearchInputV2({
const ref = useRef(null);
const [parsedEnglishQuery, setParsedEnglishQuery] = useState('');
- const autoCompleteOptions = useAutoCompleteOptions(
+ const {
+ options: autoCompleteOptions,
+ isLoadingValues,
+ tokenInfo,
+ } = useAutoCompleteOptions(
luceneLanguageFormatter,
value != null ? `${value}` : '',
{
tableConnection: tableConnection ? tableConnection : tableConnections,
additionalSuggestions,
+ dateRange,
+ inputRef: ref,
},
);
@@ -110,6 +118,8 @@ export default function SearchInputV2({
onChange={onChange}
placeholder={placeholder}
autocompleteOptions={autoCompleteOptions}
+ isLoadingValues={isLoadingValues}
+ tokenInfo={tokenInfo}
size={size}
zIndex={zIndex}
language={language}
diff --git a/packages/app/src/components/SearchInput/SearchWhereInput.tsx b/packages/app/src/components/SearchInput/SearchWhereInput.tsx
index e57fc12018..8eb82d07a6 100644
--- a/packages/app/src/components/SearchInput/SearchWhereInput.tsx
+++ b/packages/app/src/components/SearchInput/SearchWhereInput.tsx
@@ -103,6 +103,10 @@ export type SearchWhereInputProps = {
* Additional autocomplete/suggestion options (e.g. attribute names)
*/
additionalSuggestions?: string[];
+ /**
+ * Date range for autocomplete rollup queries
+ */
+ dateRange?: [Date, Date];
/**
* Form field name for the language value ('sql' | 'lucene').
* If not provided, defaults to `${name}Language` (e.g. name="where" → "whereLanguage").
@@ -152,6 +156,7 @@ export default function SearchWhereInput({
minWidth,
'data-testid': dataTestId,
additionalSuggestions,
+ dateRange,
languageName = `${name}Language`,
}: SearchWhereInputProps) {
const { field: languageField } = useController({
@@ -218,6 +223,7 @@ export default function SearchWhereInput({
size={size}
data-testid={dataTestId}
additionalSuggestions={additionalSuggestions}
+ dateRange={dateRange}
/>
)}
{enableHotkey && (
diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx
index 00eade5d9f..5192b0f497 100644
--- a/packages/app/src/hooks/useAutoCompleteOptions.tsx
+++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx
@@ -1,18 +1,44 @@
-import { useEffect, useMemo, useState } from 'react';
+import { useMemo } from 'react';
import {
Field,
TableConnection,
} from '@hyperdx/common-utils/dist/core/metadata';
-import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { NOW } from '@/config';
import {
deduplicate2dArray,
- useJsonColumns,
+ useCompleteKeyValues,
useMultipleAllFields,
- useMultipleGetKeyValues,
} from '@/hooks/useMetadata';
-import { mergePath, toArray } from '@/utils';
+import { toArray } from '@/utils';
+
+const ROLLUP_GRANULARITY = '15 minute' as const;
+
+export type TokenInfo = {
+ /** The full token at the cursor position */
+ token: string;
+ /** Index of the token in the tokens array */
+ index: number;
+ /** All tokens from splitting the input on whitespace */
+ tokens: string[];
+};
+
+/** Splits input into tokens and finds which token the cursor is in */
+function tokenizeAtCursor(value: string, cursorPos: number): TokenInfo {
+ const tokens = value.split(' ');
+ let idx = 0;
+ let pos = 0;
+ for (let i = 0; i < tokens.length; i++) {
+ pos += tokens[i].length;
+ if (pos >= cursorPos || i === tokens.length - 1) {
+ idx = i;
+ break;
+ }
+ pos++; // account for the space
+ idx = i + 1;
+ }
+ return { token: tokens[idx] ?? '', index: idx, tokens };
+}
export interface ILanguageFormatter {
formatFieldValue: (f: Field) => string;
@@ -26,19 +52,45 @@ export function useAutoCompleteOptions(
{
tableConnection,
additionalSuggestions,
+ dateRange,
+ inputRef,
}: {
tableConnection?: TableConnection | TableConnection[];
additionalSuggestions?: string[];
+ dateRange?: [Date, Date];
+ inputRef?: React.RefObject;
},
) {
- // Fetch and gather all field options
- const { data: fields } = useMultipleAllFields(
- tableConnection
- ? Array.isArray(tableConnection)
- ? tableConnection
- : [tableConnection]
- : [],
+ const tcs = useMemo(() => toArray(tableConnection), [tableConnection]);
+
+ // Add fieldsRollupView to table connections — will eventually come from Source config
+ const tcsWithRollup = useMemo(
+ () =>
+ tcs.map(tc => {
+ if (tc.tableName === 'otel_logs' || tc.tableName === 'otel_traces') {
+ return {
+ ...tc,
+ fieldsRollupView: {
+ name: `${tc.tableName}_key_rollup_15m`,
+ granularity: ROLLUP_GRANULARITY,
+ },
+ };
+ }
+ return tc;
+ }),
+ [tcs],
);
+
+ const effectiveDateRange: [Date, Date] = useMemo(
+ () => dateRange ?? [new Date(NOW - 24 * 60 * 60 * 1000), new Date(NOW)],
+ [dateRange],
+ );
+
+ // Fetch fields, using rollup for map key discovery when available
+ const { data: fields } = useMultipleAllFields(tcsWithRollup, {
+ dateRange: effectiveDateRange,
+ });
+
const { fieldCompleteOptions, fieldCompleteMap } = useMemo(() => {
const _columns = (fields ?? []).filter(c => c.jsType !== null);
@@ -63,121 +115,53 @@ export function useAutoCompleteOptions(
return { fieldCompleteOptions, fieldCompleteMap };
}, [formatter, fields, additionalSuggestions]);
- // searchField is used for the purpose of checking if a key is valid and key values should be fetched
- // TODO: Come back and refactor how this works - it's not great and wouldn't catch a person copy-pasting some text
- const [searchField, setSearchField] = useState(null);
- // check if any search field matches
- useEffect(() => {
- const v = fieldCompleteMap.get(value);
- if (v) {
- setSearchField(v);
- }
- }, [fieldCompleteMap, value]);
- // clear search field if no key matches anymore
- useEffect(() => {
- if (!searchField) return;
- if (!value.startsWith(formatter.formatFieldValue(searchField))) {
- setSearchField(null);
- }
- }, [searchField, setSearchField, value, formatter]);
- const tcForJson = Array.isArray(tableConnection)
- ? tableConnection.length > 0
- ? tableConnection[0]
- : undefined
- : tableConnection;
- const { data: jsonColumns } = useJsonColumns(
- tcForJson ?? {
- tableName: '',
- databaseName: '',
- connectionId: '',
- },
- );
- const searchKeys = useMemo(
- () =>
- searchField && jsonColumns
- ? [mergePath(searchField.path, jsonColumns)]
- : [],
- [searchField, jsonColumns],
+ // Tokenize input at cursor position
+ const tokenInfo = useMemo(() => {
+ const cursorPos = inputRef?.current?.selectionStart ?? value.length;
+ return tokenizeAtCursor(value, cursorPos);
+ }, [value, inputRef]);
+
+ // Extract the field name portion of the token (strip colon and value)
+ const fieldNameAtCursor = useMemo(() => {
+ const colonIdx = tokenInfo.token.indexOf(':');
+ return colonIdx >= 0 ? tokenInfo.token.slice(0, colonIdx) : tokenInfo.token;
+ }, [tokenInfo.token]);
+
+ // Derive the active search field from the token at cursor
+ const searchField = useMemo(
+ () => fieldCompleteMap.get(fieldNameAtCursor) ?? null,
+ [fieldCompleteMap, fieldNameAtCursor],
);
- // hooks to get key values
- const chartConfigs: BuilderChartConfigWithDateRange[] = toArray(
- tableConnection,
- ).map(({ databaseName, tableName, connectionId }) => ({
- connection: connectionId,
- from: {
- databaseName,
- tableName,
+ // Debounced fetch of values for the selected key from rollup tables
+ const firstTc = tcs.length > 0 ? tcs[0] : undefined;
+ const { data: keyValues, isFetching: isLoadingValues } = useCompleteKeyValues(
+ {
+ tableConnection: firstTc,
+ searchField,
+ dateRange: effectiveDateRange,
},
- timestampValueExpression: '',
- select: '',
- where: '',
- // TODO: Pull in date for query as arg
- // just assuming 1/2 day is okay to query over right now
- dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)],
- }));
-
- const { data: keyVals } = useMultipleGetKeyValues({
- chartConfigs,
- keys: searchKeys,
- });
+ );
+ // Build key-value pair suggestions
const keyValCompleteOptions = useMemo<
{ value: string; label: string }[]
>(() => {
- if (!keyVals || !searchField) return fieldCompleteOptions;
- const output = // TODO: Fix this hacky type assertion caused by bug in HDX-1548
- (
- keyVals as unknown as {
- key: string;
- value: (string | { [key: string]: string })[];
- }[]
- ).flatMap(kv => {
- return kv.value.flatMap(v => {
- if (typeof v === 'string') {
- const value = formatter.formatKeyValPair(
- formatter.formatFieldValue(searchField),
- v,
- );
- return [
- {
- value,
- label: value,
- },
- ];
- } else if (typeof v === 'object') {
- // TODO: Fix type issues mentioned in HDX-1548
- const output: {
- value: string;
- label: string;
- }[] = [];
- for (const [key, val] of Object.entries(v)) {
- if (typeof key !== 'string' || typeof val !== 'string') {
- console.error('unknown type for autocomplete object ', v);
- return [];
- }
- const field = structuredClone(searchField);
- field.path.push(key);
- const value = formatter.formatKeyValPair(
- formatter.formatFieldValue(field),
- val,
- );
- output.push({
- value,
- label: value,
- });
- }
- return output;
- } else {
- return [];
- }
- });
- });
- return output;
- }, [fieldCompleteOptions, keyVals, searchField, formatter]);
-
- // combine all autocomplete options
- return useMemo(() => {
+ if (!keyValues || !searchField || keyValues.length === 0) return [];
+
+ return keyValues.map(v => {
+ const formatted = formatter.formatKeyValPair(
+ formatter.formatFieldValue(searchField),
+ v,
+ );
+ return { value: formatted, label: formatted };
+ });
+ }, [keyValues, searchField, formatter]);
+
+ // Combine all autocomplete options
+ const options = useMemo(() => {
return deduplicate2dArray([fieldCompleteOptions, keyValCompleteOptions]);
}, [fieldCompleteOptions, keyValCompleteOptions]);
+
+ return { options, isLoadingValues, tokenInfo };
}
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index 86a0f79e0c..2454fcfc9f 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -123,14 +123,19 @@ export function useJsonColumns(
export function useMultipleAllFields(
tableConnections: TableConnection[],
- options?: Partial>,
+ options?: Partial> & {
+ dateRange?: [Date, Date];
+ },
) {
const metadata = useMetadataWithSettings();
const { data: me, isFetched } = api.useMe();
+ const { dateRange, ...queryOptions } = options ?? {};
return useQuery({
queryKey: [
'useMetadata.useMultipleAllFields',
...tableConnections.map(tc => ({ ...tc })),
+ dateRange?.[0]?.getTime(),
+ dateRange?.[1]?.getTime(),
],
queryFn: async () => {
const team = me?.team;
@@ -139,7 +144,7 @@ export function useMultipleAllFields(
}
const promiseResults = await Promise.allSettled(
- tableConnections.map(tc => metadata.getAllFields(tc)),
+ tableConnections.map(tc => metadata.getAllFields({ ...tc, dateRange })),
);
const fields2d: Field[][] = promiseResults.map(result => {
@@ -164,13 +169,123 @@ export function useMultipleAllFields(
tc => !!tc.databaseName && !!tc.tableName && !!tc.connectionId,
) &&
isFetched,
- ...options,
+ ...queryOptions,
+ });
+}
+
+/** Tables that have rollup tables */
+const TABLES_WITH_ROLLUPS = new Set(['otel_logs', 'otel_traces']);
+
+/**
+ * Resolves a field to the (ColumnIdentifier, Key) pair for querying the rollup table.
+ * Map fields: ColumnIdentifier = column name (e.g. 'ResourceAttributes'), Key = map key
+ * Native fields: ColumnIdentifier = 'NativeColumn', Key = column name (e.g. 'ServiceName')
+ */
+function fieldToRollupParams(
+ field: Field | null,
+ tableName: string | undefined,
+): { columnIdentifier: string; key: string } | null {
+ if (!field || !tableName) return null;
+
+ if (field.path.length >= 2) {
+ const [column, mapKey] = field.path;
+ if (getMapColumnsForTable(tableName).includes(column)) {
+ return { columnIdentifier: column, key: mapKey };
+ }
+ } else if (field.path.length === 1) {
+ if (TABLES_WITH_ROLLUPS.has(tableName)) {
+ return { columnIdentifier: 'NativeColumn', key: field.path[0] };
+ }
+ }
+
+ return null;
+}
+
+/**
+ * Debounced hook that fetches values for a specific field from rollup tables.
+ * Works for both map keys (e.g. "ResourceAttributes.http.method") and
+ * native columns (e.g. "ServiceName").
+ */
+export function useCompleteKeyValues({
+ tableConnection,
+ searchField,
+ dateRange,
+}: {
+ tableConnection: TableConnection | undefined;
+ searchField: Field | null;
+ dateRange: [Date, Date];
+}) {
+ const metadata = useMetadataWithSettings();
+
+ // Debounce: only query after the field stabilizes for 300ms
+ const [debouncedField, setDebouncedField] = useState(null);
+ useEffect(() => {
+ const timer = setTimeout(() => setDebouncedField(searchField), 300);
+ return () => clearTimeout(timer);
+ }, [searchField]);
+
+ const rollupParams = fieldToRollupParams(
+ debouncedField,
+ tableConnection?.tableName,
+ );
+
+ return useQuery({
+ queryKey: [
+ 'useCompleteKeyValues',
+ tableConnection?.databaseName,
+ tableConnection?.tableName,
+ tableConnection?.connectionId,
+ rollupParams?.columnIdentifier,
+ rollupParams?.key,
+ dateRange[0].getTime(),
+ dateRange[1].getTime(),
+ ],
+ queryFn: async ({ signal }) => {
+ if (!tableConnection || !rollupParams || !debouncedField) return [];
+
+ // Try rollup first
+ const rollupValues = await metadata.getCompleteKeyValues({
+ databaseName: tableConnection.databaseName,
+ tableName: tableConnection.tableName,
+ column: rollupParams.columnIdentifier,
+ key: rollupParams.key,
+ connectionId: tableConnection.connectionId,
+ dateRange,
+ signal,
+ });
+
+ if (rollupValues.length > 0) return rollupValues;
+
+ // Fall back to main table scan
+ if (rollupParams.columnIdentifier !== 'NativeColumn') {
+ // Map column: use getMapValues
+ return metadata.getMapValues({
+ databaseName: tableConnection.databaseName,
+ tableName: tableConnection.tableName,
+ column: rollupParams.columnIdentifier,
+ key: rollupParams.key,
+ connectionId: tableConnection.connectionId,
+ });
+ } else {
+ // Native column: use getMapValues without a key (queries column directly)
+ return metadata.getMapValues({
+ databaseName: tableConnection.databaseName,
+ tableName: tableConnection.tableName,
+ column: debouncedField.path[0],
+ connectionId: tableConnection.connectionId,
+ });
+ }
+ },
+ staleTime: 1000 * 60 * 5,
+ enabled: !!rollupParams,
});
}
export function useAllFields(
tableConnection: TableConnection | undefined,
- options?: Partial>,
+ options?: Partial> & {
+ dateRange?: [Date, Date];
+ },
) {
return useMultipleAllFields(
tableConnection ? [tableConnection] : [],
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index e3fb2ffc0a..7c1c98b29d 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -740,7 +740,7 @@ describe('Metadata', () => {
const fields = await md.getAllFields({
databaseName: 'otel',
- tableName: 'otel_logs',
+ tableName: 'test_logs',
connectionId: 'conn-1',
});
@@ -819,7 +819,7 @@ describe('Metadata', () => {
const fields = await md.getAllFields({
databaseName: 'otel',
- tableName: 'otel_logs',
+ tableName: 'test_logs',
connectionId: 'conn-1',
});
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 2d13ecc8fc..0b336406d7 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -21,13 +21,32 @@ import type {
import { SourceKind } from '@/types';
import { optimizeGetKeyValuesCalls } from './materializedViews';
-import { getDistributedTableArgs, objectHash } from './utils';
+import {
+ getAlignedDateRange,
+ getDistributedTableArgs,
+ objectHash,
+} from './utils';
// If filters initially are taking too long to load, decrease this number.
// Between 1e6 - 5e6 is a good range.
export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6;
const DEFAULT_MAX_KEYS = 1000;
+/** Tables that have rollup materialized views */
+const TABLES_WITH_ROLLUPS = new Set(['otel_logs', 'otel_traces']);
+
+/** Returns the key-only rollup table name for a base table, or undefined if not supported */
+function getKeyRollupTableName(tableName: string): string | undefined {
+ if (!TABLES_WITH_ROLLUPS.has(tableName)) return undefined;
+ return `${tableName}_key_rollup_15m`;
+}
+
+/** Returns the KV rollup table name for a base table, or undefined if not supported */
+function getKvRollupTableName(tableName: string): string | undefined {
+ if (!TABLES_WITH_ROLLUPS.has(tableName)) return undefined;
+ return `${tableName}_kv_rollup_15m`;
+}
+
export class MetadataCache {
private cache = new Map();
private pendingQueries = new Map>();
@@ -339,6 +358,8 @@ export class Metadata {
maxKeys = DEFAULT_MAX_KEYS,
connectionId,
metricName,
+ fieldsRollupView,
+ dateRange,
}: {
databaseName: string;
tableName: string;
@@ -346,16 +367,60 @@ export class Metadata {
maxKeys?: number;
connectionId: string;
metricName?: string;
+ fieldsRollupView?: FieldsRollupView;
+ dateRange?: [Date, Date];
}) {
+ // Align date range to rollup granularity for consistent cache keys
+ const alignedDateRange =
+ fieldsRollupView && dateRange
+ ? getAlignedDateRange(dateRange, fieldsRollupView.granularity)
+ : undefined;
+
const cacheKey = metricName
? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.keys`
- : `${connectionId}.${databaseName}.${tableName}.${column}.keys`;
+ : fieldsRollupView && alignedDateRange
+ ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.keys`
+ : `${connectionId}.${databaseName}.${tableName}.${column}.keys`;
const cachedKeys = this.cache.get(cacheKey);
if (cachedKeys != null) {
return cachedKeys;
}
+ // Rollup path: query the key rollup table filtered by ColumnIdentifier and date range
+ if (fieldsRollupView && alignedDateRange) {
+ return this.cache.getOrFetch(cacheKey, async () => {
+ const timeFilter = chSql`AND Timestamp >= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})) AND Timestamp <= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }}))`;
+ const sql = chSql`
+ SELECT Key
+ FROM ${tableExpr({ database: databaseName, table: fieldsRollupView.name })}
+ WHERE ColumnIdentifier = ${{ String: column }}
+ ${timeFilter}
+ GROUP BY Key
+ ORDER BY sum(count) DESC
+ LIMIT ${{ Int32: maxKeys }}
+ `;
+
+ const keys = await this.clickhouseClient
+ .query<'JSON'>({
+ query: sql.sql,
+ query_params: sql.params,
+ connectionId,
+ clickhouse_settings: {
+ ...this.getClickHouseSettings(),
+ timeout_overflow_mode: 'break',
+ max_execution_time: 15,
+ max_rows_to_read: '0',
+ },
+ })
+ .then(res => res.json<{ Key: string }>())
+ .then(d => d.data.map(row => row.Key).filter(k => k));
+
+ return keys;
+ });
+ }
+
+ // Original path: scan main table
const colMeta = await this.getColumn({
databaseName,
tableName,
@@ -586,7 +651,9 @@ export class Metadata {
tableName,
connectionId,
metricName,
- }: TableConnection) {
+ fieldsRollupView,
+ dateRange,
+ }: TableConnection & { dateRange?: [Date, Date] }) {
const fields: Field[] = [];
const columns = await this.getColumns({
databaseName,
@@ -634,6 +701,8 @@ export class Metadata {
column: column.name,
connectionId,
metricName,
+ fieldsRollupView,
+ dateRange,
});
const match = column.type.match(/Map\(.+,\s*(.+)\)/);
@@ -1132,6 +1201,70 @@ export class Metadata {
);
}
+ /**
+ * Autocomplete: fetches top values for a specific map key from the KV rollup table.
+ * Only filters by date range — no WHERE conditions. Values ordered by frequency.
+ */
+ async getCompleteKeyValues({
+ databaseName,
+ tableName,
+ column,
+ key,
+ maxValues = 1000,
+ connectionId,
+ dateRange,
+ signal,
+ }: {
+ databaseName: string;
+ tableName: string;
+ column: string;
+ key: string;
+ maxValues?: number;
+ connectionId: string;
+ dateRange: [Date, Date];
+ signal?: AbortSignal;
+ }): Promise {
+ const kvRollupTable = getKvRollupTableName(tableName);
+ if (!kvRollupTable) return [];
+
+ const timeFilter = chSql`AND Timestamp >= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})) AND Timestamp <= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }}))`;
+ const cacheKey = `${connectionId}.${databaseName}.${tableName}.${column}.${key}.${dateRange[0].getTime()}.${dateRange[1].getTime()}.completeKeyValues`;
+
+ return this.cache.getOrFetch(cacheKey, async () => {
+ try {
+ const sql = chSql`
+ SELECT Value
+ FROM ${tableExpr({ database: databaseName, table: kvRollupTable })}
+ WHERE ColumnIdentifier = ${{ String: column }}
+ AND Key = ${{ String: key }}
+ AND Value != ''
+ ${timeFilter}
+ GROUP BY Value
+ ORDER BY sum(count) DESC
+ LIMIT ${{ Int32: maxValues }}
+ `;
+
+ return await this.clickhouseClient
+ .query<'JSON'>({
+ query: sql.sql,
+ query_params: sql.params,
+ connectionId,
+ clickhouse_settings: {
+ ...this.getClickHouseSettings(),
+ timeout_overflow_mode: 'break',
+ max_execution_time: 15,
+ max_rows_to_read: '0',
+ },
+ abort_signal: signal,
+ })
+ .then(res => res.json<{ Value: string }>())
+ .then(d => d.data.map(r => r.Value));
+ } catch {
+ return [];
+ }
+ });
+ }
+
async getKeyValues({
chartConfig,
keys,
@@ -1323,11 +1456,18 @@ export type Field = {
jsType: JSDataType | null;
};
+export type FieldsRollupView = {
+ name: string;
+ granularity: string;
+};
+
+// Describes a table and potentially related views
export type TableConnection = {
databaseName: string;
tableName: string;
connectionId: string;
metricName?: string;
+ fieldsRollupView?: FieldsRollupView;
};
export type TableConnectionChoice =
From 5b47c0ef3269b8e826612bcde651ad14fbcf4130 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Wed, 15 Apr 2026 20:40:52 -0400
Subject: [PATCH 02/12] feat: add metadata materialized views to SourceForm and
also inferred config
---
docker-compose.yml | 6 +-
docker/hyperdx/entry.local.base.sh | 2 +-
packages/api/.env.development | 2 +-
packages/api/openapi.json | 44 +++++++++
packages/api/src/models/source.ts | 16 +++
.../src/routers/external-api/v2/sources.ts | 66 +++++++++++--
packages/app/src/DBSearchPage.tsx | 17 +---
.../src/components/DBSearchPageFilters.tsx | 15 +--
.../app/src/components/Sources/SourceForm.tsx | 99 +++++++++++++++++++
.../app/src/hooks/useAutoCompleteOptions.tsx | 24 +----
packages/app/src/hooks/useMetadata.tsx | 23 ++---
packages/app/src/source.ts | 32 ++++++
packages/common-utils/src/core/metadata.ts | 54 ++++------
packages/common-utils/src/types.ts | 12 +++
14 files changed, 304 insertions(+), 108 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 3e6fe3ba58..d950a54909 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -68,8 +68,10 @@ services:
'[{"name":"Local
ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]'
DEFAULT_SOURCES:
- '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local
- ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local
+ '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15
+ minute"},"connection":"Local
+ ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15
+ minute"},"connection":"Local
ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local
ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local
ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]'
diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh
index 7d350669d4..212280c5a8 100644
--- a/docker/hyperdx/entry.local.base.sh
+++ b/docker/hyperdx/entry.local.base.sh
@@ -30,7 +30,7 @@ if [ -z "${DEFAULT_CONNECTIONS}" ]; then
DEFAULT_CONNECTIONS='[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]'
fi
export DEFAULT_CONNECTIONS
-export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]'
+export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]'
# if BETA_CH_OTEL_JSON_SCHEMA_ENABLED is true then return empty default sources
if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then
export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]'
diff --git a/packages/api/.env.development b/packages/api/.env.development
index 52421a6c66..ddb1d85cb5 100644
--- a/packages/api/.env.development
+++ b/packages/api/.env.development
@@ -18,7 +18,7 @@ USAGE_STATS_ENABLED=false
NODE_OPTIONS="--max-http-header-size=131072"
ENABLE_SWAGGER=true
DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}]
-DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}]
+DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}]
INGESTION_API_KEY="super-secure-ingestion-api-key"
HYPERDX_API_KEY=$INGESTION_API_KEY
ANTHROPIC_API_KEY="your-anthropic-api-key-here"
diff --git a/packages/api/openapi.json b/packages/api/openapi.json
index 4b6dc21307..9424da0b28 100644
--- a/packages/api/openapi.json
+++ b/packages/api/openapi.json
@@ -2501,6 +2501,28 @@
"$ref": "#/components/schemas/MaterializedView"
},
"nullable": true
+ },
+ "metadataMaterializedViews": {
+ "type": "object",
+ "description": "Configure materialized views for fast field discovery and value autocomplete.",
+ "nullable": true,
+ "properties": {
+ "keyRollupTable": {
+ "type": "string",
+ "description": "ClickHouse table name for the key rollup (field discovery).",
+ "example": "otel_logs_key_rollup_15m"
+ },
+ "kvRollupTable": {
+ "type": "string",
+ "description": "ClickHouse table name for the key-value rollup (value autocomplete).",
+ "example": "otel_logs_kv_rollup_15m"
+ },
+ "granularity": {
+ "type": "string",
+ "description": "The time granularity of the rollup tables.",
+ "example": "15m"
+ }
+ }
}
}
},
@@ -2688,6 +2710,28 @@
"$ref": "#/components/schemas/MaterializedView"
},
"nullable": true
+ },
+ "metadataMaterializedViews": {
+ "type": "object",
+ "description": "Configure materialized views for fast field discovery and value autocomplete.",
+ "nullable": true,
+ "properties": {
+ "keyRollupTable": {
+ "type": "string",
+ "description": "ClickHouse table name for the key rollup (field discovery).",
+ "example": "otel_traces_key_rollup_15m"
+ },
+ "kvRollupTable": {
+ "type": "string",
+ "description": "ClickHouse table name for the key-value rollup (value autocomplete).",
+ "example": "otel_traces_kv_rollup_15m"
+ },
+ "granularity": {
+ "type": "string",
+ "description": "The time granularity of the rollup tables.",
+ "example": "15m"
+ }
+ }
}
}
},
diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts
index 25201dd0f9..c24905f4a7 100644
--- a/packages/api/src/models/source.ts
+++ b/packages/api/src/models/source.ts
@@ -144,6 +144,14 @@ export const LogSource = Source.discriminator(
materializedViews: {
type: mongoose.Schema.Types.Array,
},
+ metadataMaterializedViews: {
+ type: {
+ keyRollupTable: String,
+ kvRollupTable: String,
+ granularity: String,
+ },
+ default: undefined,
+ },
orderByExpression: String,
}),
);
@@ -184,6 +192,14 @@ export const TraceSource = Source.discriminator(
materializedViews: {
type: mongoose.Schema.Types.Array,
},
+ metadataMaterializedViews: {
+ type: {
+ keyRollupTable: String,
+ kvRollupTable: String,
+ granularity: String,
+ },
+ default: undefined,
+ },
orderByExpression: String,
}),
);
diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts
index 06d8767d6a..e61f22176f 100644
--- a/packages/api/src/routers/external-api/v2/sources.ts
+++ b/packages/api/src/routers/external-api/v2/sources.ts
@@ -30,17 +30,29 @@ export function mapGranularityToExternalFormat(granularity: string): string {
function mapSourceToExternalSource(source: TSource): TSource {
if (!('materializedViews' in source)) return source;
- if (!Array.isArray(source.materializedViews)) return source;
- return {
- ...source,
- materializedViews: source.materializedViews.map(view => {
- return {
- ...view,
- minGranularity: mapGranularityToExternalFormat(view.minGranularity),
- };
- }),
- };
+ const mapped = { ...source };
+
+ if (Array.isArray(source.materializedViews)) {
+ mapped.materializedViews = source.materializedViews.map(view => ({
+ ...view,
+ minGranularity: mapGranularityToExternalFormat(view.minGranularity),
+ }));
+ }
+
+ if (
+ 'metadataMaterializedViews' in source &&
+ source.metadataMaterializedViews
+ ) {
+ mapped.metadataMaterializedViews = {
+ ...source.metadataMaterializedViews,
+ granularity: mapGranularityToExternalFormat(
+ source.metadataMaterializedViews.granularity,
+ ),
+ };
+ }
+
+ return mapped;
}
function applyLegacyDefaults(
@@ -359,6 +371,23 @@ function formatExternalSource(source: SourceDocument) {
* items:
* $ref: '#/components/schemas/MaterializedView'
* nullable: true
+ * metadataMaterializedViews:
+ * type: object
+ * description: Configure materialized views for fast field discovery and value autocomplete.
+ * nullable: true
+ * properties:
+ * keyRollupTable:
+ * type: string
+ * description: ClickHouse table name for the key rollup (field discovery).
+ * example: otel_logs_key_rollup_15m
+ * kvRollupTable:
+ * type: string
+ * description: ClickHouse table name for the key-value rollup (value autocomplete).
+ * example: otel_logs_kv_rollup_15m
+ * granularity:
+ * type: string
+ * description: The time granularity of the rollup tables.
+ * example: 15m
* TraceSource:
* type: object
* required:
@@ -509,6 +538,23 @@ function formatExternalSource(source: SourceDocument) {
* items:
* $ref: '#/components/schemas/MaterializedView'
* nullable: true
+ * metadataMaterializedViews:
+ * type: object
+ * description: Configure materialized views for fast field discovery and value autocomplete.
+ * nullable: true
+ * properties:
+ * keyRollupTable:
+ * type: string
+ * description: ClickHouse table name for the key rollup (field discovery).
+ * example: otel_traces_key_rollup_15m
+ * kvRollupTable:
+ * type: string
+ * description: ClickHouse table name for the key-value rollup (value autocomplete).
+ * example: otel_traces_kv_rollup_15m
+ * granularity:
+ * type: string
+ * description: The time granularity of the rollup tables.
+ * example: 15m
* MetricSource:
* type: object
* required:
diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx
index c3f310ada7..099c084962 100644
--- a/packages/app/src/DBSearchPage.tsx
+++ b/packages/app/src/DBSearchPage.tsx
@@ -1612,19 +1612,10 @@ export function DBSearchPage() {
],
);
- const inputSourceTableConnection = useMemo(() => {
- const tc = tcFromSource(inputSourceObj);
- if (tc.tableName === 'otel_logs' || tc.tableName === 'otel_traces') {
- return {
- ...tc,
- fieldsRollupView: {
- name: `${tc.tableName}_key_rollup_15m`,
- granularity: '15 minute',
- },
- };
- }
- return tc;
- }, [inputSourceObj]);
+ const inputSourceTableConnection = useMemo(
+ () => tcFromSource(inputSourceObj),
+ [inputSourceObj],
+ );
const sourceSchemaPreview = useMemo(
() => ,
diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx
index a51f6e4e82..28889e3b92 100644
--- a/packages/app/src/components/DBSearchPageFilters.tsx
+++ b/packages/app/src/components/DBSearchPageFilters.tsx
@@ -1048,20 +1048,14 @@ const DBSearchPageFiltersComponent = ({
tableName: chartConfig.from.tableName,
connectionId: chartConfig.connection,
});
- const fieldsRollupView =
- chartConfig.from.tableName === 'otel_logs' ||
- chartConfig.from.tableName === 'otel_traces'
- ? {
- name: `${chartConfig.from.tableName}_key_rollup_15m`,
- granularity: '15 minute',
- }
- : undefined;
+ const { data: source } = useSource({ id: sourceId });
+ const sourceTableConnection = tcFromSource(source);
const { data, isLoading, error } = useAllFields(
{
databaseName: chartConfig.from.databaseName,
tableName: chartConfig.from.tableName,
connectionId: chartConfig.connection,
- fieldsRollupView,
+ metadataMVs: sourceTableConnection.metadataMVs,
},
{
dateRange: chartConfig.dateRange,
@@ -1073,8 +1067,7 @@ const DBSearchPageFiltersComponent = ({
connectionId: chartConfig.connection,
});
- const { data: source } = useSource({ id: sourceId });
- const { data: tableMetadata } = useTableMetadata(tcFromSource(source));
+ const { data: tableMetadata } = useTableMetadata(sourceTableConnection);
useEffect(() => {
if (error) {
diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx
index 57186fe4f4..7a0dde7a6f 100644
--- a/packages/app/src/components/Sources/SourceForm.tsx
+++ b/packages/app/src/components/Sources/SourceForm.tsx
@@ -563,6 +563,96 @@ function MaterializedViewsFormSection({ control, setValue }: TableModelProps) {
);
}
+/** Component for configuring metadata materialized views (key + KV rollups) */
+function MetadataMaterializedViewsFormSection({
+ control,
+ setValue,
+}: TableModelProps) {
+ const databaseName = useWatch({
+ control,
+ name: `from.databaseName`,
+ defaultValue: DEFAULT_DATABASE,
+ });
+ const connection = useWatch({ control, name: `connection` });
+
+ const metadataMVs = useWatch({
+ control,
+ name: 'metadataMaterializedViews',
+ });
+
+ const hasMetadataMVs = !!metadataMVs;
+
+ return (
+
+
+ {hasMetadataMVs ? (
+
+
+ setValue('metadataMaterializedViews', undefined)}
+ >
+
+
+
+
+
+
+ Key Rollup Table
+
+
+
+
+
+ KV Rollup Table
+
+
+
+
+
+
+ ) : (
+
+ )}
+
+
+ );
+}
+
/** Component for configuring a single materialized view */
function MaterializedViewFormSection({
control,
@@ -1314,6 +1404,8 @@ function LogTableModelForm(props: TableModelProps) {
+
+
+
+
{
+ if (value && typeof value === 'object' && !Array.isArray(value)) {
+ setValue(key as any, value);
+ return;
+ }
resetField(key as any, {
keepDirty: true,
defaultValue: value,
@@ -1909,6 +2007,7 @@ export function TableSourceForm({
watchedKind,
resetField,
metadata,
+ setValue,
]);
// Sets the default connection field to the first connection after the
diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx
index 5192b0f497..306234716f 100644
--- a/packages/app/src/hooks/useAutoCompleteOptions.tsx
+++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx
@@ -12,8 +12,6 @@ import {
} from '@/hooks/useMetadata';
import { toArray } from '@/utils';
-const ROLLUP_GRANULARITY = '15 minute' as const;
-
export type TokenInfo = {
/** The full token at the cursor position */
token: string;
@@ -63,31 +61,13 @@ export function useAutoCompleteOptions(
) {
const tcs = useMemo(() => toArray(tableConnection), [tableConnection]);
- // Add fieldsRollupView to table connections — will eventually come from Source config
- const tcsWithRollup = useMemo(
- () =>
- tcs.map(tc => {
- if (tc.tableName === 'otel_logs' || tc.tableName === 'otel_traces') {
- return {
- ...tc,
- fieldsRollupView: {
- name: `${tc.tableName}_key_rollup_15m`,
- granularity: ROLLUP_GRANULARITY,
- },
- };
- }
- return tc;
- }),
- [tcs],
- );
-
const effectiveDateRange: [Date, Date] = useMemo(
() => dateRange ?? [new Date(NOW - 24 * 60 * 60 * 1000), new Date(NOW)],
[dateRange],
);
// Fetch fields, using rollup for map key discovery when available
- const { data: fields } = useMultipleAllFields(tcsWithRollup, {
+ const { data: fields } = useMultipleAllFields(tcs, {
dateRange: effectiveDateRange,
});
@@ -117,7 +97,9 @@ export function useAutoCompleteOptions(
// Tokenize input at cursor position
const tokenInfo = useMemo(() => {
+ // eslint-disable-next-line react-hooks/refs
const cursorPos = inputRef?.current?.selectionStart ?? value.length;
+ // eslint-disable-next-line react-hooks/refs
return tokenizeAtCursor(value, cursorPos);
}, [value, inputRef]);
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index 2454fcfc9f..0ea7ccf5cd 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -173,29 +173,24 @@ export function useMultipleAllFields(
});
}
-/** Tables that have rollup tables */
-const TABLES_WITH_ROLLUPS = new Set(['otel_logs', 'otel_traces']);
-
/**
* Resolves a field to the (ColumnIdentifier, Key) pair for querying the rollup table.
* Map fields: ColumnIdentifier = column name (e.g. 'ResourceAttributes'), Key = map key
* Native fields: ColumnIdentifier = 'NativeColumn', Key = column name (e.g. 'ServiceName')
+ *
+ * Only returns a result when the table connection has metadataMVs configured.
*/
function fieldToRollupParams(
field: Field | null,
- tableName: string | undefined,
+ tableConnection: TableConnection | undefined,
): { columnIdentifier: string; key: string } | null {
- if (!field || !tableName) return null;
+ if (!field || !tableConnection?.metadataMVs) return null;
if (field.path.length >= 2) {
const [column, mapKey] = field.path;
- if (getMapColumnsForTable(tableName).includes(column)) {
- return { columnIdentifier: column, key: mapKey };
- }
+ return { columnIdentifier: column, key: mapKey };
} else if (field.path.length === 1) {
- if (TABLES_WITH_ROLLUPS.has(tableName)) {
- return { columnIdentifier: 'NativeColumn', key: field.path[0] };
- }
+ return { columnIdentifier: 'NativeColumn', key: field.path[0] };
}
return null;
@@ -224,10 +219,7 @@ export function useCompleteKeyValues({
return () => clearTimeout(timer);
}, [searchField]);
- const rollupParams = fieldToRollupParams(
- debouncedField,
- tableConnection?.tableName,
- );
+ const rollupParams = fieldToRollupParams(debouncedField, tableConnection);
return useQuery({
queryKey: [
@@ -250,6 +242,7 @@ export function useCompleteKeyValues({
column: rollupParams.columnIdentifier,
key: rollupParams.key,
connectionId: tableConnection.connectionId,
+ metadataMVs: tableConnection.metadataMVs,
dateRange,
signal,
});
diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts
index 504d03ce39..e361f57a2c 100644
--- a/packages/app/src/source.ts
+++ b/packages/app/src/source.ts
@@ -348,6 +348,36 @@ export async function inferTableSourceConfig({
// Check if SpanEvents column is available
const hasSpanEvents = columns.some(col => col.name === 'Events.Timestamp');
+ // Check if metadata rollup tables exist
+ const hasMetadataMVs =
+ isOtelLogSchema || isOtelSpanSchema
+ ? await (async () => {
+ const [keyMeta, kvMeta] = await Promise.all([
+ metadata.getTableMetadata({
+ databaseName,
+ tableName: `${tableName}_key_rollup_15m`,
+ connectionId,
+ }),
+ metadata.getTableMetadata({
+ databaseName,
+ tableName: `${tableName}_kv_rollup_15m`,
+ connectionId,
+ }),
+ ]);
+ return keyMeta != null && kvMeta != null;
+ })()
+ : false;
+
+ const metadataMVsConfig = hasMetadataMVs
+ ? {
+ metadataMaterializedViews: {
+ keyRollupTable: `${tableName}_key_rollup_15m`,
+ kvRollupTable: `${tableName}_kv_rollup_15m`,
+ granularity: '15 minute',
+ },
+ }
+ : {};
+
return {
...baseConfig,
...(isOtelLogSchema
@@ -365,6 +395,7 @@ export async function inferTableSourceConfig({
traceIdExpression: 'TraceId',
severityTextExpression: 'SeverityText',
+ ...metadataMVsConfig,
}
: {}),
...(isOtelSpanSchema
@@ -387,6 +418,7 @@ export async function inferTableSourceConfig({
statusCodeExpression: 'StatusCode',
statusMessageExpression: 'StatusMessage',
...(hasSpanEvents ? { spanEventsValueExpression: 'Events' } : {}),
+ ...metadataMVsConfig,
}
: {}),
};
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 0b336406d7..381c26f07d 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -16,9 +16,10 @@ import { renderChartConfig } from '@/core/renderChartConfig';
import type {
BuilderChartConfig,
BuilderChartConfigWithDateRange,
+ MetadataMaterializedViews,
TSource,
} from '@/types';
-import { SourceKind } from '@/types';
+import { isLogSource, isTraceSource, SourceKind } from '@/types';
import { optimizeGetKeyValuesCalls } from './materializedViews';
import {
@@ -32,21 +33,6 @@ import {
export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6;
const DEFAULT_MAX_KEYS = 1000;
-/** Tables that have rollup materialized views */
-const TABLES_WITH_ROLLUPS = new Set(['otel_logs', 'otel_traces']);
-
-/** Returns the key-only rollup table name for a base table, or undefined if not supported */
-function getKeyRollupTableName(tableName: string): string | undefined {
- if (!TABLES_WITH_ROLLUPS.has(tableName)) return undefined;
- return `${tableName}_key_rollup_15m`;
-}
-
-/** Returns the KV rollup table name for a base table, or undefined if not supported */
-function getKvRollupTableName(tableName: string): string | undefined {
- if (!TABLES_WITH_ROLLUPS.has(tableName)) return undefined;
- return `${tableName}_kv_rollup_15m`;
-}
-
export class MetadataCache {
private cache = new Map();
private pendingQueries = new Map>();
@@ -358,7 +344,7 @@ export class Metadata {
maxKeys = DEFAULT_MAX_KEYS,
connectionId,
metricName,
- fieldsRollupView,
+ metadataMVs,
dateRange,
}: {
databaseName: string;
@@ -367,18 +353,18 @@ export class Metadata {
maxKeys?: number;
connectionId: string;
metricName?: string;
- fieldsRollupView?: FieldsRollupView;
+ metadataMVs?: MetadataMaterializedViews;
dateRange?: [Date, Date];
}) {
// Align date range to rollup granularity for consistent cache keys
const alignedDateRange =
- fieldsRollupView && dateRange
- ? getAlignedDateRange(dateRange, fieldsRollupView.granularity)
+ metadataMVs && dateRange
+ ? getAlignedDateRange(dateRange, metadataMVs.granularity)
: undefined;
const cacheKey = metricName
? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.keys`
- : fieldsRollupView && alignedDateRange
+ : metadataMVs && alignedDateRange
? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.keys`
: `${connectionId}.${databaseName}.${tableName}.${column}.keys`;
const cachedKeys = this.cache.get(cacheKey);
@@ -388,12 +374,12 @@ export class Metadata {
}
// Rollup path: query the key rollup table filtered by ColumnIdentifier and date range
- if (fieldsRollupView && alignedDateRange) {
+ if (metadataMVs && alignedDateRange) {
return this.cache.getOrFetch(cacheKey, async () => {
const timeFilter = chSql`AND Timestamp >= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})) AND Timestamp <= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }}))`;
const sql = chSql`
SELECT Key
- FROM ${tableExpr({ database: databaseName, table: fieldsRollupView.name })}
+ FROM ${tableExpr({ database: databaseName, table: metadataMVs.keyRollupTable })}
WHERE ColumnIdentifier = ${{ String: column }}
${timeFilter}
GROUP BY Key
@@ -651,7 +637,7 @@ export class Metadata {
tableName,
connectionId,
metricName,
- fieldsRollupView,
+ metadataMVs,
dateRange,
}: TableConnection & { dateRange?: [Date, Date] }) {
const fields: Field[] = [];
@@ -701,7 +687,7 @@ export class Metadata {
column: column.name,
connectionId,
metricName,
- fieldsRollupView,
+ metadataMVs,
dateRange,
});
@@ -1212,6 +1198,7 @@ export class Metadata {
key,
maxValues = 1000,
connectionId,
+ metadataMVs,
dateRange,
signal,
}: {
@@ -1221,11 +1208,11 @@ export class Metadata {
key: string;
maxValues?: number;
connectionId: string;
+ metadataMVs?: MetadataMaterializedViews;
dateRange: [Date, Date];
signal?: AbortSignal;
}): Promise {
- const kvRollupTable = getKvRollupTableName(tableName);
- if (!kvRollupTable) return [];
+ if (!metadataMVs) return [];
const timeFilter = chSql`AND Timestamp >= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})) AND Timestamp <= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }}))`;
const cacheKey = `${connectionId}.${databaseName}.${tableName}.${column}.${key}.${dateRange[0].getTime()}.${dateRange[1].getTime()}.completeKeyValues`;
@@ -1234,7 +1221,7 @@ export class Metadata {
try {
const sql = chSql`
SELECT Value
- FROM ${tableExpr({ database: databaseName, table: kvRollupTable })}
+ FROM ${tableExpr({ database: databaseName, table: metadataMVs.kvRollupTable })}
WHERE ColumnIdentifier = ${{ String: column }}
AND Key = ${{ String: key }}
AND Value != ''
@@ -1456,18 +1443,13 @@ export type Field = {
jsType: JSDataType | null;
};
-export type FieldsRollupView = {
- name: string;
- granularity: string;
-};
-
// Describes a table and potentially related views
export type TableConnection = {
databaseName: string;
tableName: string;
connectionId: string;
metricName?: string;
- fieldsRollupView?: FieldsRollupView;
+ metadataMVs?: MetadataMaterializedViews;
};
export type TableConnectionChoice =
@@ -1495,6 +1477,10 @@ export function tcFromSource(source?: TSource): TableConnection {
databaseName: source?.from?.databaseName ?? '',
tableName: source?.from?.tableName ?? '',
connectionId: source?.connection ?? '',
+ metadataMVs:
+ source && (isLogSource(source) || isTraceSource(source))
+ ? source.metadataMaterializedViews
+ : undefined,
};
}
diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts
index 01fefd75c1..8ea69c043e 100644
--- a/packages/common-utils/src/types.ts
+++ b/packages/common-utils/src/types.ts
@@ -990,6 +990,16 @@ export type MaterializedViewConfiguration = z.infer<
typeof MaterializedViewConfigurationSchema
>;
+export const MetadataMaterializedViewsSchema = z.object({
+ keyRollupTable: z.string().min(1, 'Key rollup table name is required'),
+ kvRollupTable: z.string().min(1, 'KV rollup table name is required'),
+ granularity: SQLIntervalSchema,
+});
+
+export type MetadataMaterializedViews = z.infer<
+ typeof MetadataMaterializedViewsSchema
+>;
+
// Log source form schema
export const LogSourceSchema = BaseSourceSchema.extend({
kind: z.literal(SourceKind.Log),
@@ -1015,6 +1025,7 @@ export const LogSourceSchema = BaseSourceSchema.extend({
highlightedRowAttributeExpressions:
HighlightedAttributeExpressionsSchema.optional(),
materializedViews: z.array(MaterializedViewConfigurationSchema).optional(),
+ metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(),
orderByExpression: z.string().optional(),
});
@@ -1054,6 +1065,7 @@ export const TraceSourceSchema = BaseSourceSchema.extend({
highlightedRowAttributeExpressions:
HighlightedAttributeExpressionsSchema.optional(),
materializedViews: z.array(MaterializedViewConfigurationSchema).optional(),
+ metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(),
orderByExpression: z.string().optional(),
});
From 933d640d3511bcfb3fd5c2742b9c31fdf7d8fab8 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 16 Apr 2026 13:50:37 -0400
Subject: [PATCH 03/12] fix: update tests
---
.../__tests__/useAutoCompleteOptions.test.tsx | 83 +++----------------
1 file changed, 10 insertions(+), 73 deletions(-)
diff --git a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
index a11f497d1f..c5c6060d22 100644
--- a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
+++ b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
@@ -4,18 +4,13 @@ import { renderHook } from '@testing-library/react';
import { LuceneLanguageFormatter } from '../../components/SearchInput/SearchInputV2';
import { useAutoCompleteOptions } from '../useAutoCompleteOptions';
-import {
- useJsonColumns,
- useMultipleAllFields,
- useMultipleGetKeyValues,
-} from '../useMetadata';
+import { useCompleteKeyValues, useMultipleAllFields } from '../useMetadata';
// Mock dependencies
jest.mock('../useMetadata', () => ({
...jest.requireActual('../useMetadata.tsx'),
useMultipleAllFields: jest.fn(),
- useMultipleGetKeyValues: jest.fn(),
- useJsonColumns: jest.fn(),
+ useCompleteKeyValues: jest.fn(),
}));
const luceneFormatter = new LuceneLanguageFormatter();
@@ -54,12 +49,9 @@ describe('useAutoCompleteOptions', () => {
data: mockFields,
});
- (useMultipleGetKeyValues as jest.Mock).mockReturnValue({
- data: null,
- });
-
- (useJsonColumns as jest.Mock).mockReturnValue({
+ (useCompleteKeyValues as jest.Mock).mockReturnValue({
data: null,
+ isFetching: false,
});
});
@@ -70,7 +62,7 @@ describe('useAutoCompleteOptions', () => {
}),
);
- expect(result.current).toEqual([
+ expect(result.current.options).toEqual([
{
value: 'ResourceAttributes',
label: 'ResourceAttributes (map)',
@@ -87,15 +79,9 @@ describe('useAutoCompleteOptions', () => {
});
it('should return key value options with correct lucene formatting', () => {
- const mockKeyValues = [
- {
- key: 'ResourceAttributes.service.name',
- value: ['frontend', 'backend'],
- },
- ];
-
- (useMultipleGetKeyValues as jest.Mock).mockReturnValue({
- data: mockKeyValues,
+ (useCompleteKeyValues as jest.Mock).mockReturnValue({
+ data: ['frontend', 'backend'],
+ isFetching: false,
});
const { result } = renderHook(() =>
@@ -108,7 +94,7 @@ describe('useAutoCompleteOptions', () => {
),
);
- expect(result.current).toEqual([
+ expect(result.current.options).toEqual([
{
value: 'ResourceAttributes',
label: 'ResourceAttributes (map)',
@@ -132,55 +118,6 @@ describe('useAutoCompleteOptions', () => {
]);
});
- // TODO: Does this test case need to be removed after HDX-1548?
- it('should handle nested key value options', () => {
- const mockKeyValues = [
- {
- key: 'ResourceAttributes',
- value: [
- {
- 'service.name': 'frontend',
- 'deployment.environment': 'production',
- },
- ],
- },
- ];
-
- (useMultipleGetKeyValues as jest.Mock).mockReturnValue({
- data: mockKeyValues,
- });
-
- const { result } = renderHook(() =>
- useAutoCompleteOptions(luceneFormatter, 'ResourceAttributes', {
- tableConnection: mockTableConnection,
- }),
- );
-
- //console.log(result.current);
- expect(result.current).toEqual([
- {
- value: 'ResourceAttributes',
- label: 'ResourceAttributes (map)',
- },
- {
- value: 'ResourceAttributes.service.name',
- label: 'ResourceAttributes.service.name (string)',
- },
- {
- value: 'TraceAttributes.trace.id',
- label: 'TraceAttributes.trace.id (string)',
- },
- {
- value: 'ResourceAttributes.service.name:"frontend"',
- label: 'ResourceAttributes.service.name:"frontend"',
- },
- {
- value: 'ResourceAttributes.deployment.environment:"production"',
- label: 'ResourceAttributes.deployment.environment:"production"',
- },
- ]);
- });
-
it('should handle additional suggestions', () => {
const { result } = renderHook(() =>
useAutoCompleteOptions(luceneFormatter, 'ResourceAttributes', {
@@ -189,7 +126,7 @@ describe('useAutoCompleteOptions', () => {
}),
);
- expect(result.current).toEqual([
+ expect(result.current.options).toEqual([
{
value: 'ResourceAttributes',
label: 'ResourceAttributes (map)',
From a0515d408e0ec6b0bd97c2a67943a10b9275f347 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 16 Apr 2026 13:55:09 -0400
Subject: [PATCH 04/12] fix: debounce value
---
packages/app/src/HDXMultiSeriesTimeChart.tsx | 6 ++++--
packages/app/src/hooks/useAutoCompleteOptions.tsx | 5 +++--
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx
index 376ae42a1a..aa28e7a032 100644
--- a/packages/app/src/HDXMultiSeriesTimeChart.tsx
+++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx
@@ -367,8 +367,10 @@ export const MemoChart = memo(function MemoChart({
const [isHovered, setIsHovered] = useState(false);
- const ChartComponent =
- displayType === DisplayType.StackedBar ? BarChart : AreaChart; // LineChart;
+ const ChartComponent = useMemo(
+ () => (displayType === DisplayType.StackedBar ? BarChart : AreaChart), // LineChart;
+ [displayType],
+ );
const lines = useMemo(() => {
const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0;
diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx
index 306234716f..427ce2eb97 100644
--- a/packages/app/src/hooks/useAutoCompleteOptions.tsx
+++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx
@@ -10,7 +10,7 @@ import {
useCompleteKeyValues,
useMultipleAllFields,
} from '@/hooks/useMetadata';
-import { toArray } from '@/utils';
+import { toArray, useDebounce } from '@/utils';
export type TokenInfo = {
/** The full token at the cursor position */
@@ -46,7 +46,7 @@ export interface ILanguageFormatter {
export function useAutoCompleteOptions(
formatter: ILanguageFormatter,
- value: string,
+ _value: string,
{
tableConnection,
additionalSuggestions,
@@ -59,6 +59,7 @@ export function useAutoCompleteOptions(
inputRef?: React.RefObject;
},
) {
+ const value = useDebounce(_value, 300);
const tcs = useMemo(() => toArray(tableConnection), [tableConnection]);
const effectiveDateRange: [Date, Date] = useMemo(
From 91f36fc70b80079d5facc4d102064b70648618c1 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 16 Apr 2026 15:45:09 -0400
Subject: [PATCH 05/12] fix: inserts in integration tests
---
.../schema/seed/00006_otel_logs_rollups.sql | 34 +++++++++----------
.../schema/seed/00007_otel_traces_rollups.sql | 16 ++++-----
2 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql
index 641775d5f6..dfb448ce22 100644
--- a/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql
+++ b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql
@@ -49,7 +49,7 @@ AS WITH elements AS (
'ResourceAttributes' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
- entry.2 AS Value
+ CAST(entry.2 AS String) AS Value
FROM ${DATABASE}.otel_logs
ARRAY JOIN ResourceAttributes AS entry
UNION ALL
@@ -57,7 +57,7 @@ AS WITH elements AS (
'LogAttributes' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
- entry.2 AS Value
+ CAST(entry.2 AS String) AS Value
FROM ${DATABASE}.otel_logs
ARRAY JOIN LogAttributes AS entry
UNION ALL
@@ -65,7 +65,7 @@ AS WITH elements AS (
'ScopeAttributes' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
- entry.2 AS Value
+ CAST(entry.2 AS String) AS Value
FROM ${DATABASE}.otel_logs
ARRAY JOIN ScopeAttributes AS entry
UNION ALL
@@ -73,98 +73,98 @@ AS WITH elements AS (
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'SeverityText' as Key,
- SeverityText as Value
+ CAST(SeverityText AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ServiceName' as Key,
- ServiceName as Value
+ CAST(ServiceName AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ScopeName' as Key,
- ScopeName as Value
+ CAST(ScopeName AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ScopeVersion' as Key,
- ScopeVersion as Value
+ CAST(ScopeVersion AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ResourceSchemaUrl' as Key,
- ResourceSchemaUrl as Value
+ CAST(ResourceSchemaUrl AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ScopeSchemaUrl' as Key,
- ScopeSchemaUrl as Value
+ CAST(ScopeSchemaUrl AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_k8s.cluster.name' as Key,
- `__hdx_materialized_k8s.cluster.name` as Value
+ CAST(`__hdx_materialized_k8s.cluster.name` AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_k8s.container.name' as Key,
- `__hdx_materialized_k8s.container.name` as Value
+ CAST(`__hdx_materialized_k8s.container.name` AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_k8s.deployment.name' as Key,
- `__hdx_materialized_k8s.deployment.name` as Value
+ CAST(`__hdx_materialized_k8s.deployment.name` AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_k8s.namespace.name' as Key,
- `__hdx_materialized_k8s.namespace.name` as Value
+ CAST(`__hdx_materialized_k8s.namespace.name` AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_k8s.node.name' as Key,
- `__hdx_materialized_k8s.node.name` as Value
+ CAST(`__hdx_materialized_k8s.node.name` AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_k8s.pod.name' as Key,
- `__hdx_materialized_k8s.pod.name` as Value
+ CAST(`__hdx_materialized_k8s.pod.name` AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_k8s.pod.uid' as Key,
- `__hdx_materialized_k8s.pod.uid` as Value
+ CAST(`__hdx_materialized_k8s.pod.uid` AS String) as Value
FROM ${DATABASE}.otel_logs
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'__hdx_materialized_deployment.environment.name' as Key,
- `__hdx_materialized_deployment.environment.name` as Value
+ CAST(`__hdx_materialized_deployment.environment.name` AS String) as Value
FROM ${DATABASE}.otel_logs
)
SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements
diff --git a/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql
index ac007bfa99..2ca574dbb2 100644
--- a/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql
+++ b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql
@@ -50,7 +50,7 @@ AS WITH elements AS (
'ResourceAttributes' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
- entry.2 AS Value
+ CAST(entry.2 AS String) AS Value
FROM ${DATABASE}.otel_traces
ARRAY JOIN ResourceAttributes AS entry
UNION ALL
@@ -58,7 +58,7 @@ AS WITH elements AS (
'SpanAttributes' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key,
- entry.2 AS Value
+ CAST(entry.2 AS String) AS Value
FROM ${DATABASE}.otel_traces
ARRAY JOIN SpanAttributes AS entry
UNION ALL
@@ -66,42 +66,42 @@ AS WITH elements AS (
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ServiceName' AS Key,
- ServiceName AS Value
+ CAST(ServiceName AS String) AS Value
FROM ${DATABASE}.otel_traces
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'SpanName' AS Key,
- SpanName AS Value
+ CAST(SpanName AS String) AS Value
FROM ${DATABASE}.otel_traces
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'SpanKind' AS Key,
- SpanKind AS Value
+ CAST(SpanKind AS String) AS Value
FROM ${DATABASE}.otel_traces
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'StatusCode' AS Key,
- StatusCode AS Value
+ CAST(StatusCode AS String) AS Value
FROM ${DATABASE}.otel_traces
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ScopeName' AS Key,
- ScopeName AS Value
+ CAST(ScopeName AS String) AS Value
FROM ${DATABASE}.otel_traces
UNION ALL
SELECT
'NativeColumn' AS ColumnIdentifier,
toStartOfFifteenMinutes(Timestamp) AS Timestamp,
'ScopeVersion' AS Key,
- ScopeVersion AS Value
+ CAST(ScopeVersion AS String) AS Value
FROM ${DATABASE}.otel_traces
)
SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements
From cface4610c0a425ef4d5cda5101a742f1e4f4dd5 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 17 Apr 2026 14:13:52 -0400
Subject: [PATCH 06/12] add changeset
---
.changeset/gold-snakes-repeat.md | 7 +++++++
1 file changed, 7 insertions(+)
create mode 100644 .changeset/gold-snakes-repeat.md
diff --git a/.changeset/gold-snakes-repeat.md b/.changeset/gold-snakes-repeat.md
new file mode 100644
index 0000000000..ea3ecd413f
--- /dev/null
+++ b/.changeset/gold-snakes-repeat.md
@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+---
+
+feat: introducing rollup and source support for full autocomplete
From 41fe0b836937478cfc45c3d59bfa5de0bacfd1d9 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Fri, 17 Apr 2026 16:51:18 -0400
Subject: [PATCH 07/12] updates for PR
---
.../__tests__/useAutoCompleteOptions.test.tsx | 171 +++++++++++++++-
.../app/src/hooks/useAutoCompleteOptions.tsx | 124 +++++++++--
packages/app/src/hooks/useMetadata.tsx | 19 +-
packages/app/src/source.ts | 20 +-
.../src/__tests__/materializedViews.test.ts | 192 ++++++++++++++++++
.../src/core/materializedViews.ts | 175 +++++++++++++++-
packages/common-utils/src/core/metadata.ts | 35 +++-
7 files changed, 690 insertions(+), 46 deletions(-)
create mode 100644 packages/common-utils/src/__tests__/materializedViews.test.ts
diff --git a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
index c5c6060d22..0850e9b48f 100644
--- a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
+++ b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
@@ -4,13 +4,14 @@ import { renderHook } from '@testing-library/react';
import { LuceneLanguageFormatter } from '../../components/SearchInput/SearchInputV2';
import { useAutoCompleteOptions } from '../useAutoCompleteOptions';
-import { useCompleteKeyValues, useMultipleAllFields } from '../useMetadata';
+import { tokenizeAtCursor } from '../useAutoCompleteOptions';
+import { useAllKeyValues, useMultipleAllFields } from '../useMetadata';
// Mock dependencies
jest.mock('../useMetadata', () => ({
...jest.requireActual('../useMetadata.tsx'),
useMultipleAllFields: jest.fn(),
- useCompleteKeyValues: jest.fn(),
+ useAllKeyValues: jest.fn(),
}));
const luceneFormatter = new LuceneLanguageFormatter();
@@ -49,7 +50,7 @@ describe('useAutoCompleteOptions', () => {
data: mockFields,
});
- (useCompleteKeyValues as jest.Mock).mockReturnValue({
+ (useAllKeyValues as jest.Mock).mockReturnValue({
data: null,
isFetching: false,
});
@@ -79,7 +80,7 @@ describe('useAutoCompleteOptions', () => {
});
it('should return key value options with correct lucene formatting', () => {
- (useCompleteKeyValues as jest.Mock).mockReturnValue({
+ (useAllKeyValues as jest.Mock).mockReturnValue({
data: ['frontend', 'backend'],
isFetching: false,
});
@@ -146,3 +147,165 @@ describe('useAutoCompleteOptions', () => {
]);
});
});
+
+describe('tokenizeAtCursor', () => {
+ // Each case is tokenized with the cursor at the end of the input, so
+ // `expectedToken` is the token the user is currently typing into.
+ const cases: {
+ name: string;
+ input: string;
+ expectedToken: string;
+ expectedTokens: string[];
+ }[] = [
+ // Basic whitespace splitting
+ {
+ name: 'splits a bare whitespace-separated string into tokens',
+ input: 'foo bar baz',
+ expectedToken: 'baz',
+ expectedTokens: ['foo', 'bar', 'baz'],
+ },
+ {
+ name: 'returns a single token for input with no whitespace',
+ input: 'FieldName',
+ expectedToken: 'FieldName',
+ expectedTokens: ['FieldName'],
+ },
+ {
+ name: 'returns a single empty token for empty input',
+ input: '',
+ expectedToken: '',
+ expectedTokens: [''],
+ },
+ {
+ name: 'produces empty tokens for consecutive spaces (split-like semantics)',
+ input: 'foo bar',
+ expectedToken: 'bar',
+ expectedTokens: ['foo', '', 'bar'],
+ },
+ {
+ name: 'produces a trailing empty token when input ends in a space',
+ input: 'foo ',
+ expectedToken: '',
+ expectedTokens: ['foo', ''],
+ },
+
+ // Balanced quoted regions
+ {
+ name: 'keeps whitespace inside paired quotes as part of one token',
+ input: 'Service:"hello world"',
+ expectedToken: 'Service:"hello world"',
+ expectedTokens: ['Service:"hello world"'],
+ },
+ {
+ name: 'splits two paired-quote fields on the space between them',
+ input: 'ServiceName:"clickhouse" SeverityText:"debug"',
+ expectedToken: 'SeverityText:"debug"',
+ expectedTokens: ['ServiceName:"clickhouse"', 'SeverityText:"debug"'],
+ },
+ {
+ name: 'preserves escaped quotes inside a quoted region',
+ input: 'Service:"he said \\"hi\\"" other',
+ expectedToken: 'other',
+ expectedTokens: ['Service:"he said \\"hi\\""', 'other'],
+ },
+ {
+ name: 'treats a colon inside a quoted value as literal text',
+ input: 'URL:"http://example.com/x" x',
+ expectedToken: 'x',
+ expectedTokens: ['URL:"http://example.com/x"', 'x'],
+ },
+
+ // Unclosed quotes — reproduces the bug where `Field:" Other:"v"` (three
+ // quotes with the first unclosed) previously collapsed into one token.
+ {
+ name: 'breaks at whitespace when a quote is followed by a new field pattern',
+ input: 'ServiceName:" SeverityText:"debug"',
+ expectedToken: 'SeverityText:"debug"',
+ expectedTokens: ['ServiceName:"', 'SeverityText:"debug"'],
+ },
+ {
+ name: 'treats a single unclosed quote at end of input as a literal',
+ input: 'Service:"hel',
+ expectedToken: 'Service:"hel',
+ expectedTokens: ['Service:"hel'],
+ },
+ {
+ // No `ident:` after the space, so the quote can still legitimately
+ // extend — but there's no closing quote anywhere, so it's unclosed.
+ name: 'handles an unclosed quote followed by whitespace then bare text',
+ input: 'Service:"hello world',
+ expectedToken: 'world',
+ expectedTokens: ['Service:"hello', 'world'],
+ },
+ {
+ name: 'handles multiple unclosed quotes across fields',
+ input: 'A:" B:" C:"done"',
+ expectedToken: 'C:"done"',
+ expectedTokens: ['A:"', 'B:"', 'C:"done"'],
+ },
+
+ // Identifier-like characters after whitespace
+ {
+ // The space inside the quoted value is followed by `!`, not `ident:`,
+ // so the quote should still be able to close.
+ name: 'does not bail out on whitespace followed by a non-identifier',
+ input: 'Service:"hello !world"',
+ expectedToken: 'Service:"hello !world"',
+ expectedTokens: ['Service:"hello !world"'],
+ },
+ {
+ name: 'does not treat whitespace + ident without colon as a new field',
+ input: 'Service:"hello world done"',
+ expectedToken: 'Service:"hello world done"',
+ expectedTokens: ['Service:"hello world done"'],
+ },
+ {
+ name: 'handles dotted identifiers in the new-field pattern',
+ input: 'Foo:" ResourceAttributes.service.name:"x"',
+ expectedToken: 'ResourceAttributes.service.name:"x"',
+ expectedTokens: ['Foo:"', 'ResourceAttributes.service.name:"x"'],
+ },
+ ];
+
+ it.each(cases)('$name', ({ input, expectedToken, expectedTokens }) => {
+ const { token, tokens } = tokenizeAtCursor(input, input.length);
+ expect(tokens).toEqual(expectedTokens);
+ expect(token).toBe(expectedToken);
+ });
+
+ // Cursor-positioning is orthogonal to tokenization — keep these separate
+ // because each case exercises a different cursor offset for the same input.
+ describe('cursor positioning', () => {
+ it('returns the first token when the cursor is at position 0', () => {
+ const { token, index } = tokenizeAtCursor('foo bar baz', 0);
+ expect(token).toBe('foo');
+ expect(index).toBe(0);
+ });
+
+ it('returns the middle token when the cursor is inside it', () => {
+ // 0123456789012
+ // 'foo bar baz' — cursor at 5 is inside 'bar'
+ const { token, index } = tokenizeAtCursor('foo bar baz', 5);
+ expect(token).toBe('bar');
+ expect(index).toBe(1);
+ });
+
+ it('returns the token whose range contains the cursor in a quoted field', () => {
+ const input = 'Service:"hello world" other';
+ // cursor inside the quoted token
+ const { token, index } = tokenizeAtCursor(input, 15);
+ expect(token).toBe('Service:"hello world"');
+ expect(index).toBe(0);
+ });
+
+ it('returns the unclosed-quote token when the cursor is inside it', () => {
+ // User is typing a value — the first quote is unclosed because the
+ // next field pattern `SeverityText:` appears after whitespace. Cursor
+ // sits just after `l`, inside the in-progress `ServiceName:"cl` token.
+ const input = 'ServiceName:"cl SeverityText:"info"';
+ const { token, index } = tokenizeAtCursor(input, 15);
+ expect(token).toBe('ServiceName:"cl');
+ expect(index).toBe(0);
+ });
+ });
+});
diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx
index 427ce2eb97..d3de439887 100644
--- a/packages/app/src/hooks/useAutoCompleteOptions.tsx
+++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx
@@ -7,7 +7,7 @@ import {
import { NOW } from '@/config';
import {
deduplicate2dArray,
- useCompleteKeyValues,
+ useAllKeyValues,
useMultipleAllFields,
} from '@/hooks/useMetadata';
import { toArray, useDebounce } from '@/utils';
@@ -21,20 +21,112 @@ export type TokenInfo = {
tokens: string[];
};
-/** Splits input into tokens and finds which token the cursor is in */
-function tokenizeAtCursor(value: string, cursorPos: number): TokenInfo {
- const tokens = value.split(' ');
- let idx = 0;
- let pos = 0;
+const IDENT_RE = /[A-Za-z0-9_.]/;
+
+function findMatchingQuote(value: string, startIdx: number): number {
+ let i = startIdx + 1;
+ while (i < value.length) {
+ const ch = value[i];
+ if (ch === '\\' && i + 1 < value.length) {
+ i += 2;
+ continue;
+ }
+ if (ch === '"') return i;
+ if (ch === ' ' || ch === '\t' || ch === '\n') {
+ let k = i;
+ while (
+ k < value.length &&
+ (value[k] === ' ' || value[k] === '\t' || value[k] === '\n')
+ )
+ k++;
+ const identStart = k;
+ while (k < value.length && IDENT_RE.test(value[k])) k++;
+ if (k > identStart && k < value.length && value[k] === ':') {
+ return -1;
+ }
+ }
+ i++;
+ }
+ return -1;
+}
+
+export function tokenizeAtCursor(value: string, cursorPos: number): TokenInfo {
+ const tokens: string[] = [];
+ // Start offsets of each token in the original string
+ const starts: number[] = [];
+
+ let current = '';
+ let currentStart = -1;
+ let inQuotes = false;
+ let escaped = false;
+
+ for (let i = 0; i < value.length; i++) {
+ const ch = value[i];
+
+ if (escaped) {
+ // Always include the escaped character verbatim (along with its backslash)
+ current += ch;
+ escaped = false;
+ continue;
+ }
+
+ if (ch === '\\' && inQuotes) {
+ current += ch;
+ escaped = true;
+ continue;
+ }
+
+ if (ch === '"') {
+ if (inQuotes) {
+ // Closing an already-opened quoted region.
+ if (currentStart === -1) currentStart = i;
+ current += ch;
+ inQuotes = false;
+ continue;
+ }
+ // Only enter a quoted region if there's a matching close ahead.
+ if (findMatchingQuote(value, i) !== -1) {
+ if (currentStart === -1) currentStart = i;
+ current += ch;
+ inQuotes = true;
+ continue;
+ }
+ // Stray/unclosed quote — treat as a literal character.
+ if (currentStart === -1) currentStart = i;
+ current += ch;
+ continue;
+ }
+
+ if (!inQuotes && ch === ' ') {
+ // Boundary: flush current token (even if empty, to mirror prior `split(' ')`
+ // semantics where consecutive spaces produce empty tokens).
+ tokens.push(current);
+ starts.push(currentStart === -1 ? i : currentStart);
+ current = '';
+ currentStart = -1;
+ continue;
+ }
+
+ if (currentStart === -1) currentStart = i;
+ current += ch;
+ }
+ // Flush trailing token
+ tokens.push(current);
+ starts.push(currentStart === -1 ? value.length : currentStart);
+
+ // Locate token containing the cursor. The cursor sits *between* characters,
+ // so a token covers [start, start+len]; we pick the last token whose range
+ // contains cursorPos.
+ let idx = tokens.length - 1;
for (let i = 0; i < tokens.length; i++) {
- pos += tokens[i].length;
- if (pos >= cursorPos || i === tokens.length - 1) {
+ const start = starts[i];
+ const end = start + tokens[i].length;
+ if (cursorPos <= end) {
idx = i;
break;
}
- pos++; // account for the space
- idx = i + 1;
}
+
return { token: tokens[idx] ?? '', index: idx, tokens };
}
@@ -118,13 +210,11 @@ export function useAutoCompleteOptions(
// Debounced fetch of values for the selected key from rollup tables
const firstTc = tcs.length > 0 ? tcs[0] : undefined;
- const { data: keyValues, isFetching: isLoadingValues } = useCompleteKeyValues(
- {
- tableConnection: firstTc,
- searchField,
- dateRange: effectiveDateRange,
- },
- );
+ const { data: keyValues, isFetching: isLoadingValues } = useAllKeyValues({
+ tableConnection: firstTc,
+ searchField,
+ dateRange: effectiveDateRange,
+ });
// Build key-value pair suggestions
const keyValCompleteOptions = useMemo<
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index 0ea7ccf5cd..c81683cbc2 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -201,7 +201,7 @@ function fieldToRollupParams(
* Works for both map keys (e.g. "ResourceAttributes.http.method") and
* native columns (e.g. "ServiceName").
*/
-export function useCompleteKeyValues({
+export function useAllKeyValues({
tableConnection,
searchField,
dateRange,
@@ -212,18 +212,11 @@ export function useCompleteKeyValues({
}) {
const metadata = useMetadataWithSettings();
- // Debounce: only query after the field stabilizes for 300ms
- const [debouncedField, setDebouncedField] = useState(null);
- useEffect(() => {
- const timer = setTimeout(() => setDebouncedField(searchField), 300);
- return () => clearTimeout(timer);
- }, [searchField]);
-
- const rollupParams = fieldToRollupParams(debouncedField, tableConnection);
+ const rollupParams = fieldToRollupParams(searchField, tableConnection);
return useQuery({
queryKey: [
- 'useCompleteKeyValues',
+ 'useAllKeyValues',
tableConnection?.databaseName,
tableConnection?.tableName,
tableConnection?.connectionId,
@@ -233,10 +226,10 @@ export function useCompleteKeyValues({
dateRange[1].getTime(),
],
queryFn: async ({ signal }) => {
- if (!tableConnection || !rollupParams || !debouncedField) return [];
+ if (!tableConnection || !rollupParams || !searchField) return [];
// Try rollup first
- const rollupValues = await metadata.getCompleteKeyValues({
+ const rollupValues = await metadata.getAllKeyValues({
databaseName: tableConnection.databaseName,
tableName: tableConnection.tableName,
column: rollupParams.columnIdentifier,
@@ -264,7 +257,7 @@ export function useCompleteKeyValues({
return metadata.getMapValues({
databaseName: tableConnection.databaseName,
tableName: tableConnection.tableName,
- column: debouncedField.path[0],
+ column: searchField.path[0],
connectionId: tableConnection.connectionId,
});
}
diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts
index e361f57a2c..4713a22170 100644
--- a/packages/app/src/source.ts
+++ b/packages/app/src/source.ts
@@ -7,6 +7,7 @@ import {
filterColumnMetaByType,
JSDataType,
} from '@hyperdx/common-utils/dist/clickhouse';
+import { inferGranularityFromMVSelect } from '@hyperdx/common-utils/dist/core/materializedViews';
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards';
@@ -348,8 +349,9 @@ export async function inferTableSourceConfig({
// Check if SpanEvents column is available
const hasSpanEvents = columns.some(col => col.name === 'Events.Timestamp');
- // Check if metadata rollup tables exist
- const hasMetadataMVs =
+ // Check if metadata rollup tables exist and, if so, infer the bucketing
+ // granularity from the key-rollup view's `as_select`
+ const rollupMeta =
isOtelLogSchema || isOtelSpanSchema
? await (async () => {
const [keyMeta, kvMeta] = await Promise.all([
@@ -364,16 +366,22 @@ export async function inferTableSourceConfig({
connectionId,
}),
]);
- return keyMeta != null && kvMeta != null;
+ return keyMeta != null && kvMeta != null
+ ? { keyMeta, kvMeta }
+ : undefined;
})()
- : false;
+ : undefined;
- const metadataMVsConfig = hasMetadataMVs
+ const metadataMVsConfig = rollupMeta
? {
metadataMaterializedViews: {
keyRollupTable: `${tableName}_key_rollup_15m`,
kvRollupTable: `${tableName}_kv_rollup_15m`,
- granularity: '15 minute',
+ // Fall back to '15 minute' to preserve the prior default when the
+ // MV's `as_select` doesn't contain a recognized bucketing function.
+ granularity:
+ inferGranularityFromMVSelect(rollupMeta.keyMeta.as_select) ??
+ '15 minute',
},
}
: {};
diff --git a/packages/common-utils/src/__tests__/materializedViews.test.ts b/packages/common-utils/src/__tests__/materializedViews.test.ts
new file mode 100644
index 0000000000..c1c2bd6c0c
--- /dev/null
+++ b/packages/common-utils/src/__tests__/materializedViews.test.ts
@@ -0,0 +1,192 @@
+import { inferGranularityFromMVSelect } from '@/core/materializedViews';
+
+describe('inferGranularityFromMVSelect', () => {
+ describe('real MV schemas', () => {
+ // Shape matches the `otel_logs_attr_kv_rollup_15m_mv` view in
+ // docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql.
+ it('detects 15 minute from the otel_logs kv rollup MV select', () => {
+ const asSelect = `WITH elements AS (
+ SELECT
+ 'ResourceAttributes' AS ColumnIdentifier,
+ toStartOfFifteenMinutes(Timestamp) AS Timestamp,
+ replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key,
+ CAST(entry.2 AS String) AS Value
+ FROM default.otel_logs
+ ARRAY JOIN ResourceAttributes AS entry
+ UNION ALL
+ SELECT
+ 'LogAttributes' AS ColumnIdentifier,
+ toStartOfFifteenMinutes(Timestamp) AS Timestamp,
+ replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key,
+ CAST(entry.2 AS String) AS Value
+ FROM default.otel_logs
+ ARRAY JOIN LogAttributes AS entry
+)
+SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements
+GROUP BY Timestamp, ColumnIdentifier, Key, Value`;
+
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('15 minute');
+ });
+
+ // Shape matches the `otel_traces_kv_rollup_15m_mv` view in
+ // docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql.
+ it('detects 15 minute from the otel_traces kv rollup MV select', () => {
+ const asSelect = `WITH elements AS (
+ SELECT
+ 'ResourceAttributes' AS ColumnIdentifier,
+ toStartOfFifteenMinutes(Timestamp) AS Timestamp,
+ replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key,
+ CAST(entry.2 AS String) AS Value
+ FROM default.otel_traces
+ ARRAY JOIN ResourceAttributes AS entry
+ UNION ALL
+ SELECT
+ 'SpanAttributes' AS ColumnIdentifier,
+ toStartOfFifteenMinutes(Timestamp) AS Timestamp,
+ replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key,
+ CAST(entry.2 AS String) AS Value
+ FROM default.otel_traces
+ ARRAY JOIN SpanAttributes AS entry
+)
+SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements
+GROUP BY Timestamp, ColumnIdentifier, Key, Value`;
+
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('15 minute');
+ });
+
+ // The key-rollup MV rolls up the kv rollup, so it doesn't bucket the
+ // timestamp itself — it just selects the already-bucketed Timestamp.
+ it('returns undefined when the select contains no bucketing function', () => {
+ const asSelect = `SELECT
+ Timestamp,
+ ColumnIdentifier,
+ Key,
+ sum(count) as count
+FROM default.otel_logs_kv_rollup_15m
+GROUP BY ColumnIdentifier, Key, Timestamp`;
+
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+ });
+
+ describe('toStartOfInterval form', () => {
+ it.each([
+ ['INTERVAL 5 MINUTE', '5 minute'],
+ ['INTERVAL 1 HOUR', '1 hour'],
+ ['INTERVAL 2 hour', '2 hour'],
+ ['INTERVAL 30 SECOND', '30 second'],
+ ['INTERVAL 1 DAY', '1 day'],
+ ])('parses %s', (interval, expected) => {
+ const asSelect = `SELECT toStartOfInterval(Timestamp, ${interval}) AS ts, count() FROM t GROUP BY ts`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe(expected);
+ });
+
+ it('accepts plural units', () => {
+ const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 10 MINUTES) AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('10 minute');
+ });
+
+ it('handles extra arguments (origin, timezone)', () => {
+ const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 1 DAY, toDateTime('2025-01-01'), 'America/Los_Angeles') AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('1 day');
+ });
+
+ it('handles extra whitespace and newlines inside the call', () => {
+ const asSelect = `SELECT
+ toStartOfInterval (
+ Timestamp,
+ INTERVAL 15 MINUTE
+ ) AS ts
+FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('15 minute');
+ });
+
+ it('ignores unknown units', () => {
+ const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 1 WEEK) AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+ });
+
+ describe('named bucket functions', () => {
+ it.each([
+ ['toStartOfSecond', '1 second'],
+ ['toStartOfMinute', '1 minute'],
+ ['toStartOfFiveMinutes', '5 minute'],
+ ['toStartOfTenMinutes', '10 minute'],
+ ['toStartOfFifteenMinutes', '15 minute'],
+ ['toStartOfHour', '1 hour'],
+ ['toStartOfDay', '1 day'],
+ ])('maps %s to %s', (fn, expected) => {
+ const asSelect = `SELECT ${fn}(Timestamp) AS ts, count() FROM t GROUP BY ts`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe(expected);
+ });
+
+ it('returns undefined for unrecognized toStartOf* functions', () => {
+ // toStartOfMonth is a real CH function but not in NAMED_BUCKET_FUNCTIONS.
+ const asSelect = `SELECT toStartOfMonth(Timestamp) AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+ });
+
+ describe('first match wins', () => {
+ it('returns the granularity of the first toStartOf call encountered', () => {
+ const asSelect = `SELECT toStartOfHour(Timestamp) AS h, toStartOfMinute(Timestamp) AS m FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('1 hour');
+ });
+
+ it('returns first call even when a later call would also match', () => {
+ const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 5 MINUTE) AS a, toStartOfHour(Timestamp) AS b FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('5 minute');
+ });
+ });
+
+ describe('robustness against quoting and noise', () => {
+ it('ignores toStartOf* tokens inside single-quoted string literals', () => {
+ const asSelect = `SELECT 'toStartOfHour(Timestamp)' AS label, toStartOfMinute(Timestamp) AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('1 minute');
+ });
+
+ it('ignores toStartOf* tokens inside backtick-quoted identifiers', () => {
+ const asSelect = 'SELECT `toStartOfHour` AS col FROM t';
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+
+ it('ignores toStartOf* substrings embedded in longer identifiers', () => {
+ // my_toStartOfHour_col is a single identifier, not a function call.
+ const asSelect = `SELECT my_toStartOfHour_col AS x FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+
+ it('is not confused by commas inside nested calls', () => {
+ const asSelect = `SELECT toStartOfInterval(coalesce(Timestamp, now()), INTERVAL 5 MINUTE) AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBe('5 minute');
+ });
+
+ it('skips a toStartOf call without matching parens and stops scanning', () => {
+ // Unterminated call — we stop rather than looping forever.
+ const asSelect = `SELECT toStartOfHour(Timestamp FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+ });
+
+ describe('degenerate inputs', () => {
+ it('returns undefined for empty string', () => {
+ expect(inferGranularityFromMVSelect('')).toBeUndefined();
+ });
+
+ it('returns undefined when no toStartOf call is present', () => {
+ const asSelect = `SELECT Timestamp, count() FROM t GROUP BY Timestamp`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+
+ it('returns undefined when toStartOfInterval has no interval arg', () => {
+ const asSelect = `SELECT toStartOfInterval(Timestamp) AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+
+ it('returns undefined when toStartOfInterval has a malformed interval arg', () => {
+ const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL abc MINUTE) AS ts FROM t`;
+ expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined();
+ });
+ });
+});
diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts
index 51fac8f336..03e6272bc8 100644
--- a/packages/common-utils/src/core/materializedViews.ts
+++ b/packages/common-utils/src/core/materializedViews.ts
@@ -1,6 +1,6 @@
import { differenceInSeconds } from 'date-fns';
-import { BaseClickhouseClient } from '@/clickhouse';
+import { BaseClickhouseClient, ChSql, chSql } from '@/clickhouse';
import {
BuilderChartConfigWithOptDateRange,
CteChartConfig,
@@ -9,6 +9,7 @@ import {
isLogSource,
isTraceSource,
MaterializedViewConfiguration,
+ type SQLInterval,
TLogSource,
TSource,
TTraceSource,
@@ -25,6 +26,178 @@ import {
splitAndTrimWithBracket,
} from './utils';
+// ClickHouse named time-bucketing functions and their granularity equivalents.
+const NAMED_BUCKET_FUNCTIONS: Record = {
+ toStartOfSecond: '1 second',
+ toStartOfMinute: '1 minute',
+ toStartOfFiveMinutes: '5 minute',
+ toStartOfTenMinutes: '10 minute',
+ toStartOfFifteenMinutes: '15 minute',
+ toStartOfHour: '1 hour',
+ toStartOfDay: '1 day',
+};
+
+const VALID_INTERVAL_UNITS = new Set(['second', 'minute', 'hour', 'day']);
+
+const isIdentChar = (ch: string) =>
+ (ch >= 'a' && ch <= 'z') ||
+ (ch >= 'A' && ch <= 'Z') ||
+ (ch >= '0' && ch <= '9') ||
+ ch === '_';
+
+const isWhitespace = (ch: string) =>
+ ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r';
+
+function findToStartOfCalls(
+ input: string,
+): { fn: string; argsInner: string }[] {
+ const out: { fn: string; argsInner: string }[] = [];
+ const n = input.length;
+ let i = 0;
+
+ // Skip the rest of a quoted region starting at `input[start]`.
+ // Returns the index of the character just past the closing quote.
+ const skipQuoted = (start: number, quote: string): number => {
+ let p = start + 1;
+ while (p < n) {
+ const c = input[p];
+ if (c === '\\' && p + 1 < n) {
+ p += 2;
+ continue;
+ }
+ if (c === quote) return p + 1;
+ p++;
+ }
+ return n;
+ };
+
+ while (i < n) {
+ const ch = input[i];
+
+ if (ch === "'" || ch === '"' || ch === '`') {
+ i = skipQuoted(i, ch);
+ continue;
+ }
+
+ // Try to read an identifier starting at a word boundary. A preceding
+ // identifier character would mean we're mid-token (e.g. `fooToStartOf…`).
+ const atBoundary = i === 0 || !isIdentChar(input[i - 1]);
+ if (!atBoundary || !isIdentChar(ch)) {
+ i++;
+ continue;
+ }
+
+ let j = i;
+ while (j < n && isIdentChar(input[j])) j++;
+ const ident = input.substring(i, j);
+
+ if (!ident.startsWith('toStartOf')) {
+ i = j;
+ continue;
+ }
+
+ // Expect '(' (possibly after whitespace) for this to be a call.
+ let k = j;
+ while (k < n && isWhitespace(input[k])) k++;
+ if (input[k] !== '(') {
+ i = j;
+ continue;
+ }
+
+ // Walk to the matching ')', honoring nested parens and quoted regions.
+ const argStart = k + 1;
+ let depth = 1;
+ let p = argStart;
+ while (p < n && depth > 0) {
+ const c = input[p];
+ if (c === "'" || c === '"' || c === '`') {
+ p = skipQuoted(p, c);
+ continue;
+ }
+ if (c === '(') depth++;
+ else if (c === ')') {
+ depth--;
+ if (depth === 0) break;
+ }
+ p++;
+ }
+ if (depth !== 0) break; // unterminated call — stop scanning
+ out.push({ fn: ident, argsInner: input.substring(argStart, p) });
+ i = p + 1;
+ }
+
+ return out;
+}
+
+function parseIntervalLiteral(expr: string): SQLInterval | undefined {
+ const tokens: string[] = [];
+ let cur = '';
+ for (const ch of expr) {
+ if (isWhitespace(ch)) {
+ if (cur) tokens.push(cur);
+ cur = '';
+ } else {
+ cur += ch;
+ }
+ }
+ if (cur) tokens.push(cur);
+
+ if (tokens.length < 3) return undefined;
+ if (tokens[0].toUpperCase() !== 'INTERVAL') return undefined;
+
+ const num = Number.parseInt(tokens[1], 10);
+ if (!Number.isFinite(num) || num <= 0 || String(num) !== tokens[1]) {
+ return undefined;
+ }
+
+ // Accept both singular and plural forms (MINUTE / MINUTES).
+ let unit = tokens[2].toLowerCase();
+ if (unit.endsWith('s')) unit = unit.slice(0, -1);
+ if (!VALID_INTERVAL_UNITS.has(unit)) return undefined;
+
+ return `${num} ${unit}` as SQLInterval;
+}
+
+export function inferGranularityFromMVSelect(
+ asSelect: string,
+): SQLInterval | undefined {
+ for (const { fn, argsInner } of findToStartOfCalls(asSelect)) {
+ if (fn in NAMED_BUCKET_FUNCTIONS) {
+ return NAMED_BUCKET_FUNCTIONS[fn];
+ }
+ if (fn === 'toStartOfInterval') {
+ const args = splitAndTrimWithBracket(argsInner);
+ if (args.length < 2) continue;
+ const parsed = parseIntervalLiteral(args[1]);
+ if (parsed) return parsed;
+ }
+ }
+ return undefined;
+}
+
+export function getNamedBucketFunction(
+ granularity: SQLInterval,
+): string | undefined {
+ for (const [fn, g] of Object.entries(NAMED_BUCKET_FUNCTIONS)) {
+ if (g === granularity) return fn;
+ }
+ return undefined;
+}
+
+export function renderStartOfBucketExpr(
+ granularity: SQLInterval,
+ inner: ChSql,
+): ChSql {
+ const namedFn = getNamedBucketFunction(granularity);
+ if (namedFn) {
+ // namedFn comes from a fixed allow-list (NAMED_BUCKET_FUNCTIONS keys), so
+ // splicing it as raw SQL is safe.
+ return chSql`${{ UNSAFE_RAW_SQL: namedFn }}(${inner})`;
+ }
+ const seconds = convertGranularityToSeconds(granularity);
+ return chSql`toStartOfInterval(${inner}, INTERVAL ${{ Int64: seconds }} SECOND)`;
+}
+
type SelectItem = Exclude<
BuilderChartConfigWithOptDateRange['select'],
string
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 381c26f07d..a39bfcc1aa 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -21,7 +21,10 @@ import type {
} from '@/types';
import { isLogSource, isTraceSource, SourceKind } from '@/types';
-import { optimizeGetKeyValuesCalls } from './materializedViews';
+import {
+ optimizeGetKeyValuesCalls,
+ renderStartOfBucketExpr,
+} from './materializedViews';
import {
getAlignedDateRange,
getDistributedTableArgs,
@@ -376,7 +379,15 @@ export class Metadata {
// Rollup path: query the key rollup table filtered by ColumnIdentifier and date range
if (metadataMVs && alignedDateRange) {
return this.cache.getOrFetch(cacheKey, async () => {
- const timeFilter = chSql`AND Timestamp >= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})) AND Timestamp <= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }}))`;
+ const startExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`,
+ );
+ const endExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`,
+ );
+ const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`;
const sql = chSql`
SELECT Key
FROM ${tableExpr({ database: databaseName, table: metadataMVs.keyRollupTable })}
@@ -1191,7 +1202,7 @@ export class Metadata {
* Autocomplete: fetches top values for a specific map key from the KV rollup table.
* Only filters by date range — no WHERE conditions. Values ordered by frequency.
*/
- async getCompleteKeyValues({
+ async getAllKeyValues({
databaseName,
tableName,
column,
@@ -1214,8 +1225,22 @@ export class Metadata {
}): Promise {
if (!metadataMVs) return [];
- const timeFilter = chSql`AND Timestamp >= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})) AND Timestamp <= toStartOfFifteenMinutes(fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }}))`;
- const cacheKey = `${connectionId}.${databaseName}.${tableName}.${column}.${key}.${dateRange[0].getTime()}.${dateRange[1].getTime()}.completeKeyValues`;
+ // Align date range to rollup granularity for consistent cache keys
+ const alignedDateRange = getAlignedDateRange(
+ dateRange,
+ metadataMVs.granularity,
+ );
+
+ const startExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`,
+ );
+ const endExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`,
+ );
+ const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`;
+ const cacheKey = `${connectionId}.${databaseName}.${tableName}.${column}.${key}.${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.allKeyValues`;
return this.cache.getOrFetch(cacheKey, async () => {
try {
From 910fe156fce2276c7a09f718aca3d21069ac3af8 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Tue, 5 May 2026 20:07:15 -0400
Subject: [PATCH 08/12] refactor: lump into getKeyValues
---
.../components/DBSearchPageFilters/utils.ts | 14 +-
.../__tests__/useAutoCompleteOptions.test.tsx | 61 +++++++-
.../app/src/hooks/useAutoCompleteOptions.tsx | 68 ++++++---
packages/app/src/hooks/useMetadata.tsx | 140 ++++++------------
.../src/__tests__/metadata.test.ts | 39 ++++-
packages/common-utils/src/core/metadata.ts | 140 +++++++++++-------
packages/common-utils/src/queryParser.ts | 22 ++-
7 files changed, 293 insertions(+), 191 deletions(-)
diff --git a/packages/app/src/components/DBSearchPageFilters/utils.ts b/packages/app/src/components/DBSearchPageFilters/utils.ts
index 5c249314e5..be48a2dd9a 100644
--- a/packages/app/src/components/DBSearchPageFilters/utils.ts
+++ b/packages/app/src/components/DBSearchPageFilters/utils.ts
@@ -1,5 +1,7 @@
// Utility functions for parsing and grouping map-like field names
+import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata';
+
// Clean ClickHouse expressions to extract clean property paths
export function cleanClickHouseExpression(key: string): string {
// Remove toString() wrapper if present
@@ -16,17 +18,13 @@ export function cleanClickHouseExpression(key: string): string {
export function parseMapFieldName(
key: string,
): { baseName: string; propertyPath: string } | null {
- // First clean the ClickHouse expression
const cleanKey = cleanClickHouseExpression(key);
+ const path = parseKeyPath(cleanKey);
- // Match patterns like: ResourceAttributes['some.property'], SpanAttributes['key'], or json_column.key
- const mapPattern = /^([^[]+)\[['"]([^'"]+)['"]\]$/;
- const match = cleanKey.match(mapPattern);
-
- if (match) {
+ if (path.length >= 2) {
return {
- baseName: match[1],
- propertyPath: match[2],
+ baseName: path[0],
+ propertyPath: path.slice(1).join('.'),
};
}
diff --git a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
index 0850e9b48f..4258748f8b 100644
--- a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
+++ b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx
@@ -5,13 +5,13 @@ import { renderHook } from '@testing-library/react';
import { LuceneLanguageFormatter } from '../../components/SearchInput/SearchInputV2';
import { useAutoCompleteOptions } from '../useAutoCompleteOptions';
import { tokenizeAtCursor } from '../useAutoCompleteOptions';
-import { useAllKeyValues, useMultipleAllFields } from '../useMetadata';
+import { useGetKeyValues, useMultipleAllFields } from '../useMetadata';
// Mock dependencies
jest.mock('../useMetadata', () => ({
...jest.requireActual('../useMetadata.tsx'),
useMultipleAllFields: jest.fn(),
- useAllKeyValues: jest.fn(),
+ useGetKeyValues: jest.fn(),
}));
const luceneFormatter = new LuceneLanguageFormatter();
@@ -50,7 +50,7 @@ describe('useAutoCompleteOptions', () => {
data: mockFields,
});
- (useAllKeyValues as jest.Mock).mockReturnValue({
+ (useGetKeyValues as jest.Mock).mockReturnValue({
data: null,
isFetching: false,
});
@@ -80,8 +80,13 @@ describe('useAutoCompleteOptions', () => {
});
it('should return key value options with correct lucene formatting', () => {
- (useAllKeyValues as jest.Mock).mockReturnValue({
- data: ['frontend', 'backend'],
+ (useGetKeyValues as jest.Mock).mockReturnValue({
+ data: [
+ {
+ key: 'ResourceAttributes.service.name',
+ value: ['frontend', 'backend'],
+ },
+ ],
isFetching: false,
});
@@ -119,6 +124,52 @@ describe('useAutoCompleteOptions', () => {
]);
});
+ it('should handle nested key value options', () => {
+ (useGetKeyValues as jest.Mock).mockReturnValue({
+ data: [
+ {
+ key: 'ResourceAttributes',
+ value: [
+ {
+ 'service.name': 'frontend',
+ 'deployment.environment': 'production',
+ },
+ ],
+ },
+ ],
+ isFetching: false,
+ });
+
+ const { result } = renderHook(() =>
+ useAutoCompleteOptions(luceneFormatter, 'ResourceAttributes', {
+ tableConnection: mockTableConnection,
+ }),
+ );
+
+ expect(result.current.options).toEqual([
+ {
+ value: 'ResourceAttributes',
+ label: 'ResourceAttributes (map)',
+ },
+ {
+ value: 'ResourceAttributes.service.name',
+ label: 'ResourceAttributes.service.name (string)',
+ },
+ {
+ value: 'TraceAttributes.trace.id',
+ label: 'TraceAttributes.trace.id (string)',
+ },
+ {
+ value: 'ResourceAttributes.service.name:"frontend"',
+ label: 'ResourceAttributes.service.name:"frontend"',
+ },
+ {
+ value: 'ResourceAttributes.deployment.environment:"production"',
+ label: 'ResourceAttributes.deployment.environment:"production"',
+ },
+ ]);
+ });
+
it('should handle additional suggestions', () => {
const { result } = renderHook(() =>
useAutoCompleteOptions(luceneFormatter, 'ResourceAttributes', {
diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx
index d3de439887..474b1a330e 100644
--- a/packages/app/src/hooks/useAutoCompleteOptions.tsx
+++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx
@@ -1,16 +1,18 @@
import { useMemo } from 'react';
import {
Field,
+ parseKeyPath,
TableConnection,
} from '@hyperdx/common-utils/dist/core/metadata';
+import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { NOW } from '@/config';
import {
deduplicate2dArray,
- useAllKeyValues,
+ useGetKeyValues,
useMultipleAllFields,
} from '@/hooks/useMetadata';
-import { toArray, useDebounce } from '@/utils';
+import { mergePath, toArray, useDebounce } from '@/utils';
export type TokenInfo = {
/** The full token at the cursor position */
@@ -208,28 +210,60 @@ export function useAutoCompleteOptions(
[fieldCompleteMap, fieldNameAtCursor],
);
- // Debounced fetch of values for the selected key from rollup tables
- const firstTc = tcs.length > 0 ? tcs[0] : undefined;
- const { data: keyValues, isFetching: isLoadingValues } = useAllKeyValues({
- tableConnection: firstTc,
- searchField,
- dateRange: effectiveDateRange,
+ // Build chart config and keys for fetching values from rollup tables
+ const chartConfig = useMemo(
+ () =>
+ tcs.length > 0
+ ? {
+ connection: tcs[0].connectionId,
+ from: {
+ databaseName: tcs[0].databaseName,
+ tableName: tcs[0].tableName,
+ },
+ timestampValueExpression: '',
+ select: '',
+ where: '',
+ dateRange: effectiveDateRange,
+ }
+ : undefined,
+ [tcs, effectiveDateRange],
+ );
+
+ const searchKeys = useMemo(
+ () => (searchField ? [mergePath(searchField.path)] : []),
+ [searchField],
+ );
+
+ const { data: keyValues, isFetching: isLoadingValues } = useGetKeyValues({
+ mode: 'all',
+ chartConfig,
+ keys: searchKeys,
});
// Build key-value pair suggestions
const keyValCompleteOptions = useMemo<
{ value: string; label: string }[]
>(() => {
- if (!keyValues || !searchField || keyValues.length === 0) return [];
-
- return keyValues.map(v => {
- const formatted = formatter.formatKeyValPair(
- formatter.formatFieldValue(searchField),
- v,
- );
- return { value: formatted, label: formatted };
+ if (!keyValues || keyValues.length === 0) return [];
+
+ return keyValues.flatMap(kv => {
+ const fieldName = parseKeyPath(kv.key).join('.');
+ return kv.value.flatMap((v: string | Record) => {
+ if (typeof v === 'object' && v !== null) {
+ // Map columns can return objects like { 'service.name': 'frontend' }
+ return Object.entries(v).map(([subKey, subVal]) => {
+ const formatted = formatter.formatKeyValPair(
+ `${fieldName}.${subKey}`,
+ subVal,
+ );
+ return { value: formatted, label: formatted };
+ });
+ }
+ const formatted = formatter.formatKeyValPair(fieldName, v);
+ return { value: formatted, label: formatted };
+ });
});
- }, [keyValues, searchField, formatter]);
+ }, [keyValues, formatter]);
// Combine all autocomplete options
const options = useMemo(() => {
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index c81683cbc2..e8729a9e6e 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -10,7 +10,11 @@ import {
TableConnection,
TableMetadata,
} from '@hyperdx/common-utils/dist/core/metadata';
-import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
+import {
+ BuilderChartConfigWithDateRange,
+ isLogSource,
+ isTraceSource,
+} from '@hyperdx/common-utils/dist/types';
import {
keepPreviousData,
useQuery,
@@ -173,100 +177,6 @@ export function useMultipleAllFields(
});
}
-/**
- * Resolves a field to the (ColumnIdentifier, Key) pair for querying the rollup table.
- * Map fields: ColumnIdentifier = column name (e.g. 'ResourceAttributes'), Key = map key
- * Native fields: ColumnIdentifier = 'NativeColumn', Key = column name (e.g. 'ServiceName')
- *
- * Only returns a result when the table connection has metadataMVs configured.
- */
-function fieldToRollupParams(
- field: Field | null,
- tableConnection: TableConnection | undefined,
-): { columnIdentifier: string; key: string } | null {
- if (!field || !tableConnection?.metadataMVs) return null;
-
- if (field.path.length >= 2) {
- const [column, mapKey] = field.path;
- return { columnIdentifier: column, key: mapKey };
- } else if (field.path.length === 1) {
- return { columnIdentifier: 'NativeColumn', key: field.path[0] };
- }
-
- return null;
-}
-
-/**
- * Debounced hook that fetches values for a specific field from rollup tables.
- * Works for both map keys (e.g. "ResourceAttributes.http.method") and
- * native columns (e.g. "ServiceName").
- */
-export function useAllKeyValues({
- tableConnection,
- searchField,
- dateRange,
-}: {
- tableConnection: TableConnection | undefined;
- searchField: Field | null;
- dateRange: [Date, Date];
-}) {
- const metadata = useMetadataWithSettings();
-
- const rollupParams = fieldToRollupParams(searchField, tableConnection);
-
- return useQuery({
- queryKey: [
- 'useAllKeyValues',
- tableConnection?.databaseName,
- tableConnection?.tableName,
- tableConnection?.connectionId,
- rollupParams?.columnIdentifier,
- rollupParams?.key,
- dateRange[0].getTime(),
- dateRange[1].getTime(),
- ],
- queryFn: async ({ signal }) => {
- if (!tableConnection || !rollupParams || !searchField) return [];
-
- // Try rollup first
- const rollupValues = await metadata.getAllKeyValues({
- databaseName: tableConnection.databaseName,
- tableName: tableConnection.tableName,
- column: rollupParams.columnIdentifier,
- key: rollupParams.key,
- connectionId: tableConnection.connectionId,
- metadataMVs: tableConnection.metadataMVs,
- dateRange,
- signal,
- });
-
- if (rollupValues.length > 0) return rollupValues;
-
- // Fall back to main table scan
- if (rollupParams.columnIdentifier !== 'NativeColumn') {
- // Map column: use getMapValues
- return metadata.getMapValues({
- databaseName: tableConnection.databaseName,
- tableName: tableConnection.tableName,
- column: rollupParams.columnIdentifier,
- key: rollupParams.key,
- connectionId: tableConnection.connectionId,
- });
- } else {
- // Native column: use getMapValues without a key (queries column directly)
- return metadata.getMapValues({
- databaseName: tableConnection.databaseName,
- tableName: tableConnection.tableName,
- column: searchField.path[0],
- connectionId: tableConnection.connectionId,
- });
- }
- },
- staleTime: 1000 * 60 * 5,
- enabled: !!rollupParams,
- });
-}
-
export function useAllFields(
tableConnection: TableConnection | undefined,
options?: Partial> & {
@@ -313,6 +223,7 @@ export function useMultipleGetKeyValues(
keys,
limit,
disableRowLimit,
+ mode = 'exact',
}: {
chartConfigs:
| BuilderChartConfigWithDateRange
@@ -320,6 +231,7 @@ export function useMultipleGetKeyValues(
keys: string[];
limit?: number;
disableRowLimit?: boolean;
+ mode?: 'all' | 'exact';
},
options?: Omit, 'queryKey'>,
) {
@@ -336,12 +248,47 @@ export function useMultipleGetKeyValues(
const query = useQuery<{ key: string; value: string[] }[]>({
queryKey: [
'useMetadata.useGetKeyValues',
+ mode,
...chartConfigsArr.map(cc => ({ ...cc })),
...keys,
disableRowLimit,
maxKeys,
],
queryFn: async ({ signal }) => {
+ if (mode === 'all') {
+ // For 'all' mode, resolve source to get metadataMVs
+ const firstConfig = chartConfigsArr[0];
+ if (!firstConfig || keys.length === 0) return [];
+ const firstSource = firstConfig.source
+ ? sources?.find(s => s.id === firstConfig.source)
+ : undefined;
+ const metadataMVs =
+ firstSource &&
+ (isLogSource(firstSource) || isTraceSource(firstSource))
+ ? firstSource.metadataMaterializedViews
+ : undefined;
+
+ const { databaseName, tableName } = firstConfig.from;
+ const connectionId = firstConfig.connection;
+ const dateRange = firstConfig.dateRange as [Date, Date] | undefined;
+
+ return Promise.all(
+ keys.slice(0, maxKeys).map(async keyExpression => {
+ const value = await metadata.getAllKeyValues({
+ databaseName,
+ tableName,
+ keyExpression,
+ connectionId,
+ metadataMVs,
+ dateRange,
+ signal,
+ });
+ return { key: keyExpression, value };
+ }),
+ );
+ }
+
+ // 'exact' mode
return (
await Promise.all(
chartConfigsArr.map(chartConfig => {
@@ -413,11 +360,13 @@ export function useGetKeyValues(
keys,
limit,
disableRowLimit,
+ mode,
}: {
chartConfig?: BuilderChartConfigWithDateRange;
keys: string[];
limit?: number;
disableRowLimit?: boolean;
+ mode?: 'all' | 'exact';
},
options?: Omit, 'queryKey'>,
) {
@@ -427,6 +376,7 @@ export function useGetKeyValues(
keys,
limit,
disableRowLimit,
+ mode,
},
options,
);
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index 7c1c98b29d..f39b1d10e6 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -1,5 +1,5 @@
import { ClickhouseClient } from '../clickhouse/node';
-import { Metadata, MetadataCache } from '../core/metadata';
+import { Metadata, MetadataCache, parseKeyPath } from '../core/metadata';
import * as renderChartConfigModule from '../core/renderChartConfig';
import { isBuilderChartConfig } from '../guards';
import { BuilderChartConfigWithDateRange, SourceKind, TSource } from '../types';
@@ -1121,3 +1121,40 @@ describe('Metadata', () => {
});
});
});
+
+describe('parseKeyPath', () => {
+ it('parses single-quoted bracket notation', () => {
+ expect(parseKeyPath("ResourceAttributes['service.name']")).toEqual([
+ 'ResourceAttributes',
+ 'service.name',
+ ]);
+ });
+
+ it('parses double-quoted bracket notation', () => {
+ expect(parseKeyPath('ResourceAttributes["service.name"]')).toEqual([
+ 'ResourceAttributes',
+ 'service.name',
+ ]);
+ });
+
+ it('returns single-element path for native columns', () => {
+ expect(parseKeyPath('ServiceName')).toEqual(['ServiceName']);
+ });
+
+ it('handles keys with dots in the map key', () => {
+ expect(parseKeyPath("SpanAttributes['http.request.method']")).toEqual([
+ 'SpanAttributes',
+ 'http.request.method',
+ ]);
+ });
+
+ it('returns single-element path for empty string', () => {
+ expect(parseKeyPath('')).toEqual(['']);
+ });
+
+ it('does not parse incomplete bracket notation', () => {
+ expect(parseKeyPath("ResourceAttributes['service.name")).toEqual([
+ "ResourceAttributes['service.name",
+ ]);
+ });
+});
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index a39bfcc1aa..1528855167 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1205,8 +1205,7 @@ export class Metadata {
async getAllKeyValues({
databaseName,
tableName,
- column,
- key,
+ keyExpression,
maxValues = 1000,
connectionId,
metadataMVs,
@@ -1215,65 +1214,85 @@ export class Metadata {
}: {
databaseName: string;
tableName: string;
- column: string;
- key: string;
+ /** Bracket-notation key string, e.g. `ResourceAttributes['service.name']` or `ServiceName` */
+ keyExpression: string;
maxValues?: number;
connectionId: string;
metadataMVs?: MetadataMaterializedViews;
- dateRange: [Date, Date];
+ dateRange?: [Date, Date];
signal?: AbortSignal;
}): Promise {
- if (!metadataMVs) return [];
+ const path = parseKeyPath(keyExpression);
+ const isMapKey = path.length >= 2;
+
+ // Try rollup table first when available
+ if (metadataMVs && dateRange) {
+ const rollupColumn = isMapKey ? path[0] : 'NativeColumn';
+ const rollupKey = isMapKey ? path[1] : path[0];
+
+ // Align date range to rollup granularity for consistent cache keys
+ const alignedDateRange = getAlignedDateRange(
+ dateRange,
+ metadataMVs.granularity,
+ );
- // Align date range to rollup granularity for consistent cache keys
- const alignedDateRange = getAlignedDateRange(
- dateRange,
- metadataMVs.granularity,
- );
+ const startExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`,
+ );
+ const endExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`,
+ );
+ const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`;
+ const cacheKey = `${connectionId}.${databaseName}.${tableName}.${rollupColumn}.${rollupKey}.${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.allKeyValues`;
- const startExpr = renderStartOfBucketExpr(
- metadataMVs.granularity,
- chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`,
- );
- const endExpr = renderStartOfBucketExpr(
- metadataMVs.granularity,
- chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`,
- );
- const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`;
- const cacheKey = `${connectionId}.${databaseName}.${tableName}.${column}.${key}.${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.allKeyValues`;
+ const rollupValues = await this.cache.getOrFetch(cacheKey, async () => {
+ try {
+ const sql = chSql`
+ SELECT Value
+ FROM ${tableExpr({ database: databaseName, table: metadataMVs.kvRollupTable })}
+ WHERE ColumnIdentifier = ${{ String: rollupColumn }}
+ AND Key = ${{ String: rollupKey }}
+ AND Value != ''
+ ${timeFilter}
+ GROUP BY Value
+ ORDER BY sum(count) DESC
+ LIMIT ${{ Int32: maxValues }}
+ `;
+
+ return await this.clickhouseClient
+ .query<'JSON'>({
+ query: sql.sql,
+ query_params: sql.params,
+ connectionId,
+ clickhouse_settings: {
+ ...this.getClickHouseSettings(),
+ timeout_overflow_mode: 'break',
+ max_execution_time: 15,
+ max_rows_to_read: '0',
+ },
+ abort_signal: signal,
+ })
+ .then(res => res.json<{ Value: string }>())
+ .then(d => d.data.map(r => r.Value));
+ } catch {
+ return [];
+ }
+ });
- return this.cache.getOrFetch(cacheKey, async () => {
- try {
- const sql = chSql`
- SELECT Value
- FROM ${tableExpr({ database: databaseName, table: metadataMVs.kvRollupTable })}
- WHERE ColumnIdentifier = ${{ String: column }}
- AND Key = ${{ String: key }}
- AND Value != ''
- ${timeFilter}
- GROUP BY Value
- ORDER BY sum(count) DESC
- LIMIT ${{ Int32: maxValues }}
- `;
+ if (rollupValues.length > 0) return rollupValues;
+ }
- return await this.clickhouseClient
- .query<'JSON'>({
- query: sql.sql,
- query_params: sql.params,
- connectionId,
- clickhouse_settings: {
- ...this.getClickHouseSettings(),
- timeout_overflow_mode: 'break',
- max_execution_time: 15,
- max_rows_to_read: '0',
- },
- abort_signal: signal,
- })
- .then(res => res.json<{ Value: string }>())
- .then(d => d.data.map(r => r.Value));
- } catch {
- return [];
- }
+ // Fall back to main table scan
+ const column = path[0];
+ const mapKey = isMapKey ? path[1] : undefined;
+ return this.getMapValues({
+ databaseName,
+ tableName,
+ column,
+ key: mapKey,
+ connectionId,
});
}
@@ -1468,6 +1487,23 @@ export type Field = {
jsType: JSDataType | null;
};
+/**
+ * Parses a bracket-notation key string into a path array.
+ * e.g. `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']`
+ * `ServiceName` → `['ServiceName']`
+ */
+export function parseKeyPath(key: string): string[] {
+ const singleIdx = key.indexOf("['");
+ if (singleIdx !== -1 && key.endsWith("']")) {
+ return [key.slice(0, singleIdx), key.slice(singleIdx + 2, -2)];
+ }
+ const doubleIdx = key.indexOf('["');
+ if (doubleIdx !== -1 && key.endsWith('"]')) {
+ return [key.slice(0, doubleIdx), key.slice(doubleIdx + 2, -2)];
+ }
+ return [key];
+}
+
// Describes a table and potentially related views
export type TableConnection = {
databaseName: string;
diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts
index 11cde151bf..dce62070ad 100644
--- a/packages/common-utils/src/queryParser.ts
+++ b/packages/common-utils/src/queryParser.ts
@@ -9,7 +9,12 @@ import {
extractInnerCHArrayJSType,
JSDataType,
} from '@/clickhouse';
-import { Metadata, SkipIndexMetadata, TableConnection } from '@/core/metadata';
+import {
+ Metadata,
+ parseKeyPath,
+ SkipIndexMetadata,
+ TableConnection,
+} from '@/core/metadata';
import {
parseTokenizerFromTextIndex,
splitAndTrimWithBracket,
@@ -41,18 +46,9 @@ export function parse(query: string): lucene.AST {
}
function buildMapContains(mapField: string) {
- const splitMapKey = (
- field: string,
- ): { map: string; key: string } | undefined => {
- const bracketIndex = field.indexOf("['");
- if (bracketIndex === -1) return undefined;
- const map = field.slice(0, bracketIndex);
- const key = field.slice(bracketIndex + 2, -2);
- return { map, key };
- };
- const val = splitMapKey(mapField);
- if (!val) return undefined;
- return SqlString.format('mapContains(??, ?)', [val.map, val.key]);
+ const path = parseKeyPath(mapField);
+ if (path.length < 2) return undefined;
+ return SqlString.format('mapContains(??, ?)', [path[0], path[1]]);
}
const IMPLICIT_FIELD = '';
From 8e5c7fadde7b4f2976f5f4ef6b39df4a60dfff77 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Wed, 6 May 2026 10:21:23 -0400
Subject: [PATCH 09/12] fix: old autocomplete behavior preserved without
metadata
---
packages/app/src/hooks/useAutoCompleteOptions.tsx | 4 +++-
packages/app/src/hooks/useMetadata.tsx | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx
index 474b1a330e..48357fff46 100644
--- a/packages/app/src/hooks/useAutoCompleteOptions.tsx
+++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx
@@ -234,8 +234,10 @@ export function useAutoCompleteOptions(
[searchField],
);
+ const mode = tcs[0]?.metadataMVs ? 'all' : 'exact';
+
const { data: keyValues, isFetching: isLoadingValues } = useGetKeyValues({
- mode: 'all',
+ mode,
chartConfig,
keys: searchKeys,
});
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index e8729a9e6e..200c40300f 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -270,7 +270,7 @@ export function useMultipleGetKeyValues(
const { databaseName, tableName } = firstConfig.from;
const connectionId = firstConfig.connection;
- const dateRange = firstConfig.dateRange as [Date, Date] | undefined;
+ const dateRange = firstConfig.dateRange;
return Promise.all(
keys.slice(0, maxKeys).map(async keyExpression => {
From ed21358f7ca7de7612bdfb700a38b173fb98f789 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Wed, 6 May 2026 10:27:20 -0400
Subject: [PATCH 10/12] implement PR feedback
---
packages/app/src/hooks/useAutoCompleteOptions.tsx | 4 +++-
packages/app/src/hooks/useMetadata.tsx | 15 ++++++++++++---
packages/common-utils/src/core/metadata.ts | 3 ++-
3 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx
index 48357fff46..cdb96481ed 100644
--- a/packages/app/src/hooks/useAutoCompleteOptions.tsx
+++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx
@@ -234,12 +234,14 @@ export function useAutoCompleteOptions(
[searchField],
);
- const mode = tcs[0]?.metadataMVs ? 'all' : 'exact';
+ const metadataMVs = tcs[0]?.metadataMVs;
+ const mode = metadataMVs ? 'all' : 'exact';
const { data: keyValues, isFetching: isLoadingValues } = useGetKeyValues({
mode,
chartConfig,
keys: searchKeys,
+ metadataMVs,
});
// Build key-value pair suggestions
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index 200c40300f..7610eed324 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -14,6 +14,7 @@ import {
BuilderChartConfigWithDateRange,
isLogSource,
isTraceSource,
+ MetadataMaterializedViews,
} from '@hyperdx/common-utils/dist/types';
import {
keepPreviousData,
@@ -224,6 +225,7 @@ export function useMultipleGetKeyValues(
limit,
disableRowLimit,
mode = 'exact',
+ metadataMVs: metadataMVsOverride,
}: {
chartConfigs:
| BuilderChartConfigWithDateRange
@@ -232,6 +234,8 @@ export function useMultipleGetKeyValues(
limit?: number;
disableRowLimit?: boolean;
mode?: 'all' | 'exact';
+ /** Pass directly to skip source-based resolution (e.g. when chartConfig has no source) */
+ metadataMVs?: MetadataMaterializedViews;
},
options?: Omit, 'queryKey'>,
) {
@@ -256,17 +260,19 @@ export function useMultipleGetKeyValues(
],
queryFn: async ({ signal }) => {
if (mode === 'all') {
- // For 'all' mode, resolve source to get metadataMVs
const firstConfig = chartConfigsArr[0];
if (!firstConfig || keys.length === 0) return [];
+
+ // Use explicit override, or resolve from source
const firstSource = firstConfig.source
? sources?.find(s => s.id === firstConfig.source)
: undefined;
const metadataMVs =
- firstSource &&
+ metadataMVsOverride ??
+ (firstSource &&
(isLogSource(firstSource) || isTraceSource(firstSource))
? firstSource.metadataMaterializedViews
- : undefined;
+ : undefined);
const { databaseName, tableName } = firstConfig.from;
const connectionId = firstConfig.connection;
@@ -361,12 +367,14 @@ export function useGetKeyValues(
limit,
disableRowLimit,
mode,
+ metadataMVs,
}: {
chartConfig?: BuilderChartConfigWithDateRange;
keys: string[];
limit?: number;
disableRowLimit?: boolean;
mode?: 'all' | 'exact';
+ metadataMVs?: MetadataMaterializedViews;
},
options?: Omit, 'queryKey'>,
) {
@@ -377,6 +385,7 @@ export function useGetKeyValues(
limit,
disableRowLimit,
mode,
+ metadataMVs,
},
options,
);
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 1528855167..57c860bce5 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -1276,7 +1276,8 @@ export class Metadata {
})
.then(res => res.json<{ Value: string }>())
.then(d => d.data.map(r => r.Value));
- } catch {
+ } catch (e) {
+ console.warn('getAllKeyValues rollup query failed', e);
return [];
}
});
From 05f13bcd5fdcb3abbcf6eb46ec347d1900f39200 Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Wed, 6 May 2026 12:21:53 -0400
Subject: [PATCH 11/12] fix: allow fallback
---
packages/common-utils/src/core/metadata.ts | 78 ++++++++++++----------
1 file changed, 43 insertions(+), 35 deletions(-)
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 57c860bce5..225df1eb16 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -378,43 +378,51 @@ export class Metadata {
// Rollup path: query the key rollup table filtered by ColumnIdentifier and date range
if (metadataMVs && alignedDateRange) {
- return this.cache.getOrFetch(cacheKey, async () => {
- const startExpr = renderStartOfBucketExpr(
- metadataMVs.granularity,
- chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`,
- );
- const endExpr = renderStartOfBucketExpr(
- metadataMVs.granularity,
- chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`,
- );
- const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`;
- const sql = chSql`
- SELECT Key
- FROM ${tableExpr({ database: databaseName, table: metadataMVs.keyRollupTable })}
- WHERE ColumnIdentifier = ${{ String: column }}
- ${timeFilter}
- GROUP BY Key
- ORDER BY sum(count) DESC
- LIMIT ${{ Int32: maxKeys }}
- `;
+ const rollupKeys = await this.cache.getOrFetch(
+ cacheKey,
+ async () => {
+ try {
+ const startExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`,
+ );
+ const endExpr = renderStartOfBucketExpr(
+ metadataMVs.granularity,
+ chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`,
+ );
+ const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`;
+ const sql = chSql`
+ SELECT Key
+ FROM ${tableExpr({ database: databaseName, table: metadataMVs.keyRollupTable })}
+ WHERE ColumnIdentifier = ${{ String: column }}
+ ${timeFilter}
+ GROUP BY Key
+ ORDER BY sum(count) DESC
+ LIMIT ${{ Int32: maxKeys }}
+ `;
- const keys = await this.clickhouseClient
- .query<'JSON'>({
- query: sql.sql,
- query_params: sql.params,
- connectionId,
- clickhouse_settings: {
- ...this.getClickHouseSettings(),
- timeout_overflow_mode: 'break',
- max_execution_time: 15,
- max_rows_to_read: '0',
- },
- })
- .then(res => res.json<{ Key: string }>())
- .then(d => d.data.map(row => row.Key).filter(k => k));
+ return await this.clickhouseClient
+ .query<'JSON'>({
+ query: sql.sql,
+ query_params: sql.params,
+ connectionId,
+ clickhouse_settings: {
+ ...this.getClickHouseSettings(),
+ timeout_overflow_mode: 'break',
+ max_execution_time: 15,
+ max_rows_to_read: '0',
+ },
+ })
+ .then(res => res.json<{ Key: string }>())
+ .then(d => d.data.map(row => row.Key).filter(k => k));
+ } catch (e) {
+ console.warn('getMapKeys rollup query failed', e);
+ return [];
+ }
+ },
+ );
- return keys;
- });
+ if (rollupKeys.length > 0) return rollupKeys;
}
// Original path: scan main table
From 1c97c05963f1892f11e0cac64df03e30d978b1ca Mon Sep 17 00:00:00 2001
From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com>
Date: Thu, 7 May 2026 09:45:21 -0400
Subject: [PATCH 12/12] fix: metadataMVsOverride added to query key
---
packages/app/src/hooks/useMetadata.tsx | 1 +
1 file changed, 1 insertion(+)
diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx
index 7610eed324..6e9e64ee1d 100644
--- a/packages/app/src/hooks/useMetadata.tsx
+++ b/packages/app/src/hooks/useMetadata.tsx
@@ -253,6 +253,7 @@ export function useMultipleGetKeyValues(
queryKey: [
'useMetadata.useGetKeyValues',
mode,
+ metadataMVsOverride,
...chartConfigsArr.map(cc => ({ ...cc })),
...keys,
disableRowLimit,