diff --git a/.changeset/gold-snakes-repeat.md b/.changeset/gold-snakes-repeat.md new file mode 100644 index 0000000000..ea3ecd413f --- /dev/null +++ b/.changeset/gold-snakes-repeat.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +feat: introducing rollup and source support for full autocomplete diff --git a/docker-compose.yml b/docker-compose.yml index 58490cf5c9..e9d720f98d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,8 +72,10 @@ services: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' DEFAULT_SOURCES: - '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local - ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local + '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 + minute"},"connection":"Local + ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 + minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index 0a8ebcf632..865cbc5d53 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -30,7 +30,7 @@ if [ -z "${DEFAULT_CONNECTIONS}" ]; then DEFAULT_CONNECTIONS='[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]' fi export DEFAULT_CONNECTIONS -export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' +export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' # if BETA_CH_OTEL_JSON_SCHEMA_ENABLED is true then return empty default sources if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' diff --git a/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql new file mode 100644 index 0000000000..dfb448ce22 --- /dev/null +++ b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql @@ -0,0 +1,171 @@ +-- +goose Up + +-- Map Attributes Key Value Rollup (all map columns in one table) +CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_kv_rollup_15m +( + `Timestamp` DateTime, + `ColumnIdentifier` LowCardinality(String), + `Key` LowCardinality(String), + `Value` String, + `count` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp, Value) +TTL Timestamp + ${TABLES_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +-- Key Only Rollup (derived from KV rollup) +CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_key_rollup_15m +( + `Timestamp` DateTime, + `ColumnIdentifier` LowCardinality(String), + `Key` LowCardinality(String), + `count` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp) +TTL Timestamp + ${TABLES_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_logs_key_rollup_15m_mv TO ${DATABASE}.otel_logs_key_rollup_15m +AS SELECT + Timestamp, + ColumnIdentifier, + Key, + sum(count) as count +FROM ${DATABASE}.otel_logs_kv_rollup_15m +GROUP BY ColumnIdentifier, Key, Timestamp; + +-- Single MV: CTE with UNION ALL across all columns, then aggregate +CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_logs_attr_kv_rollup_15m_mv TO ${DATABASE}.otel_logs_kv_rollup_15m +AS WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${DATABASE}.otel_logs + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'LogAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${DATABASE}.otel_logs + ARRAY JOIN LogAttributes AS entry + UNION ALL + SELECT + 'ScopeAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${DATABASE}.otel_logs + ARRAY JOIN ScopeAttributes AS entry + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'SeverityText' as Key, + CAST(SeverityText AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ServiceName' as Key, + CAST(ServiceName AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ScopeName' as Key, + CAST(ScopeName AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ScopeVersion' as Key, + CAST(ScopeVersion AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ResourceSchemaUrl' as Key, + CAST(ResourceSchemaUrl AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ScopeSchemaUrl' as Key, + CAST(ScopeSchemaUrl AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_k8s.cluster.name' as Key, + CAST(`__hdx_materialized_k8s.cluster.name` AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_k8s.container.name' as Key, + CAST(`__hdx_materialized_k8s.container.name` AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_k8s.deployment.name' as Key, + CAST(`__hdx_materialized_k8s.deployment.name` AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_k8s.namespace.name' as Key, + CAST(`__hdx_materialized_k8s.namespace.name` AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_k8s.node.name' as Key, + CAST(`__hdx_materialized_k8s.node.name` AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_k8s.pod.name' as Key, + CAST(`__hdx_materialized_k8s.pod.name` AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_k8s.pod.uid' as Key, + CAST(`__hdx_materialized_k8s.pod.uid` AS String) as Value + FROM ${DATABASE}.otel_logs + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + '__hdx_materialized_deployment.environment.name' as Key, + CAST(`__hdx_materialized_deployment.environment.name` AS String) as Value + FROM ${DATABASE}.otel_logs +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value; diff --git a/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql new file mode 100644 index 0000000000..2ca574dbb2 --- /dev/null +++ b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql @@ -0,0 +1,108 @@ +-- +goose Up + +-- Key Value Rollup (all columns in one table) +CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m +( + `Timestamp` DateTime, + `ColumnIdentifier` LowCardinality(String), + `Key` LowCardinality(String), + `Value` String, + `count` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp, Value) +TTL Timestamp + ${TABLES_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +-- Key Only Rollup (derived from KV rollup) +CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_key_rollup_15m +( + `Timestamp` DateTime, + `ColumnIdentifier` LowCardinality(String), + `Key` LowCardinality(String), + `count` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp) +TTL Timestamp + ${TABLES_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +-- Key only rollup MV +CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_traces_key_rollup_15m_mv TO ${DATABASE}.otel_traces_key_rollup_15m +AS SELECT + Timestamp, + ColumnIdentifier, + Key, + sum(count) AS count +FROM ${DATABASE}.otel_traces_kv_rollup_15m +GROUP BY ColumnIdentifier, Key, Timestamp; + +-- Single MV: CTE with UNION ALL across all columns, then aggregate +CREATE MATERIALIZED VIEW IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m_mv TO ${DATABASE}.otel_traces_kv_rollup_15m +AS WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${DATABASE}.otel_traces + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'SpanAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\[\\d+\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${DATABASE}.otel_traces + ARRAY JOIN SpanAttributes AS entry + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ServiceName' AS Key, + CAST(ServiceName AS String) AS Value + FROM ${DATABASE}.otel_traces + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'SpanName' AS Key, + CAST(SpanName AS String) AS Value + FROM ${DATABASE}.otel_traces + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'SpanKind' AS Key, + CAST(SpanKind AS String) AS Value + FROM ${DATABASE}.otel_traces + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'StatusCode' AS Key, + CAST(StatusCode AS String) AS Value + FROM ${DATABASE}.otel_traces + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ScopeName' AS Key, + CAST(ScopeName AS String) AS Value + FROM ${DATABASE}.otel_traces + UNION ALL + SELECT + 'NativeColumn' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + 'ScopeVersion' AS Key, + CAST(ScopeVersion AS String) AS Value + FROM ${DATABASE}.otel_traces +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value; diff --git a/packages/api/.env.development b/packages/api/.env.development index be156e564a..cc6593ef21 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -22,7 +22,7 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" diff --git a/packages/api/openapi.json b/packages/api/openapi.json index bfc26a5650..265b09de1b 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -2564,6 +2564,28 @@ "$ref": "#/components/schemas/MaterializedView" }, "nullable": true + }, + "metadataMaterializedViews": { + "type": "object", + "description": "Configure materialized views for fast field discovery and value autocomplete.", + "nullable": true, + "properties": { + "keyRollupTable": { + "type": "string", + "description": "ClickHouse table name for the key rollup (field discovery).", + "example": "otel_logs_key_rollup_15m" + }, + "kvRollupTable": { + "type": "string", + "description": "ClickHouse table name for the key-value rollup (value autocomplete).", + "example": "otel_logs_kv_rollup_15m" + }, + "granularity": { + "type": "string", + "description": "The time granularity of the rollup tables.", + "example": "15m" + } + } } } }, @@ -2751,6 +2773,28 @@ "$ref": "#/components/schemas/MaterializedView" }, "nullable": true + }, + "metadataMaterializedViews": { + "type": "object", + "description": "Configure materialized views for fast field discovery and value autocomplete.", + "nullable": true, + "properties": { + "keyRollupTable": { + "type": "string", + "description": "ClickHouse table name for the key rollup (field discovery).", + "example": "otel_traces_key_rollup_15m" + }, + "kvRollupTable": { + "type": "string", + "description": "ClickHouse table name for the key-value rollup (value autocomplete).", + "example": "otel_traces_kv_rollup_15m" + }, + "granularity": { + "type": "string", + "description": "The time granularity of the rollup tables.", + "example": "15m" + } + } } } }, diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index e06602e4a7..7e5e17512c 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -148,6 +148,14 @@ export const LogSource = Source.discriminator( materializedViews: { type: mongoose.Schema.Types.Array, }, + metadataMaterializedViews: { + type: { + keyRollupTable: String, + kvRollupTable: String, + granularity: String, + }, + default: undefined, + }, orderByExpression: String, }), ); @@ -188,6 +196,14 @@ export const TraceSource = Source.discriminator( materializedViews: { type: mongoose.Schema.Types.Array, }, + metadataMaterializedViews: { + type: { + keyRollupTable: String, + kvRollupTable: String, + granularity: String, + }, + default: undefined, + }, orderByExpression: String, }), ); diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 06d8767d6a..e61f22176f 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -30,17 +30,29 @@ export function mapGranularityToExternalFormat(granularity: string): string { function mapSourceToExternalSource(source: TSource): TSource { if (!('materializedViews' in source)) return source; - if (!Array.isArray(source.materializedViews)) return source; - return { - ...source, - materializedViews: source.materializedViews.map(view => { - return { - ...view, - minGranularity: mapGranularityToExternalFormat(view.minGranularity), - }; - }), - }; + const mapped = { ...source }; + + if (Array.isArray(source.materializedViews)) { + mapped.materializedViews = source.materializedViews.map(view => ({ + ...view, + minGranularity: mapGranularityToExternalFormat(view.minGranularity), + })); + } + + if ( + 'metadataMaterializedViews' in source && + source.metadataMaterializedViews + ) { + mapped.metadataMaterializedViews = { + ...source.metadataMaterializedViews, + granularity: mapGranularityToExternalFormat( + source.metadataMaterializedViews.granularity, + ), + }; + } + + return mapped; } function applyLegacyDefaults( @@ -359,6 +371,23 @@ function formatExternalSource(source: SourceDocument) { * items: * $ref: '#/components/schemas/MaterializedView' * nullable: true + * metadataMaterializedViews: + * type: object + * description: Configure materialized views for fast field discovery and value autocomplete. + * nullable: true + * properties: + * keyRollupTable: + * type: string + * description: ClickHouse table name for the key rollup (field discovery). + * example: otel_logs_key_rollup_15m + * kvRollupTable: + * type: string + * description: ClickHouse table name for the key-value rollup (value autocomplete). + * example: otel_logs_kv_rollup_15m + * granularity: + * type: string + * description: The time granularity of the rollup tables. + * example: 15m * TraceSource: * type: object * required: @@ -509,6 +538,23 @@ function formatExternalSource(source: SourceDocument) { * items: * $ref: '#/components/schemas/MaterializedView' * nullable: true + * metadataMaterializedViews: + * type: object + * description: Configure materialized views for fast field discovery and value autocomplete. + * nullable: true + * properties: + * keyRollupTable: + * type: string + * description: ClickHouse table name for the key rollup (field discovery). + * example: otel_traces_key_rollup_15m + * kvRollupTable: + * type: string + * description: ClickHouse table name for the key-value rollup (value autocomplete). + * example: otel_traces_kv_rollup_15m + * granularity: + * type: string + * description: The time granularity of the rollup tables. + * example: 15m * MetricSource: * type: object * required: diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 939ee45ab8..11bc4842f5 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -1830,6 +1830,7 @@ export function DBSearchPage() { label="SELECT" size="xs" allowMultiline + dateRange={searchedTimeRange} /> @@ -1841,6 +1842,7 @@ export function DBSearchPage() { onSubmit={onSubmit} label="ORDER BY" size="xs" + dateRange={searchedTimeRange} /> <> @@ -1901,6 +1903,7 @@ export function DBSearchPage() { enableHotkey data-testid="search-input" minWidth="min(600px, 100%)" + dateRange={searchedTimeRange} /> (displayType === DisplayType.StackedBar ? BarChart : AreaChart), // LineChart; + [displayType], + ); const lines = useMemo(() => { const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0; diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index d07365fc8a..7cb2052aee 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -1126,19 +1126,26 @@ const DBSearchPageFiltersComponent = ({ tableName: chartConfig.from.tableName, connectionId: chartConfig.connection, }); - const { data, isLoading, error } = useAllFields({ - databaseName: chartConfig.from.databaseName, - tableName: chartConfig.from.tableName, - connectionId: chartConfig.connection, - }); + const { data: source } = useSource({ id: sourceId }); + const sourceTableConnection = tcFromSource(source); + const { data, isLoading, error } = useAllFields( + { + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + connectionId: chartConfig.connection, + metadataMVs: sourceTableConnection.metadataMVs, + }, + { + dateRange: chartConfig.dateRange, + }, + ); const { data: columns } = useColumns({ databaseName: chartConfig.from.databaseName, tableName: chartConfig.from.tableName, connectionId: chartConfig.connection, }); - const { data: source } = useSource({ id: sourceId }); - const { data: tableMetadata } = useTableMetadata(tcFromSource(source)); + const { data: tableMetadata } = useTableMetadata(sourceTableConnection); useEffect(() => { if (error) { diff --git a/packages/app/src/components/DBSearchPageFilters/utils.ts b/packages/app/src/components/DBSearchPageFilters/utils.ts index 5c249314e5..be48a2dd9a 100644 --- a/packages/app/src/components/DBSearchPageFilters/utils.ts +++ b/packages/app/src/components/DBSearchPageFilters/utils.ts @@ -1,5 +1,7 @@ // Utility functions for parsing and grouping map-like field names +import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; + // Clean ClickHouse expressions to extract clean property paths export function cleanClickHouseExpression(key: string): string { // Remove toString() wrapper if present @@ -16,17 +18,13 @@ export function cleanClickHouseExpression(key: string): string { export function parseMapFieldName( key: string, ): { baseName: string; propertyPath: string } | null { - // First clean the ClickHouse expression const cleanKey = cleanClickHouseExpression(key); + const path = parseKeyPath(cleanKey); - // Match patterns like: ResourceAttributes['some.property'], SpanAttributes['key'], or json_column.key - const mapPattern = /^([^[]+)\[['"]([^'"]+)['"]\]$/; - const match = cleanKey.match(mapPattern); - - if (match) { + if (path.length >= 2) { return { - baseName: match[1], - propertyPath: match[2], + baseName: path[0], + propertyPath: path.slice(1).join('.'), }; } diff --git a/packages/app/src/components/SQLEditor/SQLEditor.tsx b/packages/app/src/components/SQLEditor/SQLEditor.tsx index ba14096106..66ce0965e6 100644 --- a/packages/app/src/components/SQLEditor/SQLEditor.tsx +++ b/packages/app/src/components/SQLEditor/SQLEditor.tsx @@ -28,6 +28,7 @@ type SQLEditorProps = { enableLineWrapping?: boolean; tableConnections?: TableConnection[]; additionalCompletions?: SQLCompletion[]; + dateRange?: [Date, Date]; }; export default function SQLEditor({ @@ -38,12 +39,15 @@ export default function SQLEditor({ enableLineWrapping = false, tableConnections, additionalCompletions, + dateRange, }: SQLEditorProps) { const { colorScheme } = useMantineColorScheme(); const ref = useRef(null); const compartmentRef = useRef(new Compartment()); - const { data: fields } = useMultipleAllFields(tableConnections ?? []); + const { data: fields } = useMultipleAllFields(tableConnections ?? [], { + dateRange, + }); const updateAutocompleteColumns = useCallback( (viewRef: EditorView) => { diff --git a/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx b/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx index 28edd0362e..beaa354c23 100644 --- a/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx +++ b/packages/app/src/components/SQLEditor/SQLInlineEditor.tsx @@ -62,6 +62,7 @@ type SQLInlineEditorProps = { queryHistoryType?: string; parentRef?: HTMLElement | null; allowMultiline?: boolean; + dateRange?: [Date, Date]; }; const MAX_EDITOR_HEIGHT = '150px'; @@ -86,12 +87,15 @@ export default function SQLInlineEditor({ queryHistoryType, parentRef, allowMultiline = true, + dateRange, }: SQLInlineEditorProps & TableConnectionChoice) { const { colorScheme } = useMantineColorScheme(); const _tableConnections = tableConnection ? [tableConnection] : tableConnections; - const { data: fields } = useMultipleAllFields(_tableConnections ?? []); + const { data: fields } = useMultipleAllFields(_tableConnections ?? [], { + dateRange, + }); const filteredFields = useMemo(() => { return filterField ? fields?.filter(filterField) : fields; }, [fields, filterField]); diff --git a/packages/app/src/components/SearchInput/AutocompleteInput.tsx b/packages/app/src/components/SearchInput/AutocompleteInput.tsx index 906937cdeb..861c019bfc 100644 --- a/packages/app/src/components/SearchInput/AutocompleteInput.tsx +++ b/packages/app/src/components/SearchInput/AutocompleteInput.tsx @@ -1,10 +1,10 @@ import { useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react'; import cx from 'classnames'; import Fuse from 'fuse.js'; -import { Popover, Textarea, UnstyledButton } from '@mantine/core'; +import { Loader, Popover, Textarea, UnstyledButton } from '@mantine/core'; +import type { TokenInfo } from '@/hooks/useAutoCompleteOptions'; import { useQueryHistory } from '@/utils'; -import { useDebounce } from '@/utils'; import InputLanguageSwitch from './InputLanguageSwitch'; @@ -16,6 +16,8 @@ export default function AutocompleteInput({ onChange, placeholder = 'Search your events for anything...', autocompleteOptions, + isLoadingValues, + tokenInfo, size = 'sm', aboveSuggestions, belowSuggestions, @@ -35,6 +37,8 @@ export default function AutocompleteInput({ placeholder?: string; size?: 'xs' | 'sm' | 'lg'; autocompleteOptions?: { value: string; label: string }[]; + isLoadingValues?: boolean; + tokenInfo?: TokenInfo; aboveSuggestions?: React.ReactNode; belowSuggestions?: React.ReactNode; showSuggestionsOnEmpty?: boolean; @@ -94,16 +98,14 @@ export default function AutocompleteInput({ [autocompleteOptions], ); - const debouncedValue = useDebounce(value ?? '', 200); const suggestedProperties = useMemo(() => { - const tokens = debouncedValue.split(' '); - const lastToken = tokens[tokens.length - 1]; + const token = tokenInfo?.token ?? ''; - if (lastToken.length === 0 && showSuggestionsOnEmpty) { + if (token.length === 0 && showSuggestionsOnEmpty) { return autocompleteOptions ?? []; } - return fuse.search(lastToken).map(result => result.item); - }, [debouncedValue, fuse, autocompleteOptions, showSuggestionsOnEmpty]); + return fuse.search(token).map(result => result.item); + }, [tokenInfo, fuse, autocompleteOptions, showSuggestionsOnEmpty]); const onSelectSearchHistory = (query: string) => { setSelectedQueryHistoryIndex(-1); @@ -116,13 +118,30 @@ export default function AutocompleteInput({ const onAcceptSuggestion = (suggestion: string) => { setSelectedAutocompleteIndex(-1); - const newValue = - value == null - ? suggestion - : value.split(' ').slice(0, -1).join(' ') + - `${value.split(' ').length > 1 ? ' ' : ''}${suggestion}`; + if (value == null || !tokenInfo) { + onChange(suggestion); + inputRef.current?.focus(); + return; + } + + // Replace the token at cursor with the suggestion + const tokens = [...tokenInfo.tokens]; + tokens[tokenInfo.index] = suggestion; + const newValue = tokens.join(' '); + + // Place cursor right after the inserted suggestion + let newCursorPos = 0; + for (let i = 0; i <= tokenInfo.index; i++) { + newCursorPos += tokens[i].length; + if (i < tokenInfo.index) newCursorPos++; // space + } + onChange(newValue); - inputRef.current?.focus(); + + requestAnimationFrame(() => { + inputRef.current?.setSelectionRange(newCursorPos, newCursorPos); + inputRef.current?.focus(); + }); }; const ref = useRef(null); useLayoutEffect(() => { @@ -238,6 +257,7 @@ export default function AutocompleteInput({ e.target instanceof HTMLTextAreaElement ) { if (suggestedProperties.length > 0) { + e.preventDefault(); setSelectedAutocompleteIndex( Math.min( selectedAutocompleteIndex + 1, @@ -252,6 +272,7 @@ export default function AutocompleteInput({ e.target instanceof HTMLTextAreaElement ) { if (suggestedProperties.length > 0) { + e.preventDefault(); setSelectedAutocompleteIndex( Math.max(selectedAutocompleteIndex - 1, 0), ); @@ -281,6 +302,9 @@ export default function AutocompleteInput({
{suggestionsHeader} + {isLoadingValues && ( + + )}
{suggestedProperties.length > suggestionsLimit && (
diff --git a/packages/app/src/components/SearchInput/SearchInputV2.tsx b/packages/app/src/components/SearchInput/SearchInputV2.tsx index 54cad034aa..7ff4545608 100644 --- a/packages/app/src/components/SearchInput/SearchInputV2.tsx +++ b/packages/app/src/components/SearchInput/SearchInputV2.tsx @@ -44,6 +44,7 @@ export default function SearchInputV2({ onSubmit, additionalSuggestions, queryHistoryType, + dateRange, 'data-testid': dataTestId, ...props }: { @@ -56,6 +57,7 @@ export default function SearchInputV2({ onSubmit?: () => void; additionalSuggestions?: string[]; queryHistoryType?: string; + dateRange?: [Date, Date]; 'data-testid'?: string; } & UseControllerProps & TableConnectionChoice) { @@ -67,12 +69,18 @@ export default function SearchInputV2({ const ref = useRef(null); const [parsedEnglishQuery, setParsedEnglishQuery] = useState(''); - const autoCompleteOptions = useAutoCompleteOptions( + const { + options: autoCompleteOptions, + isLoadingValues, + tokenInfo, + } = useAutoCompleteOptions( luceneLanguageFormatter, value != null ? `${value}` : '', { tableConnection: tableConnection ? tableConnection : tableConnections, additionalSuggestions, + dateRange, + inputRef: ref, }, ); @@ -110,6 +118,8 @@ export default function SearchInputV2({ onChange={onChange} placeholder={placeholder} autocompleteOptions={autoCompleteOptions} + isLoadingValues={isLoadingValues} + tokenInfo={tokenInfo} size={size} zIndex={zIndex} language={language} diff --git a/packages/app/src/components/SearchInput/SearchWhereInput.tsx b/packages/app/src/components/SearchInput/SearchWhereInput.tsx index e57fc12018..8eb82d07a6 100644 --- a/packages/app/src/components/SearchInput/SearchWhereInput.tsx +++ b/packages/app/src/components/SearchInput/SearchWhereInput.tsx @@ -103,6 +103,10 @@ export type SearchWhereInputProps = { * Additional autocomplete/suggestion options (e.g. attribute names) */ additionalSuggestions?: string[]; + /** + * Date range for autocomplete rollup queries + */ + dateRange?: [Date, Date]; /** * Form field name for the language value ('sql' | 'lucene'). * If not provided, defaults to `${name}Language` (e.g. name="where" → "whereLanguage"). @@ -152,6 +156,7 @@ export default function SearchWhereInput({ minWidth, 'data-testid': dataTestId, additionalSuggestions, + dateRange, languageName = `${name}Language`, }: SearchWhereInputProps) { const { field: languageField } = useController({ @@ -218,6 +223,7 @@ export default function SearchWhereInput({ size={size} data-testid={dataTestId} additionalSuggestions={additionalSuggestions} + dateRange={dateRange} /> )} {enableHotkey && ( diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index 1fda76d195..bed02e7982 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -564,6 +564,96 @@ function MaterializedViewsFormSection({ control, setValue }: TableModelProps) { ); } +/** Component for configuring metadata materialized views (key + KV rollups) */ +function MetadataMaterializedViewsFormSection({ + control, + setValue, +}: TableModelProps) { + const databaseName = useWatch({ + control, + name: `from.databaseName`, + defaultValue: DEFAULT_DATABASE, + }); + const connection = useWatch({ control, name: `connection` }); + + const metadataMVs = useWatch({ + control, + name: 'metadataMaterializedViews', + }); + + const hasMetadataMVs = !!metadataMVs; + + return ( + + + {hasMetadataMVs ? ( + + + setValue('metadataMaterializedViews', undefined)} + > + + + + + + + Key Rollup Table + + + + + + KV Rollup Table + + + + + + + ) : ( + + )} + + + ); +} + /** Component for configuring a single materialized view */ function MaterializedViewFormSection({ control, @@ -1300,6 +1390,8 @@ function LogTableModelForm(props: TableModelProps) { + + + + { + if (value && typeof value === 'object' && !Array.isArray(value)) { + setValue(key as any, value); + return; + } resetField(key as any, { keepDirty: true, defaultValue: value, @@ -1895,6 +1993,7 @@ export function TableSourceForm({ watchedKind, resetField, metadata, + setValue, ]); // Sets the default connection field to the first connection after the diff --git a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx index a11f497d1f..4258748f8b 100644 --- a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx +++ b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx @@ -4,18 +4,14 @@ import { renderHook } from '@testing-library/react'; import { LuceneLanguageFormatter } from '../../components/SearchInput/SearchInputV2'; import { useAutoCompleteOptions } from '../useAutoCompleteOptions'; -import { - useJsonColumns, - useMultipleAllFields, - useMultipleGetKeyValues, -} from '../useMetadata'; +import { tokenizeAtCursor } from '../useAutoCompleteOptions'; +import { useGetKeyValues, useMultipleAllFields } from '../useMetadata'; // Mock dependencies jest.mock('../useMetadata', () => ({ ...jest.requireActual('../useMetadata.tsx'), useMultipleAllFields: jest.fn(), - useMultipleGetKeyValues: jest.fn(), - useJsonColumns: jest.fn(), + useGetKeyValues: jest.fn(), })); const luceneFormatter = new LuceneLanguageFormatter(); @@ -54,12 +50,9 @@ describe('useAutoCompleteOptions', () => { data: mockFields, }); - (useMultipleGetKeyValues as jest.Mock).mockReturnValue({ - data: null, - }); - - (useJsonColumns as jest.Mock).mockReturnValue({ + (useGetKeyValues as jest.Mock).mockReturnValue({ data: null, + isFetching: false, }); }); @@ -70,7 +63,7 @@ describe('useAutoCompleteOptions', () => { }), ); - expect(result.current).toEqual([ + expect(result.current.options).toEqual([ { value: 'ResourceAttributes', label: 'ResourceAttributes (map)', @@ -87,15 +80,14 @@ describe('useAutoCompleteOptions', () => { }); it('should return key value options with correct lucene formatting', () => { - const mockKeyValues = [ - { - key: 'ResourceAttributes.service.name', - value: ['frontend', 'backend'], - }, - ]; - - (useMultipleGetKeyValues as jest.Mock).mockReturnValue({ - data: mockKeyValues, + (useGetKeyValues as jest.Mock).mockReturnValue({ + data: [ + { + key: 'ResourceAttributes.service.name', + value: ['frontend', 'backend'], + }, + ], + isFetching: false, }); const { result } = renderHook(() => @@ -108,7 +100,7 @@ describe('useAutoCompleteOptions', () => { ), ); - expect(result.current).toEqual([ + expect(result.current.options).toEqual([ { value: 'ResourceAttributes', label: 'ResourceAttributes (map)', @@ -132,22 +124,20 @@ describe('useAutoCompleteOptions', () => { ]); }); - // TODO: Does this test case need to be removed after HDX-1548? it('should handle nested key value options', () => { - const mockKeyValues = [ - { - key: 'ResourceAttributes', - value: [ - { - 'service.name': 'frontend', - 'deployment.environment': 'production', - }, - ], - }, - ]; - - (useMultipleGetKeyValues as jest.Mock).mockReturnValue({ - data: mockKeyValues, + (useGetKeyValues as jest.Mock).mockReturnValue({ + data: [ + { + key: 'ResourceAttributes', + value: [ + { + 'service.name': 'frontend', + 'deployment.environment': 'production', + }, + ], + }, + ], + isFetching: false, }); const { result } = renderHook(() => @@ -156,8 +146,7 @@ describe('useAutoCompleteOptions', () => { }), ); - //console.log(result.current); - expect(result.current).toEqual([ + expect(result.current.options).toEqual([ { value: 'ResourceAttributes', label: 'ResourceAttributes (map)', @@ -189,7 +178,7 @@ describe('useAutoCompleteOptions', () => { }), ); - expect(result.current).toEqual([ + expect(result.current.options).toEqual([ { value: 'ResourceAttributes', label: 'ResourceAttributes (map)', @@ -209,3 +198,165 @@ describe('useAutoCompleteOptions', () => { ]); }); }); + +describe('tokenizeAtCursor', () => { + // Each case is tokenized with the cursor at the end of the input, so + // `expectedToken` is the token the user is currently typing into. + const cases: { + name: string; + input: string; + expectedToken: string; + expectedTokens: string[]; + }[] = [ + // Basic whitespace splitting + { + name: 'splits a bare whitespace-separated string into tokens', + input: 'foo bar baz', + expectedToken: 'baz', + expectedTokens: ['foo', 'bar', 'baz'], + }, + { + name: 'returns a single token for input with no whitespace', + input: 'FieldName', + expectedToken: 'FieldName', + expectedTokens: ['FieldName'], + }, + { + name: 'returns a single empty token for empty input', + input: '', + expectedToken: '', + expectedTokens: [''], + }, + { + name: 'produces empty tokens for consecutive spaces (split-like semantics)', + input: 'foo bar', + expectedToken: 'bar', + expectedTokens: ['foo', '', 'bar'], + }, + { + name: 'produces a trailing empty token when input ends in a space', + input: 'foo ', + expectedToken: '', + expectedTokens: ['foo', ''], + }, + + // Balanced quoted regions + { + name: 'keeps whitespace inside paired quotes as part of one token', + input: 'Service:"hello world"', + expectedToken: 'Service:"hello world"', + expectedTokens: ['Service:"hello world"'], + }, + { + name: 'splits two paired-quote fields on the space between them', + input: 'ServiceName:"clickhouse" SeverityText:"debug"', + expectedToken: 'SeverityText:"debug"', + expectedTokens: ['ServiceName:"clickhouse"', 'SeverityText:"debug"'], + }, + { + name: 'preserves escaped quotes inside a quoted region', + input: 'Service:"he said \\"hi\\"" other', + expectedToken: 'other', + expectedTokens: ['Service:"he said \\"hi\\""', 'other'], + }, + { + name: 'treats a colon inside a quoted value as literal text', + input: 'URL:"http://example.com/x" x', + expectedToken: 'x', + expectedTokens: ['URL:"http://example.com/x"', 'x'], + }, + + // Unclosed quotes — reproduces the bug where `Field:" Other:"v"` (three + // quotes with the first unclosed) previously collapsed into one token. + { + name: 'breaks at whitespace when a quote is followed by a new field pattern', + input: 'ServiceName:" SeverityText:"debug"', + expectedToken: 'SeverityText:"debug"', + expectedTokens: ['ServiceName:"', 'SeverityText:"debug"'], + }, + { + name: 'treats a single unclosed quote at end of input as a literal', + input: 'Service:"hel', + expectedToken: 'Service:"hel', + expectedTokens: ['Service:"hel'], + }, + { + // No `ident:` after the space, so the quote can still legitimately + // extend — but there's no closing quote anywhere, so it's unclosed. + name: 'handles an unclosed quote followed by whitespace then bare text', + input: 'Service:"hello world', + expectedToken: 'world', + expectedTokens: ['Service:"hello', 'world'], + }, + { + name: 'handles multiple unclosed quotes across fields', + input: 'A:" B:" C:"done"', + expectedToken: 'C:"done"', + expectedTokens: ['A:"', 'B:"', 'C:"done"'], + }, + + // Identifier-like characters after whitespace + { + // The space inside the quoted value is followed by `!`, not `ident:`, + // so the quote should still be able to close. + name: 'does not bail out on whitespace followed by a non-identifier', + input: 'Service:"hello !world"', + expectedToken: 'Service:"hello !world"', + expectedTokens: ['Service:"hello !world"'], + }, + { + name: 'does not treat whitespace + ident without colon as a new field', + input: 'Service:"hello world done"', + expectedToken: 'Service:"hello world done"', + expectedTokens: ['Service:"hello world done"'], + }, + { + name: 'handles dotted identifiers in the new-field pattern', + input: 'Foo:" ResourceAttributes.service.name:"x"', + expectedToken: 'ResourceAttributes.service.name:"x"', + expectedTokens: ['Foo:"', 'ResourceAttributes.service.name:"x"'], + }, + ]; + + it.each(cases)('$name', ({ input, expectedToken, expectedTokens }) => { + const { token, tokens } = tokenizeAtCursor(input, input.length); + expect(tokens).toEqual(expectedTokens); + expect(token).toBe(expectedToken); + }); + + // Cursor-positioning is orthogonal to tokenization — keep these separate + // because each case exercises a different cursor offset for the same input. + describe('cursor positioning', () => { + it('returns the first token when the cursor is at position 0', () => { + const { token, index } = tokenizeAtCursor('foo bar baz', 0); + expect(token).toBe('foo'); + expect(index).toBe(0); + }); + + it('returns the middle token when the cursor is inside it', () => { + // 0123456789012 + // 'foo bar baz' — cursor at 5 is inside 'bar' + const { token, index } = tokenizeAtCursor('foo bar baz', 5); + expect(token).toBe('bar'); + expect(index).toBe(1); + }); + + it('returns the token whose range contains the cursor in a quoted field', () => { + const input = 'Service:"hello world" other'; + // cursor inside the quoted token + const { token, index } = tokenizeAtCursor(input, 15); + expect(token).toBe('Service:"hello world"'); + expect(index).toBe(0); + }); + + it('returns the unclosed-quote token when the cursor is inside it', () => { + // User is typing a value — the first quote is unclosed because the + // next field pattern `SeverityText:` appears after whitespace. Cursor + // sits just after `l`, inside the in-progress `ServiceName:"cl` token. + const input = 'ServiceName:"cl SeverityText:"info"'; + const { token, index } = tokenizeAtCursor(input, 15); + expect(token).toBe('ServiceName:"cl'); + expect(index).toBe(0); + }); + }); +}); diff --git a/packages/app/src/hooks/useAutoCompleteOptions.tsx b/packages/app/src/hooks/useAutoCompleteOptions.tsx index 00eade5d9f..cdb96481ed 100644 --- a/packages/app/src/hooks/useAutoCompleteOptions.tsx +++ b/packages/app/src/hooks/useAutoCompleteOptions.tsx @@ -1,6 +1,7 @@ -import { useEffect, useMemo, useState } from 'react'; +import { useMemo } from 'react'; import { Field, + parseKeyPath, TableConnection, } from '@hyperdx/common-utils/dist/core/metadata'; import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; @@ -8,11 +9,128 @@ import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/type import { NOW } from '@/config'; import { deduplicate2dArray, - useJsonColumns, + useGetKeyValues, useMultipleAllFields, - useMultipleGetKeyValues, } from '@/hooks/useMetadata'; -import { mergePath, toArray } from '@/utils'; +import { mergePath, toArray, useDebounce } from '@/utils'; + +export type TokenInfo = { + /** The full token at the cursor position */ + token: string; + /** Index of the token in the tokens array */ + index: number; + /** All tokens from splitting the input on whitespace */ + tokens: string[]; +}; + +const IDENT_RE = /[A-Za-z0-9_.]/; + +function findMatchingQuote(value: string, startIdx: number): number { + let i = startIdx + 1; + while (i < value.length) { + const ch = value[i]; + if (ch === '\\' && i + 1 < value.length) { + i += 2; + continue; + } + if (ch === '"') return i; + if (ch === ' ' || ch === '\t' || ch === '\n') { + let k = i; + while ( + k < value.length && + (value[k] === ' ' || value[k] === '\t' || value[k] === '\n') + ) + k++; + const identStart = k; + while (k < value.length && IDENT_RE.test(value[k])) k++; + if (k > identStart && k < value.length && value[k] === ':') { + return -1; + } + } + i++; + } + return -1; +} + +export function tokenizeAtCursor(value: string, cursorPos: number): TokenInfo { + const tokens: string[] = []; + // Start offsets of each token in the original string + const starts: number[] = []; + + let current = ''; + let currentStart = -1; + let inQuotes = false; + let escaped = false; + + for (let i = 0; i < value.length; i++) { + const ch = value[i]; + + if (escaped) { + // Always include the escaped character verbatim (along with its backslash) + current += ch; + escaped = false; + continue; + } + + if (ch === '\\' && inQuotes) { + current += ch; + escaped = true; + continue; + } + + if (ch === '"') { + if (inQuotes) { + // Closing an already-opened quoted region. + if (currentStart === -1) currentStart = i; + current += ch; + inQuotes = false; + continue; + } + // Only enter a quoted region if there's a matching close ahead. + if (findMatchingQuote(value, i) !== -1) { + if (currentStart === -1) currentStart = i; + current += ch; + inQuotes = true; + continue; + } + // Stray/unclosed quote — treat as a literal character. + if (currentStart === -1) currentStart = i; + current += ch; + continue; + } + + if (!inQuotes && ch === ' ') { + // Boundary: flush current token (even if empty, to mirror prior `split(' ')` + // semantics where consecutive spaces produce empty tokens). + tokens.push(current); + starts.push(currentStart === -1 ? i : currentStart); + current = ''; + currentStart = -1; + continue; + } + + if (currentStart === -1) currentStart = i; + current += ch; + } + // Flush trailing token + tokens.push(current); + starts.push(currentStart === -1 ? value.length : currentStart); + + // Locate token containing the cursor. The cursor sits *between* characters, + // so a token covers [start, start+len]; we pick the last token whose range + // contains cursorPos. + let idx = tokens.length - 1; + for (let i = 0; i < tokens.length; i++) { + const start = starts[i]; + const end = start + tokens[i].length; + if (cursorPos <= end) { + idx = i; + break; + } + } + + return { token: tokens[idx] ?? '', index: idx, tokens }; +} export interface ILanguageFormatter { formatFieldValue: (f: Field) => string; @@ -22,23 +140,32 @@ export interface ILanguageFormatter { export function useAutoCompleteOptions( formatter: ILanguageFormatter, - value: string, + _value: string, { tableConnection, additionalSuggestions, + dateRange, + inputRef, }: { tableConnection?: TableConnection | TableConnection[]; additionalSuggestions?: string[]; + dateRange?: [Date, Date]; + inputRef?: React.RefObject; }, ) { - // Fetch and gather all field options - const { data: fields } = useMultipleAllFields( - tableConnection - ? Array.isArray(tableConnection) - ? tableConnection - : [tableConnection] - : [], + const value = useDebounce(_value, 300); + const tcs = useMemo(() => toArray(tableConnection), [tableConnection]); + + const effectiveDateRange: [Date, Date] = useMemo( + () => dateRange ?? [new Date(NOW - 24 * 60 * 60 * 1000), new Date(NOW)], + [dateRange], ); + + // Fetch fields, using rollup for map key discovery when available + const { data: fields } = useMultipleAllFields(tcs, { + dateRange: effectiveDateRange, + }); + const { fieldCompleteOptions, fieldCompleteMap } = useMemo(() => { const _columns = (fields ?? []).filter(c => c.jsType !== null); @@ -63,121 +190,89 @@ export function useAutoCompleteOptions( return { fieldCompleteOptions, fieldCompleteMap }; }, [formatter, fields, additionalSuggestions]); - // searchField is used for the purpose of checking if a key is valid and key values should be fetched - // TODO: Come back and refactor how this works - it's not great and wouldn't catch a person copy-pasting some text - const [searchField, setSearchField] = useState(null); - // check if any search field matches - useEffect(() => { - const v = fieldCompleteMap.get(value); - if (v) { - setSearchField(v); - } - }, [fieldCompleteMap, value]); - // clear search field if no key matches anymore - useEffect(() => { - if (!searchField) return; - if (!value.startsWith(formatter.formatFieldValue(searchField))) { - setSearchField(null); - } - }, [searchField, setSearchField, value, formatter]); - const tcForJson = Array.isArray(tableConnection) - ? tableConnection.length > 0 - ? tableConnection[0] - : undefined - : tableConnection; - const { data: jsonColumns } = useJsonColumns( - tcForJson ?? { - tableName: '', - databaseName: '', - connectionId: '', - }, + // Tokenize input at cursor position + const tokenInfo = useMemo(() => { + // eslint-disable-next-line react-hooks/refs + const cursorPos = inputRef?.current?.selectionStart ?? value.length; + // eslint-disable-next-line react-hooks/refs + return tokenizeAtCursor(value, cursorPos); + }, [value, inputRef]); + + // Extract the field name portion of the token (strip colon and value) + const fieldNameAtCursor = useMemo(() => { + const colonIdx = tokenInfo.token.indexOf(':'); + return colonIdx >= 0 ? tokenInfo.token.slice(0, colonIdx) : tokenInfo.token; + }, [tokenInfo.token]); + + // Derive the active search field from the token at cursor + const searchField = useMemo( + () => fieldCompleteMap.get(fieldNameAtCursor) ?? null, + [fieldCompleteMap, fieldNameAtCursor], ); - const searchKeys = useMemo( + + // Build chart config and keys for fetching values from rollup tables + const chartConfig = useMemo( () => - searchField && jsonColumns - ? [mergePath(searchField.path, jsonColumns)] - : [], - [searchField, jsonColumns], + tcs.length > 0 + ? { + connection: tcs[0].connectionId, + from: { + databaseName: tcs[0].databaseName, + tableName: tcs[0].tableName, + }, + timestampValueExpression: '', + select: '', + where: '', + dateRange: effectiveDateRange, + } + : undefined, + [tcs, effectiveDateRange], ); - // hooks to get key values - const chartConfigs: BuilderChartConfigWithDateRange[] = toArray( - tableConnection, - ).map(({ databaseName, tableName, connectionId }) => ({ - connection: connectionId, - from: { - databaseName, - tableName, - }, - timestampValueExpression: '', - select: '', - where: '', - // TODO: Pull in date for query as arg - // just assuming 1/2 day is okay to query over right now - dateRange: [new Date(NOW - (86400 * 1000) / 2), new Date(NOW)], - })); - - const { data: keyVals } = useMultipleGetKeyValues({ - chartConfigs, + const searchKeys = useMemo( + () => (searchField ? [mergePath(searchField.path)] : []), + [searchField], + ); + + const metadataMVs = tcs[0]?.metadataMVs; + const mode = metadataMVs ? 'all' : 'exact'; + + const { data: keyValues, isFetching: isLoadingValues } = useGetKeyValues({ + mode, + chartConfig, keys: searchKeys, + metadataMVs, }); + // Build key-value pair suggestions const keyValCompleteOptions = useMemo< { value: string; label: string }[] >(() => { - if (!keyVals || !searchField) return fieldCompleteOptions; - const output = // TODO: Fix this hacky type assertion caused by bug in HDX-1548 - ( - keyVals as unknown as { - key: string; - value: (string | { [key: string]: string })[]; - }[] - ).flatMap(kv => { - return kv.value.flatMap(v => { - if (typeof v === 'string') { - const value = formatter.formatKeyValPair( - formatter.formatFieldValue(searchField), - v, + if (!keyValues || keyValues.length === 0) return []; + + return keyValues.flatMap(kv => { + const fieldName = parseKeyPath(kv.key).join('.'); + return kv.value.flatMap((v: string | Record) => { + if (typeof v === 'object' && v !== null) { + // Map columns can return objects like { 'service.name': 'frontend' } + return Object.entries(v).map(([subKey, subVal]) => { + const formatted = formatter.formatKeyValPair( + `${fieldName}.${subKey}`, + subVal, ); - return [ - { - value, - label: value, - }, - ]; - } else if (typeof v === 'object') { - // TODO: Fix type issues mentioned in HDX-1548 - const output: { - value: string; - label: string; - }[] = []; - for (const [key, val] of Object.entries(v)) { - if (typeof key !== 'string' || typeof val !== 'string') { - console.error('unknown type for autocomplete object ', v); - return []; - } - const field = structuredClone(searchField); - field.path.push(key); - const value = formatter.formatKeyValPair( - formatter.formatFieldValue(field), - val, - ); - output.push({ - value, - label: value, - }); - } - return output; - } else { - return []; - } - }); + return { value: formatted, label: formatted }; + }); + } + const formatted = formatter.formatKeyValPair(fieldName, v); + return { value: formatted, label: formatted }; }); - return output; - }, [fieldCompleteOptions, keyVals, searchField, formatter]); + }); + }, [keyValues, formatter]); - // combine all autocomplete options - return useMemo(() => { + // Combine all autocomplete options + const options = useMemo(() => { return deduplicate2dArray([fieldCompleteOptions, keyValCompleteOptions]); }, [fieldCompleteOptions, keyValCompleteOptions]); + + return { options, isLoadingValues, tokenInfo }; } diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 86a0f79e0c..6e9e64ee1d 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -10,7 +10,12 @@ import { TableConnection, TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; -import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { + BuilderChartConfigWithDateRange, + isLogSource, + isTraceSource, + MetadataMaterializedViews, +} from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, useQuery, @@ -123,14 +128,19 @@ export function useJsonColumns( export function useMultipleAllFields( tableConnections: TableConnection[], - options?: Partial>, + options?: Partial> & { + dateRange?: [Date, Date]; + }, ) { const metadata = useMetadataWithSettings(); const { data: me, isFetched } = api.useMe(); + const { dateRange, ...queryOptions } = options ?? {}; return useQuery({ queryKey: [ 'useMetadata.useMultipleAllFields', ...tableConnections.map(tc => ({ ...tc })), + dateRange?.[0]?.getTime(), + dateRange?.[1]?.getTime(), ], queryFn: async () => { const team = me?.team; @@ -139,7 +149,7 @@ export function useMultipleAllFields( } const promiseResults = await Promise.allSettled( - tableConnections.map(tc => metadata.getAllFields(tc)), + tableConnections.map(tc => metadata.getAllFields({ ...tc, dateRange })), ); const fields2d: Field[][] = promiseResults.map(result => { @@ -164,13 +174,15 @@ export function useMultipleAllFields( tc => !!tc.databaseName && !!tc.tableName && !!tc.connectionId, ) && isFetched, - ...options, + ...queryOptions, }); } export function useAllFields( tableConnection: TableConnection | undefined, - options?: Partial>, + options?: Partial> & { + dateRange?: [Date, Date]; + }, ) { return useMultipleAllFields( tableConnection ? [tableConnection] : [], @@ -212,6 +224,8 @@ export function useMultipleGetKeyValues( keys, limit, disableRowLimit, + mode = 'exact', + metadataMVs: metadataMVsOverride, }: { chartConfigs: | BuilderChartConfigWithDateRange @@ -219,6 +233,9 @@ export function useMultipleGetKeyValues( keys: string[]; limit?: number; disableRowLimit?: boolean; + mode?: 'all' | 'exact'; + /** Pass directly to skip source-based resolution (e.g. when chartConfig has no source) */ + metadataMVs?: MetadataMaterializedViews; }, options?: Omit, 'queryKey'>, ) { @@ -235,12 +252,50 @@ export function useMultipleGetKeyValues( const query = useQuery<{ key: string; value: string[] }[]>({ queryKey: [ 'useMetadata.useGetKeyValues', + mode, + metadataMVsOverride, ...chartConfigsArr.map(cc => ({ ...cc })), ...keys, disableRowLimit, maxKeys, ], queryFn: async ({ signal }) => { + if (mode === 'all') { + const firstConfig = chartConfigsArr[0]; + if (!firstConfig || keys.length === 0) return []; + + // Use explicit override, or resolve from source + const firstSource = firstConfig.source + ? sources?.find(s => s.id === firstConfig.source) + : undefined; + const metadataMVs = + metadataMVsOverride ?? + (firstSource && + (isLogSource(firstSource) || isTraceSource(firstSource)) + ? firstSource.metadataMaterializedViews + : undefined); + + const { databaseName, tableName } = firstConfig.from; + const connectionId = firstConfig.connection; + const dateRange = firstConfig.dateRange; + + return Promise.all( + keys.slice(0, maxKeys).map(async keyExpression => { + const value = await metadata.getAllKeyValues({ + databaseName, + tableName, + keyExpression, + connectionId, + metadataMVs, + dateRange, + signal, + }); + return { key: keyExpression, value }; + }), + ); + } + + // 'exact' mode return ( await Promise.all( chartConfigsArr.map(chartConfig => { @@ -312,11 +367,15 @@ export function useGetKeyValues( keys, limit, disableRowLimit, + mode, + metadataMVs, }: { chartConfig?: BuilderChartConfigWithDateRange; keys: string[]; limit?: number; disableRowLimit?: boolean; + mode?: 'all' | 'exact'; + metadataMVs?: MetadataMaterializedViews; }, options?: Omit, 'queryKey'>, ) { @@ -326,6 +385,8 @@ export function useGetKeyValues( keys, limit, disableRowLimit, + mode, + metadataMVs, }, options, ); diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index 504d03ce39..4713a22170 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -7,6 +7,7 @@ import { filterColumnMetaByType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; +import { inferGranularityFromMVSelect } from '@hyperdx/common-utils/dist/core/materializedViews'; import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; @@ -348,6 +349,43 @@ export async function inferTableSourceConfig({ // Check if SpanEvents column is available const hasSpanEvents = columns.some(col => col.name === 'Events.Timestamp'); + // Check if metadata rollup tables exist and, if so, infer the bucketing + // granularity from the key-rollup view's `as_select` + const rollupMeta = + isOtelLogSchema || isOtelSpanSchema + ? await (async () => { + const [keyMeta, kvMeta] = await Promise.all([ + metadata.getTableMetadata({ + databaseName, + tableName: `${tableName}_key_rollup_15m`, + connectionId, + }), + metadata.getTableMetadata({ + databaseName, + tableName: `${tableName}_kv_rollup_15m`, + connectionId, + }), + ]); + return keyMeta != null && kvMeta != null + ? { keyMeta, kvMeta } + : undefined; + })() + : undefined; + + const metadataMVsConfig = rollupMeta + ? { + metadataMaterializedViews: { + keyRollupTable: `${tableName}_key_rollup_15m`, + kvRollupTable: `${tableName}_kv_rollup_15m`, + // Fall back to '15 minute' to preserve the prior default when the + // MV's `as_select` doesn't contain a recognized bucketing function. + granularity: + inferGranularityFromMVSelect(rollupMeta.keyMeta.as_select) ?? + '15 minute', + }, + } + : {}; + return { ...baseConfig, ...(isOtelLogSchema @@ -365,6 +403,7 @@ export async function inferTableSourceConfig({ traceIdExpression: 'TraceId', severityTextExpression: 'SeverityText', + ...metadataMVsConfig, } : {}), ...(isOtelSpanSchema @@ -387,6 +426,7 @@ export async function inferTableSourceConfig({ statusCodeExpression: 'StatusCode', statusMessageExpression: 'StatusMessage', ...(hasSpanEvents ? { spanEventsValueExpression: 'Events' } : {}), + ...metadataMVsConfig, } : {}), }; diff --git a/packages/common-utils/src/__tests__/materializedViews.test.ts b/packages/common-utils/src/__tests__/materializedViews.test.ts new file mode 100644 index 0000000000..c1c2bd6c0c --- /dev/null +++ b/packages/common-utils/src/__tests__/materializedViews.test.ts @@ -0,0 +1,192 @@ +import { inferGranularityFromMVSelect } from '@/core/materializedViews'; + +describe('inferGranularityFromMVSelect', () => { + describe('real MV schemas', () => { + // Shape matches the `otel_logs_attr_kv_rollup_15m_mv` view in + // docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql. + it('detects 15 minute from the otel_logs kv rollup MV select', () => { + const asSelect = `WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM default.otel_logs + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'LogAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM default.otel_logs + ARRAY JOIN LogAttributes AS entry +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value`; + + expect(inferGranularityFromMVSelect(asSelect)).toBe('15 minute'); + }); + + // Shape matches the `otel_traces_kv_rollup_15m_mv` view in + // docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql. + it('detects 15 minute from the otel_traces kv rollup MV select', () => { + const asSelect = `WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM default.otel_traces + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'SpanAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM default.otel_traces + ARRAY JOIN SpanAttributes AS entry +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value`; + + expect(inferGranularityFromMVSelect(asSelect)).toBe('15 minute'); + }); + + // The key-rollup MV rolls up the kv rollup, so it doesn't bucket the + // timestamp itself — it just selects the already-bucketed Timestamp. + it('returns undefined when the select contains no bucketing function', () => { + const asSelect = `SELECT + Timestamp, + ColumnIdentifier, + Key, + sum(count) as count +FROM default.otel_logs_kv_rollup_15m +GROUP BY ColumnIdentifier, Key, Timestamp`; + + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + }); + + describe('toStartOfInterval form', () => { + it.each([ + ['INTERVAL 5 MINUTE', '5 minute'], + ['INTERVAL 1 HOUR', '1 hour'], + ['INTERVAL 2 hour', '2 hour'], + ['INTERVAL 30 SECOND', '30 second'], + ['INTERVAL 1 DAY', '1 day'], + ])('parses %s', (interval, expected) => { + const asSelect = `SELECT toStartOfInterval(Timestamp, ${interval}) AS ts, count() FROM t GROUP BY ts`; + expect(inferGranularityFromMVSelect(asSelect)).toBe(expected); + }); + + it('accepts plural units', () => { + const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 10 MINUTES) AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBe('10 minute'); + }); + + it('handles extra arguments (origin, timezone)', () => { + const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 1 DAY, toDateTime('2025-01-01'), 'America/Los_Angeles') AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBe('1 day'); + }); + + it('handles extra whitespace and newlines inside the call', () => { + const asSelect = `SELECT + toStartOfInterval ( + Timestamp, + INTERVAL 15 MINUTE + ) AS ts +FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBe('15 minute'); + }); + + it('ignores unknown units', () => { + const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 1 WEEK) AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + }); + + describe('named bucket functions', () => { + it.each([ + ['toStartOfSecond', '1 second'], + ['toStartOfMinute', '1 minute'], + ['toStartOfFiveMinutes', '5 minute'], + ['toStartOfTenMinutes', '10 minute'], + ['toStartOfFifteenMinutes', '15 minute'], + ['toStartOfHour', '1 hour'], + ['toStartOfDay', '1 day'], + ])('maps %s to %s', (fn, expected) => { + const asSelect = `SELECT ${fn}(Timestamp) AS ts, count() FROM t GROUP BY ts`; + expect(inferGranularityFromMVSelect(asSelect)).toBe(expected); + }); + + it('returns undefined for unrecognized toStartOf* functions', () => { + // toStartOfMonth is a real CH function but not in NAMED_BUCKET_FUNCTIONS. + const asSelect = `SELECT toStartOfMonth(Timestamp) AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + }); + + describe('first match wins', () => { + it('returns the granularity of the first toStartOf call encountered', () => { + const asSelect = `SELECT toStartOfHour(Timestamp) AS h, toStartOfMinute(Timestamp) AS m FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBe('1 hour'); + }); + + it('returns first call even when a later call would also match', () => { + const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL 5 MINUTE) AS a, toStartOfHour(Timestamp) AS b FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBe('5 minute'); + }); + }); + + describe('robustness against quoting and noise', () => { + it('ignores toStartOf* tokens inside single-quoted string literals', () => { + const asSelect = `SELECT 'toStartOfHour(Timestamp)' AS label, toStartOfMinute(Timestamp) AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBe('1 minute'); + }); + + it('ignores toStartOf* tokens inside backtick-quoted identifiers', () => { + const asSelect = 'SELECT `toStartOfHour` AS col FROM t'; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + + it('ignores toStartOf* substrings embedded in longer identifiers', () => { + // my_toStartOfHour_col is a single identifier, not a function call. + const asSelect = `SELECT my_toStartOfHour_col AS x FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + + it('is not confused by commas inside nested calls', () => { + const asSelect = `SELECT toStartOfInterval(coalesce(Timestamp, now()), INTERVAL 5 MINUTE) AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBe('5 minute'); + }); + + it('skips a toStartOf call without matching parens and stops scanning', () => { + // Unterminated call — we stop rather than looping forever. + const asSelect = `SELECT toStartOfHour(Timestamp FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + }); + + describe('degenerate inputs', () => { + it('returns undefined for empty string', () => { + expect(inferGranularityFromMVSelect('')).toBeUndefined(); + }); + + it('returns undefined when no toStartOf call is present', () => { + const asSelect = `SELECT Timestamp, count() FROM t GROUP BY Timestamp`; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + + it('returns undefined when toStartOfInterval has no interval arg', () => { + const asSelect = `SELECT toStartOfInterval(Timestamp) AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + + it('returns undefined when toStartOfInterval has a malformed interval arg', () => { + const asSelect = `SELECT toStartOfInterval(Timestamp, INTERVAL abc MINUTE) AS ts FROM t`; + expect(inferGranularityFromMVSelect(asSelect)).toBeUndefined(); + }); + }); +}); diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index e3fb2ffc0a..f39b1d10e6 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1,5 +1,5 @@ import { ClickhouseClient } from '../clickhouse/node'; -import { Metadata, MetadataCache } from '../core/metadata'; +import { Metadata, MetadataCache, parseKeyPath } from '../core/metadata'; import * as renderChartConfigModule from '../core/renderChartConfig'; import { isBuilderChartConfig } from '../guards'; import { BuilderChartConfigWithDateRange, SourceKind, TSource } from '../types'; @@ -740,7 +740,7 @@ describe('Metadata', () => { const fields = await md.getAllFields({ databaseName: 'otel', - tableName: 'otel_logs', + tableName: 'test_logs', connectionId: 'conn-1', }); @@ -819,7 +819,7 @@ describe('Metadata', () => { const fields = await md.getAllFields({ databaseName: 'otel', - tableName: 'otel_logs', + tableName: 'test_logs', connectionId: 'conn-1', }); @@ -1121,3 +1121,40 @@ describe('Metadata', () => { }); }); }); + +describe('parseKeyPath', () => { + it('parses single-quoted bracket notation', () => { + expect(parseKeyPath("ResourceAttributes['service.name']")).toEqual([ + 'ResourceAttributes', + 'service.name', + ]); + }); + + it('parses double-quoted bracket notation', () => { + expect(parseKeyPath('ResourceAttributes["service.name"]')).toEqual([ + 'ResourceAttributes', + 'service.name', + ]); + }); + + it('returns single-element path for native columns', () => { + expect(parseKeyPath('ServiceName')).toEqual(['ServiceName']); + }); + + it('handles keys with dots in the map key', () => { + expect(parseKeyPath("SpanAttributes['http.request.method']")).toEqual([ + 'SpanAttributes', + 'http.request.method', + ]); + }); + + it('returns single-element path for empty string', () => { + expect(parseKeyPath('')).toEqual(['']); + }); + + it('does not parse incomplete bracket notation', () => { + expect(parseKeyPath("ResourceAttributes['service.name")).toEqual([ + "ResourceAttributes['service.name", + ]); + }); +}); diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index 29237ce876..810dd79063 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -1,6 +1,6 @@ import { differenceInSeconds } from 'date-fns'; -import { BaseClickhouseClient } from '@/clickhouse'; +import { BaseClickhouseClient, ChSql, chSql } from '@/clickhouse'; import { BuilderChartConfigWithOptDateRange, CteChartConfig, @@ -9,6 +9,7 @@ import { isLogSource, isTraceSource, MaterializedViewConfiguration, + type SQLInterval, TLogSource, TSource, TTraceSource, @@ -25,6 +26,178 @@ import { splitAndTrimWithBracket, } from './utils'; +// ClickHouse named time-bucketing functions and their granularity equivalents. +const NAMED_BUCKET_FUNCTIONS: Record = { + toStartOfSecond: '1 second', + toStartOfMinute: '1 minute', + toStartOfFiveMinutes: '5 minute', + toStartOfTenMinutes: '10 minute', + toStartOfFifteenMinutes: '15 minute', + toStartOfHour: '1 hour', + toStartOfDay: '1 day', +}; + +const VALID_INTERVAL_UNITS = new Set(['second', 'minute', 'hour', 'day']); + +const isIdentChar = (ch: string) => + (ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || + ch === '_'; + +const isWhitespace = (ch: string) => + ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r'; + +function findToStartOfCalls( + input: string, +): { fn: string; argsInner: string }[] { + const out: { fn: string; argsInner: string }[] = []; + const n = input.length; + let i = 0; + + // Skip the rest of a quoted region starting at `input[start]`. + // Returns the index of the character just past the closing quote. + const skipQuoted = (start: number, quote: string): number => { + let p = start + 1; + while (p < n) { + const c = input[p]; + if (c === '\\' && p + 1 < n) { + p += 2; + continue; + } + if (c === quote) return p + 1; + p++; + } + return n; + }; + + while (i < n) { + const ch = input[i]; + + if (ch === "'" || ch === '"' || ch === '`') { + i = skipQuoted(i, ch); + continue; + } + + // Try to read an identifier starting at a word boundary. A preceding + // identifier character would mean we're mid-token (e.g. `fooToStartOf…`). + const atBoundary = i === 0 || !isIdentChar(input[i - 1]); + if (!atBoundary || !isIdentChar(ch)) { + i++; + continue; + } + + let j = i; + while (j < n && isIdentChar(input[j])) j++; + const ident = input.substring(i, j); + + if (!ident.startsWith('toStartOf')) { + i = j; + continue; + } + + // Expect '(' (possibly after whitespace) for this to be a call. + let k = j; + while (k < n && isWhitespace(input[k])) k++; + if (input[k] !== '(') { + i = j; + continue; + } + + // Walk to the matching ')', honoring nested parens and quoted regions. + const argStart = k + 1; + let depth = 1; + let p = argStart; + while (p < n && depth > 0) { + const c = input[p]; + if (c === "'" || c === '"' || c === '`') { + p = skipQuoted(p, c); + continue; + } + if (c === '(') depth++; + else if (c === ')') { + depth--; + if (depth === 0) break; + } + p++; + } + if (depth !== 0) break; // unterminated call — stop scanning + out.push({ fn: ident, argsInner: input.substring(argStart, p) }); + i = p + 1; + } + + return out; +} + +function parseIntervalLiteral(expr: string): SQLInterval | undefined { + const tokens: string[] = []; + let cur = ''; + for (const ch of expr) { + if (isWhitespace(ch)) { + if (cur) tokens.push(cur); + cur = ''; + } else { + cur += ch; + } + } + if (cur) tokens.push(cur); + + if (tokens.length < 3) return undefined; + if (tokens[0].toUpperCase() !== 'INTERVAL') return undefined; + + const num = Number.parseInt(tokens[1], 10); + if (!Number.isFinite(num) || num <= 0 || String(num) !== tokens[1]) { + return undefined; + } + + // Accept both singular and plural forms (MINUTE / MINUTES). + let unit = tokens[2].toLowerCase(); + if (unit.endsWith('s')) unit = unit.slice(0, -1); + if (!VALID_INTERVAL_UNITS.has(unit)) return undefined; + + return `${num} ${unit}` as SQLInterval; +} + +export function inferGranularityFromMVSelect( + asSelect: string, +): SQLInterval | undefined { + for (const { fn, argsInner } of findToStartOfCalls(asSelect)) { + if (fn in NAMED_BUCKET_FUNCTIONS) { + return NAMED_BUCKET_FUNCTIONS[fn]; + } + if (fn === 'toStartOfInterval') { + const args = splitAndTrimWithBracket(argsInner); + if (args.length < 2) continue; + const parsed = parseIntervalLiteral(args[1]); + if (parsed) return parsed; + } + } + return undefined; +} + +export function getNamedBucketFunction( + granularity: SQLInterval, +): string | undefined { + for (const [fn, g] of Object.entries(NAMED_BUCKET_FUNCTIONS)) { + if (g === granularity) return fn; + } + return undefined; +} + +export function renderStartOfBucketExpr( + granularity: SQLInterval, + inner: ChSql, +): ChSql { + const namedFn = getNamedBucketFunction(granularity); + if (namedFn) { + // namedFn comes from a fixed allow-list (NAMED_BUCKET_FUNCTIONS keys), so + // splicing it as raw SQL is safe. + return chSql`${{ UNSAFE_RAW_SQL: namedFn }}(${inner})`; + } + const seconds = convertGranularityToSeconds(granularity); + return chSql`toStartOfInterval(${inner}, INTERVAL ${{ Int64: seconds }} SECOND)`; +} + type SelectItem = Exclude< BuilderChartConfigWithOptDateRange['select'], string diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 2d13ecc8fc..225df1eb16 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -16,12 +16,20 @@ import { renderChartConfig } from '@/core/renderChartConfig'; import type { BuilderChartConfig, BuilderChartConfigWithDateRange, + MetadataMaterializedViews, TSource, } from '@/types'; -import { SourceKind } from '@/types'; +import { isLogSource, isTraceSource, SourceKind } from '@/types'; -import { optimizeGetKeyValuesCalls } from './materializedViews'; -import { getDistributedTableArgs, objectHash } from './utils'; +import { + optimizeGetKeyValuesCalls, + renderStartOfBucketExpr, +} from './materializedViews'; +import { + getAlignedDateRange, + getDistributedTableArgs, + objectHash, +} from './utils'; // If filters initially are taking too long to load, decrease this number. // Between 1e6 - 5e6 is a good range. @@ -339,6 +347,8 @@ export class Metadata { maxKeys = DEFAULT_MAX_KEYS, connectionId, metricName, + metadataMVs, + dateRange, }: { databaseName: string; tableName: string; @@ -346,16 +356,76 @@ export class Metadata { maxKeys?: number; connectionId: string; metricName?: string; + metadataMVs?: MetadataMaterializedViews; + dateRange?: [Date, Date]; }) { + // Align date range to rollup granularity for consistent cache keys + const alignedDateRange = + metadataMVs && dateRange + ? getAlignedDateRange(dateRange, metadataMVs.granularity) + : undefined; + const cacheKey = metricName ? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.keys` - : `${connectionId}.${databaseName}.${tableName}.${column}.keys`; + : metadataMVs && alignedDateRange + ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.keys` + : `${connectionId}.${databaseName}.${tableName}.${column}.keys`; const cachedKeys = this.cache.get(cacheKey); if (cachedKeys != null) { return cachedKeys; } + // Rollup path: query the key rollup table filtered by ColumnIdentifier and date range + if (metadataMVs && alignedDateRange) { + const rollupKeys = await this.cache.getOrFetch( + cacheKey, + async () => { + try { + const startExpr = renderStartOfBucketExpr( + metadataMVs.granularity, + chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`, + ); + const endExpr = renderStartOfBucketExpr( + metadataMVs.granularity, + chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`, + ); + const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`; + const sql = chSql` + SELECT Key + FROM ${tableExpr({ database: databaseName, table: metadataMVs.keyRollupTable })} + WHERE ColumnIdentifier = ${{ String: column }} + ${timeFilter} + GROUP BY Key + ORDER BY sum(count) DESC + LIMIT ${{ Int32: maxKeys }} + `; + + return await this.clickhouseClient + .query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + clickhouse_settings: { + ...this.getClickHouseSettings(), + timeout_overflow_mode: 'break', + max_execution_time: 15, + max_rows_to_read: '0', + }, + }) + .then(res => res.json<{ Key: string }>()) + .then(d => d.data.map(row => row.Key).filter(k => k)); + } catch (e) { + console.warn('getMapKeys rollup query failed', e); + return []; + } + }, + ); + + if (rollupKeys.length > 0) return rollupKeys; + } + + // Original path: scan main table const colMeta = await this.getColumn({ databaseName, tableName, @@ -586,7 +656,9 @@ export class Metadata { tableName, connectionId, metricName, - }: TableConnection) { + metadataMVs, + dateRange, + }: TableConnection & { dateRange?: [Date, Date] }) { const fields: Field[] = []; const columns = await this.getColumns({ databaseName, @@ -634,6 +706,8 @@ export class Metadata { column: column.name, connectionId, metricName, + metadataMVs, + dateRange, }); const match = column.type.match(/Map\(.+,\s*(.+)\)/); @@ -1132,6 +1206,105 @@ export class Metadata { ); } + /** + * Autocomplete: fetches top values for a specific map key from the KV rollup table. + * Only filters by date range — no WHERE conditions. Values ordered by frequency. + */ + async getAllKeyValues({ + databaseName, + tableName, + keyExpression, + maxValues = 1000, + connectionId, + metadataMVs, + dateRange, + signal, + }: { + databaseName: string; + tableName: string; + /** Bracket-notation key string, e.g. `ResourceAttributes['service.name']` or `ServiceName` */ + keyExpression: string; + maxValues?: number; + connectionId: string; + metadataMVs?: MetadataMaterializedViews; + dateRange?: [Date, Date]; + signal?: AbortSignal; + }): Promise { + const path = parseKeyPath(keyExpression); + const isMapKey = path.length >= 2; + + // Try rollup table first when available + if (metadataMVs && dateRange) { + const rollupColumn = isMapKey ? path[0] : 'NativeColumn'; + const rollupKey = isMapKey ? path[1] : path[0]; + + // Align date range to rollup granularity for consistent cache keys + const alignedDateRange = getAlignedDateRange( + dateRange, + metadataMVs.granularity, + ); + + const startExpr = renderStartOfBucketExpr( + metadataMVs.granularity, + chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[0].getTime() }})`, + ); + const endExpr = renderStartOfBucketExpr( + metadataMVs.granularity, + chSql`fromUnixTimestamp64Milli(${{ Int64: alignedDateRange[1].getTime() }})`, + ); + const timeFilter = chSql`AND Timestamp >= ${startExpr} AND Timestamp <= ${endExpr}`; + const cacheKey = `${connectionId}.${databaseName}.${tableName}.${rollupColumn}.${rollupKey}.${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.allKeyValues`; + + const rollupValues = await this.cache.getOrFetch(cacheKey, async () => { + try { + const sql = chSql` + SELECT Value + FROM ${tableExpr({ database: databaseName, table: metadataMVs.kvRollupTable })} + WHERE ColumnIdentifier = ${{ String: rollupColumn }} + AND Key = ${{ String: rollupKey }} + AND Value != '' + ${timeFilter} + GROUP BY Value + ORDER BY sum(count) DESC + LIMIT ${{ Int32: maxValues }} + `; + + return await this.clickhouseClient + .query<'JSON'>({ + query: sql.sql, + query_params: sql.params, + connectionId, + clickhouse_settings: { + ...this.getClickHouseSettings(), + timeout_overflow_mode: 'break', + max_execution_time: 15, + max_rows_to_read: '0', + }, + abort_signal: signal, + }) + .then(res => res.json<{ Value: string }>()) + .then(d => d.data.map(r => r.Value)); + } catch (e) { + console.warn('getAllKeyValues rollup query failed', e); + return []; + } + }); + + if (rollupValues.length > 0) return rollupValues; + } + + // Fall back to main table scan + const column = path[0]; + const mapKey = isMapKey ? path[1] : undefined; + return this.getMapValues({ + databaseName, + tableName, + column, + key: mapKey, + connectionId, + }); + } + async getKeyValues({ chartConfig, keys, @@ -1323,11 +1496,30 @@ export type Field = { jsType: JSDataType | null; }; +/** + * Parses a bracket-notation key string into a path array. + * e.g. `ResourceAttributes['service.name']` → `['ResourceAttributes', 'service.name']` + * `ServiceName` → `['ServiceName']` + */ +export function parseKeyPath(key: string): string[] { + const singleIdx = key.indexOf("['"); + if (singleIdx !== -1 && key.endsWith("']")) { + return [key.slice(0, singleIdx), key.slice(singleIdx + 2, -2)]; + } + const doubleIdx = key.indexOf('["'); + if (doubleIdx !== -1 && key.endsWith('"]')) { + return [key.slice(0, doubleIdx), key.slice(doubleIdx + 2, -2)]; + } + return [key]; +} + +// Describes a table and potentially related views export type TableConnection = { databaseName: string; tableName: string; connectionId: string; metricName?: string; + metadataMVs?: MetadataMaterializedViews; }; export type TableConnectionChoice = @@ -1355,6 +1547,10 @@ export function tcFromSource(source?: TSource): TableConnection { databaseName: source?.from?.databaseName ?? '', tableName: source?.from?.tableName ?? '', connectionId: source?.connection ?? '', + metadataMVs: + source && (isLogSource(source) || isTraceSource(source)) + ? source.metadataMaterializedViews + : undefined, }; } diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 11cde151bf..dce62070ad 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -9,7 +9,12 @@ import { extractInnerCHArrayJSType, JSDataType, } from '@/clickhouse'; -import { Metadata, SkipIndexMetadata, TableConnection } from '@/core/metadata'; +import { + Metadata, + parseKeyPath, + SkipIndexMetadata, + TableConnection, +} from '@/core/metadata'; import { parseTokenizerFromTextIndex, splitAndTrimWithBracket, @@ -41,18 +46,9 @@ export function parse(query: string): lucene.AST { } function buildMapContains(mapField: string) { - const splitMapKey = ( - field: string, - ): { map: string; key: string } | undefined => { - const bracketIndex = field.indexOf("['"); - if (bracketIndex === -1) return undefined; - const map = field.slice(0, bracketIndex); - const key = field.slice(bracketIndex + 2, -2); - return { map, key }; - }; - const val = splitMapKey(mapField); - if (!val) return undefined; - return SqlString.format('mapContains(??, ?)', [val.map, val.key]); + const path = parseKeyPath(mapField); + if (path.length < 2) return undefined; + return SqlString.format('mapContains(??, ?)', [path[0], path[1]]); } const IMPLICIT_FIELD = ''; diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dfd31e500e..40faa15437 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1130,6 +1130,16 @@ export type MaterializedViewConfiguration = z.infer< typeof MaterializedViewConfigurationSchema >; +export const MetadataMaterializedViewsSchema = z.object({ + keyRollupTable: z.string().min(1, 'Key rollup table name is required'), + kvRollupTable: z.string().min(1, 'KV rollup table name is required'), + granularity: SQLIntervalSchema, +}); + +export type MetadataMaterializedViews = z.infer< + typeof MetadataMaterializedViewsSchema +>; + // Log source form schema export const LogSourceSchema = BaseSourceSchema.extend({ kind: z.literal(SourceKind.Log), @@ -1163,6 +1173,7 @@ export const LogSourceSchema = BaseSourceSchema.extend({ highlightedRowAttributeExpressions: HighlightedAttributeExpressionsSchema.optional(), materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), + metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), orderByExpression: z.string().optional(), }); @@ -1202,6 +1213,7 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ highlightedRowAttributeExpressions: HighlightedAttributeExpressionsSchema.optional(), materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), + metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), orderByExpression: z.string().optional(), });