From 93c530d8ede89ac633584471105af31bb0c173ef Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:52:20 -0400 Subject: [PATCH 01/10] feat: optimized default otel-logs schema --- .changeset/calm-tips-repeat.md | 12 ++++++++++ .../schema/seed/00002_otel_logs.sql | 23 ++++++++----------- 2 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 .changeset/calm-tips-repeat.md diff --git a/.changeset/calm-tips-repeat.md b/.changeset/calm-tips-repeat.md new file mode 100644 index 0000000000..c39633b07c --- /dev/null +++ b/.changeset/calm-tips-repeat.md @@ -0,0 +1,12 @@ +--- +"@hyperdx/otel-collector": patch +--- + +feat: new optimized otel schema based on weeks of benchmarks. + +The Primary Key is now grouped by `toStartOfFiveMinutes`. At extremely large +data sizes, it may be helpful to reduce granularity to 1 minute instead of 5. +Bloom Filter indexes can be used instead, but full text search performs better +across the board. Additionally, tests show that TimestampTime is effectively +not necessary, which is especially true with data grouped by 5 minute +boundaries by default. diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 5cc414ed76..3c4b13b596 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -2,7 +2,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs ( `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), `TraceId` String CODEC(ZSTD(1)), `SpanId` String CODEC(ZSTD(1)), `TraceFlags` UInt8, @@ -25,19 +24,17 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY (ServiceName, TimestampTime, Timestamp) +PARTITION BY toDate(Timestamp) +ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL TimestampTime + ${TABLES_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; - From 32fdb05ed89267889e72322c963b383a49048bf8 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:15:11 -0400 Subject: [PATCH 02/10] fix: upgrade clickhouse version; update DEFAULT_SOURCES --- docker-compose.ci.yml | 2 +- docker-compose.dev.yml | 2 +- docker-compose.yml | 4 ++-- docker/clickhouse/local/init-db-e2e.sh | 24 +++++++++---------- docker/hyperdx/Dockerfile | 2 +- docker/hyperdx/entry.local.base.sh | 2 +- .../schema/seed/00002_otel_logs.sql | 2 +- packages/api/.env.development | 2 +- packages/app/src/config.ts | 2 +- packages/app/tests/e2e/docker-compose.yml | 2 +- .../app/tests/e2e/fixtures/e2e-fixtures.json | 4 ++-- 11 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 525209c8e6..ef3e2d1a22 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -22,7 +22,7 @@ services: depends_on: - ch-server ch-server: - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine environment: # default settings CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dd2e57475c..df1ba0e00b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -100,7 +100,7 @@ services: hdx.dev.service: clickhouse hdx.dev.port: '${HDX_DEV_CH_HTTP_PORT:-8123}' hdx.dev.url: 'http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}' - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine ports: - '${HDX_DEV_CH_HTTP_PORT:-8123}:8123' # http api - '${HDX_DEV_CH_NATIVE_PORT:-9000}:9000' # native diff --git a/docker-compose.yml b/docker-compose.yml index 3e6fe3ba58..415276ac78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,7 +68,7 @@ services: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' DEFAULT_SOURCES: - '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local + '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local @@ -79,7 +79,7 @@ services: - ch-server - db ch-server: - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine # WARNING: Exposing the database port will make it accessible from outside the container, # potentially allowing unauthorized access. If you uncomment the ports below, # ensure to secure your database (e.g., with strong authentication, proper network rules, and firewalls). diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh index 7b6fe87889..7bbd4b2b2a 100755 --- a/docker/clickhouse/local/init-db-e2e.sh +++ b/docker/clickhouse/local/init-db-e2e.sh @@ -17,7 +17,6 @@ CREATE DATABASE IF NOT EXISTS ${DATABASE}; CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs ( \`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - \`TimestampTime\` DateTime DEFAULT toDateTime(Timestamp), \`TraceId\` String CODEC(ZSTD(1)), \`SpanId\` String CODEC(ZSTD(1)), \`TraceFlags\` UInt8, @@ -40,20 +39,19 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + toIntervalDay(30) +PARTITION BY toDate(Timestamp) +ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDateTime(Timestamp) + toIntervalDay(30) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index d7ffaea764..24f52b1c5e 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -8,7 +8,7 @@ # - Otel Collector (otelcol) ARG NODE_VERSION=22.22 -ARG CLICKHOUSE_VERSION=26.1 +ARG CLICKHOUSE_VERSION=26.2 ARG OTEL_COLLECTOR_VERSION=0.149.0 ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0 diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index 7d350669d4..0a8ebcf632 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -30,7 +30,7 @@ if [ -z "${DEFAULT_CONNECTIONS}" ]; then DEFAULT_CONNECTIONS='[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]' fi export DEFAULT_CONNECTIONS -export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' +export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' # if BETA_CH_OTEL_JSON_SCHEMA_ENABLED is true then return empty default sources if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 3c4b13b596..67bab8b5db 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -36,5 +36,5 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) -TTL TimestampTime + ${TABLES_TTL} +TTL toDateTime(Timestamp) + ${TABLES_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; diff --git a/packages/api/.env.development b/packages/api/.env.development index 52421a6c66..9be84e062b 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -18,7 +18,7 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 653aed63d7..ec6b522494 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -1,7 +1,7 @@ import { env } from 'next-runtime-env'; // ONLY USED IN LOCAL MODE -// ex: NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS='[{"id":"local","name":"Demo","host":"https://demo-ch.hyperdx.io","username":"demo","password":"demo"}]' NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES='[{"id":"l701179602","kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"default","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"l-758211293","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage"},{"id":"l-758211293","kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"default","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","traceSourceId":"l701179602"}]' yarn dev:local +// ex: NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS='[{"id":"local","name":"Demo","host":"https://demo-ch.hyperdx.io","username":"demo","password":"demo"}]' NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES='[{"id":"l701179602","kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"default","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"l-758211293","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage"},{"id":"l-758211293","kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"default","tableName":"otel_logs"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","traceSourceId":"l701179602"}]' yarn dev:local export const HDX_LOCAL_DEFAULT_CONNECTIONS = env( 'NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS', ); diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml index 69990ecbc7..e2ebb61c5f 100644 --- a/packages/app/tests/e2e/docker-compose.yml +++ b/packages/app/tests/e2e/docker-compose.yml @@ -15,7 +15,7 @@ services: memory: 512M cpus: '0.5' ch-server: - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine ports: - ${HDX_E2E_CH_PORT:-20500}:8123 # http api - ${HDX_E2E_CH_NATIVE_PORT:-20600}:9000 # native diff --git a/packages/app/tests/e2e/fixtures/e2e-fixtures.json b/packages/app/tests/e2e/fixtures/e2e-fixtures.json index d4e3f31b3d..e275d1d096 100644 --- a/packages/app/tests/e2e/fixtures/e2e-fixtures.json +++ b/packages/app/tests/e2e/fixtures/e2e-fixtures.json @@ -15,7 +15,7 @@ "name": "E2E Logs", "connection": "local", "from": { "databaseName": "default", "tableName": "e2e_otel_logs" }, - "timestampValueExpression": "TimestampTime", + "timestampValueExpression": "Timestamp", "defaultTableSelectExpression": "Timestamp, ServiceName, SeverityText, Body", "serviceNameExpression": "ServiceName", "severityTextExpression": "SeverityText", @@ -99,7 +99,7 @@ "name": "E2E K8s Logs No Metrics", "connection": "local", "from": { "databaseName": "default", "tableName": "e2e_otel_logs" }, - "timestampValueExpression": "TimestampTime", + "timestampValueExpression": "Timestamp", "defaultTableSelectExpression": "Timestamp, ServiceName, SeverityText, Body", "serviceNameExpression": "ServiceName", "severityTextExpression": "SeverityText", From db5a56ddfd0d76c75a2284502bbd4fb8540d8ca9 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:30:20 -0400 Subject: [PATCH 03/10] fix: remove TimestampTime from order by in smoke tests --- .../otel-collector/data/auto-parse/default/assert_query.sql | 2 +- .../otel-collector/data/auto-parse/json-string/assert_query.sql | 2 +- .../otel-collector/data/auto-parse/otel-map/assert_query.sql | 2 +- .../data/normalize-severity/text-case/assert_query.sql | 2 +- .../data/severity-inference/infer-debug/assert_query.sql | 2 +- .../data/severity-inference/infer-error/assert_query.sql | 2 +- .../data/severity-inference/infer-fatal/assert_query.sql | 2 +- .../data/severity-inference/infer-info/assert_query.sql | 2 +- .../data/severity-inference/infer-superstring/assert_query.sql | 2 +- .../data/severity-inference/infer-trace/assert_query.sql | 2 +- .../data/severity-inference/infer-warn/assert_query.sql | 2 +- .../data/severity-inference/no-infer-substring/assert_query.sql | 2 +- .../data/severity-inference/skip-infer/assert_query.sql | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql index 614ec16f13..ab095cb9aa 100644 --- a/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql +++ b/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql @@ -1 +1 @@ -SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'default' ORDER BY TimestampTime FORMAT CSV +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'default' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql index 895efbc50d..13822cca83 100644 --- a/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql +++ b/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql @@ -1 +1 @@ -SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'json-string' ORDER BY TimestampTime FORMAT CSV +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'json-string' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql index 1332626501..0f4b2c82b5 100644 --- a/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql +++ b/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql @@ -1 +1 @@ -SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'otel-map' ORDER BY TimestampTime FORMAT CSV +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'otel-map' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql b/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql index f1524232bd..e9f972f124 100644 --- a/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql +++ b/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'normalize-severity' AND ResourceAttributes['test-id'] = 'text-case' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'normalize-severity' AND ResourceAttributes['test-id'] = 'text-case' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql index 37fc65a0c2..42aa2617d3 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-debug' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-debug' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql index b8a32fa127..8bafa94a32 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-error' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-error' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql index 81137c7ac6..48fead52a6 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-fatal' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-fatal' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql index cc77ad843f..cf9feb8484 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-info' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-info' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql index 7add4b48f5..c166dc0645 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-superstring' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-superstring' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql index 702238aacf..50df1b1694 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-trace' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-trace' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql index 6568fcd0b4..0ea3cc8d7e 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-warn' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-warn' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql index 24dc118f13..aea383f7b6 100644 --- a/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'no-infer-substring' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'no-infer-substring' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql index d1ce54c491..7c61c797d1 100644 --- a/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'skip-infer' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'skip-infer' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV From 12584572f871e5e604230c63f50f01bd56a174ec Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Wed, 15 Apr 2026 17:03:00 -0400 Subject: [PATCH 04/10] fix: e2e tests --- packages/app/tests/e2e/features/dashboard.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 1c90b95ad1..51d449156a 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -651,13 +651,13 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { ); test.describe('Raw SQL Dashboard Tiles', () => { - const LINE_SQL = `SELECT toStartOfInterval(TimestampTime, INTERVAL {intervalSeconds:Int64} SECOND) AS ts, count() AS count FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ts ORDER BY ts ASC`; + const LINE_SQL = `SELECT toStartOfInterval(Timestamp, INTERVAL {intervalSeconds:Int64} SECOND) AS ts, count() AS count FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ts ORDER BY ts ASC`; - const TABLE_SQL = `SELECT ServiceName, count() AS count FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName LIMIT 200`; + const TABLE_SQL = `SELECT ServiceName, count() AS count FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName LIMIT 200`; - const NUMBER_SQL = `SELECT 1234 FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})`; + const NUMBER_SQL = `SELECT 1234 FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})`; - const PIE_SQL = `SELECT ServiceName, count() FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName`; + const PIE_SQL = `SELECT ServiceName, count() FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName`; test.beforeEach(async () => { await dashboardPage.createNewDashboard(); @@ -836,7 +836,7 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { name: DELETABLE_SOURCE_NAME, connection, from, - timestampValueExpression: 'TimestampTime', + timestampValueExpression: 'Timestamp', defaultTableSelectExpression: 'Timestamp, ServiceName, SeverityText, Body', serviceNameExpression: 'ServiceName', From 9ad1d751d9c828fc2992f60e43dc5e8afe0ed473 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:27:08 -0400 Subject: [PATCH 05/10] feat: only use full text index if clickhouse version is greater than 26.2 --- .../schema/seed/00002_otel_logs_compat.sql | 41 ++++++++++ packages/otel-collector/cmd/migrate/main.go | 82 ++++++++++++++++++- 2 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 docker/otel-collector/schema/seed/00002_otel_logs_compat.sql diff --git a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql new file mode 100644 index 0000000000..66b5aa3e6f --- /dev/null +++ b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql @@ -0,0 +1,41 @@ +-- +goose Up +-- Compatibility schema for ClickHouse < 26.2 (no full text search indexes) +CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `TraceFlags` UInt8, + `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityNumber` UInt8, + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Body` String CODEC(ZSTD(1)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), + `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDateTime(Timestamp) + ${TABLES_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index 3e3db44f3f..b3ec358f46 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -23,7 +23,6 @@ import ( "github.com/pressly/goose/v3" ) - // Config holds all configuration for the migration tool type Config struct { // ClickHouse connection settings @@ -318,7 +317,6 @@ func processSchemaDir(schemaDir, database, tablesTTLExpr string) (string, error) return tempDir, nil } - // runMigrationWithRetry runs goose seed with exponential backoff retry func runMigrationWithRetry(ctx context.Context, db *sql.DB, migrationsDir string, maxRetries int) error { var lastErr error @@ -350,6 +348,68 @@ func runMigrationWithRetry(ctx context.Context, db *sql.DB, migrationsDir string return fmt.Errorf("seed failed after %d attempts: %w", maxRetries, lastErr) } +// getClickHouseVersion queries the ClickHouse server version and returns the +// major and minor version numbers (e.g. 26, 2 for version "26.2.1.0"). +func getClickHouseVersion(ctx context.Context, db *sql.DB) (major, minor int, err error) { + var version string + if err := db.QueryRowContext(ctx, "SELECT version()").Scan(&version); err != nil { + return 0, 0, fmt.Errorf("failed to query ClickHouse version: %w", err) + } + + parts := strings.SplitN(version, ".", 3) + if len(parts) < 2 { + return 0, 0, fmt.Errorf("unexpected version format: %s", version) + } + + major, err = strconv.Atoi(parts[0]) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse major version %q: %w", parts[0], err) + } + minor, err = strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse minor version %q: %w", parts[1], err) + } + + return major, minor, nil +} + +// supportsFullTextSearch returns true if the ClickHouse version supports +// full text search indexes (TYPE text). This requires ClickHouse >= 26.2. +func supportsFullTextSearch(major, minor int) bool { + return major >= 26 && minor >= 2 +} + +// swapLogsSchemaForCompat replaces the full-text-search logs schema with the +// compatibility variant (bloom_filter indexes) in the processed temp directory. +// It removes 00002_otel_logs.sql and renames 00002_otel_logs_compat.sql to +// take its place, so goose runs the compat schema instead. +func swapLogsSchemaForCompat(tempDir string) error { + fullTextPath := filepath.Join(tempDir, "00002_otel_logs.sql") + compatPath := filepath.Join(tempDir, "00002_otel_logs_compat.sql") + + // Remove the full-text-search schema + if err := os.Remove(fullTextPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove full text logs schema: %w", err) + } + + // Rename compat schema to the original name so goose picks it up in order + if err := os.Rename(compatPath, fullTextPath); err != nil { + return fmt.Errorf("failed to rename compat logs schema: %w", err) + } + + return nil +} + +// removeCompatLogsSchema removes the compat schema file from the temp directory +// when full text search is supported, so goose doesn't run both schemas. +func removeCompatLogsSchema(tempDir string) error { + compatPath := filepath.Join(tempDir, "00002_otel_logs_compat.sql") + if err := os.Remove(compatPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove compat logs schema: %w", err) + } + return nil +} + // listSQLFiles lists all SQL files in a directory for logging purposes func listSQLFiles(dir string) ([]string, error) { entries, err := os.ReadDir(dir) @@ -397,6 +457,12 @@ func main() { } log.Println("Successfully connected to ClickHouse") + // Check ClickHouse version for feature support + chMajor, chMinor, err := getClickHouseVersion(ctx, db) + if err != nil { + log.Fatalf("Failed to determine ClickHouse version: %v", err) + } + // Parse tables TTL tablesTTLExpr, err := ttlToClickHouseInterval(cfg.TablesTTL) if err != nil { @@ -412,6 +478,18 @@ func main() { } defer os.RemoveAll(tempDir) + // Select the appropriate logs schema based on ClickHouse version + if supportsFullTextSearch(chMajor, chMinor) { + if err := removeCompatLogsSchema(tempDir); err != nil { + log.Fatalf("Failed to remove compat schema: %v", err) + } + } else { + log.Printf("ClickHouse %d.%d < 26.2, falling back to compatibility logs schema", chMajor, chMinor) + if err := swapLogsSchemaForCompat(tempDir); err != nil { + log.Fatalf("Failed to swap logs schema: %v", err) + } + } + // List SQL files sqlFiles, err := listSQLFiles(tempDir) if err != nil { From 508f369338f7240decc15f367b86982fff32aff3 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:40:42 -0400 Subject: [PATCH 06/10] chore: add testing for compatibility schema --- packages/otel-collector/cmd/migrate/main.go | 2 +- .../otel-collector/cmd/migrate/main_test.go | 159 ++++++++++++++++++ 2 files changed, 160 insertions(+), 1 deletion(-) diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index b3ec358f46..8af809b190 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -376,7 +376,7 @@ func getClickHouseVersion(ctx context.Context, db *sql.DB) (major, minor int, er // supportsFullTextSearch returns true if the ClickHouse version supports // full text search indexes (TYPE text). This requires ClickHouse >= 26.2. func supportsFullTextSearch(major, minor int) bool { - return major >= 26 && minor >= 2 + return major > 26 || (major == 26 && minor >= 2) } // swapLogsSchemaForCompat replaces the full-text-search logs schema with the diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index 64d0239a30..39a9a5ae3f 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -720,6 +720,165 @@ func TestTTLToClickHouseInterval(t *testing.T) { } } +// --------------------------------------------------------------------------- +// supportsFullTextSearch +// --------------------------------------------------------------------------- + +func TestSupportsFullTextSearch(t *testing.T) { + tests := []struct { + name string + major int + minor int + expected bool + }{ + {"26.2 exact threshold", 26, 2, true}, + {"26.3 above minor", 26, 3, true}, + {"27.0 above major", 27, 0, true}, + {"27.1 above both", 27, 1, true}, + {"26.1 below minor", 26, 1, false}, + {"26.0 below minor", 26, 0, false}, + {"25.9 below major", 25, 9, false}, + {"24.8 old version", 24, 8, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := supportsFullTextSearch(tt.major, tt.minor) + if got != tt.expected { + t.Errorf("supportsFullTextSearch(%d, %d) = %v, want %v", tt.major, tt.minor, got, tt.expected) + } + }) + } +} + +// --------------------------------------------------------------------------- +// swapLogsSchemaForCompat +// --------------------------------------------------------------------------- + +func TestSwapLogsSchemaForCompat(t *testing.T) { + t.Run("swaps compat over full text", func(t *testing.T) { + dir := t.TempDir() + fullTextPath := filepath.Join(dir, "00002_otel_logs.sql") + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(fullTextPath, []byte("FULL TEXT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(compatPath, []byte("COMPAT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := swapLogsSchemaForCompat(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // The full text path should now contain the compat content + got, err := os.ReadFile(fullTextPath) + if err != nil { + t.Fatalf("failed to read swapped file: %v", err) + } + if string(got) != "COMPAT SCHEMA" { + t.Errorf("expected compat content, got %q", string(got)) + } + + // The compat file should no longer exist + if _, err := os.Stat(compatPath); !os.IsNotExist(err) { + t.Error("compat file should have been renamed away") + } + }) + + t.Run("works when full text file is missing", func(t *testing.T) { + dir := t.TempDir() + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(compatPath, []byte("COMPAT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := swapLogsSchemaForCompat(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got, err := os.ReadFile(filepath.Join(dir, "00002_otel_logs.sql")) + if err != nil { + t.Fatalf("failed to read swapped file: %v", err) + } + if string(got) != "COMPAT SCHEMA" { + t.Errorf("expected compat content, got %q", string(got)) + } + }) + + t.Run("errors when compat file is missing", func(t *testing.T) { + dir := t.TempDir() + fullTextPath := filepath.Join(dir, "00002_otel_logs.sql") + + if err := os.WriteFile(fullTextPath, []byte("FULL TEXT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + err := swapLogsSchemaForCompat(dir) + if err == nil { + t.Fatal("expected error when compat file is missing") + } + }) +} + +// --------------------------------------------------------------------------- +// removeCompatLogsSchema +// --------------------------------------------------------------------------- + +func TestRemoveCompatLogsSchema(t *testing.T) { + t.Run("removes existing compat file", func(t *testing.T) { + dir := t.TempDir() + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(compatPath, []byte("COMPAT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := removeCompatLogsSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(compatPath); !os.IsNotExist(err) { + t.Error("compat file should have been removed") + } + }) + + t.Run("no error when compat file does not exist", func(t *testing.T) { + dir := t.TempDir() + + if err := removeCompatLogsSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("preserves other files", func(t *testing.T) { + dir := t.TempDir() + otherPath := filepath.Join(dir, "00001_other.sql") + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(otherPath, []byte("OTHER"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(compatPath, []byte("COMPAT"), 0644); err != nil { + t.Fatal(err) + } + + if err := removeCompatLogsSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(otherPath); err != nil { + t.Error("other file should still exist") + } + }) +} + +// --------------------------------------------------------------------------- +// processSchemaDir +// --------------------------------------------------------------------------- + func TestProcessSchemaDir(t *testing.T) { // Create a temp schema directory with a test SQL file schemaDir := t.TempDir() From 2ddf9fd35063a643114b5c43c5784f8eda0da73f Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:09:48 -0400 Subject: [PATCH 07/10] feat: add EventName column (from upstream otel-collector-contrib) --- .changeset/calm-tips-repeat.md | 2 +- docker/clickhouse/local/init-db-e2e.sh | 1 + docker/otel-collector/schema/seed/00002_otel_logs.sql | 1 + docker/otel-collector/schema/seed/00002_otel_logs_compat.sql | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.changeset/calm-tips-repeat.md b/.changeset/calm-tips-repeat.md index c39633b07c..c81908c211 100644 --- a/.changeset/calm-tips-repeat.md +++ b/.changeset/calm-tips-repeat.md @@ -1,5 +1,5 @@ --- -"@hyperdx/otel-collector": patch +"@hyperdx/otel-collector": minor --- feat: new optimized otel schema based on weeks of benchmarks. diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh index 7bbd4b2b2a..b28676aa64 100755 --- a/docker/clickhouse/local/init-db-e2e.sh +++ b/docker/clickhouse/local/init-db-e2e.sh @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs \`ScopeVersion\` LowCardinality(String) CODEC(ZSTD(1)), \`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)), \`LogAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + \`EventName\` String CODEC(ZSTD(1)), \`__hdx_materialized_k8s.cluster.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.container.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.deployment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 67bab8b5db..3bc666f62c 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `EventName` String CODEC(ZSTD(1)), `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), diff --git a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql index 66b5aa3e6f..b6d7f21f54 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `EventName` String CODEC(ZSTD(1)), `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), From bf582c67841ee247e9c417be41ae1efce084e532 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Mon, 4 May 2026 16:53:15 -0400 Subject: [PATCH 08/10] add compat schema tests --- .github/workflows/main.yml | 2 +- smoke-tests/otel-collector/compat-schema.bats | 14 +++++ .../basic-insert/assert_query.sql | 1 + .../compat-schema/basic-insert/expected.snap | 2 + .../compat-schema/basic-insert/input.json | 51 +++++++++++++++++++ .../index-types/assert_query.sql | 7 +++ .../compat-schema/index-types/expected.snap | 2 + .../otel-collector/docker-compose.yaml | 46 +++++++++++++++++ smoke-tests/otel-collector/setup_suite.bash | 1 + .../test_helpers/assertions.bash | 13 ++++- 10 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 smoke-tests/otel-collector/compat-schema.bats create mode 100644 smoke-tests/otel-collector/data/compat-schema/basic-insert/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/compat-schema/basic-insert/expected.snap create mode 100644 smoke-tests/otel-collector/data/compat-schema/basic-insert/input.json create mode 100644 smoke-tests/otel-collector/data/compat-schema/index-types/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/compat-schema/index-types/expected.snap diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c7c8486215..f5af70bc8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,7 +104,7 @@ jobs: with: files: | docker/otel-collector/** - smoke-tests/otel-ccollector/** + smoke-tests/otel-collector/** - name: Install required tooling if: steps.changed-files.outputs.any_changed == 'true' env: diff --git a/smoke-tests/otel-collector/compat-schema.bats b/smoke-tests/otel-collector/compat-schema.bats new file mode 100644 index 0000000000..8ebafa6e44 --- /dev/null +++ b/smoke-tests/otel-collector/compat-schema.bats @@ -0,0 +1,14 @@ +#!/usr/bin/env bats + +load 'test_helpers/utilities.bash' +load 'test_helpers/assertions.bash' + +@test "compat schema should use bloom_filter indexes instead of full text on ClickHouse < 26.2" { + assert_test_data_compat "data/compat-schema/index-types" +} + +@test "compat schema should insert and query log data correctly" { + emit_otel_data "http://localhost:24318" "data/compat-schema/basic-insert" + sleep 1 + assert_test_data_compat "data/compat-schema/basic-insert" +} diff --git a/smoke-tests/otel-collector/data/compat-schema/basic-insert/assert_query.sql b/smoke-tests/otel-collector/data/compat-schema/basic-insert/assert_query.sql new file mode 100644 index 0000000000..051bd0f5e2 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/basic-insert/assert_query.sql @@ -0,0 +1 @@ +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'compat-schema' AND ResourceAttributes['test-id'] = 'basic-insert' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/compat-schema/basic-insert/expected.snap b/smoke-tests/otel-collector/data/compat-schema/basic-insert/expected.snap new file mode 100644 index 0000000000..cf778479d9 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/basic-insert/expected.snap @@ -0,0 +1,2 @@ +"info",9,"compat schema test log entry" +"warn",13,"compat schema warning message" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/compat-schema/basic-insert/input.json b/smoke-tests/otel-collector/data/compat-schema/basic-insert/input.json new file mode 100644 index 0000000000..1b14b19bf9 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/basic-insert/input.json @@ -0,0 +1,51 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "compat-schema" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "basic-insert" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "compat-test-service" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999580000000000", + "severityNumber": 9, + "severityText": "INFO", + "body": { + "stringValue": "compat schema test log entry" + } + }, + { + "timeUnixNano": "1901999580000000001", + "severityNumber": 13, + "severityText": "WARN", + "body": { + "stringValue": "compat schema warning message" + } + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/data/compat-schema/index-types/assert_query.sql b/smoke-tests/otel-collector/data/compat-schema/index-types/assert_query.sql new file mode 100644 index 0000000000..4e7b27b1da --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/index-types/assert_query.sql @@ -0,0 +1,7 @@ +SELECT name, type +FROM system.data_skipping_indices +WHERE database = 'default' + AND table = 'otel_logs' + AND name IN ('idx_trace_id', 'idx_lower_body') +ORDER BY name +FORMAT CSV diff --git a/smoke-tests/otel-collector/data/compat-schema/index-types/expected.snap b/smoke-tests/otel-collector/data/compat-schema/index-types/expected.snap new file mode 100644 index 0000000000..99cc5b5ed9 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/index-types/expected.snap @@ -0,0 +1,2 @@ +"idx_lower_body","tokenbf_v1" +"idx_trace_id","bloom_filter" \ No newline at end of file diff --git a/smoke-tests/otel-collector/docker-compose.yaml b/smoke-tests/otel-collector/docker-compose.yaml index 5df6e6edcb..b9c668477c 100644 --- a/smoke-tests/otel-collector/docker-compose.yaml +++ b/smoke-tests/otel-collector/docker-compose.yaml @@ -68,6 +68,52 @@ services: ch-server: condition: service_healthy + # Older ClickHouse (< 26.2) to test the compatibility schema (bloom filters + # instead of full text indexes) + ch-server-compat: + image: clickhouse/clickhouse-server:25.12-alpine + environment: + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + volumes: + - ../../docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml + - ../../docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml + ports: + - 19000:9000 + - 18123:8123 + networks: + - internal + healthcheck: + test: + wget -O /dev/null --no-verbose --tries=1 http://127.0.0.1:8123/ping || + exit 1 + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + otel-collector-compat: + build: + context: ../.. + dockerfile: docker/otel-collector/Dockerfile + target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.149.0} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.55.0} + environment: + - CLICKHOUSE_ENDPOINT=tcp://ch-server-compat:9000?dial_timeout=10s + - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server-compat:9363 + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD= + - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default + - HYPERDX_LOG_LEVEL=info + # OPAMP_SERVER_URL is intentionally not set to run in standalone mode + ports: + - 24318:4318 # OTLP http receiver + networks: + - internal + depends_on: + ch-server-compat: + condition: service_healthy + networks: internal: name: 'smoke-test-internal-network' diff --git a/smoke-tests/otel-collector/setup_suite.bash b/smoke-tests/otel-collector/setup_suite.bash index ae3869646a..a5569526eb 100644 --- a/smoke-tests/otel-collector/setup_suite.bash +++ b/smoke-tests/otel-collector/setup_suite.bash @@ -8,6 +8,7 @@ setup_suite() { docker compose up --build --detach wait_for_ready "otel-collector" wait_for_ready "otel-collector-json" + wait_for_ready "otel-collector-compat" } teardown_suite() { diff --git a/smoke-tests/otel-collector/test_helpers/assertions.bash b/smoke-tests/otel-collector/test_helpers/assertions.bash index 999a1e82a1..46d70d5364 100644 --- a/smoke-tests/otel-collector/test_helpers/assertions.bash +++ b/smoke-tests/otel-collector/test_helpers/assertions.bash @@ -1,5 +1,14 @@ assert_test_data() { - local testdir=$1 + _assert_test_data_on_port "9000" "$@" +} + +assert_test_data_compat() { + _assert_test_data_on_port "19000" "$@" +} + +_assert_test_data_on_port() { + local port=$1 + local testdir=$2 local query_file="${testdir}/assert_query.sql" local expected_file="${testdir}/expected.snap" @@ -26,7 +35,7 @@ assert_test_data() { fi # Execute the query using clickhouse-client and capture the results - local query_result=$(clickhouse-client --queries-file="$query_file" 2>&1) + local query_result=$(clickhouse-client --port="$port" --queries-file="$query_file" 2>&1) # Check if the clickhouse-client command succeeded if [ $? -ne 0 ]; then From 613f82f1e16543125acc53616b65bd5092ccaf3a Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Mon, 4 May 2026 18:08:55 -0400 Subject: [PATCH 09/10] chore: fix tests --- smoke-tests/otel-collector/compat-schema.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smoke-tests/otel-collector/compat-schema.bats b/smoke-tests/otel-collector/compat-schema.bats index 8ebafa6e44..57055aab50 100644 --- a/smoke-tests/otel-collector/compat-schema.bats +++ b/smoke-tests/otel-collector/compat-schema.bats @@ -9,6 +9,6 @@ load 'test_helpers/assertions.bash' @test "compat schema should insert and query log data correctly" { emit_otel_data "http://localhost:24318" "data/compat-schema/basic-insert" - sleep 1 + sleep 3 assert_test_data_compat "data/compat-schema/basic-insert" } From ef954a7a59e0b97c0804715607638ba39636e711 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Tue, 5 May 2026 11:24:07 -0400 Subject: [PATCH 10/10] fix: add unique row settings to compat schema --- docker/otel-collector/schema/seed/00002_otel_logs_compat.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql index b6d7f21f54..e4c18b17ff 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql @@ -39,4 +39,4 @@ ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + ${TABLES_TTL} -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1;