diff --git a/.changeset/calm-tips-repeat.md b/.changeset/calm-tips-repeat.md new file mode 100644 index 0000000000..c81908c211 --- /dev/null +++ b/.changeset/calm-tips-repeat.md @@ -0,0 +1,12 @@ +--- +"@hyperdx/otel-collector": minor +--- + +feat: new optimized otel schema based on weeks of benchmarks. + +The Primary Key is now grouped by `toStartOfFiveMinutes`. At extremely large +data sizes, it may be helpful to reduce granularity to 1 minute instead of 5. +Bloom Filter indexes can be used instead, but full text search performs better +across the board. Additionally, tests show that TimestampTime is effectively +not necessary, which is especially true with data grouped by 5 minute +boundaries by default. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c7c8486215..f5af70bc8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,7 +104,7 @@ jobs: with: files: | docker/otel-collector/** - smoke-tests/otel-ccollector/** + smoke-tests/otel-collector/** - name: Install required tooling if: steps.changed-files.outputs.any_changed == 'true' env: diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 525209c8e6..ef3e2d1a22 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -22,7 +22,7 @@ services: depends_on: - ch-server ch-server: - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine environment: # default settings CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dd2e57475c..df1ba0e00b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -100,7 +100,7 @@ services: hdx.dev.service: clickhouse hdx.dev.port: '${HDX_DEV_CH_HTTP_PORT:-8123}' hdx.dev.url: 'http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}' - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine ports: - '${HDX_DEV_CH_HTTP_PORT:-8123}:8123' # http api - '${HDX_DEV_CH_NATIVE_PORT:-9000}:9000' # native diff --git a/docker-compose.yml b/docker-compose.yml index 6c143496d2..58490cf5c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' DEFAULT_SOURCES: - '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local + '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local @@ -83,7 +83,7 @@ services: - ch-server - db ch-server: - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine # WARNING: Exposing the database port will make it accessible from outside the container, # potentially allowing unauthorized access. If you uncomment the ports below, # ensure to secure your database (e.g., with strong authentication, proper network rules, and firewalls). diff --git a/docker/clickhouse/local/init-db-e2e.sh b/docker/clickhouse/local/init-db-e2e.sh index 7b6fe87889..b28676aa64 100755 --- a/docker/clickhouse/local/init-db-e2e.sh +++ b/docker/clickhouse/local/init-db-e2e.sh @@ -17,7 +17,6 @@ CREATE DATABASE IF NOT EXISTS ${DATABASE}; CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs ( \`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - \`TimestampTime\` DateTime DEFAULT toDateTime(Timestamp), \`TraceId\` String CODEC(ZSTD(1)), \`SpanId\` String CODEC(ZSTD(1)), \`TraceFlags\` UInt8, @@ -32,6 +31,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs \`ScopeVersion\` LowCardinality(String) CODEC(ZSTD(1)), \`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)), \`LogAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + \`EventName\` String CODEC(ZSTD(1)), \`__hdx_materialized_k8s.cluster.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.container.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.deployment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), @@ -40,20 +40,19 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_logs \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + toIntervalDay(30) +PARTITION BY toDate(Timestamp) +ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDateTime(Timestamp) + toIntervalDay(30) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; CREATE TABLE IF NOT EXISTS ${DATABASE}.e2e_otel_traces diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index d7ffaea764..24f52b1c5e 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -8,7 +8,7 @@ # - Otel Collector (otelcol) ARG NODE_VERSION=22.22 -ARG CLICKHOUSE_VERSION=26.1 +ARG CLICKHOUSE_VERSION=26.2 ARG OTEL_COLLECTOR_VERSION=0.149.0 ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0 diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index 7d350669d4..0a8ebcf632 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -30,7 +30,7 @@ if [ -z "${DEFAULT_CONNECTIONS}" ]; then DEFAULT_CONNECTIONS='[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]' fi export DEFAULT_CONNECTIONS -export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' +export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' # if BETA_CH_OTEL_JSON_SCHEMA_ENABLED is true then return empty default sources if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponentialHistogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]' diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 3f7fb36669..c5e96b260c 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -2,7 +2,6 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs ( `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), `TraceId` String CODEC(ZSTD(1)), `SpanId` String CODEC(ZSTD(1)), `TraceFlags` UInt8, @@ -17,6 +16,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `EventName` String CODEC(ZSTD(1)), `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), @@ -25,19 +25,17 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + ${TABLES_TTL} +PARTITION BY toDate(Timestamp) +ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDateTime(Timestamp) + ${TABLES_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1; - diff --git a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql new file mode 100644 index 0000000000..e4c18b17ff --- /dev/null +++ b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql @@ -0,0 +1,42 @@ +-- +goose Up +-- Compatibility schema for ClickHouse < 26.2 (no full text search indexes) +CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `TraceFlags` UInt8, + `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityNumber` UInt8, + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Body` String CODEC(ZSTD(1)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `EventName` String CODEC(ZSTD(1)), + `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), + `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDateTime(Timestamp) + ${TABLES_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1; diff --git a/packages/api/.env.development b/packages/api/.env.development index c6563f782f..be156e564a 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -22,7 +22,7 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 8b7b037f06..aef6428117 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -1,7 +1,7 @@ import { env } from 'next-runtime-env'; // ONLY USED IN LOCAL MODE -// ex: NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS='[{"id":"local","name":"Demo","host":"https://demo-ch.hyperdx.io","username":"demo","password":"demo"}]' NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES='[{"id":"l701179602","kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"default","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"l-758211293","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage"},{"id":"l-758211293","kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"default","tableName":"otel_logs"},"timestampValueExpression":"TimestampTime","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","traceSourceId":"l701179602"}]' yarn dev:local +// ex: NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS='[{"id":"local","name":"Demo","host":"https://demo-ch.hyperdx.io","username":"demo","password":"demo"}]' NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES='[{"id":"l701179602","kind":"trace","name":"Demo Traces","connection":"local","from":{"databaseName":"default","tableName":"otel_traces"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, StatusCode, round(Duration / 1e6), SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"SpanName","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanKindExpression":"SpanKind","spanNameExpression":"SpanName","logSourceId":"l-758211293","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage"},{"id":"l-758211293","kind":"log","name":"Demo Logs","connection":"local","from":{"databaseName":"default","tableName":"otel_logs"},"timestampValueExpression":"Timestamp","defaultTableSelectExpression":"Timestamp, ServiceName, SeverityText, Body","serviceNameExpression":"ServiceName","severityTextExpression":"SeverityText","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","traceIdExpression":"TraceId","spanIdExpression":"SpanId","implicitColumnExpression":"Body","traceSourceId":"l701179602"}]' yarn dev:local export const HDX_LOCAL_DEFAULT_CONNECTIONS = env( 'NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS', ); diff --git a/packages/app/tests/e2e/docker-compose.yml b/packages/app/tests/e2e/docker-compose.yml index 69990ecbc7..e2ebb61c5f 100644 --- a/packages/app/tests/e2e/docker-compose.yml +++ b/packages/app/tests/e2e/docker-compose.yml @@ -15,7 +15,7 @@ services: memory: 512M cpus: '0.5' ch-server: - image: clickhouse/clickhouse-server:26.1-alpine + image: clickhouse/clickhouse-server:26.2-alpine ports: - ${HDX_E2E_CH_PORT:-20500}:8123 # http api - ${HDX_E2E_CH_NATIVE_PORT:-20600}:9000 # native diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index a3984fab45..e70acb74f3 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -651,13 +651,13 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { ); test.describe('Raw SQL Dashboard Tiles', () => { - const LINE_SQL = `SELECT toStartOfInterval(TimestampTime, INTERVAL {intervalSeconds:Int64} SECOND) AS ts, count() AS count FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ts ORDER BY ts ASC`; + const LINE_SQL = `SELECT toStartOfInterval(Timestamp, INTERVAL {intervalSeconds:Int64} SECOND) AS ts, count() AS count FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ts ORDER BY ts ASC`; - const TABLE_SQL = `SELECT ServiceName, count() AS count FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName LIMIT 200`; + const TABLE_SQL = `SELECT ServiceName, count() AS count FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp <= fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName LIMIT 200`; - const NUMBER_SQL = `SELECT 1234 FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})`; + const NUMBER_SQL = `SELECT 1234 FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})`; - const PIE_SQL = `SELECT ServiceName, count() FROM default.e2e_otel_logs WHERE TimestampTime >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND TimestampTime < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName`; + const PIE_SQL = `SELECT ServiceName, count() FROM default.e2e_otel_logs WHERE Timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) AND Timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) GROUP BY ServiceName`; test.beforeEach(async () => { await dashboardPage.createNewDashboard(); @@ -836,7 +836,7 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { name: DELETABLE_SOURCE_NAME, connection, from, - timestampValueExpression: 'TimestampTime', + timestampValueExpression: 'Timestamp', defaultTableSelectExpression: 'Timestamp, ServiceName, SeverityText, Body', serviceNameExpression: 'ServiceName', diff --git a/packages/app/tests/e2e/fixtures/e2e-fixtures.json b/packages/app/tests/e2e/fixtures/e2e-fixtures.json index d4e3f31b3d..e275d1d096 100644 --- a/packages/app/tests/e2e/fixtures/e2e-fixtures.json +++ b/packages/app/tests/e2e/fixtures/e2e-fixtures.json @@ -15,7 +15,7 @@ "name": "E2E Logs", "connection": "local", "from": { "databaseName": "default", "tableName": "e2e_otel_logs" }, - "timestampValueExpression": "TimestampTime", + "timestampValueExpression": "Timestamp", "defaultTableSelectExpression": "Timestamp, ServiceName, SeverityText, Body", "serviceNameExpression": "ServiceName", "severityTextExpression": "SeverityText", @@ -99,7 +99,7 @@ "name": "E2E K8s Logs No Metrics", "connection": "local", "from": { "databaseName": "default", "tableName": "e2e_otel_logs" }, - "timestampValueExpression": "TimestampTime", + "timestampValueExpression": "Timestamp", "defaultTableSelectExpression": "Timestamp, ServiceName, SeverityText, Body", "serviceNameExpression": "ServiceName", "severityTextExpression": "SeverityText", diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index 3e3db44f3f..8af809b190 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -23,7 +23,6 @@ import ( "github.com/pressly/goose/v3" ) - // Config holds all configuration for the migration tool type Config struct { // ClickHouse connection settings @@ -318,7 +317,6 @@ func processSchemaDir(schemaDir, database, tablesTTLExpr string) (string, error) return tempDir, nil } - // runMigrationWithRetry runs goose seed with exponential backoff retry func runMigrationWithRetry(ctx context.Context, db *sql.DB, migrationsDir string, maxRetries int) error { var lastErr error @@ -350,6 +348,68 @@ func runMigrationWithRetry(ctx context.Context, db *sql.DB, migrationsDir string return fmt.Errorf("seed failed after %d attempts: %w", maxRetries, lastErr) } +// getClickHouseVersion queries the ClickHouse server version and returns the +// major and minor version numbers (e.g. 26, 2 for version "26.2.1.0"). +func getClickHouseVersion(ctx context.Context, db *sql.DB) (major, minor int, err error) { + var version string + if err := db.QueryRowContext(ctx, "SELECT version()").Scan(&version); err != nil { + return 0, 0, fmt.Errorf("failed to query ClickHouse version: %w", err) + } + + parts := strings.SplitN(version, ".", 3) + if len(parts) < 2 { + return 0, 0, fmt.Errorf("unexpected version format: %s", version) + } + + major, err = strconv.Atoi(parts[0]) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse major version %q: %w", parts[0], err) + } + minor, err = strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse minor version %q: %w", parts[1], err) + } + + return major, minor, nil +} + +// supportsFullTextSearch returns true if the ClickHouse version supports +// full text search indexes (TYPE text). This requires ClickHouse >= 26.2. +func supportsFullTextSearch(major, minor int) bool { + return major > 26 || (major == 26 && minor >= 2) +} + +// swapLogsSchemaForCompat replaces the full-text-search logs schema with the +// compatibility variant (bloom_filter indexes) in the processed temp directory. +// It removes 00002_otel_logs.sql and renames 00002_otel_logs_compat.sql to +// take its place, so goose runs the compat schema instead. +func swapLogsSchemaForCompat(tempDir string) error { + fullTextPath := filepath.Join(tempDir, "00002_otel_logs.sql") + compatPath := filepath.Join(tempDir, "00002_otel_logs_compat.sql") + + // Remove the full-text-search schema + if err := os.Remove(fullTextPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove full text logs schema: %w", err) + } + + // Rename compat schema to the original name so goose picks it up in order + if err := os.Rename(compatPath, fullTextPath); err != nil { + return fmt.Errorf("failed to rename compat logs schema: %w", err) + } + + return nil +} + +// removeCompatLogsSchema removes the compat schema file from the temp directory +// when full text search is supported, so goose doesn't run both schemas. +func removeCompatLogsSchema(tempDir string) error { + compatPath := filepath.Join(tempDir, "00002_otel_logs_compat.sql") + if err := os.Remove(compatPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove compat logs schema: %w", err) + } + return nil +} + // listSQLFiles lists all SQL files in a directory for logging purposes func listSQLFiles(dir string) ([]string, error) { entries, err := os.ReadDir(dir) @@ -397,6 +457,12 @@ func main() { } log.Println("Successfully connected to ClickHouse") + // Check ClickHouse version for feature support + chMajor, chMinor, err := getClickHouseVersion(ctx, db) + if err != nil { + log.Fatalf("Failed to determine ClickHouse version: %v", err) + } + // Parse tables TTL tablesTTLExpr, err := ttlToClickHouseInterval(cfg.TablesTTL) if err != nil { @@ -412,6 +478,18 @@ func main() { } defer os.RemoveAll(tempDir) + // Select the appropriate logs schema based on ClickHouse version + if supportsFullTextSearch(chMajor, chMinor) { + if err := removeCompatLogsSchema(tempDir); err != nil { + log.Fatalf("Failed to remove compat schema: %v", err) + } + } else { + log.Printf("ClickHouse %d.%d < 26.2, falling back to compatibility logs schema", chMajor, chMinor) + if err := swapLogsSchemaForCompat(tempDir); err != nil { + log.Fatalf("Failed to swap logs schema: %v", err) + } + } + // List SQL files sqlFiles, err := listSQLFiles(tempDir) if err != nil { diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index 64d0239a30..39a9a5ae3f 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -720,6 +720,165 @@ func TestTTLToClickHouseInterval(t *testing.T) { } } +// --------------------------------------------------------------------------- +// supportsFullTextSearch +// --------------------------------------------------------------------------- + +func TestSupportsFullTextSearch(t *testing.T) { + tests := []struct { + name string + major int + minor int + expected bool + }{ + {"26.2 exact threshold", 26, 2, true}, + {"26.3 above minor", 26, 3, true}, + {"27.0 above major", 27, 0, true}, + {"27.1 above both", 27, 1, true}, + {"26.1 below minor", 26, 1, false}, + {"26.0 below minor", 26, 0, false}, + {"25.9 below major", 25, 9, false}, + {"24.8 old version", 24, 8, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := supportsFullTextSearch(tt.major, tt.minor) + if got != tt.expected { + t.Errorf("supportsFullTextSearch(%d, %d) = %v, want %v", tt.major, tt.minor, got, tt.expected) + } + }) + } +} + +// --------------------------------------------------------------------------- +// swapLogsSchemaForCompat +// --------------------------------------------------------------------------- + +func TestSwapLogsSchemaForCompat(t *testing.T) { + t.Run("swaps compat over full text", func(t *testing.T) { + dir := t.TempDir() + fullTextPath := filepath.Join(dir, "00002_otel_logs.sql") + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(fullTextPath, []byte("FULL TEXT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(compatPath, []byte("COMPAT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := swapLogsSchemaForCompat(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // The full text path should now contain the compat content + got, err := os.ReadFile(fullTextPath) + if err != nil { + t.Fatalf("failed to read swapped file: %v", err) + } + if string(got) != "COMPAT SCHEMA" { + t.Errorf("expected compat content, got %q", string(got)) + } + + // The compat file should no longer exist + if _, err := os.Stat(compatPath); !os.IsNotExist(err) { + t.Error("compat file should have been renamed away") + } + }) + + t.Run("works when full text file is missing", func(t *testing.T) { + dir := t.TempDir() + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(compatPath, []byte("COMPAT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := swapLogsSchemaForCompat(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got, err := os.ReadFile(filepath.Join(dir, "00002_otel_logs.sql")) + if err != nil { + t.Fatalf("failed to read swapped file: %v", err) + } + if string(got) != "COMPAT SCHEMA" { + t.Errorf("expected compat content, got %q", string(got)) + } + }) + + t.Run("errors when compat file is missing", func(t *testing.T) { + dir := t.TempDir() + fullTextPath := filepath.Join(dir, "00002_otel_logs.sql") + + if err := os.WriteFile(fullTextPath, []byte("FULL TEXT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + err := swapLogsSchemaForCompat(dir) + if err == nil { + t.Fatal("expected error when compat file is missing") + } + }) +} + +// --------------------------------------------------------------------------- +// removeCompatLogsSchema +// --------------------------------------------------------------------------- + +func TestRemoveCompatLogsSchema(t *testing.T) { + t.Run("removes existing compat file", func(t *testing.T) { + dir := t.TempDir() + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(compatPath, []byte("COMPAT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := removeCompatLogsSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(compatPath); !os.IsNotExist(err) { + t.Error("compat file should have been removed") + } + }) + + t.Run("no error when compat file does not exist", func(t *testing.T) { + dir := t.TempDir() + + if err := removeCompatLogsSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("preserves other files", func(t *testing.T) { + dir := t.TempDir() + otherPath := filepath.Join(dir, "00001_other.sql") + compatPath := filepath.Join(dir, "00002_otel_logs_compat.sql") + + if err := os.WriteFile(otherPath, []byte("OTHER"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(compatPath, []byte("COMPAT"), 0644); err != nil { + t.Fatal(err) + } + + if err := removeCompatLogsSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(otherPath); err != nil { + t.Error("other file should still exist") + } + }) +} + +// --------------------------------------------------------------------------- +// processSchemaDir +// --------------------------------------------------------------------------- + func TestProcessSchemaDir(t *testing.T) { // Create a temp schema directory with a test SQL file schemaDir := t.TempDir() diff --git a/smoke-tests/otel-collector/compat-schema.bats b/smoke-tests/otel-collector/compat-schema.bats new file mode 100644 index 0000000000..57055aab50 --- /dev/null +++ b/smoke-tests/otel-collector/compat-schema.bats @@ -0,0 +1,14 @@ +#!/usr/bin/env bats + +load 'test_helpers/utilities.bash' +load 'test_helpers/assertions.bash' + +@test "compat schema should use bloom_filter indexes instead of full text on ClickHouse < 26.2" { + assert_test_data_compat "data/compat-schema/index-types" +} + +@test "compat schema should insert and query log data correctly" { + emit_otel_data "http://localhost:24318" "data/compat-schema/basic-insert" + sleep 3 + assert_test_data_compat "data/compat-schema/basic-insert" +} diff --git a/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql index 614ec16f13..ab095cb9aa 100644 --- a/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql +++ b/smoke-tests/otel-collector/data/auto-parse/default/assert_query.sql @@ -1 +1 @@ -SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'default' ORDER BY TimestampTime FORMAT CSV +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'default' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql index 895efbc50d..13822cca83 100644 --- a/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql +++ b/smoke-tests/otel-collector/data/auto-parse/json-string/assert_query.sql @@ -1 +1 @@ -SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'json-string' ORDER BY TimestampTime FORMAT CSV +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'json-string' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql b/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql index 1332626501..0f4b2c82b5 100644 --- a/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql +++ b/smoke-tests/otel-collector/data/auto-parse/otel-map/assert_query.sql @@ -1 +1 @@ -SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'otel-map' ORDER BY TimestampTime FORMAT CSV +SELECT ResourceAttributes, LogAttributes FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'auto-parse' AND ResourceAttributes['test-id'] = 'otel-map' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/compat-schema/basic-insert/assert_query.sql b/smoke-tests/otel-collector/data/compat-schema/basic-insert/assert_query.sql new file mode 100644 index 0000000000..051bd0f5e2 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/basic-insert/assert_query.sql @@ -0,0 +1 @@ +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'compat-schema' AND ResourceAttributes['test-id'] = 'basic-insert' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/compat-schema/basic-insert/expected.snap b/smoke-tests/otel-collector/data/compat-schema/basic-insert/expected.snap new file mode 100644 index 0000000000..cf778479d9 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/basic-insert/expected.snap @@ -0,0 +1,2 @@ +"info",9,"compat schema test log entry" +"warn",13,"compat schema warning message" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/compat-schema/basic-insert/input.json b/smoke-tests/otel-collector/data/compat-schema/basic-insert/input.json new file mode 100644 index 0000000000..1b14b19bf9 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/basic-insert/input.json @@ -0,0 +1,51 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "compat-schema" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "basic-insert" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "compat-test-service" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999580000000000", + "severityNumber": 9, + "severityText": "INFO", + "body": { + "stringValue": "compat schema test log entry" + } + }, + { + "timeUnixNano": "1901999580000000001", + "severityNumber": 13, + "severityText": "WARN", + "body": { + "stringValue": "compat schema warning message" + } + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/data/compat-schema/index-types/assert_query.sql b/smoke-tests/otel-collector/data/compat-schema/index-types/assert_query.sql new file mode 100644 index 0000000000..4e7b27b1da --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/index-types/assert_query.sql @@ -0,0 +1,7 @@ +SELECT name, type +FROM system.data_skipping_indices +WHERE database = 'default' + AND table = 'otel_logs' + AND name IN ('idx_trace_id', 'idx_lower_body') +ORDER BY name +FORMAT CSV diff --git a/smoke-tests/otel-collector/data/compat-schema/index-types/expected.snap b/smoke-tests/otel-collector/data/compat-schema/index-types/expected.snap new file mode 100644 index 0000000000..99cc5b5ed9 --- /dev/null +++ b/smoke-tests/otel-collector/data/compat-schema/index-types/expected.snap @@ -0,0 +1,2 @@ +"idx_lower_body","tokenbf_v1" +"idx_trace_id","bloom_filter" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql b/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql index f1524232bd..e9f972f124 100644 --- a/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql +++ b/smoke-tests/otel-collector/data/normalize-severity/text-case/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'normalize-severity' AND ResourceAttributes['test-id'] = 'text-case' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'normalize-severity' AND ResourceAttributes['test-id'] = 'text-case' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql index 37fc65a0c2..42aa2617d3 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-debug/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-debug' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-debug' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql index b8a32fa127..8bafa94a32 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-error/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-error' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-error' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql index 81137c7ac6..48fead52a6 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-fatal/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-fatal' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-fatal' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql index cc77ad843f..cf9feb8484 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-info/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-info' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-info' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql index 7add4b48f5..c166dc0645 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-superstring/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-superstring' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-superstring' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql index 702238aacf..50df1b1694 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-trace/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-trace' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-trace' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql index 6568fcd0b4..0ea3cc8d7e 100644 --- a/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/infer-warn/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-warn' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-warn' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql index 24dc118f13..aea383f7b6 100644 --- a/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/no-infer-substring/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'no-infer-substring' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'no-infer-substring' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql index d1ce54c491..7c61c797d1 100644 --- a/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql +++ b/smoke-tests/otel-collector/data/severity-inference/skip-infer/assert_query.sql @@ -1 +1 @@ -SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'skip-infer' ORDER BY TimestampTime FORMAT CSV +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'skip-infer' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/docker-compose.yaml b/smoke-tests/otel-collector/docker-compose.yaml index 5499b72fff..a7d8bf80cb 100644 --- a/smoke-tests/otel-collector/docker-compose.yaml +++ b/smoke-tests/otel-collector/docker-compose.yaml @@ -76,6 +76,52 @@ services: ch-server: condition: service_healthy + # Older ClickHouse (< 26.2) to test the compatibility schema (bloom filters + # instead of full text indexes) + ch-server-compat: + image: clickhouse/clickhouse-server:25.12-alpine + environment: + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + volumes: + - ../../docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml + - ../../docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml + ports: + - 29000:9000 + - 28123:8123 + networks: + - internal + healthcheck: + test: + wget -O /dev/null --no-verbose --tries=1 http://127.0.0.1:8123/ping || + exit 1 + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + otel-collector-compat: + build: + context: ../.. + dockerfile: docker/otel-collector/Dockerfile + target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.149.0} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.55.0} + environment: + - CLICKHOUSE_ENDPOINT=tcp://ch-server-compat:9000?dial_timeout=10s + - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server-compat:9363 + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD= + - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default + - HYPERDX_LOG_LEVEL=info + # OPAMP_SERVER_URL is intentionally not set to run in standalone mode + ports: + - 24318:4318 # OTLP http receiver + networks: + - internal + depends_on: + ch-server-compat: + condition: service_healthy + networks: internal: name: 'smoke-test-internal-network' diff --git a/smoke-tests/otel-collector/setup_suite.bash b/smoke-tests/otel-collector/setup_suite.bash index ae3869646a..a5569526eb 100644 --- a/smoke-tests/otel-collector/setup_suite.bash +++ b/smoke-tests/otel-collector/setup_suite.bash @@ -8,6 +8,7 @@ setup_suite() { docker compose up --build --detach wait_for_ready "otel-collector" wait_for_ready "otel-collector-json" + wait_for_ready "otel-collector-compat" } teardown_suite() { diff --git a/smoke-tests/otel-collector/test_helpers/assertions.bash b/smoke-tests/otel-collector/test_helpers/assertions.bash index 999a1e82a1..a4ed90c0ca 100644 --- a/smoke-tests/otel-collector/test_helpers/assertions.bash +++ b/smoke-tests/otel-collector/test_helpers/assertions.bash @@ -1,5 +1,14 @@ assert_test_data() { - local testdir=$1 + _assert_test_data_on_port "9000" "$@" +} + +assert_test_data_compat() { + _assert_test_data_on_port "29000" "$@" +} + +_assert_test_data_on_port() { + local port=$1 + local testdir=$2 local query_file="${testdir}/assert_query.sql" local expected_file="${testdir}/expected.snap" @@ -26,7 +35,7 @@ assert_test_data() { fi # Execute the query using clickhouse-client and capture the results - local query_result=$(clickhouse-client --queries-file="$query_file" 2>&1) + local query_result=$(clickhouse-client --port="$port" --queries-file="$query_file" 2>&1) # Check if the clickhouse-client command succeeded if [ $? -ne 0 ]; then