From 1d4ef076cb5d87ecd430a6c0ce6ea3b59d0a994d Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Tue, 20 Oct 2026 15:31:47 -0700 Subject: [PATCH] feat: SeverityNumber column on Logs Source --- .../source-severity-number-expression.md | 10 +++++ docker-compose.yml | 2 +- docker/hyperdx/entry.local.base.sh | 4 +- packages/api/.env.development | 2 +- packages/api/openapi.json | 6 +++ .../src/mcp/tools/sources/describeSource.ts | 1 + .../api/src/mcp/tools/sources/listSources.ts | 1 + packages/api/src/mcp/tools/sources/schemas.ts | 7 ++++ packages/api/src/models/source.ts | 1 + .../routers/api/__tests__/sources.int.test.ts | 15 +++++++ .../src/routers/external-api/v2/sources.ts | 5 +++ packages/app/src/__tests__/source.test.ts | 42 +++++++++++++++++++ .../app/src/components/OnboardingModal.tsx | 1 + .../Sources/SourceForm/LogTableModelForm.tsx | 12 ++++++ .../Sources/SourceForm/constants.ts | 3 ++ packages/app/src/source.ts | 10 +++++ .../app/src/utils/sourceFieldSuggestions.tsx | 7 ++++ .../app/tests/e2e/features/sources.spec.ts | 1 + packages/common-utils/src/clickhouse/index.ts | 8 ++++ packages/common-utils/src/types.ts | 5 +++ 20 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 .changeset/source-severity-number-expression.md diff --git a/.changeset/source-severity-number-expression.md b/.changeset/source-severity-number-expression.md new file mode 100644 index 0000000000..6ac30b61ca --- /dev/null +++ b/.changeset/source-severity-number-expression.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +'@hyperdx/api': minor +--- + +Add an optional `severityNumberExpression` to log sources, pointing at the +numeric OpenTelemetry SeverityNumber (1-24). Source auto-detection sets it when +an OTel log table has an integer `SeverityNumber` column. The field is stored +and editable only: log levels still come from the severity text. diff --git a/docker-compose.yml b/docker-compose.yml index bf1a89133f..5532825d1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,7 +95,7 @@ services: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' DEFAULT_SOURCES: - '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 + '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index 2c464a247d..0d8a0c9dfe 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -36,10 +36,10 @@ if [ -z "${DEFAULT_CONNECTIONS}" ]; then DEFAULT_CONNECTIONS='[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]' fi export DEFAULT_CONNECTIONS -export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' +export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' # if BETA_CH_OTEL_JSON_SCHEMA_ENABLED is true then return empty default sources if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then - export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' + export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' fi # Simulate Docker Service DNS diff --git a/packages/api/.env.development b/packages/api/.env.development index 1b71f80e83..550cbc2e59 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -22,7 +22,7 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"JSON Logs"},{"kind":"promql","name":"PromQL","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"JSON Logs"},{"kind":"promql","name":"PromQL","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY diff --git a/packages/api/openapi.json b/packages/api/openapi.json index b6263b7423..5b97af8486 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4597,6 +4597,12 @@ "nullable": true, "example": "SeverityText" }, + "severityNumberExpression": { + "type": "string", + "description": "Expression for the numeric OpenTelemetry SeverityNumber (1-24).", + "nullable": true, + "example": "SeverityNumber" + }, "bodyExpression": { "type": "string", "description": "Expression to extract the log message body.", diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 4be29904da..1a27095b58 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -146,6 +146,7 @@ async function describeSourceSchema( body: source.bodyExpression, serviceName: source.serviceNameExpression, severityText: source.severityTextExpression, + severityNumber: source.severityNumberExpression, traceId: source.traceIdExpression, }; } else if (source.kind === SourceKind.Metric) { diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts index 0cc36b9a5e..b833bf3ec6 100644 --- a/packages/api/src/mcp/tools/sources/listSources.ts +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -285,6 +285,7 @@ export function registerListSources({ body: s.bodyExpression, serviceName: s.serviceNameExpression, severityText: s.severityTextExpression, + severityNumber: s.severityNumberExpression, traceId: s.traceIdExpression, }; } else if (s.kind === SourceKind.Metric) { diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 0b52a6fb57..852da76f6e 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -280,6 +280,13 @@ export const mcpSaveSourceSchema = z.object({ .describe( 'Log sources. Expression for the log level / severity text, e.g. SeverityText.', ), + severityNumberExpression: z + .string() + .optional() + .describe( + 'Log sources. Expression for the numeric OpenTelemetry SeverityNumber ' + + '(1-24), e.g. SeverityNumber. Omit if the table has no such column.', + ), // ── Trace (required for trace kind) ── durationExpression: z diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 664874df5b..95045e017a 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -157,6 +157,7 @@ export const LogSource = Source.discriminator( serviceNameExpression: String, serviceVersionExpression: String, severityTextExpression: String, + severityNumberExpression: String, bodyExpression: String, eventAttributesExpression: String, resourceAttributesExpression: String, diff --git a/packages/api/src/routers/api/__tests__/sources.int.test.ts b/packages/api/src/routers/api/__tests__/sources.int.test.ts index 531916af56..fad658e73f 100644 --- a/packages/api/src/routers/api/__tests__/sources.int.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.int.test.ts @@ -644,6 +644,21 @@ describe('sources router', () => { }); }); + // Same regression guard as serviceVersionExpression above. + it('POST / - round-trips severityNumberExpression on a log source', async () => { + const { agent } = await getLoggedInAgent(server); + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, severityNumberExpression: 'SeverityNumber' }) + .expect(200); + + const response = await agent.get('/sources').expect(200); + + expect(response.body[0]).toMatchObject({ + severityNumberExpression: 'SeverityNumber', + }); + }); + describe('seriesTable', () => { it('POST / - creates a metric source with seriesTable and it round-trips', async () => { const { agent } = await getLoggedInAgent(server); diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index bf8a485e46..fce6007c00 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -408,6 +408,11 @@ function formatExternalSource(source: SourceDocument) { * description: Expression to extract the severity/log level text. * nullable: true * example: SeverityText + * severityNumberExpression: + * type: string + * description: Expression for the numeric OpenTelemetry SeverityNumber (1-24). + * nullable: true + * example: SeverityNumber * bodyExpression: * type: string * description: Expression to extract the log message body. diff --git a/packages/app/src/__tests__/source.test.ts b/packages/app/src/__tests__/source.test.ts index 731669c3db..c7681deaa7 100644 --- a/packages/app/src/__tests__/source.test.ts +++ b/packages/app/src/__tests__/source.test.ts @@ -1,4 +1,5 @@ import { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; +import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; import { ChartConfigWithOptTimestamp, NumberFormat, @@ -14,6 +15,7 @@ import { getEventBody, getSourceValidationNotificationId, getTraceDurationNumberFormat, + inferTableSourceConfig, pickTimeSeriesTable, useChartNumberFormats, useSingleSeriesNumberFormat, @@ -110,6 +112,46 @@ describe('pickTimeSeriesTable', () => { }); }); +describe('inferTableSourceConfig', () => { + const inferOtelLogConfig = (severityNumberType: string) => { + const columns = [ + 'Timestamp', + 'Body', + 'SeverityText', + 'TraceId', + 'SpanId', + 'ServiceName', + 'LogAttributes', + 'ResourceAttributes', + ].map(name => ({ name, type: 'String' })); + columns.push({ name: 'SeverityNumber', type: severityNumberType }); + + return inferTableSourceConfig({ + databaseName: 'default', + tableName: 'otel_logs', + connectionId: 'test-connection', + kind: SourceKind.Log, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + metadata: { + getColumns: async () => columns, + getTableMetadata: async () => undefined, + } as unknown as Metadata, + }); + }; + + it('sets severityNumberExpression only for an integer SeverityNumber column', async () => { + for (const type of ['UInt8', 'Int32', 'LowCardinality(Nullable(UInt8))']) { + expect(await inferOtelLogConfig(type)).toHaveProperty( + 'severityNumberExpression', + 'SeverityNumber', + ); + } + expect(await inferOtelLogConfig('String')).not.toHaveProperty( + 'severityNumberExpression', + ); + }); +}); + describe('getEventBody', () => { it('returns spanNameExpression for trace kind source when both bodyExpression and spanNameExpression are present', () => { const result = getEventBody(TRACE_SOURCE); diff --git a/packages/app/src/components/OnboardingModal.tsx b/packages/app/src/components/OnboardingModal.tsx index dfccade434..f26f4b3358 100644 --- a/packages/app/src/components/OnboardingModal.tsx +++ b/packages/app/src/components/OnboardingModal.tsx @@ -101,6 +101,7 @@ async function addOtelDemoSources({ 'Timestamp, ServiceName, SeverityText, Body', serviceNameExpression: 'ServiceName', severityTextExpression: 'SeverityText', + severityNumberExpression: 'SeverityNumber', eventAttributesExpression: 'LogAttributes', resourceAttributesExpression: 'ResourceAttributes', traceIdExpression: 'TraceId', diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index fb2cbac79c..9dfb107266 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -13,6 +13,7 @@ import { DEFAULT_DATABASE, KNOWN_COLUMNS_EXPRESSION_HELP_TEXT, SERVICE_VERSION_EXPRESSION_HELP_TEXT, + SEVERITY_NUMBER_EXPRESSION_HELP_TEXT, } from './constants'; import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; @@ -140,6 +141,17 @@ export function LogTableModelForm(props: TableModelProps) { sourceKind={SourceKind.Log} tableConnection={tableConnection} /> + col.name === 'Links.TraceId'); + // Old exporter tables store SeverityNumber as Int32, current ones as UInt8. + // A String column of that name is not the OTel number. + const hasSeverityNumber = columns.some( + col => col.name === 'SeverityNumber' && isCHIntegerType(col.type), + ); + // Check if metadata rollup tables exist and, if so, infer the bucketing // granularity from the key-rollup view's `as_select` const rollupMeta = @@ -461,6 +468,9 @@ export async function inferTableSourceConfig({ traceIdExpression: 'TraceId', severityTextExpression: 'SeverityText', + ...(hasSeverityNumber + ? { severityNumberExpression: 'SeverityNumber' } + : {}), ...metadataMVsConfig, } : {}), diff --git a/packages/app/src/utils/sourceFieldSuggestions.tsx b/packages/app/src/utils/sourceFieldSuggestions.tsx index b0f9fab58a..7d1ba6edf8 100644 --- a/packages/app/src/utils/sourceFieldSuggestions.tsx +++ b/packages/app/src/utils/sourceFieldSuggestions.tsx @@ -2,6 +2,7 @@ import { JSX } from 'react'; import { ColumnMetaType, convertCHDataTypeToJSType, + isCHIntegerType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; @@ -12,6 +13,7 @@ export type SourceFieldKind = | 'serviceNameExpression' | 'serviceVersionExpression' | 'severityTextExpression' + | 'severityNumberExpression' | 'eventAttributesExpression' | 'resourceAttributesExpression' | 'traceIdExpression' @@ -122,6 +124,11 @@ const FIELD_RULES: Record = { typeMatches: isStringy, recommendStrategy: 'name-only', }, + severityNumberExpression: { + canonicalNames: ['SeverityNumber', 'severity_number'], + typeMatches: isCHIntegerType, + recommendStrategy: 'name-only', + }, eventAttributesExpression: { canonicalNames: ['LogAttributes', 'Attributes', 'attributes', 'tags'], canonicalNamesByKind: { diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index 0ee087888a..869e8f6cae 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -24,6 +24,7 @@ const LOG_FIELDS = [ 'Service Name Expression', 'Service Version Expression', 'Log Level Expression', + 'Severity Number Expression', 'Body Expression', 'Log Attributes Expression', 'Resource Attributes Expression', diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 1f1009bda1..0f719d413a 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -158,6 +158,14 @@ export const isCHFixedStringType = (dataType: string): boolean => { return stripTypeWrappers(dataType).startsWith('FixedString'); }; +/** + * True when the ClickHouse type is a signed or unsigned integer, including + * Nullable and LowCardinality wrappers. JSDataType.Number also covers Float. + */ +export const isCHIntegerType = (dataType: string): boolean => { + return /^U?Int(8|16|32|64|128|256)$/.test(stripTypeWrappers(dataType)); +}; + export const isJSDataTypeJSONStringifiable = ( dataType: JSDataType | null | undefined, ) => { diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 4c71dbdd34..3fe5513a82 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2708,6 +2708,11 @@ export const LogSourceSchema = BaseSourceSchema.extend({ */ serviceVersionExpression: z.string().optional(), severityTextExpression: z.string().optional(), + /** + * Expression for the numeric OpenTelemetry SeverityNumber (1-24). No default: + * many log tables have no such column, so unset must stay distinguishable. + */ + severityNumberExpression: z.string().optional(), bodyExpression: z.string().optional(), eventAttributesExpression: z.string().optional(), resourceAttributesExpression: z.string().optional(),