From 9b527cb0d41a9169c2abe3a0fc4fe4a8fd4380c3 Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Wed, 21 Oct 2026 15:47:02 -0700 Subject: [PATCH] feat: EventName column on Logs Source --- .changeset/source-event-name-expression.md | 10 +++++++++ docker-compose.yml | 2 +- docker/hyperdx/entry.local.base.sh | 4 ++-- packages/api/.env.development | 2 +- packages/api/openapi.json | 6 ++++++ .../src/mcp/tools/sources/describeSource.ts | 1 + .../api/src/mcp/tools/sources/listSources.ts | 1 + packages/api/src/mcp/tools/sources/schemas.ts | 7 +++++++ packages/api/src/models/source.ts | 1 + .../routers/api/__tests__/sources.int.test.ts | 14 +++++++++++++ .../src/routers/external-api/v2/sources.ts | 5 +++++ packages/app/src/__tests__/source.test.ts | 21 ++++++++++++++++++- .../Sources/SourceForm/LogTableModelForm.tsx | 12 +++++++++++ .../Sources/SourceForm/constants.ts | 3 +++ packages/app/src/source.ts | 8 +++++++ .../app/src/utils/sourceFieldSuggestions.tsx | 6 ++++++ .../app/tests/e2e/features/sources.spec.ts | 1 + packages/common-utils/src/types.ts | 5 +++++ 18 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 .changeset/source-event-name-expression.md diff --git a/.changeset/source-event-name-expression.md b/.changeset/source-event-name-expression.md new file mode 100644 index 0000000000..9aabbcf7af --- /dev/null +++ b/.changeset/source-event-name-expression.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +'@hyperdx/api': minor +--- + +Add an optional `eventNameExpression` to log sources, pointing at the +OpenTelemetry log EventName. Source auto-detection sets it when an OTel log +table has a string `EventName` column. The field is stored and editable only: +nothing reads it yet. HDX-5787 diff --git a/docker-compose.yml b/docker-compose.yml index 5532825d1c..9f741fe8d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,7 +95,7 @@ services: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' DEFAULT_SOURCES: - '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 + '[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","eventNameExpression":"EventName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index 0d8a0c9dfe..82ceb0aebf 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -36,10 +36,10 @@ if [ -z "${DEFAULT_CONNECTIONS}" ]; then DEFAULT_CONNECTIONS='[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]' fi export DEFAULT_CONNECTIONS -export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' +export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","eventNameExpression":"EventName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' # if BETA_CH_OTEL_JSON_SCHEMA_ENABLED is true then return empty default sources if [ "$BETA_CH_OTEL_JSON_SCHEMA_ENABLED" = "true" ]; then - export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' + export DEFAULT_SOURCES='[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","eventNameExpression":"EventName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"Timestamp","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"}]' fi # Simulate Docker Service DNS diff --git a/packages/api/.env.development b/packages/api/.env.development index 550cbc2e59..d97b0ff888 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -22,7 +22,7 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"JSON Logs"},{"kind":"promql","name":"PromQL","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","eventNameExpression":"EventName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","metadataMaterializedViews":{"kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"Logs"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","resourceAttributesExpression":"ResourceAttributes","connection":"Local ClickHouse","traceSourceId":"Traces"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","severityNumberExpression":"SeverityNumber","eventNameExpression":"EventName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","spanEventsValueExpression":"Events","spanLinksValueExpression":"Links","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","summary":"otel_metrics_summary","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"JSON Logs"},{"kind":"promql","name":"PromQL","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 5b97af8486..de21409f23 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4609,6 +4609,12 @@ "nullable": true, "example": "Body" }, + "eventNameExpression": { + "type": "string", + "description": "Expression for the OpenTelemetry log EventName.", + "nullable": true, + "example": "EventName" + }, "eventAttributesExpression": { "type": "string", "description": "Expression to extract event-level attributes.", diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 1a27095b58..ff52479143 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -147,6 +147,7 @@ async function describeSourceSchema( serviceName: source.serviceNameExpression, severityText: source.severityTextExpression, severityNumber: source.severityNumberExpression, + eventName: source.eventNameExpression, traceId: source.traceIdExpression, }; } else if (source.kind === SourceKind.Metric) { diff --git a/packages/api/src/mcp/tools/sources/listSources.ts b/packages/api/src/mcp/tools/sources/listSources.ts index b833bf3ec6..ce65fa822d 100644 --- a/packages/api/src/mcp/tools/sources/listSources.ts +++ b/packages/api/src/mcp/tools/sources/listSources.ts @@ -286,6 +286,7 @@ export function registerListSources({ serviceName: s.serviceNameExpression, severityText: s.severityTextExpression, severityNumber: s.severityNumberExpression, + eventName: s.eventNameExpression, traceId: s.traceIdExpression, }; } else if (s.kind === SourceKind.Metric) { diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 852da76f6e..b3be96b1b6 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -287,6 +287,13 @@ export const mcpSaveSourceSchema = z.object({ 'Log sources. Expression for the numeric OpenTelemetry SeverityNumber ' + '(1-24), e.g. SeverityNumber. Omit if the table has no such column.', ), + eventNameExpression: z + .string() + .optional() + .describe( + 'Log sources. Expression for the OpenTelemetry log EventName, e.g. ' + + 'EventName. Omit if the table has no such column.', + ), // ── Trace (required for trace kind) ── durationExpression: z diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 95045e017a..19902a41a0 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -159,6 +159,7 @@ export const LogSource = Source.discriminator( severityTextExpression: String, severityNumberExpression: String, bodyExpression: String, + eventNameExpression: String, eventAttributesExpression: String, resourceAttributesExpression: String, displayedTimestampValueExpression: String, diff --git a/packages/api/src/routers/api/__tests__/sources.int.test.ts b/packages/api/src/routers/api/__tests__/sources.int.test.ts index fad658e73f..7cb4f072d6 100644 --- a/packages/api/src/routers/api/__tests__/sources.int.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.int.test.ts @@ -659,6 +659,20 @@ describe('sources router', () => { }); }); + it('POST / - round-trips eventNameExpression on a log source', async () => { + const { agent } = await getLoggedInAgent(server); + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, eventNameExpression: 'EventName' }) + .expect(200); + + const response = await agent.get('/sources').expect(200); + + expect(response.body[0]).toMatchObject({ + eventNameExpression: 'EventName', + }); + }); + describe('seriesTable', () => { it('POST / - creates a metric source with seriesTable and it round-trips', async () => { const { agent } = await getLoggedInAgent(server); diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index fce6007c00..38c9d2b2f5 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -418,6 +418,11 @@ function formatExternalSource(source: SourceDocument) { * description: Expression to extract the log message body. * nullable: true * example: Body + * eventNameExpression: + * type: string + * description: Expression for the OpenTelemetry log EventName. + * nullable: true + * example: EventName * eventAttributesExpression: * type: string * description: Expression to extract event-level attributes. diff --git a/packages/app/src/__tests__/source.test.ts b/packages/app/src/__tests__/source.test.ts index c7681deaa7..fbc43e78ee 100644 --- a/packages/app/src/__tests__/source.test.ts +++ b/packages/app/src/__tests__/source.test.ts @@ -113,7 +113,10 @@ describe('pickTimeSeriesTable', () => { }); describe('inferTableSourceConfig', () => { - const inferOtelLogConfig = (severityNumberType: string) => { + const inferOtelLogConfig = ( + severityNumberType: string, + extraColumns: ColumnMetaType[] = [], + ) => { const columns = [ 'Timestamp', 'Body', @@ -125,6 +128,7 @@ describe('inferTableSourceConfig', () => { 'ResourceAttributes', ].map(name => ({ name, type: 'String' })); columns.push({ name: 'SeverityNumber', type: severityNumberType }); + columns.push(...extraColumns); return inferTableSourceConfig({ databaseName: 'default', @@ -150,6 +154,21 @@ describe('inferTableSourceConfig', () => { 'severityNumberExpression', ); }); + + it('sets eventNameExpression only for a string EventName column', async () => { + for (const type of [ + 'String', + 'LowCardinality(String)', + 'Nullable(String)', + ]) { + expect( + await inferOtelLogConfig('UInt8', [{ name: 'EventName', type }]), + ).toHaveProperty('eventNameExpression', 'EventName'); + } + expect( + await inferOtelLogConfig('UInt8', [{ name: 'EventName', type: 'UInt8' }]), + ).not.toHaveProperty('eventNameExpression'); + }); }); describe('getEventBody', () => { diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index 9dfb107266..52d918fb92 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -11,6 +11,7 @@ import { useBrandDisplayName } from '@/theme/ThemeProvider'; import { DEFAULT_DATABASE, + EVENT_NAME_EXPRESSION_HELP_TEXT, KNOWN_COLUMNS_EXPRESSION_HELP_TEXT, SERVICE_VERSION_EXPRESSION_HELP_TEXT, SEVERITY_NUMBER_EXPRESSION_HELP_TEXT, @@ -162,6 +163,17 @@ export function LogTableModelForm(props: TableModelProps) { sourceKind={SourceKind.Log} tableConnection={tableConnection} /> + col.name === 'SeverityNumber' && isCHIntegerType(col.type), ); + const hasEventName = columns.some( + col => + col.name === 'EventName' && + convertCHDataTypeToJSType(col.type) === JSDataType.String, + ); + // Check if metadata rollup tables exist and, if so, infer the bucketing // granularity from the key-rollup view's `as_select` const rollupMeta = @@ -471,6 +478,7 @@ export async function inferTableSourceConfig({ ...(hasSeverityNumber ? { severityNumberExpression: 'SeverityNumber' } : {}), + ...(hasEventName ? { eventNameExpression: 'EventName' } : {}), ...metadataMVsConfig, } : {}), diff --git a/packages/app/src/utils/sourceFieldSuggestions.tsx b/packages/app/src/utils/sourceFieldSuggestions.tsx index 7d1ba6edf8..3ea4ee45b8 100644 --- a/packages/app/src/utils/sourceFieldSuggestions.tsx +++ b/packages/app/src/utils/sourceFieldSuggestions.tsx @@ -14,6 +14,7 @@ export type SourceFieldKind = | 'serviceVersionExpression' | 'severityTextExpression' | 'severityNumberExpression' + | 'eventNameExpression' | 'eventAttributesExpression' | 'resourceAttributesExpression' | 'traceIdExpression' @@ -129,6 +130,11 @@ const FIELD_RULES: Record = { typeMatches: isCHIntegerType, recommendStrategy: 'name-only', }, + eventNameExpression: { + canonicalNames: ['EventName', 'event_name'], + typeMatches: isStringy, + recommendStrategy: 'name-only', + }, eventAttributesExpression: { canonicalNames: ['LogAttributes', 'Attributes', 'attributes', 'tags'], canonicalNamesByKind: { diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index 869e8f6cae..327a03d17c 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -26,6 +26,7 @@ const LOG_FIELDS = [ 'Log Level Expression', 'Severity Number Expression', 'Body Expression', + 'Event Name Expression', 'Log Attributes Expression', 'Resource Attributes Expression', 'Displayed Timestamp Column', diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 3fe5513a82..f1c006a299 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2714,6 +2714,11 @@ export const LogSourceSchema = BaseSourceSchema.extend({ */ severityNumberExpression: z.string().optional(), bodyExpression: z.string().optional(), + /** + * Expression for the OpenTelemetry log EventName (e.g. browser.web_vital). + * No default: older and non-OTel log tables have no such column. + */ + eventNameExpression: z.string().optional(), eventAttributesExpression: z.string().optional(), resourceAttributesExpression: z.string().optional(), displayedTimestampValueExpression: z.string().optional(),