diff --git a/.changeset/source-service-version-expression.md b/.changeset/source-service-version-expression.md new file mode 100644 index 0000000000..aa2d0bdb02 --- /dev/null +++ b/.changeset/source-service-version-expression.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/app': minor +'@hyperdx/api': minor +--- + +Add an optional `serviceVersionExpression` to log and trace sources, identifying +the running release of a service. Defaults to the OpenTelemetry +`service.version` resource attribute; teams whose release identifier lives +elsewhere, such as a container image tag under GitOps, can point it there +instead of changing instrumentation. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index a72d5e5fe8..341c19fba7 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -3821,6 +3821,12 @@ "nullable": true, "example": "ServiceName" }, + "serviceVersionExpression": { + "type": "string", + "description": "Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')).", + "nullable": true, + "example": "ResourceAttributes['service.version']" + }, "severityTextExpression": { "type": "string", "description": "Expression to extract the severity/log level text.", @@ -4095,6 +4101,12 @@ "nullable": true, "example": "ServiceName" }, + "serviceVersionExpression": { + "type": "string", + "description": "Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')).", + "nullable": true, + "example": "ResourceAttributes['service.version']" + }, "resourceAttributesExpression": { "type": "string", "description": "Expression to extract resource-level attributes.", diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 0de9634a48..45d80cb072 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -139,6 +139,7 @@ export const LogSource = Source.discriminator( new Schema({ defaultTableSelectExpression: String, serviceNameExpression: String, + serviceVersionExpression: String, severityTextExpression: String, bodyExpression: String, eventAttributesExpression: String, @@ -199,6 +200,7 @@ export const TraceSource = Source.discriminator( statusCodeExpression: String, statusMessageExpression: String, serviceNameExpression: String, + serviceVersionExpression: String, resourceAttributesExpression: String, eventAttributesExpression: String, spanEventsValueExpression: String, diff --git a/packages/api/src/routers/api/__tests__/sources.int.test.ts b/packages/api/src/routers/api/__tests__/sources.int.test.ts index a85d1b216f..85ad0ed75c 100644 --- a/packages/api/src/routers/api/__tests__/sources.int.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.int.test.ts @@ -526,6 +526,112 @@ describe('sources router', () => { expect(updatedSource.severityTextExpression).toBe('SeverityText'); }); + // Regression guard: a field present in the Zod schema but missing from the + // Mongoose discriminator is silently dropped on write, with no error. Only a + // round-trip through the database catches that. + describe('serviceVersionExpression', () => { + const VERSION_EXPR = "ResourceAttributes['container.image.tag']"; + + /** The single persisted source, narrowed to a kind that carries the field. */ + const persisted = async (kind: SourceKind.Log | SourceKind.Trace) => { + const [source] = await Source.find({}).lean(); + if (source?.kind !== kind) { + throw new Error(`Expected a ${kind} source, got ${source?.kind}`); + } + return source; + }; + + it('POST / - persists the expression on a log source', async () => { + const { agent } = await getLoggedInAgent(server); + + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, serviceVersionExpression: VERSION_EXPR }) + .expect(200); + + expect((await persisted(SourceKind.Log)).serviceVersionExpression).toBe( + VERSION_EXPR, + ); + }); + + it('POST / - persists the expression on a trace source', async () => { + const { agent } = await getLoggedInAgent(server); + + const traceSource: Omit, 'id'> = { + kind: SourceKind.Trace, + name: 'Test Trace Source', + // The suite provisions a real Connection for this id; POST now 400s on + // one that doesn't exist for the team. + connection: MOCK_SOURCE.connection, + from: { databaseName: 'test_db', tableName: 'otel_traces' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp, SpanName', + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + serviceVersionExpression: VERSION_EXPR, + }; + + await agent.post('/sources').send(traceSource).expect(200); + + expect((await persisted(SourceKind.Trace)).serviceVersionExpression).toBe( + VERSION_EXPR, + ); + }); + + it('GET / - returns the expression', async () => { + const { agent } = await getLoggedInAgent(server); + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, serviceVersionExpression: VERSION_EXPR }) + .expect(200); + + const response = await agent.get('/sources').expect(200); + + expect(response.body[0]).toMatchObject({ + serviceVersionExpression: VERSION_EXPR, + }); + }); + + it('PUT /:id - updates the expression, and clears it when omitted', async () => { + const { agent } = await getLoggedInAgent(server); + const created = await agent + .post('/sources') + .send({ ...MOCK_SOURCE, serviceVersionExpression: VERSION_EXPR }) + .expect(200); + const id = created.body.id; + + await agent + .put(`/sources/${id}`) + .send({ ...MOCK_SOURCE, id, serviceVersionExpression: 'Version' }) + .expect(200); + expect((await persisted(SourceKind.Log)).serviceVersionExpression).toBe( + 'Version', + ); + + await agent + .put(`/sources/${id}`) + .send({ ...MOCK_SOURCE, id }) + .expect(200); + expect( + (await persisted(SourceKind.Log)).serviceVersionExpression, + ).toBeUndefined(); + }); + + it('POST / - is optional', async () => { + const { agent } = await getLoggedInAgent(server); + await agent.post('/sources').send(MOCK_SOURCE).expect(200); + + expect( + (await persisted(SourceKind.Log)).serviceVersionExpression, + ).toBeUndefined(); + }); + }); + describe('seriesTable', () => { it('POST / - creates a metric source with seriesTable and it round-trips', async () => { const { agent } = await getLoggedInAgent(server); diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 345065af84..f4233bd75f 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -398,6 +398,11 @@ function formatExternalSource(source: SourceDocument) { * description: Expression to extract the service name from log rows. * nullable: true * example: ServiceName + * serviceVersionExpression: + * type: string + * description: Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')). + * nullable: true + * example: ResourceAttributes['service.version'] * severityTextExpression: * type: string * description: Expression to extract the severity/log level text. @@ -616,6 +621,11 @@ function formatExternalSource(source: SourceDocument) { * description: Expression to extract the service name from trace rows. * nullable: true * example: ServiceName + * serviceVersionExpression: + * type: string + * description: Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')). + * nullable: true + * example: ResourceAttributes['service.version'] * resourceAttributesExpression: * type: string * description: Expression to extract resource-level attributes. diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index 0ebb3c96a9..fb2cbac79c 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -12,6 +12,7 @@ import { useBrandDisplayName } from '@/theme/ThemeProvider'; import { DEFAULT_DATABASE, KNOWN_COLUMNS_EXPRESSION_HELP_TEXT, + SERVICE_VERSION_EXPRESSION_HELP_TEXT, } from './constants'; import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; @@ -118,6 +119,17 @@ export function LogTableModelForm(props: TableModelProps) { sourceKind={SourceKind.Log} tableConnection={tableConnection} /> + + = { typeMatches: isStringy, recommendStrategy: 'name-only', }, + // Usually a map lookup rather than a column, so these only hit on schemas + // that materialize the version out. No match just leaves the placeholder. + serviceVersionExpression: { + canonicalNames: [ + 'ServiceVersion', + 'service.version', + 'service_version', + 'version', + ], + typeMatches: isStringy, + recommendStrategy: 'name-only', + }, severityTextExpression: { canonicalNames: [ 'SeverityText', diff --git a/packages/app/tests/e2e/features/sources.spec.ts b/packages/app/tests/e2e/features/sources.spec.ts index 9029916365..b8b6d045a0 100644 --- a/packages/app/tests/e2e/features/sources.spec.ts +++ b/packages/app/tests/e2e/features/sources.spec.ts @@ -22,6 +22,7 @@ const COMMON_FIELDS = [ const LOG_FIELDS = [ ...COMMON_FIELDS, 'Service Name Expression', + 'Service Version Expression', 'Log Level Expression', 'Body Expression', 'Log Attributes Expression', @@ -50,6 +51,7 @@ const TRACE_FIELDS = [ 'Status Code Expression', 'Status Message Expression', 'Service Name Expression', + 'Service Version Expression', 'Resource Attributes Expression', 'Event Attributes Expression', 'Span Events Expression', diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 96a9883bff..329c6db7c6 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1878,6 +1878,13 @@ export const LogSourceSchema = BaseSourceSchema.extend({ .min(1, 'Default Select Expression is required'), // Optional fields for logs serviceNameExpression: z.string().optional(), + /** + * Expression identifying the running release of a service. Defaults to the + * OpenTelemetry `service.version` resource attribute when unset; teams whose + * version lives elsewhere - a container image tag under GitOps, a custom + * attribute - point it there instead of changing instrumentation. + */ + serviceVersionExpression: z.string().optional(), severityTextExpression: z.string().optional(), bodyExpression: z.string().optional(), eventAttributesExpression: z.string().optional(), @@ -1935,6 +1942,8 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ statusCodeExpression: z.string().optional(), statusMessageExpression: z.string().optional(), serviceNameExpression: z.string().optional(), + /** See `serviceVersionExpression` on `LogSourceSchema`. */ + serviceVersionExpression: z.string().optional(), resourceAttributesExpression: z.string().optional(), eventAttributesExpression: z.string().optional(), spanEventsValueExpression: z.string().optional(),