feat(sources): configurable service version expression

.changeset/source-service-version-expression.md

@@ -0,0 +1,10 @@
+---
+'@hyperdx/app': minor
+'@hyperdx/api': minor
+---
+
+Add an optional `serviceVersionExpression` to log and trace sources, identifying
+the running release of a service. Defaults to the OpenTelemetry
+`service.version` resource attribute; teams whose release identifier lives
+elsewhere, such as a container image tag under GitOps, can point it there
+instead of changing instrumentation.

packages/api/openapi.json

@@ -3821,6 +3821,12 @@
             "nullable": true,
             "example": "ServiceName"
           },
+          "serviceVersionExpression": {
+            "type": "string",
+            "description": "Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')).",
+            "nullable": true,
+            "example": "ResourceAttributes['service.version']"
+          },
           "severityTextExpression": {
             "type": "string",
             "description": "Expression to extract the severity/log level text.",
@@ -4095,6 +4101,12 @@
             "nullable": true,
             "example": "ServiceName"
           },
+          "serviceVersionExpression": {
+            "type": "string",
+            "description": "Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')).",
+            "nullable": true,
+            "example": "ResourceAttributes['service.version']"
+          },
           "resourceAttributesExpression": {
             "type": "string",
             "description": "Expression to extract resource-level attributes.",

packages/api/src/models/source.ts

@@ -139,6 +139,7 @@ export const LogSource = Source.discriminator<ILogSource>(
   new Schema<ILogSource>({
     defaultTableSelectExpression: String,
     serviceNameExpression: String,
+    serviceVersionExpression: String,
     severityTextExpression: String,
     bodyExpression: String,
     eventAttributesExpression: String,
@@ -199,6 +200,7 @@ export const TraceSource = Source.discriminator<ITraceSource>(
     statusCodeExpression: String,
     statusMessageExpression: String,
     serviceNameExpression: String,
+    serviceVersionExpression: String,
     resourceAttributesExpression: String,
     eventAttributesExpression: String,
     spanEventsValueExpression: String,

packages/api/src/routers/api/__tests__/sources.int.test.ts

@@ -526,6 +526,112 @@ describe('sources router', () => {
     expect(updatedSource.severityTextExpression).toBe('SeverityText');
   });
 
+  // Regression guard: a field present in the Zod schema but missing from the
+  // Mongoose discriminator is silently dropped on write, with no error. Only a
+  // round-trip through the database catches that.
+  describe('serviceVersionExpression', () => {
+    const VERSION_EXPR = "ResourceAttributes['container.image.tag']";
+
+    /** The single persisted source, narrowed to a kind that carries the field. */
+    const persisted = async (kind: SourceKind.Log | SourceKind.Trace) => {
+      const [source] = await Source.find({}).lean();
+      if (source?.kind !== kind) {
+        throw new Error(`Expected a ${kind} source, got ${source?.kind}`);
+      }
+      return source;
+    };
+
+    it('POST / - persists the expression on a log source', async () => {
+      const { agent } = await getLoggedInAgent(server);
+
+      await agent
+        .post('/sources')
+        .send({ ...MOCK_SOURCE, serviceVersionExpression: VERSION_EXPR })
+        .expect(200);
+
+      expect((await persisted(SourceKind.Log)).serviceVersionExpression).toBe(
+        VERSION_EXPR,
+      );
+    });
+
+    it('POST / - persists the expression on a trace source', async () => {
+      const { agent } = await getLoggedInAgent(server);
+
+      const traceSource: Omit<Extract<TSource, { kind: 'trace' }>, 'id'> = {
+        kind: SourceKind.Trace,
+        name: 'Test Trace Source',
+        // The suite provisions a real Connection for this id; POST now 400s on
+        // one that doesn't exist for the team.
+        connection: MOCK_SOURCE.connection,
+        from: { databaseName: 'test_db', tableName: 'otel_traces' },
+        timestampValueExpression: 'Timestamp',
+        defaultTableSelectExpression: 'Timestamp, SpanName',
+        durationExpression: 'Duration',
+        durationPrecision: 9,
+        traceIdExpression: 'TraceId',
+        spanIdExpression: 'SpanId',
+        parentSpanIdExpression: 'ParentSpanId',
+        spanNameExpression: 'SpanName',
+        spanKindExpression: 'SpanKind',
+        serviceVersionExpression: VERSION_EXPR,
+      };
+
+      await agent.post('/sources').send(traceSource).expect(200);
+
+      expect((await persisted(SourceKind.Trace)).serviceVersionExpression).toBe(
+        VERSION_EXPR,
+      );
+    });
+
+    it('GET / - returns the expression', async () => {
+      const { agent } = await getLoggedInAgent(server);
+      await agent
+        .post('/sources')
+        .send({ ...MOCK_SOURCE, serviceVersionExpression: VERSION_EXPR })
+        .expect(200);
+
+      const response = await agent.get('/sources').expect(200);
+
+      expect(response.body[0]).toMatchObject({
+        serviceVersionExpression: VERSION_EXPR,
+      });
+    });
+
+    it('PUT /:id - updates the expression, and clears it when omitted', async () => {
+      const { agent } = await getLoggedInAgent(server);
+      const created = await agent
+        .post('/sources')
+        .send({ ...MOCK_SOURCE, serviceVersionExpression: VERSION_EXPR })
+        .expect(200);
+      const id = created.body.id;
+
+      await agent
+        .put(`/sources/${id}`)
+        .send({ ...MOCK_SOURCE, id, serviceVersionExpression: 'Version' })
+        .expect(200);
+      expect((await persisted(SourceKind.Log)).serviceVersionExpression).toBe(
+        'Version',
+      );
+
+      await agent
+        .put(`/sources/${id}`)
+        .send({ ...MOCK_SOURCE, id })
+        .expect(200);
+      expect(
+        (await persisted(SourceKind.Log)).serviceVersionExpression,
+      ).toBeUndefined();
+    });
+
+    it('POST / - is optional', async () => {
+      const { agent } = await getLoggedInAgent(server);
+      await agent.post('/sources').send(MOCK_SOURCE).expect(200);
+
+      expect(
+        (await persisted(SourceKind.Log)).serviceVersionExpression,
+      ).toBeUndefined();
+    });
+  });
+
   describe('seriesTable', () => {
     it('POST / - creates a metric source with seriesTable and it round-trips', async () => {
       const { agent } = await getLoggedInAgent(server);

packages/api/src/routers/external-api/v2/sources.ts

@@ -398,6 +398,11 @@ function formatExternalSource(source: SourceDocument) {
  *           description: Expression to extract the service name from log rows.
  *           nullable: true
  *           example: ServiceName
+ *         serviceVersionExpression:
+ *           type: string
+ *           description: Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')).
+ *           nullable: true
+ *           example: ResourceAttributes['service.version']
  *         severityTextExpression:
  *           type: string
  *           description: Expression to extract the severity/log level text.
@@ -616,6 +621,11 @@ function formatExternalSource(source: SourceDocument) {
  *           description: Expression to extract the service name from trace rows.
  *           nullable: true
  *           example: ServiceName
+ *         serviceVersionExpression:
+ *           type: string
+ *           description: Expression identifying the running release of a service. Defaults to the OpenTelemetry service.version resource attribute when unset. Where services carry the release on different attributes, fall back across them with coalesce(nullIf(a, ''), nullIf(b, '')).
+ *           nullable: true
+ *           example: ResourceAttributes['service.version']
  *         resourceAttributesExpression:
  *           type: string
  *           description: Expression to extract resource-level attributes.

packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx

@@ -12,6 +12,7 @@ import { useBrandDisplayName } from '@/theme/ThemeProvider';
 import {
   DEFAULT_DATABASE,
   KNOWN_COLUMNS_EXPRESSION_HELP_TEXT,
+  SERVICE_VERSION_EXPRESSION_HELP_TEXT,
 } from './constants';
 import { ExpressionFormRow } from './ExpressionFormRow';
 import { FormRow } from './FormRow';
@@ -118,6 +119,17 @@ export function LogTableModelForm(props: TableModelProps) {
           sourceKind={SourceKind.Log}
           tableConnection={tableConnection}
         />
+        <ExpressionFormRow
+          control={control}
+          setValue={setValue}
+          name="serviceVersionExpression"
+          label="Service Version Expression"
+          placeholder="ResourceAttributes['service.version']"
+          helpText={SERVICE_VERSION_EXPRESSION_HELP_TEXT}
+          columns={columns}
+          sourceKind={SourceKind.Log}
+          tableConnection={tableConnection}
+        />
         <ExpressionFormRow
           control={control}
           setValue={setValue}

packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx

@@ -10,6 +10,7 @@ import { useBrandDisplayName } from '@/theme/ThemeProvider';
 import {
   DEFAULT_DATABASE,
   KNOWN_COLUMNS_EXPRESSION_HELP_TEXT,
+  SERVICE_VERSION_EXPRESSION_HELP_TEXT,
 } from './constants';
 import { ExpressionFormRow } from './ExpressionFormRow';
 import { FormRow } from './FormRow';
@@ -274,6 +275,17 @@ export function TraceTableModelForm(props: TableModelProps) {
         sourceKind={SourceKind.Trace}
         tableConnection={tableConnection}
       />
+      <ExpressionFormRow
+        control={control}
+        setValue={setValue}
+        name="serviceVersionExpression"
+        label="Service Version Expression"
+        placeholder="ResourceAttributes['service.version']"
+        helpText={SERVICE_VERSION_EXPRESSION_HELP_TEXT}
+        columns={columns}
+        sourceKind={SourceKind.Trace}
+        tableConnection={tableConnection}
+      />
       <ExpressionFormRow
         control={control}
         setValue={setValue}

packages/app/src/components/Sources/SourceForm/constants.ts

@@ -6,6 +6,9 @@ export const DEFAULT_DATABASE = 'default';
 export const KNOWN_COLUMNS_EXPRESSION_HELP_TEXT =
   'For Distributed table sources whose target tables have non-matching column sets. Provide a list of columns supported across all target tables; it is used instead of SELECT * when fetching full row data (e.g. the row side panel). Leave blank to select all columns. This should be a comma-separated list of column names - do not include non-column expressions or aliases.';
 
+export const SERVICE_VERSION_EXPRESSION_HELP_TEXT =
+  "Identifies the running release of a service. Defaults to the OpenTelemetry service.version resource attribute. Point it elsewhere if your release identifier lives in another attribute - under GitOps it is often the container image tag. If services in this table use different attributes, fall back across them: coalesce(nullIf(ResourceAttributes['service.version'], ''), nullIf(ResourceAttributes['container.image.tag'], '')).";
+
 // Placeholder written into from.databaseName / from.tableName when the
 // selected connection is Prometheus-only.
 export const PROMETHEUS_PLACEHOLDER = 'prometheus';

packages/app/src/utils/sourceFieldSuggestions.tsx

@@ -10,6 +10,7 @@ export type SourceFieldKind =
   | 'bodyExpression'
   | 'implicitColumnExpression'
   | 'serviceNameExpression'
+  | 'serviceVersionExpression'
   | 'severityTextExpression'
   | 'eventAttributesExpression'
   | 'resourceAttributesExpression'
@@ -98,6 +99,18 @@ const FIELD_RULES: Record<SourceFieldKind, FieldRule> = {
     typeMatches: isStringy,
     recommendStrategy: 'name-only',
   },
+  // Usually a map lookup rather than a column, so these only hit on schemas
+  // that materialize the version out. No match just leaves the placeholder.
+  serviceVersionExpression: {
+    canonicalNames: [
+      'ServiceVersion',
+      'service.version',
+      'service_version',
+      'version',
+    ],
+    typeMatches: isStringy,
+    recommendStrategy: 'name-only',
+  },
   severityTextExpression: {
     canonicalNames: [
       'SeverityText',

packages/app/tests/e2e/features/sources.spec.ts

@@ -22,6 +22,7 @@ const COMMON_FIELDS = [
 const LOG_FIELDS = [
   ...COMMON_FIELDS,
   'Service Name Expression',
+  'Service Version Expression',
   'Log Level Expression',
   'Body Expression',
   'Log Attributes Expression',
@@ -50,6 +51,7 @@ const TRACE_FIELDS = [
   'Status Code Expression',
   'Status Message Expression',
   'Service Name Expression',
+  'Service Version Expression',
   'Resource Attributes Expression',
   'Event Attributes Expression',
   'Span Events Expression',

packages/common-utils/src/types.ts

@@ -1878,6 +1878,13 @@ export const LogSourceSchema = BaseSourceSchema.extend({
     .min(1, 'Default Select Expression is required'),
   // Optional fields for logs
   serviceNameExpression: z.string().optional(),
+  /**
+   * Expression identifying the running release of a service. Defaults to the
+   * OpenTelemetry `service.version` resource attribute when unset; teams whose
+   * version lives elsewhere - a container image tag under GitOps, a custom
+   * attribute - point it there instead of changing instrumentation.
+   */
+  serviceVersionExpression: z.string().optional(),
   severityTextExpression: z.string().optional(),
   bodyExpression: z.string().optional(),
   eventAttributesExpression: z.string().optional(),
@@ -1935,6 +1942,8 @@ export const TraceSourceSchema = BaseSourceSchema.extend({
   statusCodeExpression: z.string().optional(),
   statusMessageExpression: z.string().optional(),
   serviceNameExpression: z.string().optional(),
+  /** See `serviceVersionExpression` on `LogSourceSchema`. */
+  serviceVersionExpression: z.string().optional(),
   resourceAttributesExpression: z.string().optional(),
   eventAttributesExpression: z.string().optional(),
   spanEventsValueExpression: z.string().optional(),