[HDX-3929] Migrate OTel Collector build to use OCB (OpenTelemetry Collector Builder)

.changeset/bump-otel-collector-to-0149.md

@@ -0,0 +1,8 @@
+---
+'@hyperdx/otel-collector': minor
+---
+
+feat: Bump OTel Collector from 0.147.0 to 0.149.0
+
+Upgrade the OpenTelemetry Collector and all its components from v0.147.0 to
+v0.149.0 (core providers from v1.53.0 to v1.55.0).

.changeset/migrate-otel-collector-to-ocb.md

@@ -0,0 +1,12 @@
+---
+'@hyperdx/otel-collector': minor
+---
+
+feat: Migrate OTel Collector build to use OCB (OpenTelemetry Collector Builder)
+
+Replace the pre-built otel/opentelemetry-collector-contrib image with a custom
+binary built via OCB. This enables adding custom receiver/processor components
+in the future while including only the components HyperDX needs. The collector
+version is now centralized in `.env` via `OTEL_COLLECTOR_VERSION` and
+`OTEL_COLLECTOR_CORE_VERSION`, with `builder-config.yaml` using templatized
+placeholders substituted at Docker build time.

.env

@@ -38,5 +38,11 @@ HDX_DEV_OTEL_HTTP_PORT=4318
 HDX_DEV_OTEL_METRICS_PORT=8888
 HDX_DEV_OTEL_JSON_HTTP_PORT=14318
 
+# Otel Collector version (used as Docker build arg for image tags and component versions)
+# When bumping, look up the core version from the upstream manifest:
+# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml
+OTEL_COLLECTOR_VERSION=0.149.0
+OTEL_COLLECTOR_CORE_VERSION=1.55.0
+
 # Otel/Clickhouse config
 HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default

docker-compose.ci.yml

@@ -5,6 +5,9 @@ services:
       context: .
       dockerfile: docker/otel-collector/Dockerfile
       target: dev
+      args:
+        OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
+        OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
     environment:
       CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
       HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}

docker-compose.dev.yml

@@ -26,6 +26,9 @@ services:
       context: .
       dockerfile: docker/otel-collector/Dockerfile
       target: dev
+      args:
+        OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
+        OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
     environment:
       CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
       CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
@@ -63,6 +66,9 @@ services:
       context: .
       dockerfile: docker/otel-collector/Dockerfile
       target: dev
+      args:
+        OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
+        OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
     environment:
       CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
       CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'

docker/hyperdx/Dockerfile

@@ -9,15 +9,29 @@
 
 ARG NODE_VERSION=22.22
 ARG CLICKHOUSE_VERSION=26.1
-ARG OTEL_COLLECTOR_VERSION=0.147.0
-ARG OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION=0.147.0
+ARG OTEL_COLLECTOR_VERSION=0.149.0
+ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0
 
 # base #############################################################################################
 # == Otel Collector Image ==
-FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base
-FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION} AS otel_collector_opampsupervisor_base
+FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS otel_collector_opampsupervisor_base
 FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose
 
+# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder).
+# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can
+# include custom receiver/processor components alongside the standard contrib ones.
+# Note: The official OCB image may ship an older Go than the contrib modules require,
+# so we copy the ocb binary into a golang base with a newer toolchain.
+FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin
+FROM golang:1.26-alpine AS ocb-builder
+ARG OTEL_COLLECTOR_VERSION
+ARG OTEL_COLLECTOR_CORE_VERSION
+COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb
+WORKDIR /build
+COPY packages/otel-collector/builder-config.yaml .
+RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \
+    CGO_ENABLED=0 ocb --config=builder-config.yaml
+
 # Build the Go migration tool with full TLS support for ClickHouse
 FROM golang:1.26-alpine AS migrate-builder
 WORKDIR /build
@@ -132,7 +146,7 @@ ARG USER_GID=10001
 ENV CODE_VERSION=$CODE_VERSION
 ENV OTEL_RESOURCE_ATTRIBUTES="service.version=$CODE_VERSION"
 # Copy from otel collector bases
-COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol
+COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol
 COPY --from=otel_collector_opampsupervisor_base --chmod=755 /usr/local/bin/opampsupervisor /usr/local/bin/opampsupervisor
 
 # Copy Node.js runtime from node base

docker/otel-collector/Dockerfile

@@ -1,12 +1,29 @@
 ## base #############################################################################################
-FROM otel/opentelemetry-collector-contrib:0.147.0 AS col
-FROM otel/opentelemetry-collector-opampsupervisor:0.147.0 AS supervisor
+ARG OTEL_COLLECTOR_VERSION=0.149.0
+ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0
+
+FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS supervisor
 FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate
 FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose
 
-# Build the Go migration tool with full TLS support for ClickHouse
+# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder).
+# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can
+# include custom receiver/processor components alongside the standard contrib ones.
 # Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .)
-FROM golang:1.25-alpine AS migrate-builder
+# Note: The official OCB image may ship an older Go than the contrib modules require,
+# so we copy the ocb binary into a golang base with a newer toolchain.
+FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin
+FROM golang:1.26-alpine AS ocb-builder
+ARG OTEL_COLLECTOR_VERSION
+ARG OTEL_COLLECTOR_CORE_VERSION
+COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb
+WORKDIR /build
+COPY packages/otel-collector/builder-config.yaml .
+RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \
+    CGO_ENABLED=0 ocb --config=builder-config.yaml
+
+# Build the Go migration tool with full TLS support for ClickHouse
+FROM golang:1.26-alpine AS migrate-builder
 WORKDIR /build
 COPY packages/otel-collector/go.mod packages/otel-collector/go.sum ./
 RUN go mod download
@@ -38,7 +55,7 @@ COPY --from=migrate-builder /migrate /usr/local/bin/migrate
 USER ${USER_UID}:${USER_GID}
 
 COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor
-COPY --from=col --chmod=755 /otelcol-contrib /otelcontribcol
+COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol
 
 # Copy entrypoint and log tail wrapper scripts
 COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh