From 8539d73fd85464cbfe8e39d64a3be061fcb63456 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 12 Apr 2026 22:41:48 -0700 Subject: [PATCH 1/9] feat: migrate OTel Collector build to use OCB (OpenTelemetry Collector Builder) Replace the pre-built otel/opentelemetry-collector-contrib image with a custom binary built via OCB. This enables adding custom receiver/processor components in the future while including only the components HyperDX needs. - Add packages/otel-collector/builder-config.yaml with all required components - Update docker/otel-collector/Dockerfile to use ocb-builder stage - Update docker/hyperdx/Dockerfile (all-in-one) to use ocb-builder stage --- docker/hyperdx/Dockerfile | 13 +++- docker/otel-collector/Dockerfile | 14 +++- packages/otel-collector/builder-config.yaml | 84 +++++++++++++++++++++ 3 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 packages/otel-collector/builder-config.yaml diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 0447dda7dd..f670524e45 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -9,15 +9,22 @@ ARG NODE_VERSION=22.22 ARG CLICKHOUSE_VERSION=26.1 -ARG OTEL_COLLECTOR_VERSION=0.147.0 ARG OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION=0.147.0 # base ############################################################################################# # == Otel Collector Image == -FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION} AS otel_collector_opampsupervisor_base FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose +# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder). +# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can +# include custom receiver/processor components alongside the standard contrib ones. +FROM golang:1.26-alpine AS ocb-builder +RUN go install go.opentelemetry.io/collector/cmd/builder@v0.147.0 +WORKDIR /build +COPY packages/otel-collector/builder-config.yaml . +RUN CGO_ENABLED=0 builder --config=builder-config.yaml + # Build the Go migration tool with full TLS support for ClickHouse FROM golang:1.26-alpine AS migrate-builder WORKDIR /build @@ -132,7 +139,7 @@ ARG USER_GID=10001 ENV CODE_VERSION=$CODE_VERSION ENV OTEL_RESOURCE_ATTRIBUTES="service.version=$CODE_VERSION" # Copy from otel collector bases -COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol +COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol COPY --from=otel_collector_opampsupervisor_base --chmod=755 /usr/local/bin/opampsupervisor /usr/local/bin/opampsupervisor # Copy Node.js runtime from node base diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index b5ab29065c..2d89c32200 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -1,11 +1,19 @@ ## base ############################################################################################# -FROM otel/opentelemetry-collector-contrib:0.147.0 AS col FROM otel/opentelemetry-collector-opampsupervisor:0.147.0 AS supervisor FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose -# Build the Go migration tool with full TLS support for ClickHouse +# Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder). +# This replaces the pre-built otel/opentelemetry-collector-contrib image so we can +# include custom receiver/processor components alongside the standard contrib ones. # Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .) +FROM golang:1.25-alpine AS ocb-builder +RUN go install go.opentelemetry.io/collector/cmd/builder@v0.147.0 +WORKDIR /build +COPY packages/otel-collector/builder-config.yaml . +RUN CGO_ENABLED=0 builder --config=builder-config.yaml + +# Build the Go migration tool with full TLS support for ClickHouse FROM golang:1.25-alpine AS migrate-builder WORKDIR /build COPY packages/otel-collector/go.mod packages/otel-collector/go.sum ./ @@ -38,7 +46,7 @@ COPY --from=migrate-builder /migrate /usr/local/bin/migrate USER ${USER_UID}:${USER_GID} COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor -COPY --from=col --chmod=755 /otelcol-contrib /otelcontribcol +COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol # Copy entrypoint and log tail wrapper scripts COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml new file mode 100644 index 0000000000..67c47248f1 --- /dev/null +++ b/packages/otel-collector/builder-config.yaml @@ -0,0 +1,84 @@ +# OpenTelemetry Collector Builder (OCB) manifest for HyperDX. +# This replaces the pre-built otel/opentelemetry-collector-contrib image with a +# custom-built binary that includes only the components HyperDX needs, plus any +# custom receivers/processors we add in the future. +# +# To update the OTel Collector version, change the version numbers below. +# The upstream contrib manifest for reference: +# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml + +dist: + name: otelcol-hyperdx + description: HyperDX OpenTelemetry Collector + version: 0.147.0 + output_path: /build/output + build_tags: 'grpcnotrace' + +receivers: + # Core + - gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.147.0 + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.147.0 + # Contrib + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver + v0.147.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver + v0.147.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver + v0.147.0 + +processors: + # Core + - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.147.0 + - gomod: + go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.147.0 + # Contrib + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor + v0.147.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor + v0.147.0 + +exporters: + # Core + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.147.0 + - gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.147.0 + - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.147.0 + - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.147.0 + # Contrib + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter + v0.147.0 + +connectors: + # Core + - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.147.0 + # Contrib + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector + v0.147.0 + +extensions: + # Contrib + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension + v0.147.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension + v0.147.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension + v0.147.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension + v0.147.0 + +providers: + - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.53.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.53.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v1.53.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.53.0 + - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.53.0 From c3619d03f029f7e2e1d985fb4011d03bcb2a54e2 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 12 Apr 2026 22:47:26 -0700 Subject: [PATCH 2/9] refactor: use official OCB image instead of go install for collector builder Copy the version-pinned ocb binary from otel/opentelemetry-collector-builder:0.147.0 into the golang base image. This avoids the go install step while still using a newer Go toolchain (the official OCB image ships Go 1.24 which is too old for the contrib modules that require Go 1.25+). --- docker/hyperdx/Dockerfile | 7 +++++-- docker/otel-collector/Dockerfile | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index f670524e45..a9b62a7e94 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -19,11 +19,14 @@ FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f # Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder). # This replaces the pre-built otel/opentelemetry-collector-contrib image so we can # include custom receiver/processor components alongside the standard contrib ones. +# Note: The official OCB 0.147.0 image ships Go 1.24 which is too old for the contrib +# modules, so we copy the ocb binary into a golang:1.26-alpine base with a newer toolchain. +FROM otel/opentelemetry-collector-builder:0.147.0 AS ocb-bin FROM golang:1.26-alpine AS ocb-builder -RUN go install go.opentelemetry.io/collector/cmd/builder@v0.147.0 +COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build COPY packages/otel-collector/builder-config.yaml . -RUN CGO_ENABLED=0 builder --config=builder-config.yaml +RUN CGO_ENABLED=0 ocb --config=builder-config.yaml # Build the Go migration tool with full TLS support for ClickHouse FROM golang:1.26-alpine AS migrate-builder diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index 2d89c32200..4e84b679e5 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -7,11 +7,14 @@ FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f # This replaces the pre-built otel/opentelemetry-collector-contrib image so we can # include custom receiver/processor components alongside the standard contrib ones. # Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .) +# Note: The official OCB 0.147.0 image ships Go 1.24 which is too old for the contrib +# modules, so we copy the ocb binary into a golang:1.25-alpine base with a newer toolchain. +FROM otel/opentelemetry-collector-builder:0.147.0 AS ocb-bin FROM golang:1.25-alpine AS ocb-builder -RUN go install go.opentelemetry.io/collector/cmd/builder@v0.147.0 +COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build COPY packages/otel-collector/builder-config.yaml . -RUN CGO_ENABLED=0 builder --config=builder-config.yaml +RUN CGO_ENABLED=0 ocb --config=builder-config.yaml # Build the Go migration tool with full TLS support for ClickHouse FROM golang:1.25-alpine AS migrate-builder From 5a8331aaa7c9f1e053544cf6a0df333dc55ae9fc Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 12 Apr 2026 22:58:50 -0700 Subject: [PATCH 3/9] refactor: centralize OTel Collector version into OTEL_COLLECTOR_VERSION Add OTEL_COLLECTOR_VERSION to .env and use it as a Docker build ARG to control the opampsupervisor and OCB builder image tags. This replaces hardcoded 0.147.0 references across both Dockerfiles and consolidates the previous OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION ARG. Docker Compose files pass the version through as a build arg so that changing the version only requires updating .env (for image tags) and builder-config.yaml (for component module versions). --- .env | 3 +++ docker-compose.ci.yml | 2 ++ docker-compose.dev.yml | 4 ++++ docker/hyperdx/Dockerfile | 10 +++++----- docker/otel-collector/Dockerfile | 10 ++++++---- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 7d0a5bb53a..c05dc31981 100644 --- a/.env +++ b/.env @@ -38,5 +38,8 @@ HDX_DEV_OTEL_HTTP_PORT=4318 HDX_DEV_OTEL_METRICS_PORT=8888 HDX_DEV_OTEL_JSON_HTTP_PORT=14318 +# Otel Collector version (used as Docker build arg for image tags) +OTEL_COLLECTOR_VERSION=0.147.0 + # Otel/Clickhouse config HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 42a3511178..5109d0c03f 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -5,6 +5,8 @@ services: context: . dockerfile: docker/otel-collector/Dockerfile target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 35e640a042..02987a0652 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,6 +26,8 @@ services: context: . dockerfile: docker/otel-collector/Dockerfile target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' @@ -63,6 +65,8 @@ services: context: . dockerfile: docker/otel-collector/Dockerfile target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index a9b62a7e94..28679b68df 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -9,19 +9,19 @@ ARG NODE_VERSION=22.22 ARG CLICKHOUSE_VERSION=26.1 -ARG OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION=0.147.0 +ARG OTEL_COLLECTOR_VERSION=0.147.0 # base ############################################################################################# # == Otel Collector Image == -FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION} AS otel_collector_opampsupervisor_base +FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS otel_collector_opampsupervisor_base FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose # Build the custom HyperDX collector binary using OCB (OpenTelemetry Collector Builder). # This replaces the pre-built otel/opentelemetry-collector-contrib image so we can # include custom receiver/processor components alongside the standard contrib ones. -# Note: The official OCB 0.147.0 image ships Go 1.24 which is too old for the contrib -# modules, so we copy the ocb binary into a golang:1.26-alpine base with a newer toolchain. -FROM otel/opentelemetry-collector-builder:0.147.0 AS ocb-bin +# Note: The official OCB image may ship an older Go than the contrib modules require, +# so we copy the ocb binary into a golang base with a newer toolchain. +FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin FROM golang:1.26-alpine AS ocb-builder COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index 4e84b679e5..e20e0105b2 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -1,5 +1,7 @@ ## base ############################################################################################# -FROM otel/opentelemetry-collector-opampsupervisor:0.147.0 AS supervisor +ARG OTEL_COLLECTOR_VERSION=0.147.0 + +FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS supervisor FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f6ef1a316209058e23 AS goose @@ -7,9 +9,9 @@ FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f # This replaces the pre-built otel/opentelemetry-collector-contrib image so we can # include custom receiver/processor components alongside the standard contrib ones. # Note: Build context must be repo root (use: docker build -f docker/otel-collector/Dockerfile .) -# Note: The official OCB 0.147.0 image ships Go 1.24 which is too old for the contrib -# modules, so we copy the ocb binary into a golang:1.25-alpine base with a newer toolchain. -FROM otel/opentelemetry-collector-builder:0.147.0 AS ocb-bin +# Note: The official OCB image may ship an older Go than the contrib modules require, +# so we copy the ocb binary into a golang base with a newer toolchain. +FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin FROM golang:1.25-alpine AS ocb-builder COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build From 0f25fc7580f97c6eb8a1a7e5cd67dc0d2dc7c6ba Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 12 Apr 2026 23:21:23 -0700 Subject: [PATCH 4/9] refactor: templatize builder-config.yaml and bump OTel Collector to 0.149.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace hardcoded versions in builder-config.yaml with placeholders (__OTEL_COLLECTOR_VERSION__ and __OTEL_COLLECTOR_CORE_VERSION__) that are substituted via sed at Docker build time from ARGs sourced from .env. Bump the collector from 0.147.0 to 0.149.0 (core providers 1.53.0 → 1.55.0). To bump the collector version now only requires changing two values in .env: OTEL_COLLECTOR_VERSION=0.149.0 OTEL_COLLECTOR_CORE_VERSION=1.55.0 --- .env | 7 +- docker-compose.ci.yml | 1 + docker-compose.dev.yml | 2 + docker/hyperdx/Dockerfile | 8 +- docker/otel-collector/Dockerfile | 8 +- packages/otel-collector/builder-config.yaml | 85 ++++++++++++++------- 6 files changed, 78 insertions(+), 33 deletions(-) diff --git a/.env b/.env index c05dc31981..5769d0b4d5 100644 --- a/.env +++ b/.env @@ -38,8 +38,11 @@ HDX_DEV_OTEL_HTTP_PORT=4318 HDX_DEV_OTEL_METRICS_PORT=8888 HDX_DEV_OTEL_JSON_HTTP_PORT=14318 -# Otel Collector version (used as Docker build arg for image tags) -OTEL_COLLECTOR_VERSION=0.147.0 +# Otel Collector version (used as Docker build arg for image tags and component versions) +# When bumping, look up the core version from the upstream manifest: +# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml +OTEL_COLLECTOR_VERSION=0.149.0 +OTEL_COLLECTOR_CORE_VERSION=1.55.0 # Otel/Clickhouse config HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 5109d0c03f..525209c8e6 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -7,6 +7,7 @@ services: target: dev args: OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 02987a0652..6dfc36b403 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -28,6 +28,7 @@ services: target: dev args: OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' @@ -67,6 +68,7 @@ services: target: dev args: OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 28679b68df..9f670f3047 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -9,7 +9,8 @@ ARG NODE_VERSION=22.22 ARG CLICKHOUSE_VERSION=26.1 -ARG OTEL_COLLECTOR_VERSION=0.147.0 +ARG OTEL_COLLECTOR_VERSION +ARG OTEL_COLLECTOR_CORE_VERSION # base ############################################################################################# # == Otel Collector Image == @@ -23,10 +24,13 @@ FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f # so we copy the ocb binary into a golang base with a newer toolchain. FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin FROM golang:1.26-alpine AS ocb-builder +ARG OTEL_COLLECTOR_VERSION +ARG OTEL_COLLECTOR_CORE_VERSION COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build COPY packages/otel-collector/builder-config.yaml . -RUN CGO_ENABLED=0 ocb --config=builder-config.yaml +RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ + CGO_ENABLED=0 ocb --config=builder-config.yaml # Build the Go migration tool with full TLS support for ClickHouse FROM golang:1.26-alpine AS migrate-builder diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index e20e0105b2..0f158a8bde 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -1,5 +1,6 @@ ## base ############################################################################################# -ARG OTEL_COLLECTOR_VERSION=0.147.0 +ARG OTEL_COLLECTOR_VERSION +ARG OTEL_COLLECTOR_CORE_VERSION FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS supervisor FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate @@ -13,10 +14,13 @@ FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f # so we copy the ocb binary into a golang base with a newer toolchain. FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin FROM golang:1.25-alpine AS ocb-builder +ARG OTEL_COLLECTOR_VERSION +ARG OTEL_COLLECTOR_CORE_VERSION COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb WORKDIR /build COPY packages/otel-collector/builder-config.yaml . -RUN CGO_ENABLED=0 ocb --config=builder-config.yaml +RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_COLLECTOR_CORE_VERSION__/${OTEL_COLLECTOR_CORE_VERSION}/g" builder-config.yaml && \ + CGO_ENABLED=0 ocb --config=builder-config.yaml # Build the Go migration tool with full TLS support for ClickHouse FROM golang:1.25-alpine AS migrate-builder diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index 67c47248f1..1b2acdbfac 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -3,82 +3,113 @@ # custom-built binary that includes only the components HyperDX needs, plus any # custom receivers/processors we add in the future. # -# To update the OTel Collector version, change the version numbers below. +# Version placeholders are replaced at Docker build time via sed: +# __OTEL_COLLECTOR_VERSION__ -> contrib/core component version (e.g. 0.149.0) +# __OTEL_COLLECTOR_CORE_VERSION__ -> core confmap provider version (e.g. 1.55.0) +# Both values are defined in the root .env file. +# # The upstream contrib manifest for reference: # https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml dist: name: otelcol-hyperdx description: HyperDX OpenTelemetry Collector - version: 0.147.0 + version: __OTEL_COLLECTOR_VERSION__ output_path: /build/output build_tags: 'grpcnotrace' receivers: # Core - - gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.147.0 - - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.147.0 + - gomod: + go.opentelemetry.io/collector/receiver/nopreceiver + v__OTEL_COLLECTOR_VERSION__ + - gomod: + go.opentelemetry.io/collector/receiver/otlpreceiver + v__OTEL_COLLECTOR_VERSION__ # Contrib - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ processors: # Core - - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.147.0 - gomod: - go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.147.0 + go.opentelemetry.io/collector/processor/batchprocessor + v__OTEL_COLLECTOR_VERSION__ + - gomod: + go.opentelemetry.io/collector/processor/memorylimiterprocessor + v__OTEL_COLLECTOR_VERSION__ # Contrib - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ exporters: # Core - - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.147.0 - - gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.147.0 - - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.147.0 - - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.147.0 + - gomod: + go.opentelemetry.io/collector/exporter/debugexporter + v__OTEL_COLLECTOR_VERSION__ + - gomod: + go.opentelemetry.io/collector/exporter/nopexporter + v__OTEL_COLLECTOR_VERSION__ + - gomod: + go.opentelemetry.io/collector/exporter/otlpexporter + v__OTEL_COLLECTOR_VERSION__ + - gomod: + go.opentelemetry.io/collector/exporter/otlphttpexporter + v__OTEL_COLLECTOR_VERSION__ # Contrib - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ connectors: # Core - - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.147.0 + - gomod: + go.opentelemetry.io/collector/connector/forwardconnector + v__OTEL_COLLECTOR_VERSION__ # Contrib - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ extensions: # Contrib - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension - v0.147.0 + v__OTEL_COLLECTOR_VERSION__ providers: - - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.53.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.53.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v1.53.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.53.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.53.0 + - gomod: + go.opentelemetry.io/collector/confmap/provider/envprovider + v__OTEL_COLLECTOR_CORE_VERSION__ + - gomod: + go.opentelemetry.io/collector/confmap/provider/fileprovider + v__OTEL_COLLECTOR_CORE_VERSION__ + - gomod: + go.opentelemetry.io/collector/confmap/provider/httpprovider + v__OTEL_COLLECTOR_CORE_VERSION__ + - gomod: + go.opentelemetry.io/collector/confmap/provider/httpsprovider + v__OTEL_COLLECTOR_CORE_VERSION__ + - gomod: + go.opentelemetry.io/collector/confmap/provider/yamlprovider + v__OTEL_COLLECTOR_CORE_VERSION__ From f60a36711162f6cd69d9eafd1b695def23bda45a Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 12 Apr 2026 23:21:53 -0700 Subject: [PATCH 5/9] chore: add changesets for OCB migration and collector version bump --- .changeset/bump-otel-collector-to-0149.md | 8 ++++++++ .changeset/migrate-otel-collector-to-ocb.md | 12 ++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 .changeset/bump-otel-collector-to-0149.md create mode 100644 .changeset/migrate-otel-collector-to-ocb.md diff --git a/.changeset/bump-otel-collector-to-0149.md b/.changeset/bump-otel-collector-to-0149.md new file mode 100644 index 0000000000..b78297e3b9 --- /dev/null +++ b/.changeset/bump-otel-collector-to-0149.md @@ -0,0 +1,8 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: Bump OTel Collector from 0.147.0 to 0.149.0 + +Upgrade the OpenTelemetry Collector and all its components from v0.147.0 to +v0.149.0 (core providers from v1.53.0 to v1.55.0). diff --git a/.changeset/migrate-otel-collector-to-ocb.md b/.changeset/migrate-otel-collector-to-ocb.md new file mode 100644 index 0000000000..b9f7c0b2fb --- /dev/null +++ b/.changeset/migrate-otel-collector-to-ocb.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: Migrate OTel Collector build to use OCB (OpenTelemetry Collector Builder) + +Replace the pre-built otel/opentelemetry-collector-contrib image with a custom +binary built via OCB. This enables adding custom receiver/processor components +in the future while including only the components HyperDX needs. The collector +version is now centralized in `.env` via `OTEL_COLLECTOR_VERSION` and +`OTEL_COLLECTOR_CORE_VERSION`, with `builder-config.yaml` using templatized +placeholders substituted at Docker build time. From 7578504e5b633a351e76313f23e84cbc570b870c Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 12 Apr 2026 23:35:31 -0700 Subject: [PATCH 6/9] fix: add default ARG values and smoke test build args for OTel Collector version The smoke test docker-compose was missing OTEL_COLLECTOR_VERSION and OTEL_COLLECTOR_CORE_VERSION build args, causing the Docker build to fail with an empty image tag. Add defaults to the Dockerfile ARG declarations so builds work standalone without requiring callers to pass args, and pass the args explicitly in the smoke test docker-compose. --- docker/hyperdx/Dockerfile | 4 ++-- docker/otel-collector/Dockerfile | 4 ++-- smoke-tests/otel-collector/docker-compose.yaml | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 9f670f3047..7e8a53d872 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -9,8 +9,8 @@ ARG NODE_VERSION=22.22 ARG CLICKHOUSE_VERSION=26.1 -ARG OTEL_COLLECTOR_VERSION -ARG OTEL_COLLECTOR_CORE_VERSION +ARG OTEL_COLLECTOR_VERSION=0.149.0 +ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0 # base ############################################################################################# # == Otel Collector Image == diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index 0f158a8bde..bbaf032e2c 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -1,6 +1,6 @@ ## base ############################################################################################# -ARG OTEL_COLLECTOR_VERSION -ARG OTEL_COLLECTOR_CORE_VERSION +ARG OTEL_COLLECTOR_VERSION=0.149.0 +ARG OTEL_COLLECTOR_CORE_VERSION=1.55.0 FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_VERSION} AS supervisor FROM hairyhenderson/gomplate:v4.3.3-alpine AS gomplate diff --git a/smoke-tests/otel-collector/docker-compose.yaml b/smoke-tests/otel-collector/docker-compose.yaml index bbdf753313..070bdb254a 100644 --- a/smoke-tests/otel-collector/docker-compose.yaml +++ b/smoke-tests/otel-collector/docker-compose.yaml @@ -23,6 +23,9 @@ services: context: ../.. dockerfile: docker/otel-collector/Dockerfile target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.149.0} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.55.0} environment: - CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=ch-server:9363 From c86a496ff3ba85e5d9f1f3587f779ad6b4bbb917 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 12 Apr 2026 23:39:18 -0700 Subject: [PATCH 7/9] docs: add README for otel-collector package with upgrade instructions Document the OCB build architecture, included components, version upgrade steps, and how to add custom components in the future. --- packages/otel-collector/README.md | 188 ++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 packages/otel-collector/README.md diff --git a/packages/otel-collector/README.md b/packages/otel-collector/README.md new file mode 100644 index 0000000000..42de904fae --- /dev/null +++ b/packages/otel-collector/README.md @@ -0,0 +1,188 @@ +# HyperDX OTel Collector + +Custom-built OpenTelemetry Collector for HyperDX, compiled via +[OCB (OpenTelemetry Collector Builder)](https://github.com/open-telemetry/opentelemetry-collector/tree/main/cmd/builder). +This replaces the pre-built `otel/opentelemetry-collector-contrib` image with a +binary that includes only the components HyperDX needs, plus any custom +receivers/processors added in this package. + +## Architecture + +### Build process + +The collector binary is built during `docker build` via a multi-stage +Dockerfile: + +1. The `ocb` binary is copied from the official + `otel/opentelemetry-collector-builder:` image +2. It's placed into a `golang:-alpine` base (the official OCB image may + ship an older Go than the contrib modules require) +3. Version placeholders in `builder-config.yaml` are substituted via `sed` +4. `ocb` compiles the custom binary from the resolved manifest + +### Key files + +| File | Purpose | +| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `builder-config.yaml` | OCB manifest — declares which components to include. Uses `__OTEL_COLLECTOR_VERSION__` and `__OTEL_COLLECTOR_CORE_VERSION__` placeholders substituted at build time. | +| `cmd/migrate/main.go` | Go-based ClickHouse migration tool (goose) with full TLS support | +| `go.mod` / `go.sum` | Go module for the migration tool | + +### Dockerfiles that build this collector + +| Dockerfile | Description | +| ---------------------------------- | -------------------------------------------------------------------- | +| `docker/otel-collector/Dockerfile` | Standalone collector image (dev + prod targets) | +| `docker/hyperdx/Dockerfile` | All-in-one image (includes collector, ClickHouse, MongoDB, API, App) | + +### Version configuration + +The collector version is controlled by two variables: + +- **`OTEL_COLLECTOR_VERSION`** — The contrib/core component version (e.g. + `0.149.0`) +- **`OTEL_COLLECTOR_CORE_VERSION`** — The core confmap provider version (e.g. + `1.55.0`) + +These are defined in the root `.env` file and passed as Docker build args. Both +Dockerfiles also have matching `ARG` defaults as fallbacks for standalone +builds. + +## Included components + +All components referenced in config files and the OpAMP controller: + +### Receivers + +| Component | Module | Used in | +| --------------- | ------- | ------------------------------------------------- | +| `nop` | core | OpAMP controller | +| `otlp` | core | standalone configs, OpAMP controller, smoke tests | +| `fluentforward` | contrib | standalone configs, OpAMP controller, smoke tests | +| `hostmetrics` | contrib | custom.config.yaml | +| `prometheus` | contrib | OpAMP controller, smoke tests | + +### Processors + +| Component | Module | Used in | +| ------------------- | ------- | ------------------------------------------------- | +| `batch` | core | config.yaml, standalone configs, OpAMP controller | +| `memory_limiter` | core | config.yaml, standalone configs, OpAMP controller | +| `resourcedetection` | contrib | config.yaml | +| `transform` | contrib | config.yaml, standalone configs, OpAMP controller | + +### Exporters + +| Component | Module | Used in | +| ------------ | ------- | ------------------------------------ | +| `clickhouse` | contrib | standalone configs, OpAMP controller | +| `debug` | core | OpAMP controller | +| `nop` | core | OpAMP controller | +| `otlp` | core | included for utility | +| `otlphttp` | core | custom.config.yaml | + +### Connectors + +| Component | Module | Used in | +| --------- | ------- | ------------------------------------ | +| `forward` | core | included for utility | +| `routing` | contrib | standalone configs, OpAMP controller | + +### Extensions + +| Component | Module | Used in | +| ----------------- | ------- | ---------------------------------------- | +| `bearertokenauth` | contrib | standalone-auth config, OpAMP controller | +| `health_check` | contrib | config.yaml, standalone-auth config | +| `opamp` | contrib | used by OpAMP supervisor | +| `pprof` | contrib | included for debugging/profiling | + +### Confmap Providers + +| Provider | Module | +| -------- | ------ | +| `env` | core | +| `file` | core | +| `http` | core | +| `https` | core | +| `yaml` | core | + +## Upgrading the OTel Collector version + +### Step 1: Look up the core version + +Check the upstream contrib manifest to find the core provider version that +corresponds to the new contrib version: + +https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml + +Look at the `providers:` section — the core version follows a different scheme +(e.g. contrib `0.149.0` corresponds to core `1.55.0`). + +### Step 2: Update these files + +**`.env`** (root) — primary source of truth: + +``` +OTEL_COLLECTOR_VERSION= +OTEL_COLLECTOR_CORE_VERSION= +``` + +**`docker/otel-collector/Dockerfile`** — ARG defaults: + +```dockerfile +ARG OTEL_COLLECTOR_VERSION= +ARG OTEL_COLLECTOR_CORE_VERSION= +``` + +**`docker/hyperdx/Dockerfile`** — ARG defaults: + +```dockerfile +ARG OTEL_COLLECTOR_VERSION= +ARG OTEL_COLLECTOR_CORE_VERSION= +``` + +**`smoke-tests/otel-collector/docker-compose.yaml`** — fallback defaults: + +```yaml +args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-} +``` + +### Files you do NOT need to change + +- `packages/otel-collector/builder-config.yaml` — uses placeholders, substituted + at build time +- `docker-compose.dev.yml` — reads from `.env` automatically +- `docker-compose.ci.yml` — reads from `.env` automatically + +### Step 3: Verify + +Build the image and check the version: + +```bash +docker build -f docker/otel-collector/Dockerfile --target dev -t hdx-otel-test . +docker run --rm --entrypoint /otelcontribcol hdx-otel-test --version +docker rmi hdx-otel-test +``` + +## Adding a custom component + +Once a custom receiver, processor, or exporter is implemented: + +1. Create the Go module under this package (e.g. + `packages/otel-collector/receiver/myreceiver/`) +2. Add an entry to `builder-config.yaml` under the appropriate section with a + `replaces` directive pointing to the local module: + ```yaml + receivers: + - gomod: + github.com/hyperdxio/hyperdx/packages/otel-collector/receiver/myreceiver + v0.0.0 + path: ./receiver/myreceiver + ``` +3. The Dockerfile already copies `packages/otel-collector/` into the build + context, so the local module will be available during the OCB build. +4. Add the component to the relevant OTel config files and/or the OpAMP + controller. From 61e17674e94fb02f2e31e8f0455f12506fdd16a8 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Mon, 13 Apr 2026 09:31:24 -0700 Subject: [PATCH 8/9] fix: align Go version to 1.26-alpine across both Dockerfiles The standalone otel-collector Dockerfile used golang:1.25-alpine while the all-in-one Dockerfile used golang:1.26-alpine, causing the collector binary to be built with different Go toolchains. Align both to 1.26. --- docker/otel-collector/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index bbaf032e2c..15bfc95dc2 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -13,7 +13,7 @@ FROM kukymbr/goose-docker@sha256:0cd025636df126e7f66472861ca4db3683bc649be46cd1f # Note: The official OCB image may ship an older Go than the contrib modules require, # so we copy the ocb binary into a golang base with a newer toolchain. FROM otel/opentelemetry-collector-builder:${OTEL_COLLECTOR_VERSION} AS ocb-bin -FROM golang:1.25-alpine AS ocb-builder +FROM golang:1.26-alpine AS ocb-builder ARG OTEL_COLLECTOR_VERSION ARG OTEL_COLLECTOR_CORE_VERSION COPY --from=ocb-bin /usr/local/bin/ocb /usr/local/bin/ocb @@ -23,7 +23,7 @@ RUN sed -i "s/__OTEL_COLLECTOR_VERSION__/${OTEL_COLLECTOR_VERSION}/g; s/__OTEL_C CGO_ENABLED=0 ocb --config=builder-config.yaml # Build the Go migration tool with full TLS support for ClickHouse -FROM golang:1.25-alpine AS migrate-builder +FROM golang:1.26-alpine AS migrate-builder WORKDIR /build COPY packages/otel-collector/go.mod packages/otel-collector/go.sum ./ RUN go mod download From b966f50dca65eb44b3978944f3afbcf40991c1c7 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Mon, 13 Apr 2026 15:28:54 -0700 Subject: [PATCH 9/9] docs: document both local and remote approaches for adding custom OTel components Update the README to clarify that local components require expanding the COPY in both Dockerfiles, and document the alternative remote gomod approach for published modules that needs no Dockerfile changes. --- packages/otel-collector/README.md | 77 +++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/packages/otel-collector/README.md b/packages/otel-collector/README.md index 42de904fae..e788d7fd6a 100644 --- a/packages/otel-collector/README.md +++ b/packages/otel-collector/README.md @@ -169,12 +169,21 @@ docker rmi hdx-otel-test ## Adding a custom component -Once a custom receiver, processor, or exporter is implemented: +There are two approaches for including custom components, depending on whether +the component source lives in this monorepo or is published as a standalone Go +module. + +### Approach A: Local source (monorepo development) + +Best for active development — iterate on the component alongside the rest of the +codebase without needing to publish a Go module tag on every change. 1. Create the Go module under this package (e.g. - `packages/otel-collector/receiver/myreceiver/`) + `packages/otel-collector/receiver/myreceiver/`) with its own `go.mod`. + 2. Add an entry to `builder-config.yaml` under the appropriate section with a - `replaces` directive pointing to the local module: + `path:` directive pointing to the local module: + ```yaml receivers: - gomod: @@ -182,7 +191,65 @@ Once a custom receiver, processor, or exporter is implemented: v0.0.0 path: ./receiver/myreceiver ``` -3. The Dockerfile already copies `packages/otel-collector/` into the build - context, so the local module will be available during the OCB build. + +3. **Update the Dockerfiles** to copy the component source into the OCB build + stage. Currently, only `builder-config.yaml` is copied. In both + `docker/otel-collector/Dockerfile` and `docker/hyperdx/Dockerfile`, expand + the `COPY` line in the `ocb-builder` stage: + + ```dockerfile + # Before (only copies the manifest): + COPY packages/otel-collector/builder-config.yaml . + + # After (also copies custom component source): + COPY packages/otel-collector/builder-config.yaml . + COPY packages/otel-collector/receiver/ ./receiver/ + ``` + + Or, to copy everything at once (simpler, but any change to files in + `packages/otel-collector/` will invalidate the Docker layer cache for the OCB + build stage): + + ```dockerfile + COPY packages/otel-collector/ . + ``` + 4. Add the component to the relevant OTel config files and/or the OpAMP + controller (`packages/api/src/opamp/controllers/opampController.ts`). + +### Approach B: Published Go module (remote reference) + +Best for stable, versioned components — especially useful for components shared +across repos or distributed to external users. No Dockerfile `COPY` changes +needed. + +1. Publish the component as a Go module with a tagged version (e.g. + `github.com/hyperdxio/my-otel-receiver v0.1.0`). For modules in a monorepo + subdirectory, the Git tag must include the module path prefix (e.g. + `packages/otel-collector/receiver/myreceiver/v0.1.0`). + +2. Add an entry to `builder-config.yaml` **without** a `path:` directive — OCB + will fetch it via the Go module proxy, just like contrib components: + + ```yaml + receivers: + - gomod: github.com/hyperdxio/my-otel-receiver v0.1.0 + ``` + + No Dockerfile changes needed — OCB downloads it during `go mod download`. + +3. Add the component to the relevant OTel config files and/or the OpAMP controller. + +### Which approach to use? + +| | Local (`path:`) | Remote (`gomod:`) | +| --------------------------- | -------------------- | --------------------------------- | +| Dockerfile COPY needed | Yes | No | +| Works with unpublished code | Yes | No — must be tagged and published | +| Dev iteration speed | Fast — rebuild image | Must push, tag, wait for proxy | +| Monorepo friendly | Yes | Requires proper Go module tagging | +| Best for | Active development | Stable/shared components | + +For most HyperDX development, **Approach A (local source)** is recommended. Use +Approach B when the component is mature and independently versioned.