From 5fa2ab992fc6f0147433a5ec1b42681488f4aa55 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 13:17:03 -0700 Subject: [PATCH 01/18] style: rename hyperdx docker dirs --- docker/{local => hyperdx-all-in-one}/Dockerfile | 4 ++-- docker/{local => hyperdx-all-in-one}/build.sh | 0 docker/{local => hyperdx-all-in-one}/clickhouseConfig.xml | 0 docker/{local => hyperdx-all-in-one}/entry.sh | 0 docker/{fullstack => hyperdx}/Dockerfile | 0 docker/{fullstack => hyperdx}/entry.sh | 0 6 files changed, 2 insertions(+), 2 deletions(-) rename docker/{local => hyperdx-all-in-one}/Dockerfile (98%) rename docker/{local => hyperdx-all-in-one}/build.sh (100%) rename docker/{local => hyperdx-all-in-one}/clickhouseConfig.xml (100%) rename docker/{local => hyperdx-all-in-one}/entry.sh (100%) rename docker/{fullstack => hyperdx}/Dockerfile (100%) rename docker/{fullstack => hyperdx}/entry.sh (100%) diff --git a/docker/local/Dockerfile b/docker/hyperdx-all-in-one/Dockerfile similarity index 98% rename from docker/local/Dockerfile rename to docker/hyperdx-all-in-one/Dockerfile index 7ca80f485a..47e1b03874 100644 --- a/docker/local/Dockerfile +++ b/docker/hyperdx-all-in-one/Dockerfile @@ -49,8 +49,8 @@ COPY --from=app ./public ./public COPY --from=app ./styles ./styles ENV NEXT_TELEMETRY_DISABLED 1 -ENV NEXT_OUTPUT_STANDALONE true -ENV NEXT_PUBLIC_IS_LOCAL_MODE true +ENV NEXT_OUTPUT_STANDALONE false +ENV NEXT_PUBLIC_IS_LOCAL_MODE false RUN yarn build && rm -rf node_modules && yarn workspaces focus --production # == Clickhouse/Base Image == diff --git a/docker/local/build.sh b/docker/hyperdx-all-in-one/build.sh similarity index 100% rename from docker/local/build.sh rename to docker/hyperdx-all-in-one/build.sh diff --git a/docker/local/clickhouseConfig.xml b/docker/hyperdx-all-in-one/clickhouseConfig.xml similarity index 100% rename from docker/local/clickhouseConfig.xml rename to docker/hyperdx-all-in-one/clickhouseConfig.xml diff --git a/docker/local/entry.sh b/docker/hyperdx-all-in-one/entry.sh similarity index 100% rename from docker/local/entry.sh rename to docker/hyperdx-all-in-one/entry.sh diff --git a/docker/fullstack/Dockerfile b/docker/hyperdx/Dockerfile similarity index 100% rename from docker/fullstack/Dockerfile rename to docker/hyperdx/Dockerfile diff --git a/docker/fullstack/entry.sh b/docker/hyperdx/entry.sh similarity index 100% rename from docker/fullstack/entry.sh rename to docker/hyperdx/entry.sh From f54b05c3b0dcb158afa9bfb104bafb871a9df101 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 16:58:27 -0700 Subject: [PATCH 02/18] feat: introduce all-one-one multi-stage build --- docker/hyperdx/Dockerfile | 48 +++++++-- docker/hyperdx/clickhouseConfig.xml | 150 ++++++++++++++++++++++++++++ docker/hyperdx/entry.sh | 15 +-- 3 files changed, 189 insertions(+), 24 deletions(-) create mode 100644 docker/hyperdx/clickhouseConfig.xml diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 10598017d9..65ba600434 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -3,9 +3,16 @@ # - App (Frontend) ARG NODE_VERSION=22.14.0 +ARG CLICKHOUSE_VERSION=24 +ARG OTEL_COLLECTOR_VERSION=0.120.0 # base ############################################################################################# -FROM node:${NODE_VERSION}-alpine AS base +# == Clickhouse/Base Image == +FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS clickhouse_base +# == Otel Collector Image == +FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base + +FROM node:${NODE_VERSION}-alpine AS node_base WORKDIR /app/api @@ -24,7 +31,7 @@ RUN yarn install && yarn cache clean ## API Builder Image ############################################################################### -FROM base AS api_builder +FROM node_base AS api_builder WORKDIR /app/api @@ -34,7 +41,7 @@ RUN yarn build && rm -rf node_modules && yarn workspaces focus --production # APP Builder Image ############################################################################### -FROM base AS app_builder +FROM node_base AS app_builder WORKDIR /app/app @@ -74,13 +81,26 @@ COPY --from=app_builder /app/app/node_modules ./node_modules COPY --from=app_builder /app/app/package.json ./package.json # Set up start script -COPY --chown=node:node --from=fullstack ./entry.sh /etc/local/entry.sh +COPY --chown=node:node --from=hyperdx ./entry.sh /etc/local/entry.sh ENTRYPOINT ["sh", "/etc/local/entry.sh"] # prod-extended ############################################################################################ -FROM prod AS prod-extended - -USER root +FROM scratch AS all-in-one + +# Copy base +COPY --from=clickhouse_base / / +COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib +COPY --from=node_base /usr/lib /usr/lib +COPY --from=node_base /usr/local/lib /usr/local/lib +COPY --from=node_base /usr/local/include /usr/local/include +COPY --from=node_base /usr/local/bin /usr/local/bin + +# Set up Clickhouse +COPY --from=clickhouse ./local/*.xml /etc/clickhouse-server +# overwrite default config +COPY --from=hyperdx ./clickhouseConfig.xml /etc/clickhouse-server/config.xml +# Set up Otel Collector +COPY --from=otel-collector ./config.yaml /etc/otelcol-contrib/config.yaml # == Install MongoDB v4 Deps == RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories @@ -89,10 +109,16 @@ RUN apk update RUN apk add mongodb yaml-cpp=0.6.2-r2 # Set up Mongo directories with proper permissions -RUN mkdir -p /data/db && \ - chown -R node:node /data/db && \ - chown -R node:node /var/log +RUN mkdir -p /data/db -USER node +ENV NODE_ENV production +# Install libs used for the start script +RUN npm install -g concurrently@9.1.0 +# Set up App +COPY --from=prod /app/api /app/api +COPY --from=prod /app/app /app/app +COPY --from=prod /etc/local/entry.sh /etc/local/entry.sh + +WORKDIR /app/app ENTRYPOINT ["sh", "/etc/local/entry.sh"] diff --git a/docker/hyperdx/clickhouseConfig.xml b/docker/hyperdx/clickhouseConfig.xml new file mode 100644 index 0000000000..a5650fa601 --- /dev/null +++ b/docker/hyperdx/clickhouseConfig.xml @@ -0,0 +1,150 @@ + + + + + false + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + + 0.0.0.0 + 8123 + 9000 + ch-server + 9009 + + 4096 + 64 + 100 + 0 + 0 + + 0 + + + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + + users.xml + default + default + UTC + false + + + + system + query_log
+ 7500 + + + + + system + metric_log
+ 7500 + 1000 + + + + + system + asynchronous_metric_log
+ + 7000 + + + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + + + + + + system + crash_log
+ + + 1000 + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + part_log
+ toYYYYMM(event_date) + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 + + + + /clickhouse/task_queue/ddl + + + /var/lib/clickhouse/format_schemas/ + diff --git a/docker/hyperdx/entry.sh b/docker/hyperdx/entry.sh index 9494ed18b7..5e99c348ec 100644 --- a/docker/hyperdx/entry.sh +++ b/docker/hyperdx/entry.sh @@ -2,24 +2,13 @@ export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}" -# Internal Services -export MONGO_URI="${MONGO_URI:-mongodb://127.0.0.1:27017/hyperdx}" - echo "Visit the HyperDX UI at $FRONTEND_URL" echo "" -# Start Mongo Server if it exists (will run in prod-extended but not in prod) -if command -v mongod &> /dev/null; then - echo "Starting MongoDB server..." - mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 & - # Wait for MongoDB to start - sleep 2 -fi - # Use concurrently to run both the API and App servers npx concurrently \ "--kill-others" \ "--names=API,APP,ALERT-TASK" \ - "MONGO_URI=${MONGO_URI} PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \ + "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \ "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \ - "MONGO_URI=${MONGO_URI} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/tasks/index.js check-alerts" + "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/tasks/index.js check-alerts" From e66fb7ee8326c581ebb801b3d9b9680d75af48dd Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 17:16:22 -0700 Subject: [PATCH 03/18] feat: add all-in-on-entry.sh --- docker/hyperdx/Dockerfile | 14 ++++--- docker/hyperdx/all-in-one-entry.sh | 61 ++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 docker/hyperdx/all-in-one-entry.sh diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 65ba600434..7a89ebbe73 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -90,10 +90,11 @@ FROM scratch AS all-in-one # Copy base COPY --from=clickhouse_base / / COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib +COPY --from=node_base --link /usr/local/bin /usr/local/bin +COPY --from=node_base --link /usr/local/lib /usr/local/lib COPY --from=node_base /usr/lib /usr/lib -COPY --from=node_base /usr/local/lib /usr/local/lib COPY --from=node_base /usr/local/include /usr/local/include -COPY --from=node_base /usr/local/bin /usr/local/bin + # Set up Clickhouse COPY --from=clickhouse ./local/*.xml /etc/clickhouse-server @@ -108,17 +109,20 @@ RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repos RUN apk update RUN apk add mongodb yaml-cpp=0.6.2-r2 +# Install libs used for the start script +RUN apk add curl + # Set up Mongo directories with proper permissions RUN mkdir -p /data/db ENV NODE_ENV production -# Install libs used for the start script -RUN npm install -g concurrently@9.1.0 # Set up App COPY --from=prod /app/api /app/api COPY --from=prod /app/app /app/app -COPY --from=prod /etc/local/entry.sh /etc/local/entry.sh +COPY --from=hyperdx ./all-in-one-entry.sh /etc/local/entry.sh WORKDIR /app/app +# Expose ports +EXPOSE 8080 4317 4318 13133 8123 9000 ENTRYPOINT ["sh", "/etc/local/entry.sh"] diff --git a/docker/hyperdx/all-in-one-entry.sh b/docker/hyperdx/all-in-one-entry.sh new file mode 100644 index 0000000000..a3fe418479 --- /dev/null +++ b/docker/hyperdx/all-in-one-entry.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +export HYPERDX_LOG_LEVEL="error" +# https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#logger +export CLICKHOUSE_LOG_LEVEL="error" + +# User can specify either an entire SERVER_URL, or override slectively the +# HYPERDX_API_URL or HYPERDX_API_PORT from the defaults +# Same applies to the frontend/app +export SERVER_URL="http://127.0.0.1:${HYPERDX_API_PORT:-8000}" +export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}" + +# Internal Services +export CLICKHOUSE_SERVER_ENDPOINT="ch-server:9000" +export MONGO_URI="mongodb://db:27017/hyperdx" + +export EXPRESS_SESSION_SECRET="hyperdx is cool đź‘‹" +export IS_LOCAL_APP_MODE="DANGEROUSLY_is_local_app_modeđź’€" +export NEXT_TELEMETRY_DISABLED="1" + + +# Simulate Docker Service DNS +echo "127.0.0.1 ch-server" >> /etc/hosts +echo "127.0.0.1 db" >> /etc/hosts + +echo "Visit the HyperDX UI at $FRONTEND_URL/search" +echo "" +echo "Send OpenTelemetry data via" +echo "http/protobuf: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318" +echo "gRPC: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317" +echo "" +echo "" + +# Start Clickhouse Server +/entrypoint.sh & + +# Start Mongo Server +mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 & + +# Wait for Clickhouse to be ready +while ! curl -s "http://ch-server:8123" > /dev/null; do + echo "Waiting for Clickhouse to be ready..." + sleep 1 +done +# Start Otel Collector +otelcol-contrib --config /etc/otelcol-contrib/config.yaml & + +# Start HyperDX app +npx concurrently \ + "--kill-others" \ + "--names=API,APP,ALERT-TASK" \ + "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \ + "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \ + "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/tasks/index.js check-alerts" \ + > /var/log/app.log 2>&1 & + +# Wait for any process to exit +wait -n + +# Exit with status of process that exited first +exit $? From 2464456b444de21317edc00ceff4674869ad60af Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 17:23:49 -0700 Subject: [PATCH 04/18] style: deprecate old all-in-one dir --- docker/hyperdx-all-in-one/Dockerfile | 132 --------------- docker/hyperdx-all-in-one/build.sh | 10 -- .../hyperdx-all-in-one/clickhouseConfig.xml | 150 ------------------ docker/hyperdx-all-in-one/entry.sh | 68 -------- docker/hyperdx/build.sh | 10 ++ 5 files changed, 10 insertions(+), 360 deletions(-) delete mode 100644 docker/hyperdx-all-in-one/Dockerfile delete mode 100755 docker/hyperdx-all-in-one/build.sh delete mode 100644 docker/hyperdx-all-in-one/clickhouseConfig.xml delete mode 100644 docker/hyperdx-all-in-one/entry.sh create mode 100755 docker/hyperdx/build.sh diff --git a/docker/hyperdx-all-in-one/Dockerfile b/docker/hyperdx-all-in-one/Dockerfile deleted file mode 100644 index 47e1b03874..0000000000 --- a/docker/hyperdx-all-in-one/Dockerfile +++ /dev/null @@ -1,132 +0,0 @@ -# Starts several services in a single container for local use -# - Clickhouse -# - Mongo -# - Otel Collector (otelcol) -# - API (Node) -# - App (Frontend) -# - Redis Cache - -# TODO: -# - Customize ports, need to set up env vars or relax CORS and other port requirements -# - Have otel collector listen to a directory users can mount logs into -# - Allow persisting settings on disk -# - Limiting persisted data with some auto rotation - -ARG NODE_VERSION=22.14.0 -ARG CLICKHOUSE_VERSION=24 -ARG OTEL_COLLECTOR_VERSION=0.120.0 -# Get Node base image to copy over Node binaries -FROM node:${NODE_VERSION}-alpine AS node - -# == API Builder Image == -# FROM node:${NODE_VERSION}-alpine AS api_builder - -# WORKDIR /app/api -# COPY ./yarn.lock ./.yarnrc.yml . -# COPY ./.yarn ./.yarn -# COPY --from=api ./package.json . -# RUN yarn install && yarn cache clean - -# COPY --from=api ./tsconfig.json . -# COPY --from=api ./src ./src -# RUN yarn run build - -# == App Builder Image == -FROM node:${NODE_VERSION}-alpine AS app_builder - -WORKDIR /app/app - -COPY ./yarn.lock ./.yarnrc.yml . -COPY ./.yarn ./.yarn -COPY --from=app ./package.json . - -RUN yarn install && yarn cache clean - -COPY --from=app ./.eslintrc.js ./next.config.js ./tsconfig.json ./next.config.js ./mdx.d.ts ./.eslintrc.js ./ -COPY --from=app ./src ./src -COPY --from=app ./pages ./pages -COPY --from=app ./public ./public -COPY --from=app ./styles ./styles - -ENV NEXT_TELEMETRY_DISABLED 1 -ENV NEXT_OUTPUT_STANDALONE false -ENV NEXT_PUBLIC_IS_LOCAL_MODE false -RUN yarn build && rm -rf node_modules && yarn workspaces focus --production - -# == Clickhouse/Base Image == -FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS clickhouse_base - -# == Otel Collector Image == -FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base - -FROM scratch as base -COPY --from=clickhouse_base / / -COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib - -# === -# === Install Deps -# === - -# == Install Node Deps == -COPY --from=node /usr/lib /usr/lib -COPY --from=node /usr/local/lib /usr/local/lib -COPY --from=node /usr/local/include /usr/local/include -COPY --from=node /usr/local/bin /usr/local/bin - -# RUN npm install -g yarn --force - -# == Install MongoDB v4 Deps == -# RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories -# RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories -# RUN apk update -# RUN apk add mongodb yaml-cpp=0.6.2-r2 -RUN apk add curl - -# == Install Redis == -# If this version pinning fails, this is likely due to the version being dropped -# from APK -# RUN apk add 'redis<7.0.14' - -# === -# === Set Up Services -# === - -# Set up Clickhouse -COPY --from=clickhouse ./local/*.xml /etc/clickhouse-server -# overwrite default config -COPY --from=local ./clickhouseConfig.xml /etc/clickhouse-server/config.xml - -# Set up Mongo -# RUN mkdir -p /data/db - -# Set up Otel Collector -COPY --from=otel-collector ./config.yaml /etc/otelcol-contrib/config.yaml - -# Set up API (NOT USED YET) -# WORKDIR /app/api - -# COPY --from=api_builder ./app/api/build ./build -# COPY ./yarn.lock ./.yarnrc.yml . -# COPY ./.yarn ./.yarn -# COPY --from=api_builder ./app/api/package.json . - -# # Only install prod dependencies -# RUN yarn workspaces focus --production && yarn cache clean - -# # Remove dev dependencies -# RUN rm -rf ./.yarn - -# Set up App -WORKDIR /app/app -COPY --from=app_builder /app/app/next.config.js ./ -COPY --from=app_builder /app/app/public ./public -COPY --from=app_builder /app/app/.next ./.next -COPY --from=app_builder /app/app/node_modules ./node_modules -COPY --from=app_builder /app/app/package.json ./package.json - -# Expose ports -EXPOSE 8000 8080 4317 4318 13133 8123 9000 - -# Set up start script -COPY --from=local ./entry.sh /etc/local/entry.sh -CMD sh /etc/local/entry.sh diff --git a/docker/hyperdx-all-in-one/build.sh b/docker/hyperdx-all-in-one/build.sh deleted file mode 100755 index b46a401ba0..0000000000 --- a/docker/hyperdx-all-in-one/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Meant to be run from the root of the repo - -docker build --squash -t hdx-oss-dev-local -f ./docker/local/Dockerfile \ - --build-context clickhouse=./docker/clickhouse \ - --build-context otel-collector=./docker/otel-collector \ - --build-context local=./docker/local \ - --build-context api=./packages/api \ - --build-context app=./packages/app \ - . diff --git a/docker/hyperdx-all-in-one/clickhouseConfig.xml b/docker/hyperdx-all-in-one/clickhouseConfig.xml deleted file mode 100644 index a5650fa601..0000000000 --- a/docker/hyperdx-all-in-one/clickhouseConfig.xml +++ /dev/null @@ -1,150 +0,0 @@ - - - - - false - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - - 0.0.0.0 - 8123 - 9000 - ch-server - 9009 - - 4096 - 64 - 100 - 0 - 0 - - 0 - - - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - /var/lib/clickhouse/user_files/ - - users.xml - default - default - UTC - false - - - - system - query_log
- 7500 - - - - - system - metric_log
- 7500 - 1000 - - - - - system - asynchronous_metric_log
- - 7000 - - - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 - - - - - - system - crash_log
- - - 1000 - - - - - system - processors_profile_log
- - toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - /clickhouse/task_queue/ddl - - - /var/lib/clickhouse/format_schemas/ - diff --git a/docker/hyperdx-all-in-one/entry.sh b/docker/hyperdx-all-in-one/entry.sh deleted file mode 100644 index 88f74d8044..0000000000 --- a/docker/hyperdx-all-in-one/entry.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -export HYPERDX_LOG_LEVEL="error" -# https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#logger -export CLICKHOUSE_LOG_LEVEL="error" - -# User can specify either an entire SERVER_URL, or override slectively the -# HYPERDX_API_URL or HYPERDX_API_PORT from the defaults -# Same applies to the frontend/app -export SERVER_URL="http://127.0.0.1:${HYPERDX_API_PORT:-8000}" -export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}" - -# Internal Services -export CLICKHOUSE_SERVER_ENDPOINT="ch-server:9000" -export MONGO_URI="mongodb://db:27017/hyperdx" -export REDIS_URI="redis://redis:6379" - -export EXPRESS_SESSION_SECRET="hyperdx is cool 👋" -export IS_LOCAL_APP_MODE="DANGEROUSLY_is_local_app_mode💀" -export NEXT_TELEMETRY_DISABLED="1" - - -# Simulate Docker Service DNS -echo "127.0.0.1 ch-server" >> /etc/hosts -echo "127.0.0.1 db" >> /etc/hosts -echo "127.0.0.1 redis" >> /etc/hosts - -echo "Visit the HyperDX UI at $FRONTEND_URL/search" -echo "" -echo "Send OpenTelemetry data via" -echo "http/protobuf: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318" -echo "gRPC: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317" -echo "" -echo "" - -# Start Clickhouse Server -/entrypoint.sh & - -# Start Redis Server -# redis-server > /var/log/redis.log 2>&1 & - -# Start Mongo Server -# mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 & - -# Wait for Clickhouse to be ready -while ! curl -s "http://ch-server:8123" > /dev/null; do - echo "Waiting for Clickhouse to be ready..." - sleep 1 -done -# Start Otel Collector -otelcol-contrib --config /etc/otelcol-contrib/config.yaml & - -# Api -# CLICKHOUSE_USER=api \ -# CLICKHOUSE_PASSWORD=api \ -# PORT=8000 \ -# node /app/api/build/index.js > /var/log/api.log 2>&1 & - -# App -NODE_ENV=production \ -NEXT_PUBLIC_SERVER_URL="${SERVER_URL}" \ -/app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080} > /var/log/app.log 2>&1 & - -# Wait for any process to exit -wait -n - -# Exit with status of process that exited first -exit $? diff --git a/docker/hyperdx/build.sh b/docker/hyperdx/build.sh new file mode 100755 index 0000000000..df6e442518 --- /dev/null +++ b/docker/hyperdx/build.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Meant to be run from the root of the repo + +docker build --squash . -f ./docker/hyperdx/Dockerfile \ + --build-context clickhouse=./docker/clickhouse \ + --build-context otel-collector=./docker/otel-collector \ + --build-context hyperdx=./docker/hyperdx \ + --build-context api=./packages/api \ + --build-context app=./packages/app \ + --target all-in-one -t hyperdx/dev-all-in-one From 5691a4a805bfefdf19f467f26a93e774798fc21c Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 17:44:13 -0700 Subject: [PATCH 05/18] style + fix: rename entry files + enforce auth in prod --- docker/hyperdx/Dockerfile | 6 +++--- docker/hyperdx/{all-in-one-entry.sh => entry.local.sh} | 0 docker/hyperdx/{entry.sh => entry.prod.sh} | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) rename docker/hyperdx/{all-in-one-entry.sh => entry.local.sh} (100%) rename docker/hyperdx/{entry.sh => entry.prod.sh} (83%) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 7a89ebbe73..3f0afed086 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -81,7 +81,7 @@ COPY --from=app_builder /app/app/node_modules ./node_modules COPY --from=app_builder /app/app/package.json ./package.json # Set up start script -COPY --chown=node:node --from=hyperdx ./entry.sh /etc/local/entry.sh +COPY --chown=node:node --from=hyperdx ./entry.prod.sh /etc/local/entry.sh ENTRYPOINT ["sh", "/etc/local/entry.sh"] # prod-extended ############################################################################################ @@ -90,7 +90,7 @@ FROM scratch AS all-in-one # Copy base COPY --from=clickhouse_base / / COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib -COPY --from=node_base --link /usr/local/bin /usr/local/bin +COPY --from=node_base --link /usrlocal./local/bin /usr/local/bin COPY --from=node_base --link /usr/local/lib /usr/local/lib COPY --from=node_base /usr/lib /usr/lib COPY --from=node_base /usr/local/include /usr/local/include @@ -120,7 +120,7 @@ ENV NODE_ENV production # Set up App COPY --from=prod /app/api /app/api COPY --from=prod /app/app /app/app -COPY --from=hyperdx ./all-in-one-entry.sh /etc/local/entry.sh +COPY --from=hyperdx ./entry.local.sh /etc/local/entry.sh WORKDIR /app/app # Expose ports diff --git a/docker/hyperdx/all-in-one-entry.sh b/docker/hyperdx/entry.local.sh similarity index 100% rename from docker/hyperdx/all-in-one-entry.sh rename to docker/hyperdx/entry.local.sh diff --git a/docker/hyperdx/entry.sh b/docker/hyperdx/entry.prod.sh similarity index 83% rename from docker/hyperdx/entry.sh rename to docker/hyperdx/entry.prod.sh index 5e99c348ec..ef370aca52 100644 --- a/docker/hyperdx/entry.sh +++ b/docker/hyperdx/entry.prod.sh @@ -2,6 +2,10 @@ export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}" +# Set to "REQUIRED_AUTH" to enforce API authentication. +# ⚠️ Do not change this value !!!! +export IS_LOCAL_APP_MODE="REQUIRED_AUTH" + echo "Visit the HyperDX UI at $FRONTEND_URL" echo "" From ebcc9f2e61294070843cb06d81c6d3a5c05b5fbc Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 20:04:15 -0700 Subject: [PATCH 06/18] cleaup Makefile --- .env | 2 ++ Makefile | 75 ++++++++++++++++++++++++-------------------------------- 2 files changed, 34 insertions(+), 43 deletions(-) diff --git a/.env b/.env index 2c34179a88..541bcb09b2 100644 --- a/.env +++ b/.env @@ -4,6 +4,8 @@ IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx IMAGE_NAME_HDX=docker.hyperdx.io/hyperdx/hyperdx LOCAL_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-local LOCAL_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-local +ALL_IN_ONE_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-all-in-one +ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one OTEL_COLLECTOR_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-otel-collector OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector CHANGESET_TAG=2.0.0-beta.16 diff --git a/Makefile b/Makefile index ab6665745b..7028285bca 100644 --- a/Makefile +++ b/Makefile @@ -71,12 +71,24 @@ dev-migrate-db: version: sh ./version.sh +.PHONY: release-otel-collector +release-otel-collector: + docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector \ + -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ + -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ + --target prod \ + --push \ + --cache-from=type=gha \ + --cache-to=type=gha,mode=max + .PHONY: release-local release-local: - docker buildx build --squash . -f ./docker/local/Dockerfile \ + docker buildx build --squash . -f ./docker/hyperdx/Dockerfile \ --build-context clickhouse=./docker/clickhouse \ --build-context otel-collector=./docker/otel-collector \ - --build-context local=./docker/local \ + --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ --platform ${BUILD_PLATFORMS} \ @@ -84,41 +96,15 @@ release-local: -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ -t ${LOCAL_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ -t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ - --push \ - --cache-from=type=gha \ - --cache-to=type=gha,mode=max - -.PHONY: release-local-ui -release-local-ui: - docker buildx build . -f ./packages/app/Dockerfile \ - --build-arg IS_LOCAL_MODE=true \ - --build-arg PORT=${HYPERDX_APP_PORT} \ - --target prod \ - --platform ${BUILD_PLATFORMS} \ - -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ - -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG}-ui \ - -t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ - -t ${LOCAL_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG}-ui \ - --push \ - --cache-from=type=gha \ - --cache-to=type=gha,mode=max - -.PHONY: release-otel-collector -release-otel-collector: - docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector \ - -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ - -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ - -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ - -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ - --target prod \ + --target all-in-one \ --push \ --cache-from=type=gha \ --cache-to=type=gha,mode=max .PHONY: release-app release-app: - docker buildx build --squash . -f ./docker/fullstack/Dockerfile \ - --build-context fullstack=./docker/fullstack \ + docker buildx build --squash . -f ./docker/hyperdx/Dockerfile \ + --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ --platform ${BUILD_PLATFORMS} \ @@ -131,15 +117,18 @@ release-app: --cache-from=type=gha \ --cache-to=type=gha,mode=max -.PHONY: release-extended-app -release-extended-app: - docker buildx build --squash . -f ./docker/fullstack/Dockerfile \ - --build-context fullstack=./docker/fullstack \ - --build-context api=./packages/api \ - --build-context app=./packages/app \ - --platform ${BUILD_PLATFORMS} \ - -t ${IMAGE_NAME_DOCKERHUB}:latest \ - --target prod-extended \ - --push \ - --cache-from=type=gha \ - --cache-to=type=gha,mode=max +# .PHONY: release-local-ui +# release-local-ui: +# docker buildx build . -f ./packages/app/Dockerfile \ +# --build-arg IS_LOCAL_MODE=true \ +# --build-arg PORT=${HYPERDX_APP_PORT} \ +# --target prod \ +# --platform ${BUILD_PLATFORMS} \ +# -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ +# -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG}-ui \ +# -t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ +# -t ${LOCAL_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG}-ui \ +# --push \ +# --cache-from=type=gha \ +# --cache-to=type=gha,mode=max + From 08b9f796b3ca0f482b930d119ed2b8f2815b989a Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 21:06:50 -0700 Subject: [PATCH 07/18] feat: introduce 'all-in-one-auth' and 'all-in-one-noauth' stages --- docker/hyperdx/Dockerfile | 18 ++++++++++++++---- docker/hyperdx/build.sh | 3 ++- docker/hyperdx/entry.local.auth.sh | 7 +++++++ docker/hyperdx/entry.local.noauth.sh | 7 +++++++ docker/hyperdx/entry.local.sh | 7 ++++++- 5 files changed, 36 insertions(+), 6 deletions(-) create mode 100755 docker/hyperdx/entry.local.auth.sh create mode 100755 docker/hyperdx/entry.local.noauth.sh diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 3f0afed086..90f5deb490 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -84,13 +84,13 @@ COPY --from=app_builder /app/app/package.json ./package.json COPY --chown=node:node --from=hyperdx ./entry.prod.sh /etc/local/entry.sh ENTRYPOINT ["sh", "/etc/local/entry.sh"] -# prod-extended ############################################################################################ -FROM scratch AS all-in-one +# all-in-one base ############################################################################################ +FROM scratch AS all-in-one-base # Copy base COPY --from=clickhouse_base / / COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib -COPY --from=node_base --link /usrlocal./local/bin /usr/local/bin +COPY --from=node_base --link /usr/local/bin /usr/local/bin COPY --from=node_base --link /usr/local/lib /usr/local/lib COPY --from=node_base /usr/lib /usr/lib COPY --from=node_base /usr/local/include /usr/local/include @@ -120,9 +120,19 @@ ENV NODE_ENV production # Set up App COPY --from=prod /app/api /app/api COPY --from=prod /app/app /app/app -COPY --from=hyperdx ./entry.local.sh /etc/local/entry.sh +COPY --from=hyperdx ./entry.local.sh /etc/local/entry.base.sh WORKDIR /app/app # Expose ports EXPOSE 8080 4317 4318 13133 8123 9000 + +# all-in-one no auth ############################################################################################ +FROM all-in-one-base AS all-in-one-noauth +COPY --from=hyperdx ./entry.local.noauth.sh /etc/local/entry.sh +ENTRYPOINT ["sh", "/etc/local/entry.sh"] + +# all-in-one with auth ############################################################################################ +FROM all-in-one-base AS all-in-one-auth +COPY --from=hyperdx ./entry.local.auth.sh /etc/local/entry.sh ENTRYPOINT ["sh", "/etc/local/entry.sh"] + diff --git a/docker/hyperdx/build.sh b/docker/hyperdx/build.sh index df6e442518..63eb6e699c 100755 --- a/docker/hyperdx/build.sh +++ b/docker/hyperdx/build.sh @@ -1,10 +1,11 @@ #!/bin/bash # Meant to be run from the root of the repo +# No Auth docker build --squash . -f ./docker/hyperdx/Dockerfile \ --build-context clickhouse=./docker/clickhouse \ --build-context otel-collector=./docker/otel-collector \ --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ - --target all-in-one -t hyperdx/dev-all-in-one + --target all-in-one-noauth -t hyperdx/dev-all-in-one-noauth diff --git a/docker/hyperdx/entry.local.auth.sh b/docker/hyperdx/entry.local.auth.sh new file mode 100755 index 0000000000..a6d88f10ea --- /dev/null +++ b/docker/hyperdx/entry.local.auth.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Set auth mode +export IS_LOCAL_APP_MODE="REQUIRED_AUTH" + +# Source the common entry script +source "/etc/local/entry.base.sh" diff --git a/docker/hyperdx/entry.local.noauth.sh b/docker/hyperdx/entry.local.noauth.sh new file mode 100755 index 0000000000..ec34afa369 --- /dev/null +++ b/docker/hyperdx/entry.local.noauth.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Set no auth mode +export IS_LOCAL_APP_MODE="DANGEROUSLY_is_local_app_mode💀" + +# Source the common entry script +source "/etc/local/entry.base.sh" diff --git a/docker/hyperdx/entry.local.sh b/docker/hyperdx/entry.local.sh index a3fe418479..1a77201d1c 100644 --- a/docker/hyperdx/entry.local.sh +++ b/docker/hyperdx/entry.local.sh @@ -15,7 +15,9 @@ export CLICKHOUSE_SERVER_ENDPOINT="ch-server:9000" export MONGO_URI="mongodb://db:27017/hyperdx" export EXPRESS_SESSION_SECRET="hyperdx is cool 👋" -export IS_LOCAL_APP_MODE="DANGEROUSLY_is_local_app_mode💀" +# IS_LOCAL_APP_MODE should be set by the calling script +# Default to dangerous mode if not set +export IS_LOCAL_APP_MODE="${IS_LOCAL_APP_MODE}" export NEXT_TELEMETRY_DISABLED="1" @@ -25,6 +27,8 @@ echo "127.0.0.1 db" >> /etc/hosts echo "Visit the HyperDX UI at $FRONTEND_URL/search" echo "" +echo "Local App Mode: $IS_LOCAL_APP_MODE" +echo "" echo "Send OpenTelemetry data via" echo "http/protobuf: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318" echo "gRPC: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317" @@ -42,6 +46,7 @@ while ! curl -s "http://ch-server:8123" > /dev/null; do echo "Waiting for Clickhouse to be ready..." sleep 1 done + # Start Otel Collector otelcol-contrib --config /etc/otelcol-contrib/config.yaml & From 8dbfa41a3d313f985d5cf447f109db5e691cb0c0 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 21:17:28 -0700 Subject: [PATCH 08/18] feat: update Makefile --- Makefile | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7028285bca..e524bc6a7d 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,25 @@ release-local: -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ -t ${LOCAL_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ -t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ - --target all-in-one \ + --target all-in-one-noauth \ + --push \ + --cache-from=type=gha \ + --cache-to=type=gha,mode=max + +.PHONY: release-all-in-one +release-all-in-one: + docker buildx build --squash . -f ./docker/hyperdx/Dockerfile \ + --build-context clickhouse=./docker/clickhouse \ + --build-context otel-collector=./docker/otel-collector \ + --build-context hyperdx=./docker/hyperdx \ + --build-context api=./packages/api \ + --build-context app=./packages/app \ + --platform ${BUILD_PLATFORMS} \ + -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ + -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + -t ${ALL_IN_ONE_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ + -t ${ALL_IN_ONE_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + --target all-in-one-auth \ --push \ --cache-from=type=gha \ --cache-to=type=gha,mode=max @@ -117,6 +135,7 @@ release-app: --cache-from=type=gha \ --cache-to=type=gha,mode=max +# DEPRECATED # .PHONY: release-local-ui # release-local-ui: # docker buildx build . -f ./packages/app/Dockerfile \ From ed0beb7a3a0042944c1699653ea736172a142cd7 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 21:41:16 -0700 Subject: [PATCH 09/18] feat: command to release stable to dockerhub --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index e524bc6a7d..3bbdbefd28 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,21 @@ release-app: --cache-from=type=gha \ --cache-to=type=gha,mode=max +.PHONY: release-stable +release-stable: + # hyperdx/hyperdx-all-in-one + docker pull ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} + docker tag ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} + docker push ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} + # hyperdx/hyperdx + docker pull ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} + docker tag ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} + docker push ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} + # hyperdx/hyperdx-local + docker pull ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} + docker tag ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} + docker push ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} + # DEPRECATED # .PHONY: release-local-ui # release-local-ui: From 3823877ba5044fa2489b4b031986c8f4b332b9e9 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 21:42:25 -0700 Subject: [PATCH 10/18] feat: update release workflow --- .github/workflows/release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c6f353cf92..503f2ac0b8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,11 +50,10 @@ jobs: strategy: matrix: release: + - release-all-in-one - release-app - - release-otel-collector - release-local - - release-local-ui - # - release-extended-app + - release-otel-collector steps: - name: Checkout uses: actions/checkout@v4 From b5c9ab429790b8fd7cd30bd65a978d14b4969cbd Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 15 May 2025 21:58:19 -0700 Subject: [PATCH 11/18] docs: add changeset --- .changeset/tricky-bees-hide.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/tricky-bees-hide.md diff --git a/.changeset/tricky-bees-hide.md b/.changeset/tricky-bees-hide.md new file mode 100644 index 0000000000..98aa53e5bc --- /dev/null +++ b/.changeset/tricky-bees-hide.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/app": patch +"@hyperdx/api": patch +"@hyperdx/common-utils": patch +--- + +feat: introduce all-one-one (auth vs noauth) multi-stage build From 84eb17577cd1778831926620745d162dc199e9df Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 16 May 2025 09:44:38 -0700 Subject: [PATCH 12/18] feat: attach env info to health check --- packages/api/src/routers/api/root.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/api/root.ts b/packages/api/src/routers/api/root.ts index e3d8fae528..7b9e31ba24 100644 --- a/packages/api/src/routers/api/root.ts +++ b/packages/api/src/routers/api/root.ts @@ -44,7 +44,12 @@ const registrationSchema = z const router = express.Router(); router.get('/health', async (req, res) => { - res.send({ data: 'OK', version: config.CODE_VERSION, ip: req.ip }); + res.send({ + data: 'OK', + version: config.CODE_VERSION, + ip: req.ip, + env: config.NODE_ENV, + }); }); router.get('/installation', async (req, res, next) => { From d2456be454572fed3ca24cbebeeb0bcbc5cc31c1 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 16 May 2025 09:58:04 -0700 Subject: [PATCH 13/18] perf: cleanup all-in-one-base stage build --- docker/hyperdx/Dockerfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 90f5deb490..2024c2da32 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -87,42 +87,42 @@ ENTRYPOINT ["sh", "/etc/local/entry.sh"] # all-in-one base ############################################################################################ FROM scratch AS all-in-one-base -# Copy base +# Copy from clickhouse and otel collector bases COPY --from=clickhouse_base / / COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib + +# Copy Node.js runtime from node base COPY --from=node_base --link /usr/local/bin /usr/local/bin COPY --from=node_base --link /usr/local/lib /usr/local/lib COPY --from=node_base /usr/lib /usr/lib COPY --from=node_base /usr/local/include /usr/local/include - # Set up Clickhouse COPY --from=clickhouse ./local/*.xml /etc/clickhouse-server -# overwrite default config COPY --from=hyperdx ./clickhouseConfig.xml /etc/clickhouse-server/config.xml + # Set up Otel Collector COPY --from=otel-collector ./config.yaml /etc/otelcol-contrib/config.yaml -# == Install MongoDB v4 Deps == -RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories -RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories -RUN apk update -RUN apk add mongodb yaml-cpp=0.6.2-r2 - -# Install libs used for the start script -RUN apk add curl - -# Set up Mongo directories with proper permissions -RUN mkdir -p /data/db +# Install MongoDB and other dependencies (consolidated into a single RUN command) +RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories && \ + echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories && \ + apk update && \ + apk add --no-cache mongodb yaml-cpp=0.6.2-r2 curl && \ + mkdir -p /data/db && \ + rm -rf /var/cache/apk/* ENV NODE_ENV production -# Set up App +# Set up App (copy from prod stage) COPY --from=prod /app/api /app/api COPY --from=prod /app/app /app/app COPY --from=hyperdx ./entry.local.sh /etc/local/entry.base.sh WORKDIR /app/app + +# Add hosts entry in entrypoint script instead of here + # Expose ports EXPOSE 8080 4317 4318 13133 8123 9000 From e1358e0471289f466959828abca2640882fa05b2 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Fri, 16 May 2025 11:13:18 -0700 Subject: [PATCH 14/18] docs: update comment --- docker/hyperdx/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 2024c2da32..2697f3a028 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -1,6 +1,11 @@ -# Starts several services in a single container for local use +# Starts several services in a single container +# For production build: # - API (Node) # - App (Frontend) +# For all-in-one build: +# - Clickhouse +# - Mongo +# - Otel Collector (otelcol) ARG NODE_VERSION=22.14.0 ARG CLICKHOUSE_VERSION=24 From 10b75e39d460d7597f2ab3e291fdd35a60ee93b0 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Fri, 16 May 2025 16:54:58 -0700 Subject: [PATCH 15/18] fix: alerting in local mode --- packages/api/src/controllers/team.ts | 2 +- packages/api/src/tasks/checkAlerts.ts | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/api/src/controllers/team.ts b/packages/api/src/controllers/team.ts index a65418a773..ee5c26ae18 100644 --- a/packages/api/src/controllers/team.ts +++ b/packages/api/src/controllers/team.ts @@ -7,7 +7,7 @@ import { SavedSearch } from '@/models/savedSearch'; import Team from '@/models/team'; const LOCAL_APP_TEAM_ID = '_local_team_'; -const LOCAL_APP_TEAM = { +export const LOCAL_APP_TEAM = { _id: LOCAL_APP_TEAM_ID, id: LOCAL_APP_TEAM_ID, name: 'Local App Team', diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index 67b50d4633..942c916b82 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -21,6 +21,7 @@ import { URLSearchParams } from 'url'; import * as config from '@/config'; import { AlertInput } from '@/controllers/alerts'; import { getConnectionById } from '@/controllers/connection'; +import { LOCAL_APP_TEAM } from '@/controllers/team'; import Alert, { AlertSource, AlertState, @@ -41,11 +42,20 @@ const NOTIFY_FN_NAME = '__hdx_notify_channel__'; const IS_MATCH_FN_NAME = 'is_match'; // TODO(perf): no need to populate the team -const getAlerts = () => - Alert.find({}).populate<{ +const getAlerts = async () => { + const alerts = await Alert.find({}).populate<{ team: ITeam; }>(['team']); + return config.IS_LOCAL_APP_MODE + ? alerts.map(_alert => { + // @ts-ignore + _alert.team = LOCAL_APP_TEAM; + return _alert; + }) + : alerts; +}; + type EnhancedAlert = Awaited>[0]; export const buildLogSearchLink = ({ From 816f7c113cef47b060178162abcbd8e5ad980ae5 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Mon, 19 May 2025 14:54:30 -0700 Subject: [PATCH 16/18] feat: build common-utils on the fly --- docker/hyperdx/Dockerfile | 66 +++++++++++++++-------------------- docker/hyperdx/entry.local.sh | 4 +-- docker/hyperdx/entry.prod.sh | 4 +-- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 2697f3a028..3afbc01336 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -19,47 +19,36 @@ FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_coll FROM node:${NODE_VERSION}-alpine AS node_base -WORKDIR /app/api - -COPY ./yarn.lock ./.yarnrc.yml ./ -COPY ./.yarn ./.yarn -COPY --from=api ./package.json . -RUN yarn install && yarn cache clean +WORKDIR /app -WORKDIR /app/app +COPY .yarn ./.yarn +COPY .yarnrc.yml yarn.lock package.json nx.json .prettierrc .prettierignore ./ +COPY ./packages/common-utils ./packages/common-utils +COPY ./packages/api/jest.config.js ./packages/api/tsconfig.json ./packages/api/package.json ./packages/api/ +COPY ./packages/app/jest.config.js ./packages/app/tsconfig.json ./packages/app/tsconfig.test.json ./packages/app/package.json ./packages/app/next.config.js ./packages/app/mdx.d.ts ./packages/app/.eslintrc.js ./packages/app/ -COPY ./yarn.lock ./.yarnrc.yml ./ -COPY ./.yarn ./.yarn -COPY --from=app ./package.json ./ +# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. +RUN apk add --no-cache libc6-compat -RUN yarn install && yarn cache clean +RUN yarn install --mode=skip-build && yarn cache clean -## API Builder Image ############################################################################### -FROM node_base AS api_builder - -WORKDIR /app/api +## API/APP Builder Image ########################################################################## +FROM node_base AS builder -COPY --from=api ./tsconfig.json ./ -COPY --from=api ./src ./src -RUN yarn build && rm -rf node_modules && yarn workspaces focus --production - - -# APP Builder Image ############################################################################### -FROM node_base AS app_builder - -WORKDIR /app/app +WORKDIR /app -COPY --from=app ./.eslintrc.js ./next.config.js ./tsconfig.json ./next.config.js ./mdx.d.ts ./.eslintrc.js ./ -COPY --from=app ./src ./src -COPY --from=app ./pages ./pages -COPY --from=app ./public ./public -COPY --from=app ./styles ./styles +COPY --from=api ./src ./packages/api/src +COPY --from=app ./src ./packages/app/src +COPY --from=app ./pages ./packages/app/pages +COPY --from=app ./public ./packages/app/public +COPY --from=app ./styles ./packages/app/styles ENV NEXT_TELEMETRY_DISABLED 1 ENV NEXT_OUTPUT_STANDALONE false ENV NEXT_PUBLIC_IS_LOCAL_MODE false -RUN yarn build && rm -rf node_modules && yarn workspaces focus --production +RUN npx nx run-many --target=build --projects=@hyperdx/common-utils,@hyperdx/api,@hyperdx/app +RUN rm -rf node_modules && yarn workspaces focus @hyperdx/api @hyperdx/app --production # prod ############################################################################################ @@ -74,16 +63,19 @@ USER node # Set up API WORKDIR /app/api -COPY --chown=node:node --from=api_builder ./app/api/build ./build -COPY --chown=node:node --from=api_builder ./app/api/node_modules ./node_modules +COPY --chown=node:node --from=builder /app/node_modules ./node_modules +COPY --chown=node:node --from=builder /app/packages/api/build ./packages/api/build +COPY --chown=node:node --from=builder /app/packages/common-utils/dist ./packages/common-utils/dist +COPY --chown=node:node --from=node_base /app/packages/common-utils/node_modules ./packages/common-utils/node_modules # Set up App WORKDIR /app/app -COPY --from=app_builder /app/app/next.config.js ./ -COPY --chown=node:node --from=app_builder /app/app/public ./public -COPY --chown=node:node --from=app_builder /app/app/.next ./.next -COPY --from=app_builder /app/app/node_modules ./node_modules -COPY --from=app_builder /app/app/package.json ./package.json +COPY --chown=node:node --from=builder /app/node_modules ./node_modules +COPY --chown=node:node --from=builder /app/packages/app/public ./public +COPY --chown=node:node --from=builder /app/packages/app/.next ./.next +# COPY --chown=node:node --from=builder /app/packages/app/node_modules ./packages/app/node_modules +COPY --from=builder /app/packages/app/next.config.js ./next.config.js +COPY --from=builder /app/packages/app/package.json ./package.json # Set up start script COPY --chown=node:node --from=hyperdx ./entry.prod.sh /etc/local/entry.sh diff --git a/docker/hyperdx/entry.local.sh b/docker/hyperdx/entry.local.sh index 1a77201d1c..1bd106ff18 100644 --- a/docker/hyperdx/entry.local.sh +++ b/docker/hyperdx/entry.local.sh @@ -54,9 +54,9 @@ otelcol-contrib --config /etc/otelcol-contrib/config.yaml & npx concurrently \ "--kill-others" \ "--names=API,APP,ALERT-TASK" \ - "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \ + "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/index.js" \ "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \ - "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/tasks/index.js check-alerts" \ + "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/tasks/index.js check-alerts" \ > /var/log/app.log 2>&1 & # Wait for any process to exit diff --git a/docker/hyperdx/entry.prod.sh b/docker/hyperdx/entry.prod.sh index ef370aca52..0480f94b14 100644 --- a/docker/hyperdx/entry.prod.sh +++ b/docker/hyperdx/entry.prod.sh @@ -13,6 +13,6 @@ echo "" npx concurrently \ "--kill-others" \ "--names=API,APP,ALERT-TASK" \ - "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \ + "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/index.js" \ "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \ - "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/tasks/index.js check-alerts" + "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/tasks/index.js check-alerts" From 605a3fc1c848434d53ec1702f17e763199238db3 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Mon, 19 May 2025 15:08:22 -0700 Subject: [PATCH 17/18] feedback: rename base entry file --- docker/hyperdx/Dockerfile | 2 +- docker/hyperdx/{entry.local.sh => entry.local.base.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docker/hyperdx/{entry.local.sh => entry.local.base.sh} (100%) diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile index 3afbc01336..81be09c946 100644 --- a/docker/hyperdx/Dockerfile +++ b/docker/hyperdx/Dockerfile @@ -114,7 +114,7 @@ ENV NODE_ENV production # Set up App (copy from prod stage) COPY --from=prod /app/api /app/api COPY --from=prod /app/app /app/app -COPY --from=hyperdx ./entry.local.sh /etc/local/entry.base.sh +COPY --from=hyperdx ./entry.local.base.sh /etc/local/entry.base.sh WORKDIR /app/app diff --git a/docker/hyperdx/entry.local.sh b/docker/hyperdx/entry.local.base.sh similarity index 100% rename from docker/hyperdx/entry.local.sh rename to docker/hyperdx/entry.local.base.sh From f04669b04384b1860c666f82e951a2fb3781e573 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Mon, 19 May 2025 15:28:32 -0700 Subject: [PATCH 18/18] style: remove release-stable --- Makefile | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Makefile b/Makefile index 3bbdbefd28..e524bc6a7d 100644 --- a/Makefile +++ b/Makefile @@ -135,21 +135,6 @@ release-app: --cache-from=type=gha \ --cache-to=type=gha,mode=max -.PHONY: release-stable -release-stable: - # hyperdx/hyperdx-all-in-one - docker pull ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} - docker tag ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} - docker push ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} - # hyperdx/hyperdx - docker pull ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} - docker tag ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} - docker push ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} - # hyperdx/hyperdx-local - docker pull ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} - docker tag ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} - docker push ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} - # DEPRECATED # .PHONY: release-local-ui # release-local-ui: