diff --git a/.changeset/tricky-bees-hide.md b/.changeset/tricky-bees-hide.md new file mode 100644 index 0000000000..98aa53e5bc --- /dev/null +++ b/.changeset/tricky-bees-hide.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/app": patch +"@hyperdx/api": patch +"@hyperdx/common-utils": patch +--- + +feat: introduce all-one-one (auth vs noauth) multi-stage build diff --git a/.env b/.env index 2c34179a88..541bcb09b2 100644 --- a/.env +++ b/.env @@ -4,6 +4,8 @@ IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx IMAGE_NAME_HDX=docker.hyperdx.io/hyperdx/hyperdx LOCAL_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-local LOCAL_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-local +ALL_IN_ONE_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-all-in-one +ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one OTEL_COLLECTOR_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-otel-collector OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector CHANGESET_TAG=2.0.0-beta.16 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c6f353cf92..503f2ac0b8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,11 +50,10 @@ jobs: strategy: matrix: release: + - release-all-in-one - release-app - - release-otel-collector - release-local - - release-local-ui - # - release-extended-app + - release-otel-collector steps: - name: Checkout uses: actions/checkout@v4 diff --git a/Makefile b/Makefile index ab6665745b..e524bc6a7d 100644 --- a/Makefile +++ b/Makefile @@ -71,12 +71,24 @@ dev-migrate-db: version: sh ./version.sh +.PHONY: release-otel-collector +release-otel-collector: + docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector \ + -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ + -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ + --target prod \ + --push \ + --cache-from=type=gha \ + --cache-to=type=gha,mode=max + .PHONY: release-local release-local: - docker buildx build --squash . -f ./docker/local/Dockerfile \ + docker buildx build --squash . -f ./docker/hyperdx/Dockerfile \ --build-context clickhouse=./docker/clickhouse \ --build-context otel-collector=./docker/otel-collector \ - --build-context local=./docker/local \ + --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ --platform ${BUILD_PLATFORMS} \ @@ -84,41 +96,33 @@ release-local: -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ -t ${LOCAL_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ -t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + --target all-in-one-noauth \ --push \ --cache-from=type=gha \ --cache-to=type=gha,mode=max -.PHONY: release-local-ui -release-local-ui: - docker buildx build . -f ./packages/app/Dockerfile \ - --build-arg IS_LOCAL_MODE=true \ - --build-arg PORT=${HYPERDX_APP_PORT} \ - --target prod \ +.PHONY: release-all-in-one +release-all-in-one: + docker buildx build --squash . -f ./docker/hyperdx/Dockerfile \ + --build-context clickhouse=./docker/clickhouse \ + --build-context otel-collector=./docker/otel-collector \ + --build-context hyperdx=./docker/hyperdx \ + --build-context api=./packages/api \ + --build-context app=./packages/app \ --platform ${BUILD_PLATFORMS} \ - -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ - -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG}-ui \ - -t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ - -t ${LOCAL_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG}-ui \ - --push \ - --cache-from=type=gha \ - --cache-to=type=gha,mode=max - -.PHONY: release-otel-collector -release-otel-collector: - docker buildx build --platform ${BUILD_PLATFORMS} ./docker/otel-collector \ - -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ - -t ${OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ - -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ - -t ${OTEL_COLLECTOR_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ - --target prod \ + -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \ + -t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + -t ${ALL_IN_ONE_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG} \ + -t ${ALL_IN_ONE_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \ + --target all-in-one-auth \ --push \ --cache-from=type=gha \ --cache-to=type=gha,mode=max .PHONY: release-app release-app: - docker buildx build --squash . -f ./docker/fullstack/Dockerfile \ - --build-context fullstack=./docker/fullstack \ + docker buildx build --squash . -f ./docker/hyperdx/Dockerfile \ + --build-context hyperdx=./docker/hyperdx \ --build-context api=./packages/api \ --build-context app=./packages/app \ --platform ${BUILD_PLATFORMS} \ @@ -131,15 +135,19 @@ release-app: --cache-from=type=gha \ --cache-to=type=gha,mode=max -.PHONY: release-extended-app -release-extended-app: - docker buildx build --squash . -f ./docker/fullstack/Dockerfile \ - --build-context fullstack=./docker/fullstack \ - --build-context api=./packages/api \ - --build-context app=./packages/app \ - --platform ${BUILD_PLATFORMS} \ - -t ${IMAGE_NAME_DOCKERHUB}:latest \ - --target prod-extended \ - --push \ - --cache-from=type=gha \ - --cache-to=type=gha,mode=max +# DEPRECATED +# .PHONY: release-local-ui +# release-local-ui: +# docker buildx build . -f ./packages/app/Dockerfile \ +# --build-arg IS_LOCAL_MODE=true \ +# --build-arg PORT=${HYPERDX_APP_PORT} \ +# --target prod \ +# --platform ${BUILD_PLATFORMS} \ +# -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ +# -t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG}-ui \ +# -t ${LOCAL_IMAGE_NAME}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG}-ui \ +# -t ${LOCAL_IMAGE_NAME}:${IMAGE_NIGHTLY_TAG}-ui \ +# --push \ +# --cache-from=type=gha \ +# --cache-to=type=gha,mode=max + diff --git a/docker/fullstack/Dockerfile b/docker/fullstack/Dockerfile deleted file mode 100644 index 10598017d9..0000000000 --- a/docker/fullstack/Dockerfile +++ /dev/null @@ -1,98 +0,0 @@ -# Starts several services in a single container for local use -# - API (Node) -# - App (Frontend) - -ARG NODE_VERSION=22.14.0 - -# base ############################################################################################# -FROM node:${NODE_VERSION}-alpine AS base - -WORKDIR /app/api - -COPY ./yarn.lock ./.yarnrc.yml ./ -COPY ./.yarn ./.yarn -COPY --from=api ./package.json . -RUN yarn install && yarn cache clean - -WORKDIR /app/app - -COPY ./yarn.lock ./.yarnrc.yml ./ -COPY ./.yarn ./.yarn -COPY --from=app ./package.json ./ - -RUN yarn install && yarn cache clean - - -## API Builder Image ############################################################################### -FROM base AS api_builder - -WORKDIR /app/api - -COPY --from=api ./tsconfig.json ./ -COPY --from=api ./src ./src -RUN yarn build && rm -rf node_modules && yarn workspaces focus --production - - -# APP Builder Image ############################################################################### -FROM base AS app_builder - -WORKDIR /app/app - -COPY --from=app ./.eslintrc.js ./next.config.js ./tsconfig.json ./next.config.js ./mdx.d.ts ./.eslintrc.js ./ -COPY --from=app ./src ./src -COPY --from=app ./pages ./pages -COPY --from=app ./public ./public -COPY --from=app ./styles ./styles - -ENV NEXT_TELEMETRY_DISABLED 1 -ENV NEXT_OUTPUT_STANDALONE false -ENV NEXT_PUBLIC_IS_LOCAL_MODE false -RUN yarn build && rm -rf node_modules && yarn workspaces focus --production - - -# prod ############################################################################################ -FROM node:${NODE_VERSION}-alpine AS prod - -ENV NODE_ENV production - -# Install libs used for the start script -RUN npm install -g concurrently@9.1.0 - -USER node - -# Set up API -WORKDIR /app/api -COPY --chown=node:node --from=api_builder ./app/api/build ./build -COPY --chown=node:node --from=api_builder ./app/api/node_modules ./node_modules - -# Set up App -WORKDIR /app/app -COPY --from=app_builder /app/app/next.config.js ./ -COPY --chown=node:node --from=app_builder /app/app/public ./public -COPY --chown=node:node --from=app_builder /app/app/.next ./.next -COPY --from=app_builder /app/app/node_modules ./node_modules -COPY --from=app_builder /app/app/package.json ./package.json - -# Set up start script -COPY --chown=node:node --from=fullstack ./entry.sh /etc/local/entry.sh -ENTRYPOINT ["sh", "/etc/local/entry.sh"] - -# prod-extended ############################################################################################ -FROM prod AS prod-extended - -USER root - -# == Install MongoDB v4 Deps == -RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories -RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories -RUN apk update -RUN apk add mongodb yaml-cpp=0.6.2-r2 - -# Set up Mongo directories with proper permissions -RUN mkdir -p /data/db && \ - chown -R node:node /data/db && \ - chown -R node:node /var/log - -USER node - -ENTRYPOINT ["sh", "/etc/local/entry.sh"] diff --git a/docker/fullstack/entry.sh b/docker/fullstack/entry.sh deleted file mode 100644 index 9494ed18b7..0000000000 --- a/docker/fullstack/entry.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}" - -# Internal Services -export MONGO_URI="${MONGO_URI:-mongodb://127.0.0.1:27017/hyperdx}" - -echo "Visit the HyperDX UI at $FRONTEND_URL" -echo "" - -# Start Mongo Server if it exists (will run in prod-extended but not in prod) -if command -v mongod &> /dev/null; then - echo "Starting MongoDB server..." - mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 & - # Wait for MongoDB to start - sleep 2 -fi - -# Use concurrently to run both the API and App servers -npx concurrently \ - "--kill-others" \ - "--names=API,APP,ALERT-TASK" \ - "MONGO_URI=${MONGO_URI} PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/index.js" \ - "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \ - "MONGO_URI=${MONGO_URI} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/build/tasks/index.js check-alerts" diff --git a/docker/hyperdx/Dockerfile b/docker/hyperdx/Dockerfile new file mode 100644 index 0000000000..81be09c946 --- /dev/null +++ b/docker/hyperdx/Dockerfile @@ -0,0 +1,135 @@ +# Starts several services in a single container +# For production build: +# - API (Node) +# - App (Frontend) +# For all-in-one build: +# - Clickhouse +# - Mongo +# - Otel Collector (otelcol) + +ARG NODE_VERSION=22.14.0 +ARG CLICKHOUSE_VERSION=24 +ARG OTEL_COLLECTOR_VERSION=0.120.0 + +# base ############################################################################################# +# == Clickhouse/Base Image == +FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS clickhouse_base +# == Otel Collector Image == +FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base + +FROM node:${NODE_VERSION}-alpine AS node_base + +WORKDIR /app + +COPY .yarn ./.yarn +COPY .yarnrc.yml yarn.lock package.json nx.json .prettierrc .prettierignore ./ +COPY ./packages/common-utils ./packages/common-utils +COPY ./packages/api/jest.config.js ./packages/api/tsconfig.json ./packages/api/package.json ./packages/api/ +COPY ./packages/app/jest.config.js ./packages/app/tsconfig.json ./packages/app/tsconfig.test.json ./packages/app/package.json ./packages/app/next.config.js ./packages/app/mdx.d.ts ./packages/app/.eslintrc.js ./packages/app/ + +# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. +RUN apk add --no-cache libc6-compat + +RUN yarn install --mode=skip-build && yarn cache clean + + +## API/APP Builder Image ########################################################################## +FROM node_base AS builder + +WORKDIR /app + +COPY --from=api ./src ./packages/api/src +COPY --from=app ./src ./packages/app/src +COPY --from=app ./pages ./packages/app/pages +COPY --from=app ./public ./packages/app/public +COPY --from=app ./styles ./packages/app/styles + +ENV NEXT_TELEMETRY_DISABLED 1 +ENV NEXT_OUTPUT_STANDALONE false +ENV NEXT_PUBLIC_IS_LOCAL_MODE false +RUN npx nx run-many --target=build --projects=@hyperdx/common-utils,@hyperdx/api,@hyperdx/app +RUN rm -rf node_modules && yarn workspaces focus @hyperdx/api @hyperdx/app --production + + +# prod ############################################################################################ +FROM node:${NODE_VERSION}-alpine AS prod + +ENV NODE_ENV production + +# Install libs used for the start script +RUN npm install -g concurrently@9.1.0 + +USER node + +# Set up API +WORKDIR /app/api +COPY --chown=node:node --from=builder /app/node_modules ./node_modules +COPY --chown=node:node --from=builder /app/packages/api/build ./packages/api/build +COPY --chown=node:node --from=builder /app/packages/common-utils/dist ./packages/common-utils/dist +COPY --chown=node:node --from=node_base /app/packages/common-utils/node_modules ./packages/common-utils/node_modules + +# Set up App +WORKDIR /app/app +COPY --chown=node:node --from=builder /app/node_modules ./node_modules +COPY --chown=node:node --from=builder /app/packages/app/public ./public +COPY --chown=node:node --from=builder /app/packages/app/.next ./.next +# COPY --chown=node:node --from=builder /app/packages/app/node_modules ./packages/app/node_modules +COPY --from=builder /app/packages/app/next.config.js ./next.config.js +COPY --from=builder /app/packages/app/package.json ./package.json + +# Set up start script +COPY --chown=node:node --from=hyperdx ./entry.prod.sh /etc/local/entry.sh +ENTRYPOINT ["sh", "/etc/local/entry.sh"] + +# all-in-one base ############################################################################################ +FROM scratch AS all-in-one-base + +# Copy from clickhouse and otel collector bases +COPY --from=clickhouse_base / / +COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib + +# Copy Node.js runtime from node base +COPY --from=node_base --link /usr/local/bin /usr/local/bin +COPY --from=node_base --link /usr/local/lib /usr/local/lib +COPY --from=node_base /usr/lib /usr/lib +COPY --from=node_base /usr/local/include /usr/local/include + +# Set up Clickhouse +COPY --from=clickhouse ./local/*.xml /etc/clickhouse-server +COPY --from=hyperdx ./clickhouseConfig.xml /etc/clickhouse-server/config.xml + +# Set up Otel Collector +COPY --from=otel-collector ./config.yaml /etc/otelcol-contrib/config.yaml + +# Install MongoDB and other dependencies (consolidated into a single RUN command) +RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories && \ + echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories && \ + apk update && \ + apk add --no-cache mongodb yaml-cpp=0.6.2-r2 curl && \ + mkdir -p /data/db && \ + rm -rf /var/cache/apk/* + +ENV NODE_ENV production + +# Set up App (copy from prod stage) +COPY --from=prod /app/api /app/api +COPY --from=prod /app/app /app/app +COPY --from=hyperdx ./entry.local.base.sh /etc/local/entry.base.sh + +WORKDIR /app/app + +# Add hosts entry in entrypoint script instead of here + +# Expose ports +EXPOSE 8080 4317 4318 13133 8123 9000 + +# all-in-one no auth ############################################################################################ +FROM all-in-one-base AS all-in-one-noauth +COPY --from=hyperdx ./entry.local.noauth.sh /etc/local/entry.sh +ENTRYPOINT ["sh", "/etc/local/entry.sh"] + +# all-in-one with auth ############################################################################################ +FROM all-in-one-base AS all-in-one-auth +COPY --from=hyperdx ./entry.local.auth.sh /etc/local/entry.sh +ENTRYPOINT ["sh", "/etc/local/entry.sh"] + diff --git a/docker/hyperdx/build.sh b/docker/hyperdx/build.sh new file mode 100755 index 0000000000..63eb6e699c --- /dev/null +++ b/docker/hyperdx/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Meant to be run from the root of the repo + +# No Auth +docker build --squash . -f ./docker/hyperdx/Dockerfile \ + --build-context clickhouse=./docker/clickhouse \ + --build-context otel-collector=./docker/otel-collector \ + --build-context hyperdx=./docker/hyperdx \ + --build-context api=./packages/api \ + --build-context app=./packages/app \ + --target all-in-one-noauth -t hyperdx/dev-all-in-one-noauth diff --git a/docker/local/clickhouseConfig.xml b/docker/hyperdx/clickhouseConfig.xml similarity index 100% rename from docker/local/clickhouseConfig.xml rename to docker/hyperdx/clickhouseConfig.xml diff --git a/docker/hyperdx/entry.local.auth.sh b/docker/hyperdx/entry.local.auth.sh new file mode 100755 index 0000000000..a6d88f10ea --- /dev/null +++ b/docker/hyperdx/entry.local.auth.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Set auth mode +export IS_LOCAL_APP_MODE="REQUIRED_AUTH" + +# Source the common entry script +source "/etc/local/entry.base.sh" diff --git a/docker/local/entry.sh b/docker/hyperdx/entry.local.base.sh similarity index 64% rename from docker/local/entry.sh rename to docker/hyperdx/entry.local.base.sh index 88f74d8044..1bd106ff18 100644 --- a/docker/local/entry.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -13,20 +13,22 @@ export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPE # Internal Services export CLICKHOUSE_SERVER_ENDPOINT="ch-server:9000" export MONGO_URI="mongodb://db:27017/hyperdx" -export REDIS_URI="redis://redis:6379" export EXPRESS_SESSION_SECRET="hyperdx is cool 👋" -export IS_LOCAL_APP_MODE="DANGEROUSLY_is_local_app_mode💀" +# IS_LOCAL_APP_MODE should be set by the calling script +# Default to dangerous mode if not set +export IS_LOCAL_APP_MODE="${IS_LOCAL_APP_MODE}" export NEXT_TELEMETRY_DISABLED="1" # Simulate Docker Service DNS echo "127.0.0.1 ch-server" >> /etc/hosts echo "127.0.0.1 db" >> /etc/hosts -echo "127.0.0.1 redis" >> /etc/hosts echo "Visit the HyperDX UI at $FRONTEND_URL/search" echo "" +echo "Local App Mode: $IS_LOCAL_APP_MODE" +echo "" echo "Send OpenTelemetry data via" echo "http/protobuf: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318" echo "gRPC: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317" @@ -36,30 +38,26 @@ echo "" # Start Clickhouse Server /entrypoint.sh & -# Start Redis Server -# redis-server > /var/log/redis.log 2>&1 & - # Start Mongo Server -# mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 & +mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 & # Wait for Clickhouse to be ready while ! curl -s "http://ch-server:8123" > /dev/null; do echo "Waiting for Clickhouse to be ready..." sleep 1 done + # Start Otel Collector otelcol-contrib --config /etc/otelcol-contrib/config.yaml & -# Api -# CLICKHOUSE_USER=api \ -# CLICKHOUSE_PASSWORD=api \ -# PORT=8000 \ -# node /app/api/build/index.js > /var/log/api.log 2>&1 & - -# App -NODE_ENV=production \ -NEXT_PUBLIC_SERVER_URL="${SERVER_URL}" \ -/app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080} > /var/log/app.log 2>&1 & +# Start HyperDX app +npx concurrently \ + "--kill-others" \ + "--names=API,APP,ALERT-TASK" \ + "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/index.js" \ + "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \ + "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/tasks/index.js check-alerts" \ + > /var/log/app.log 2>&1 & # Wait for any process to exit wait -n diff --git a/docker/hyperdx/entry.local.noauth.sh b/docker/hyperdx/entry.local.noauth.sh new file mode 100755 index 0000000000..ec34afa369 --- /dev/null +++ b/docker/hyperdx/entry.local.noauth.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Set no auth mode +export IS_LOCAL_APP_MODE="DANGEROUSLY_is_local_app_mode💀" + +# Source the common entry script +source "/etc/local/entry.base.sh" diff --git a/docker/hyperdx/entry.prod.sh b/docker/hyperdx/entry.prod.sh new file mode 100644 index 0000000000..0480f94b14 --- /dev/null +++ b/docker/hyperdx/entry.prod.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}" + +# Set to "REQUIRED_AUTH" to enforce API authentication. +# ⚠️ Do not change this value !!!! +export IS_LOCAL_APP_MODE="REQUIRED_AUTH" + +echo "Visit the HyperDX UI at $FRONTEND_URL" +echo "" + +# Use concurrently to run both the API and App servers +npx concurrently \ + "--kill-others" \ + "--names=API,APP,ALERT-TASK" \ + "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/index.js" \ + "HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} /app/app/node_modules/.bin/next start -p ${HYPERDX_APP_PORT:-8080}" \ + "node -r /app/api/node_modules/@hyperdx/node-opentelemetry/build/src/tracing /app/api/packages/api/build/tasks/index.js check-alerts" diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile deleted file mode 100644 index 7ca80f485a..0000000000 --- a/docker/local/Dockerfile +++ /dev/null @@ -1,132 +0,0 @@ -# Starts several services in a single container for local use -# - Clickhouse -# - Mongo -# - Otel Collector (otelcol) -# - API (Node) -# - App (Frontend) -# - Redis Cache - -# TODO: -# - Customize ports, need to set up env vars or relax CORS and other port requirements -# - Have otel collector listen to a directory users can mount logs into -# - Allow persisting settings on disk -# - Limiting persisted data with some auto rotation - -ARG NODE_VERSION=22.14.0 -ARG CLICKHOUSE_VERSION=24 -ARG OTEL_COLLECTOR_VERSION=0.120.0 -# Get Node base image to copy over Node binaries -FROM node:${NODE_VERSION}-alpine AS node - -# == API Builder Image == -# FROM node:${NODE_VERSION}-alpine AS api_builder - -# WORKDIR /app/api -# COPY ./yarn.lock ./.yarnrc.yml . -# COPY ./.yarn ./.yarn -# COPY --from=api ./package.json . -# RUN yarn install && yarn cache clean - -# COPY --from=api ./tsconfig.json . -# COPY --from=api ./src ./src -# RUN yarn run build - -# == App Builder Image == -FROM node:${NODE_VERSION}-alpine AS app_builder - -WORKDIR /app/app - -COPY ./yarn.lock ./.yarnrc.yml . -COPY ./.yarn ./.yarn -COPY --from=app ./package.json . - -RUN yarn install && yarn cache clean - -COPY --from=app ./.eslintrc.js ./next.config.js ./tsconfig.json ./next.config.js ./mdx.d.ts ./.eslintrc.js ./ -COPY --from=app ./src ./src -COPY --from=app ./pages ./pages -COPY --from=app ./public ./public -COPY --from=app ./styles ./styles - -ENV NEXT_TELEMETRY_DISABLED 1 -ENV NEXT_OUTPUT_STANDALONE true -ENV NEXT_PUBLIC_IS_LOCAL_MODE true -RUN yarn build && rm -rf node_modules && yarn workspaces focus --production - -# == Clickhouse/Base Image == -FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS clickhouse_base - -# == Otel Collector Image == -FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base - -FROM scratch as base -COPY --from=clickhouse_base / / -COPY --from=otel_collector_base /otelcol-contrib /usr/local/bin/otelcol-contrib - -# === -# === Install Deps -# === - -# == Install Node Deps == -COPY --from=node /usr/lib /usr/lib -COPY --from=node /usr/local/lib /usr/local/lib -COPY --from=node /usr/local/include /usr/local/include -COPY --from=node /usr/local/bin /usr/local/bin - -# RUN npm install -g yarn --force - -# == Install MongoDB v4 Deps == -# RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/main' >> /etc/apk/repositories -# RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.9/community' >> /etc/apk/repositories -# RUN apk update -# RUN apk add mongodb yaml-cpp=0.6.2-r2 -RUN apk add curl - -# == Install Redis == -# If this version pinning fails, this is likely due to the version being dropped -# from APK -# RUN apk add 'redis<7.0.14' - -# === -# === Set Up Services -# === - -# Set up Clickhouse -COPY --from=clickhouse ./local/*.xml /etc/clickhouse-server -# overwrite default config -COPY --from=local ./clickhouseConfig.xml /etc/clickhouse-server/config.xml - -# Set up Mongo -# RUN mkdir -p /data/db - -# Set up Otel Collector -COPY --from=otel-collector ./config.yaml /etc/otelcol-contrib/config.yaml - -# Set up API (NOT USED YET) -# WORKDIR /app/api - -# COPY --from=api_builder ./app/api/build ./build -# COPY ./yarn.lock ./.yarnrc.yml . -# COPY ./.yarn ./.yarn -# COPY --from=api_builder ./app/api/package.json . - -# # Only install prod dependencies -# RUN yarn workspaces focus --production && yarn cache clean - -# # Remove dev dependencies -# RUN rm -rf ./.yarn - -# Set up App -WORKDIR /app/app -COPY --from=app_builder /app/app/next.config.js ./ -COPY --from=app_builder /app/app/public ./public -COPY --from=app_builder /app/app/.next ./.next -COPY --from=app_builder /app/app/node_modules ./node_modules -COPY --from=app_builder /app/app/package.json ./package.json - -# Expose ports -EXPOSE 8000 8080 4317 4318 13133 8123 9000 - -# Set up start script -COPY --from=local ./entry.sh /etc/local/entry.sh -CMD sh /etc/local/entry.sh diff --git a/docker/local/build.sh b/docker/local/build.sh deleted file mode 100755 index b46a401ba0..0000000000 --- a/docker/local/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Meant to be run from the root of the repo - -docker build --squash -t hdx-oss-dev-local -f ./docker/local/Dockerfile \ - --build-context clickhouse=./docker/clickhouse \ - --build-context otel-collector=./docker/otel-collector \ - --build-context local=./docker/local \ - --build-context api=./packages/api \ - --build-context app=./packages/app \ - . diff --git a/packages/api/src/controllers/team.ts b/packages/api/src/controllers/team.ts index 70f3c2389f..1a6e21cabb 100644 --- a/packages/api/src/controllers/team.ts +++ b/packages/api/src/controllers/team.ts @@ -7,7 +7,7 @@ import { SavedSearch } from '@/models/savedSearch'; import Team from '@/models/team'; const LOCAL_APP_TEAM_ID = '_local_team_'; -const LOCAL_APP_TEAM = { +export const LOCAL_APP_TEAM = { _id: LOCAL_APP_TEAM_ID, id: LOCAL_APP_TEAM_ID, name: 'Local App Team', diff --git a/packages/api/src/routers/api/root.ts b/packages/api/src/routers/api/root.ts index 3fe2064e05..9fb3bb5bfa 100644 --- a/packages/api/src/routers/api/root.ts +++ b/packages/api/src/routers/api/root.ts @@ -45,7 +45,12 @@ const registrationSchema = z const router = express.Router(); router.get('/health', async (req, res) => { - res.send({ data: 'OK', version: config.CODE_VERSION, ip: req.ip }); + res.send({ + data: 'OK', + version: config.CODE_VERSION, + ip: req.ip, + env: config.NODE_ENV, + }); }); router.get('/installation', async (req, res, next) => { diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts index 315f7d9712..9e7099596f 100644 --- a/packages/api/src/tasks/checkAlerts.ts +++ b/packages/api/src/tasks/checkAlerts.ts @@ -22,6 +22,7 @@ import { URLSearchParams } from 'url'; import * as config from '@/config'; import { AlertInput } from '@/controllers/alerts'; import { getConnectionById } from '@/controllers/connection'; +import { LOCAL_APP_TEAM } from '@/controllers/team'; import Alert, { AlertSource, AlertState, @@ -42,11 +43,20 @@ const NOTIFY_FN_NAME = '__hdx_notify_channel__'; const IS_MATCH_FN_NAME = 'is_match'; // TODO(perf): no need to populate the team -const getAlerts = () => - Alert.find({}).populate<{ +const getAlerts = async () => { + const alerts = await Alert.find({}).populate<{ team: ITeam; }>(['team']); + return config.IS_LOCAL_APP_MODE + ? alerts.map(_alert => { + // @ts-ignore + _alert.team = LOCAL_APP_TEAM; + return _alert; + }) + : alerts; +}; + type EnhancedAlert = Awaited>[0]; export const buildLogSearchLink = ({