diff --git a/.changeset/express-session-secret-no-default.md b/.changeset/express-session-secret-no-default.md new file mode 100644 index 0000000000..7f12b91da4 --- /dev/null +++ b/.changeset/express-session-secret-no-default.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/api': minor +--- + +The API no longer falls back to the public session secret `hyperdx is cool πŸ‘‹`. +If `EXPRESS_SESSION_SECRET` is unset, the API generates a random secret and +reports it once at startup when authentication is enabled. Authenticated +deployments that explicitly set the old public value also get a random secret +and the same warning; the public value remains available only in the no-auth +local demo. Set it to a random string (`openssl rand -hex 32`) in any deployment +with authentication enabled β€” without it, restarting the API signs every user +out, and replicas do not share sessions. diff --git a/.env b/.env index aab1af1728..69ee2caa60 100644 --- a/.env +++ b/.env @@ -34,6 +34,10 @@ WEBHOOK_HOSTNAME_ALLOWLIST= # Generate with `openssl rand -base64 32`. Unset means plain text. TOKEN_ENCRYPTION_KEY= +# Generate once with openssl rand -hex 32; reuse across restarts and replicas. +# Configure in .env.local with docker compose --env-file .env --env-file .env.local up -d. +# EXPRESS_SESSION_SECRET= + # Docker service ports (overridden by scripts/dev-env.sh for isolation) HDX_DEV_MONGO_PORT=27017 HDX_DEV_CH_HTTP_PORT=8123 diff --git a/DEPLOY.md b/DEPLOY.md index d399049227..52735ada6f 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -27,8 +27,11 @@ You'll need to set the `MONGO_URI` environment variable to the URI of your MongoDB instance. Afterwards, you'll want to visit http://localhost:8080 to set up your connection with ClickHouse. -Before deploying into production, you'll want to set the -`EXPRESS_SESSION_SECRET` environment variable to a random string. +Before deploying into production, set the `EXPRESS_SESSION_SECRET` environment +variable to a random string (e.g. `openssl rand -hex 32`). Without it the API +generates one per process, which signs users out on every restart and prevents +replicas from sharing sessions. The old public value `hyperdx is cool πŸ‘‹` is +also replaced with a per-process secret when authentication is enabled. To customize the frontend URL, set the `FRONTEND_URL` environment variable to the URL your HyperDX instance is hosted on. @@ -55,8 +58,18 @@ such as ufw. See the [Docker docs](https://docs.docker.com/engine/network/packet-filtering-firewalls/#docker-and-ufw) for more information. -Additionally, you'll want to set the `EXPRESS_SESSION_SECRET` environment -variable to a random string. +Generate a secret once with `openssl rand -hex 32`, then set +`EXPRESS_SESSION_SECRET=` in `.env.local`, which Git ignores. +Do not put the secret in the tracked root `.env`. Compose forwards this value to +the app container when you load `.env.local` after `.env`: + +```bash +docker compose --env-file .env --env-file .env.local up -d +``` + +Reuse the same secret across restarts and all replicas. Without it the API +generates one per process, which signs users out on every restart and prevents +replicas from sharing sessions. The old public value is not a production secret. ## Local Development Mode diff --git a/agent_docs/development.md b/agent_docs/development.md index 8b53d3154a..305c0d9380 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -323,7 +323,7 @@ serves both the app and the API. | -------------------------------------------- | ----------------------------------------------------------------- | | `HDX_PREVIEW_INLINE_API` | `true` β€” turns on the inline path | | `MONGO_URI` | Preview MongoDB connection string | -| `EXPRESS_SESSION_SECRET` | Random 32+ char string | +| `EXPRESS_SESSION_SECRET` | Random 32+ char string shared by all instances; unset generates one per process, invalidating sessions on restart and between instances | | `DISABLED_AUTH_METHODS` | `google,saml` (avoids per-preview OAuth callback URL config) | | `AI_PROVIDER`, `AI_API_KEY`, `AI_MODEL_NAME` | Optional, only if AI features should work | | `FRONTEND_URL` | Optional. Leave unset to use host-only cookies on the preview URL | diff --git a/docker-compose.yml b/docker-compose.yml index ada6f5c0d9..14593c179d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,7 @@ services: # 32-byte base64 or hex key. Setting it encrypts stored Slack and OAuth # tokens; unset means they are stored in plain text. TOKEN_ENCRYPTION_KEY: ${TOKEN_ENCRYPTION_KEY:-} + EXPRESS_SESSION_SECRET: ${EXPRESS_SESSION_SECRET:-} # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics) # ENABLE_PROMQL: 'true' # NEXT_PUBLIC_ENABLE_PROMQL: 'true' diff --git a/docker/hyperdx/entry.local.base.sh b/docker/hyperdx/entry.local.base.sh index c0ad13b416..b46fad6b96 100644 --- a/docker/hyperdx/entry.local.base.sh +++ b/docker/hyperdx/entry.local.base.sh @@ -22,7 +22,10 @@ export CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT="${CLICKHOUSE_PROMETHEUS_METRICS_E # (IS_LOCAL_IMAGE / IS_ALL_IN_ONE_IMAGE). The auth variant must report # "all-in-one-auth" so that the INGESTION_API_KEY bootstrap path is enabled. export HYPERDX_IMAGE=$([[ "${IS_LOCAL_APP_MODE}" == "DANGEROUSLY_is_local_app_modeπŸ’€" ]] && echo "all-in-one-noauth" || echo "all-in-one-auth") -export EXPRESS_SESSION_SECRET="hyperdx is cool πŸ‘‹" +# Keep the public demo default only when authentication is disabled. +if [[ "${IS_LOCAL_APP_MODE}" == "DANGEROUSLY_is_local_app_modeπŸ’€" ]]; then + export EXPRESS_SESSION_SECRET="${EXPRESS_SESSION_SECRET:-hyperdx is cool πŸ‘‹}" +fi # IS_LOCAL_APP_MODE should be set by the calling script # Default to dangerous mode if not set export IS_LOCAL_APP_MODE="${IS_LOCAL_APP_MODE}" diff --git a/packages/api/.env.development b/packages/api/.env.development index 8ba0a6c0c8..9e15745242 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -1,6 +1,7 @@ # Ports come from scripts/dev-env.sh (worktree isolation) or root .env (defaults) HYPERDX_LOG_LEVEL=debug -EXPRESS_SESSION_SECRET="hyperdx is cool πŸ‘‹" +# Development only: keep sessions valid across nodemon restarts. +EXPRESS_SESSION_SECRET="local-development-only" FRONTEND_URL="http://localhost:${HYPERDX_APP_PORT}" HDX_NODE_ADVANCED_NETWORK_CAPTURE=1 HDX_NODE_BETA_MODE=1 diff --git a/packages/api/.env.test b/packages/api/.env.test index 0b67f560d1..45ee1e9b43 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -6,7 +6,6 @@ CLICKHOUSE_HOST=http://localhost:${HDX_CI_CH_PORT:-18123} CLICKHOUSE_PASSWORD=api CLICKHOUSE_USER=api RUN_SCHEDULED_TASKS_EXTERNALLY=true -EXPRESS_SESSION_SECRET="hyperdx is cool πŸ‘‹" FRONTEND_URL=http://app:8080 MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test NODE_ENV=test @@ -16,4 +15,4 @@ OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320} # Note: the logger module is mocked in jest.setup.ts to suppress expected # operational noise (validation errors, MCP tool errors, etc.) during tests. HYPERDX_LOG_LEVEL=error -ENABLE_PROMQL=true \ No newline at end of file +ENABLE_PROMQL=true diff --git a/packages/api/src/__tests__/config.test.ts b/packages/api/src/__tests__/config.test.ts index 6375b969ac..88f17a9d71 100644 --- a/packages/api/src/__tests__/config.test.ts +++ b/packages/api/src/__tests__/config.test.ts @@ -123,4 +123,85 @@ describe('config', () => { }, ); }); + + describe('EXPRESS_SESSION_SECRET', () => { + const ORIGINAL_SECRET = process.env.EXPRESS_SESSION_SECRET; + const ORIGINAL_LOCAL_APP_MODE = process.env.IS_LOCAL_APP_MODE; + + beforeEach(() => { + delete process.env.IS_LOCAL_APP_MODE; + }); + + afterEach(() => { + if (ORIGINAL_SECRET === undefined) { + delete process.env.EXPRESS_SESSION_SECRET; + } else { + process.env.EXPRESS_SESSION_SECRET = ORIGINAL_SECRET; + } + if (ORIGINAL_LOCAL_APP_MODE === undefined) { + delete process.env.IS_LOCAL_APP_MODE; + } else { + process.env.IS_LOCAL_APP_MODE = ORIGINAL_LOCAL_APP_MODE; + } + jest.resetModules(); + }); + + // The `jest` object's isolateModules returns `jest` itself rather than the + // callback's value, so the module has to be captured from inside it. + const loadConfig = () => { + let loaded!: typeof import('@/config'); + jest.isolateModules(() => { + loaded = jest.requireActual('@/config'); + }); + return loaded; + }; + + it('uses a configured secret verbatim', () => { + process.env.EXPRESS_SESSION_SECRET = 'a-deployment-specific-secret'; + + const config = loadConfig(); + expect(config.EXPRESS_SESSION_SECRET).toBe( + 'a-deployment-specific-secret', + ); + expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(false); + }); + + it('preserves a configured secret verbatim', () => { + process.env.EXPRESS_SESSION_SECRET = ' verbatim secret πŸ‘‹ '; + const config = loadConfig(); + expect(config.EXPRESS_SESSION_SECRET).toBe(' verbatim secret πŸ‘‹ '); + expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(false); + }); + + it('generates a secret when unset, empty, or publicly known', () => { + for (const value of [undefined, '', 'hyperdx is cool πŸ‘‹']) { + if (value === undefined) { + delete process.env.EXPRESS_SESSION_SECRET; + } else { + process.env.EXPRESS_SESSION_SECRET = value; + } + + const config = loadConfig(); + expect(config.EXPRESS_SESSION_SECRET).toMatch(/^[0-9a-f]{64}$/); + expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(true); + } + }); + + it('keeps the public demo secret only without authentication', () => { + process.env.IS_LOCAL_APP_MODE = 'DANGEROUSLY_is_local_app_modeπŸ’€'; + process.env.EXPRESS_SESSION_SECRET = 'hyperdx is cool πŸ‘‹'; + + const config = loadConfig(); + expect(config.EXPRESS_SESSION_SECRET).toBe('hyperdx is cool πŸ‘‹'); + expect(config.IS_EXPRESS_SESSION_SECRET_GENERATED).toBe(false); + }); + + it('generates a different secret on each load', () => { + delete process.env.EXPRESS_SESSION_SECRET; + + expect(loadConfig().EXPRESS_SESSION_SECRET).not.toBe( + loadConfig().EXPRESS_SESSION_SECRET, + ); + }); + }); }); diff --git a/packages/api/src/__tests__/sessionSecretStartup.test.ts b/packages/api/src/__tests__/sessionSecretStartup.test.ts new file mode 100644 index 0000000000..9f17c1864e --- /dev/null +++ b/packages/api/src/__tests__/sessionSecretStartup.test.ts @@ -0,0 +1,92 @@ +import { IncomingMessage, ServerResponse } from 'http'; +import { Socket } from 'net'; + +jest.mock('@/api-app', () => ({ __esModule: true, default: jest.fn() })); +jest.mock('@/controllers/team', () => ({ + LOCAL_APP_TEAM: { _id: { toString: () => 'local-team' } }, +})); +jest.mock('@/migrations', () => ({ runStartupMigrations: jest.fn() })); +jest.mock('@/models', () => ({ + connectDB: jest.fn().mockResolvedValue(undefined), + connectDBWithRetry: jest.fn().mockResolvedValue(undefined), +})); +jest.mock('@/opamp/app', () => ({ __esModule: true, default: jest.fn() })); +jest.mock('@/setupDefaults', () => ({ setupTeamDefaults: jest.fn() })); +jest.mock('@/utils/tokenEncryption', () => ({ + verifyTokenEncryption: jest.fn(), +})); +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { error: jest.fn(), info: jest.fn() }, +})); +jest.mock('@/utils/instrumentation', () => ({ + getCounter: jest.fn(() => ({ add: jest.fn() })), +})); +jest.mock('http-graceful-shutdown', () => jest.fn()); +jest.mock('http', () => ({ + ...jest.requireActual('http'), + createServer: jest.fn(() => ({ listen: jest.fn() })), +})); + +describe.each(['server', 'serverless'])( + 'session secret startup diagnostics (%s first)', + first => { + const originalEnv = { ...process.env }; + + afterEach(() => { + process.env = { ...originalEnv }; + jest.resetModules(); + }); + + it.each([ + [false, undefined, 1], + [false, '', 1], + [false, 'hyperdx is cool πŸ‘‹', 1], + [false, 'startup-secret-sentinel', 0], + [true, undefined, 0], + [true, 'hyperdx is cool πŸ‘‹', 0], + [true, 'startup-secret-sentinel', 0], + ])( + 'local=%s, secret=%s logs %s times across both entrypoints', + async (local, secret, count) => { + process.env.APP_TYPE = 'api'; + if (local) { + process.env.IS_LOCAL_APP_MODE = 'DANGEROUSLY_is_local_app_modeπŸ’€'; + } else { + delete process.env.IS_LOCAL_APP_MODE; + } + if (typeof secret === 'undefined') + delete process.env.EXPRESS_SESSION_SECRET; + else process.env.EXPRESS_SESSION_SECRET = secret; + + const { default: logger } = + jest.requireMock('@/utils/logger'); + const { getCounter } = jest.requireMock< + typeof import('@/utils/instrumentation') + >('@/utils/instrumentation'); + const { default: Server } = + jest.requireActual('@/server'); + const { serverlessHandler } = + jest.requireActual('@/serverless'); + const config = + jest.requireActual('@/config'); + expect(logger.error).not.toHaveBeenCalled(); + + const req = new IncomingMessage(new Socket()); + req.url = '/api/health'; + const res = new ServerResponse(req); + if (first === 'server') await new Server().start(); + await serverlessHandler(req, res); + await new Server().start(); + await serverlessHandler(req, res); + + expect(logger.error).toHaveBeenCalledTimes(count); + const counter = jest.mocked(getCounter).mock.results[0].value; + expect(counter.add).toHaveBeenCalledTimes(count); + expect( + JSON.stringify(jest.mocked(logger.error).mock.calls), + ).not.toContain(config.EXPRESS_SESSION_SECRET); + }, + ); + }, +); diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 60045ffa75..fb920c9159 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -1,8 +1,9 @@ +import crypto from 'crypto'; + const env = process.env; // DEFAULTS const DEFAULT_APP_TYPE = 'api'; -const DEFAULT_EXPRESS_SESSION = 'hyperdx is cool πŸ‘‹'; const DEFAULT_FRONTEND_URL = env.HYPERDX_APP_PORT ? `http://localhost:${env.HYPERDX_APP_PORT}` : ''; @@ -13,8 +14,20 @@ export const APP_TYPE = (env.APP_TYPE || DEFAULT_APP_TYPE) as | 'api' | 'scheduled-task'; export const CODE_VERSION = env.CODE_VERSION ?? ''; -export const EXPRESS_SESSION_SECRET = (env.EXPRESS_SESSION_SECRET || - DEFAULT_EXPRESS_SESSION) as string; +// Only for single container local deployments, disable authentication +export const IS_LOCAL_APP_MODE = + env.IS_LOCAL_APP_MODE === 'DANGEROUSLY_is_local_app_modeπŸ’€'; +const LEGACY_PUBLIC_SESSION_SECRET = 'hyperdx is cool πŸ‘‹'; +// Authenticated deployments never use the public demo secret. Without a safe +// configured value, generate one per process (sessions end on restart). +export const IS_EXPRESS_SESSION_SECRET_GENERATED = + !env.EXPRESS_SESSION_SECRET || + (!IS_LOCAL_APP_MODE && + env.EXPRESS_SESSION_SECRET === LEGACY_PUBLIC_SESSION_SECRET); +export const EXPRESS_SESSION_SECRET = + !IS_EXPRESS_SESSION_SECRET_GENERATED && env.EXPRESS_SESSION_SECRET + ? env.EXPRESS_SESSION_SECRET + : crypto.randomBytes(32).toString('hex'); export const FRONTEND_URL = (env.FRONTEND_URL || DEFAULT_FRONTEND_URL) as string; const HYPERDX_IMAGE = env.HYPERDX_IMAGE; @@ -52,10 +65,6 @@ export const RUN_SCHEDULED_TASKS_EXTERNALLY = // utils/tokenEncryption.ts. export const TOKEN_ENCRYPTION_KEY = env.TOKEN_ENCRYPTION_KEY; -// Only for single container local deployments, disable authentication -export const IS_LOCAL_APP_MODE = - env.IS_LOCAL_APP_MODE === 'DANGEROUSLY_is_local_app_modeπŸ’€'; - // Only used to bootstrap empty instances export const DEFAULT_CONNECTIONS = env.DEFAULT_CONNECTIONS; export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 6193765d9f..a6a1c3d732 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -10,6 +10,7 @@ import { connectDBWithRetry, mongooseConnection } from '@/models'; import opampApp from '@/opamp/app'; import { setupTeamDefaults } from '@/setupDefaults'; import logger from '@/utils/logger'; +import { verifySessionSecret } from '@/utils/sessionSecret'; import { verifyTokenEncryption } from '@/utils/tokenEncryption'; export default class Server { @@ -90,6 +91,7 @@ export default class Server { // Checked before Mongo so a bad encryption key or KMS policy is reported // even while the Mongo connect below is still retrying. await verifyTokenEncryption(); + verifySessionSecret(); // The HTTP servers above are already listening so that `/health` // (liveness) responds while we connect; `/ready` (readiness) stays 503 diff --git a/packages/api/src/serverless.ts b/packages/api/src/serverless.ts index b1753f13b3..abeb92e57e 100644 --- a/packages/api/src/serverless.ts +++ b/packages/api/src/serverless.ts @@ -28,6 +28,7 @@ import app from './api-app'; import * as config from './config'; import { connectDB } from './models'; import logger from './utils/logger'; +import { verifySessionSecret } from './utils/sessionSecret'; // Guard against misconfigured Vercel previews. The serverless entrypoint only // supports the `api` app type β€” OPAMP runs as its own long-lived server and @@ -76,6 +77,7 @@ export async function serverlessHandler( req: IncomingMessage, res: ServerResponse, ): Promise { + verifySessionSecret(); try { await ensureDb(); } catch { diff --git a/packages/api/src/utils/sessionSecret.ts b/packages/api/src/utils/sessionSecret.ts new file mode 100644 index 0000000000..794d57b560 --- /dev/null +++ b/packages/api/src/utils/sessionSecret.ts @@ -0,0 +1,23 @@ +import { + IS_EXPRESS_SESSION_SECRET_GENERATED, + IS_LOCAL_APP_MODE, +} from '@/config'; +import { getCounter } from '@/utils/instrumentation'; +import logger from '@/utils/logger'; + +const generatedSecretCounter = getCounter('hyperdx.session_secret.generated', { + description: + 'API processes using a generated session secret with authentication enabled.', +}); + +let reported = false; + +export function verifySessionSecret(): void { + if (reported || IS_LOCAL_APP_MODE || !IS_EXPRESS_SESSION_SECRET_GENERATED) + return; + reported = true; + generatedSecretCounter.add(1); + logger.error( + 'EXPRESS_SESSION_SECRET is missing or uses the public legacy value: generated a random secret for this process. Users are signed out whenever it restarts, and replicas do not share sessions. Configure a random secret shared by all replicas to keep sessions.', + ); +}